Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Layers lab, inpaint with colours and click-to-select

Gabriele Lozuponefd7e32c56275
diff --git a/.gitignore b/.gitignore
index f064ed0..767ff31 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,5 @@ __pycache__/
# Swift build output of the Mac engine
/engine-mac/.build
/engine-mac/.swiftpm
+# Layers lab: MobileSAM ONNX files served by the vite dev server (not committed)
+/lab-models/
diff --git a/crates/qls-core/src/engine.rs b/crates/qls-core/src/engine.rs
index 2bc2ade..ded5ccd 100644
--- a/crates/qls-core/src/engine.rs
+++ b/crates/qls-core/src/engine.rs
@@ -981,6 +981,7 @@ impl Engine {
"private": private,
}),
pipeline: None,
+ hidden: false,
};
rec.duration_ms = started.elapsed().as_millis() as u64;
write_atomic(&file, &encode_output(&out, format, &rec, img.icc_profile.as_deref())?)?;
@@ -994,6 +995,73 @@ impl Engine {
Ok(rec)
}
+ /// Saves the layers lab's flattened image (`image`, an "input:" upload) as a gallery result
+ /// made from `from` (the image that was split): mode "layers", PNG. Private when asked or
+ /// when `from` is private.
+ pub fn save_layers(&self, image: &str, from: Option<&str>, private: bool) -> AppResult<GenerationRecord> {
+ let parent = match from.and_then(|f| f.strip_prefix("gen:")) {
+ Some(id) => self.gen_store(id).get_generation(id).ok(),
+ None => None,
+ };
+ let private = private || parent.as_ref().is_some_and(|p| Arc::ptr_eq(self.gen_store(&p.id), &self.private_store));
+ let (img, input) = self.resolve_source(image, None, private)?;
+ let cfg = self.config();
+ let id = new_id();
+ let now = chrono::Local::now();
+ let store = if private { &self.private_store } else { &self.store };
+ let dir = if private { self.private_dir.join("out") } else { cfg.output_dir.join(now.format("%Y-%m-%d").to_string()) };
+ std::fs::create_dir_all(&dir).map_err(|e| AppError::io("create the output folder", &dir, e))?;
+ let file = paths::unique_path(dir.join(format!("{}-layers.png", now.format("%H%M%S"))), &[]);
+ let out = img.pixels;
+ let rec = GenerationRecord {
+ id: id.clone(),
+ created_at: now_rfc3339(),
+ mode: "layers".into(),
+ prompt: String::new(),
+ original_prompt: None,
+ loras: vec![],
+ negative_prompt: String::new(),
+ seed: parent.as_ref().map(|p| p.seed).unwrap_or(0),
+ width: out.width(),
+ height: out.height(),
+ working_width: out.width(),
+ working_height: out.height(),
+ model: String::new(),
+ sampler: String::new(),
+ scheduler: None,
+ steps: 0,
+ cfg: 0.0,
+ strength: None,
+ quality: String::new(),
+ resolution: String::new(),
+ references: vec![ReferenceRecord { index: 0, input_id: input.id.clone(), original_path: None, width: input.width, height: input.height }],
+ mask_input_id: None,
+ outpaint: None,
+ file_path: file.to_string_lossy().into_owned(),
+ extra_paths: vec![],
+ favorite: false,
+ parent_id: parent.as_ref().map(|p| p.id.clone()),
+ duration_ms: 0,
+ notes: vec!["Saved from the Layers lab.".into()],
+ app_version: APP_VERSION.into(),
+ backend_version: None,
+ client: None,
+ // `images`: what it was made from, as for other versions (the project's lineage).
+ request: serde_json::json!({ "mode": "layers", "images": [from.unwrap_or(image)], "layers_image": image, "parent_id": parent.as_ref().map(|p| p.id.clone()), "private": private }),
+ pipeline: None,
+ hidden: false,
+ };
+ write_atomic(&file, &encode_output(&out, OutputFormat::Png, &rec, img.icc_profile.as_deref())?)?;
+ if let Ok(j) = imaging::encode_jpeg(&imaging::thumbnail(&out, 512), 85) {
+ let _ = std::fs::write(store.thumb_path(&id), j);
+ }
+ store.insert_generation(&rec)?;
+ if !private {
+ let _ = self.events.send(Event::Gallery { id: id.clone(), change: "added" });
+ }
+ Ok(rec)
+ }
+
/// How this machine's measured speed compares with the time model (None: not measured).
fn speed_factor(&self, cfg: &AppConfig) -> Option<f32> {
self.speed.get(&crate::speed::machine_key(cfg, self.gpu().as_ref())).map(|c| c.factor)
@@ -1038,7 +1106,7 @@ impl Engine {
let this = self.clone();
let req2 = request.clone();
let ws = workspace.clone();
- let (images, inputs_meta, mask, ref_seeds) = tokio::task::spawn_blocking(move || -> AppResult<_> {
+ let (images, inputs_meta, mask, marks, ref_seeds) = tokio::task::spawn_blocking(move || -> AppResult<_> {
let mut images = Vec::new();
let mut metas = Vec::new();
let mut seeds = Vec::new();
@@ -1055,7 +1123,11 @@ impl Engine {
}
None => None,
};
- Ok((images, metas, mask, seeds))
+ let marks = match &req2.marks {
+ Some(m) => Some(this.resolve_source(m, ws.as_deref(), req2.private)?.0.pixels),
+ None => None,
+ };
+ Ok((images, metas, mask, marks, seeds))
})
.await
.map_err(AppError::internal)??;
@@ -1142,7 +1214,7 @@ impl Engine {
let plan2 = plan.clone();
let prompt = request.prompt.clone();
let mask_img: Option<GrayImage> = mask.as_ref().map(|(m, _)| m.clone());
- let opts = pipeline::BackendOptions { server_vae_tile_size: cfg.server_vae_tile_size() };
+ let opts = pipeline::BackendOptions { server_vae_tile_size: cfg.server_vae_tile_size(), marks };
let prepared = tokio::task::spawn_blocking(move || pipeline::prepare(&plan2, &prompt, seed, &images, mask_img.as_ref(), &opts))
.await
.map_err(AppError::internal)??;
@@ -1154,8 +1226,10 @@ impl Engine {
// 4b. Prompt enhancer: rewrites the user's prompt, looking at the references
// exactly as the image model gets them.
- if request.enhance && enhancer::applies(plan.mode) {
- enhanced = self.enhance_prompt(id, &client, &caps, &cfg, plan.mode, &request.prompt, &sd_req.ref_images, seed).await?;
+ if request.enhance && enhancer::applies_to(plan.mode, plan.marks) {
+ // Marks: an edit of the marked region (its one reference).
+ let as_mode = if plan.marks { ResolvedMode::Edit } else { plan.mode };
+ enhanced = self.enhance_prompt(id, &client, &caps, &cfg, as_mode, &request.prompt, &sd_req.ref_images, seed).await?;
if let Some(text) = &enhanced {
sd_req.prompt = pipeline::enhanced_prompt(&request.prompt, text);
// The rewrite understood a transparency request the keywords missed: keep the alpha.
@@ -1700,6 +1774,7 @@ impl Engine {
client: req.client.clone(),
request: serde_json::to_value(req).unwrap_or_default(),
pipeline,
+ hidden: req.hidden,
};
rec.notes.extend(hdr_note);
let bytes = match &hdr {
diff --git a/crates/qls-core/src/enhancer.rs b/crates/qls-core/src/enhancer.rs
index e377d8f..854439a 100644
--- a/crates/qls-core/src/enhancer.rs
+++ b/crates/qls-core/src/enhancer.rs
@@ -32,6 +32,7 @@ pub const EDIT_SYSTEM_PROMPT: &str = r#"You write prompts for an image editing m
- Base every detail on what is visible in the image and do not invent objects. Do only what the user asked: no extra fixes or additions.
- When something is removed or moved, its shadow and reflection go with it; say briefly what fills the empty area so the result looks natural.
- Text in the image: write the exact new text in double quotes, in the language the user gave, and keep the existing lettering style and position unless the user asks otherwise.
+- Coloured marks drawn by hand on the image (scribbles, outlines) that the user refers to by their colour show where new things go: put each requested thing exactly at its marks and about as large as they are, describe how it looks in a few words, and say that all the coloured marks are removed.
- For a style or mood change (painting, season, night, ...), apply it fully to the whole image and end with "Keep the composition and the subjects unchanged." instead.
- With several input images, refer to them as <image1>, <image2>, ...: start with Place or Combine, say what is taken from each image and where it goes in the result.
- Write in English whatever language the user writes in, as one paragraph with no resolution or aspect ratio: about 20 to 60 words for one change; when the user asks for several changes, one sentence for each, up to about 150 words.
@@ -87,6 +88,12 @@ pub fn applies(mode: ResolvedMode) -> bool {
matches!(mode, ResolvedMode::Generate | ResolvedMode::Compose | ResolvedMode::Edit | ResolvedMode::MultiReference)
}
+/// [`applies`], and inpaints with coloured marks: written as an edit of the marked region (the
+/// edit prompt knows marks named by colour).
+pub fn applies_to(mode: ResolvedMode, marks: bool) -> bool {
+ applies(mode) || (marks && mode == ResolvedMode::Inpaint)
+}
+
/// The system prompt for `mode`: the one saved in Settings, or the built-in one.
pub fn system_prompt(cfg: &AppConfig, mode: ResolvedMode) -> String {
let (custom, default) = match mode {
@@ -192,6 +199,8 @@ mod tests {
}
}
assert!(T2I_SYSTEM_PROMPT.contains("Otherwise write nothing about text."));
+ // The layers lab's New objects: the user names marks by colour; Prompt+ places things there.
+ assert!(EDIT_SYSTEM_PROMPT.contains("Coloured marks drawn by hand"));
}
#[test]
diff --git a/crates/qls-core/src/examples.rs b/crates/qls-core/src/examples.rs
index 0ce1583..cc17db4 100644
--- a/crates/qls-core/src/examples.rs
+++ b/crates/qls-core/src/examples.rs
@@ -176,6 +176,7 @@ impl Engine {
client: Some("example".into()),
request: serde_json::to_value(&request).unwrap_or_default(),
pipeline: None,
+ hidden: false,
};
if let Ok(j) = imaging::encode_jpeg(&imaging::thumbnail(&img, 512), 85) {
let _ = std::fs::write(store.thumb_path(&id), j);
diff --git a/crates/qls-core/src/pipeline.rs b/crates/qls-core/src/pipeline.rs
index 3c96cd4..bf0893e 100644
--- a/crates/qls-core/src/pipeline.rs
+++ b/crates/qls-core/src/pipeline.rs
@@ -17,11 +17,13 @@ use crate::sdcpp::{Guidance, ImgGenRequest, SampleParams, VaeTilingParams, b64};
pub struct BackendOptions {
/// `--vae-tile-size` the server was launched with (0 = no server-side tiling).
pub server_vae_tile_size: u32,
+ /// An inpaint's coloured marks (`GenerationRequest::marks`, any size: stretched over the image).
+ pub marks: Option<RgbaImage>,
}
impl Default for BackendOptions {
fn default() -> Self {
- Self { server_vae_tile_size: 0 }
+ Self { server_vae_tile_size: 0, marks: None }
}
}
@@ -156,6 +158,18 @@ pub fn inpaint_prompt(action: InpaintAction, prompt: &str, color: &str) -> Strin
}
}
+/// `top` drawn over `base` by its alpha (the same size).
+fn draw_over(base: &RgbaImage, top: &RgbaImage) -> RgbaImage {
+ let mut out = base.clone();
+ for (o, t) in out.pixels_mut().zip(top.pixels()) {
+ let a = t[3] as u32;
+ for c in 0..3 {
+ o[c] = ((o[c] as u32 * (255 - a) + t[c] as u32 * a + 127) / 255) as u8;
+ }
+ }
+ out
+}
+
/// Prompt sent to the model. Transparency requests use the prompt format recommended for
/// Qwen-Image 2.1 RGBA output; outpainting gets an instruction prefix so short prompts
/// like "extend left" still describe the task.
@@ -301,26 +315,46 @@ pub fn prepare(plan: &Plan, prompt: &str, seed: i64, images: &[LoadedImage], mas
// blurred away, so the model rebuilds the area instead of keeping the old object
// (a replaced bowl otherwise gets the new thing put *in* it).
let pad = |img: &RgbaImage| imaging::pad_edge(img, wc.canvas, wc.content_offset);
- let image1 = if matches!(action, InpaintAction::Remove | InpaintAction::Replace) { imaging::fill_masked(&content, &m_work) } else { content.clone() };
+ // Coloured marks: drawn on image 1, which is then the model's only reference (the
+ // marks show the area, the prompt names them); the result is pasted on the unmarked source.
+ let image1 = if plan.marks {
+ let marks = opts.marks.as_ref().ok_or_else(|| AppError::invalid("The coloured marks are missing."))?;
+ let marks = imaging::resize(marks, src_size.width, src_size.height);
+ draw_over(&content, &imaging::resize(&imaging::crop(&marks, region), wc.content.width, wc.content.height))
+ } else if matches!(action, InpaintAction::Remove | InpaintAction::Replace) {
+ imaging::fill_masked(&content, &m_work)
+ } else {
+ content.clone()
+ };
// Image 2 (guide): the area marked in a color named in the prompt. The model gets
// no mask otherwise; the latent mask only discards changes outside it.
- let gc = plan.references.get(1).map(|r| r.canvas).ok_or_else(|| AppError::internal("missing inpaint guide plan"))?;
- let (color_name, color) = imaging::pick_highlight(&content, &m_content);
- let guide_mask = imaging::resize_mask(&m_work, gc.content.width, gc.content.height);
- let guide = imaging::highlight(&imaging::resize(&image1, gc.content.width, gc.content.height), &guide_mask, color);
+ let guide = if plan.marks {
+ None
+ } else {
+ let gc = plan.references.get(1).map(|r| r.canvas).ok_or_else(|| AppError::internal("missing inpaint guide plan"))?;
+ let (color_name, color) = imaging::pick_highlight(&content, &m_content);
+ let guide_mask = imaging::resize_mask(&m_work, gc.content.width, gc.content.height);
+ let guide = imaging::highlight(&imaging::resize(&image1, gc.content.width, gc.content.height), &guide_mask, color);
+ Some((imaging::pad_edge(&guide, gc.canvas, gc.content_offset), color_name))
+ };
drop(content);
let image1_b64 = png_b64(&pad(&image1))?;
req.init_image = Some(image1_b64.clone());
let same = rc.canvas == wc.canvas && rc.content == wc.content && rc.content_offset == wc.content_offset;
req.ref_images.push(if same { image1_b64 } else { png_b64(&imaging::pad_edge(&imaging::resize(&image1, rc.content.width, rc.content.height), rc.canvas, rc.content_offset))? });
- req.ref_images.push(png_b64(&imaging::pad_edge(&guide, gc.canvas, gc.content_offset))?);
+ if let Some((g, _)) = &guide {
+ req.ref_images.push(png_b64(g)?);
+ }
// Standard detail: the references are encoded at their planned sizes (image 1 ≤ 1 MP,
// the guide a quarter of the canvas); Full lets the server resize both to the output's area.
if plan.reference_detail == Some(ReferenceDetail::Standard) {
req.ref_image_args = Some("resize_before_vae=false".into());
}
- req.prompt = inpaint_prompt(action, prompt, color_name);
+ req.prompt = match &guide {
+ Some((_, color_name)) => inpaint_prompt(action, prompt, color_name),
+ None => effective_prompt(plan, prompt),
+ };
req.mask_image = Some(gray_png_b64(&m_canvas)?);
req.strength = plan.strength;
req.image_preprocess = vec!["target=init,mode=none".into(), "target=mask,mode=none,filter=nearest-exact".into(), "target=ref,mode=none".into()];
@@ -657,7 +691,7 @@ mod tests {
PlanInputs { image_sizes: sizes, vram_total_mib: Some(16303), upscaler: Some("4xNomos2_hq_mosr".into()), ..Default::default() }
}
fn esrgan_opts() -> BackendOptions {
- BackendOptions { server_vae_tile_size: 768 }
+ BackendOptions { server_vae_tile_size: 768, ..Default::default() }
}
#[test]
@@ -919,6 +953,50 @@ mod tests {
assert!(changed_inside);
}
+ #[test]
+ fn marks_inpaint_sends_one_marked_reference() {
+ let img = photo(900, 700);
+ let mut m = GrayImage::new(900, 700);
+ // Red marks (drawn at half size: stretched over the image) where the mask is.
+ let mut marks = RgbaImage::new(450, 350);
+ for y in 300..360 {
+ for x in 400..480 {
+ m.put_pixel(x, y, Luma([255]));
+ marks.put_pixel(x / 2, y / 2, Rgba([255, 32, 32, 255]));
+ }
+ }
+ let mut r = req("Remove a lamp where the red marks are");
+ r.mask = Some("input:m".into());
+ r.marks = Some("input:k".into());
+ let inputs = PlanInputs {
+ image_sizes: vec![img.size()],
+ mask: Some((Size::new(900, 700), imaging::mask_bbox(&m, 127), imaging::mask_coverage(&m, 127))),
+ vram_total_mib: Some(16303),
+ ..Default::default()
+ };
+ let p = plan(&r, &inputs, &RecommendedSettings::default()).unwrap();
+ assert!(p.marks);
+ assert_eq!(p.references.len(), 1, "no guide image");
+ assert_eq!(p.inpaint_action, Some(InpaintAction::Edit), "never a removal that blurs the marks away");
+ let opts = BackendOptions { marks: Some(marks), ..Default::default() };
+ let j = prepare(&p, &r.prompt, 1, std::slice::from_ref(&img), Some(&m), &opts).unwrap();
+ assert_eq!(j.request.ref_images.len(), 1);
+ assert_eq!(j.request.prompt, r.prompt, "the prompt names the colours itself");
+ let ref1 = imaging::decode_image(&b64_decode(&j.request.ref_images[0]).unwrap()).unwrap().pixels;
+ let red = ref1.pixels().filter(|p| p[0] > 200 && p[1] < 80 && p[2] < 80).count();
+ assert!(red > 100, "the marks are drawn on image 1 ({red} red px)");
+ // Pasted on the unmarked source: nothing red outside the region the model painted.
+ let out = postprocess(fake_backend(p.working.width, p.working.height), &j.post).unwrap();
+ let PostKind::Inpaint { blend, region, .. } = &j.post.kind else { panic!() };
+ for (x, y, px) in out.enumerate_pixels() {
+ let inside = x >= region.x && x < region.right() && y >= region.y && y < region.bottom();
+ if !inside || blend.get_pixel(x - region.x, y - region.y)[0] == 0 {
+ assert_eq!(px, img.pixels.get_pixel(x, y));
+ }
+ }
+ assert!(prepare(&p, &r.prompt, 1, std::slice::from_ref(&img), Some(&m), &BackendOptions::default()).is_err(), "marks missing");
+ }
+
fn inpaint_job(prompt: &str, action: InpaintAction) -> (LoadedImage, GrayImage, PreparedJob) {
// A white box, painted over.
let mut img = photo(900, 700);
diff --git a/crates/qls-core/src/planner.rs b/crates/qls-core/src/planner.rs
index 5fb9948..9e3ccfc 100644
--- a/crates/qls-core/src/planner.rs
+++ b/crates/qls-core/src/planner.rs
@@ -293,6 +293,14 @@ pub struct GenerationRequest {
/// no LoRA (MCP and the HTTP API don't use LoRAs for now). Not used by upscales.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub loras: Option<Vec<String>>,
+ /// Inpaint only: coloured marks drawn by the user, an image source the mask's size (transparent
+ /// where nothing is drawn). The model's only reference is the region with the marks drawn on,
+ /// and the prompt names what goes at each colour; no guide image (see pipeline).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub marks: Option<String>,
+ /// The result is left out of the gallery list (a layers lab step, see `GenerationRecord::hidden`).
+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+ pub hidden: bool,
}
// ---------------------------------------------------------------------------------------
@@ -337,6 +345,9 @@ pub struct Plan {
pub outpaint: Option<OutpaintPlan>,
/// Region of the source sent to the model for inpainting.
pub inpaint_region: Option<Rect>,
+ /// Inpaint with coloured marks (`GenerationRequest::marks`): one reference, no guide.
+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+ pub marks: bool,
/// Resolved inpaint action (see [`InpaintAction::resolve`]).
#[serde(default)]
pub inpaint_action: Option<InpaintAction>,
@@ -904,10 +915,12 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
return Err(AppError::invalid(format!("Use at most {} reference images.", rec.max_references)));
}
let (mode, auto_mode, mode_reason) = detect_mode(req, inputs)?;
- if req.enhance_only && !(req.enhance && crate::enhancer::applies(mode)) {
+ let marks = mode == ResolvedMode::Inpaint && req.marks.is_some();
+ if req.enhance_only && !(req.enhance && crate::enhancer::applies_to(mode, marks)) {
return Err(AppError::invalid("Prompt+ writes prompts for new images, edits and combines only."));
}
- let inpaint_action = (mode == ResolvedMode::Inpaint).then(|| req.inpaint_action.resolve(&req.prompt));
+ // Marks: the prompt names things by colour, never an action of its own (a Remove would blur the marks away).
+ let inpaint_action = (mode == ResolvedMode::Inpaint).then(|| if marks { InpaintAction::Edit } else { req.inpaint_action.resolve(&req.prompt) });
// Upscales and removals need no description.
let prompt_optional = mode == ResolvedMode::Upscale || inpaint_action == Some(InpaintAction::Remove);
if req.prompt.trim().is_empty() && !prompt_optional {
@@ -1091,10 +1104,11 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
);
// References: the region itself (at most the reference cap, as in edits), plus a
// smaller copy with the area marked. Full: both at the output's area (the server's
- // own resize), as before 0.9.4.
+ // own resize), as before 0.9.4. Coloured marks show the area themselves: no guide.
+ let guide_share = if marks { 0.0 } else { INPAINT_GUIDE_AREA };
let wanted = tier_area.unwrap_or(src.area());
let image1_share = if wanted > ref_cap.saturating_add(ref_cap / 10) { ref_cap as f64 / wanted as f64 } else { 1.0 };
- let area = cap_for_refs(wanted, image1_share + INPAINT_GUIDE_AREA, &mut notes);
+ let area = cap_for_refs(wanted, image1_share + guide_share, &mut notes);
// At least the working size of context around the mask, so the model sees the scene.
let min_ctx = (area as f64).sqrt() as u32;
let region = geometry::inpaint_region(src, bbox, min_ctx, 0.55);
@@ -1109,8 +1123,10 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
// The image being painted stays at the working size; only the model's view of it shrinks.
let image1 = if working.canvas.area() <= ref_cap.saturating_add(ref_cap / 10) { working } else { ref_canvas(region.size(), ref_cap, rec) };
references.push(RefPlan { index: 0, source: region.size(), canvas: image1 });
- let guide_area = (working.canvas.area() as f64 * INPAINT_GUIDE_AREA) as u64;
- references.push(RefPlan { index: 0, source: region.size(), canvas: ref_canvas(region.size(), guide_area, rec) });
+ if !marks {
+ let guide_area = (working.canvas.area() as f64 * INPAINT_GUIDE_AREA) as u64;
+ references.push(RefPlan { index: 0, source: region.size(), canvas: ref_canvas(region.size(), guide_area, rec) });
+ }
strength = Some(adv.strength.unwrap_or(1.0).clamp(0.05, 1.0));
}
ResolvedMode::Outpaint => {
@@ -1227,6 +1243,7 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
references,
outpaint,
inpaint_region,
+ marks,
inpaint_action,
quality,
resolution: req.resolution,
diff --git a/crates/qls-core/src/projects.rs b/crates/qls-core/src/projects.rs
index 347839b..fad2bfe 100644
--- a/crates/qls-core/src/projects.rs
+++ b/crates/qls-core/src/projects.rs
@@ -167,13 +167,16 @@ pub fn merge(stored: Option<&Project>, incoming: Project, known: &HashSet<String
} else {
items.last().map(|i| i.key.clone())
};
+ // Fields a saving device did not send (one that knows nothing of the layers lab) are kept.
+ let mut extra = stored.extra.clone();
+ extra.extend(incoming.extra);
let mut p = Project {
meta,
items,
selected,
loras: incoming.loras.clone(),
rev: stored.rev + 1,
- extra: incoming.extra,
+ extra,
};
refresh_meta(&mut p);
p
@@ -567,6 +570,19 @@ mod tests {
);
}
+ #[test]
+ fn a_save_without_a_field_keeps_the_stored_one() {
+ // The layers lab travels with the project; a device that does not send it keeps it.
+ let mut stored = project("p", vec![img("a")]);
+ stored.extra.insert("lab".into(), Value::from("doc"));
+ let stored = merge(None, stored, &known(&[]));
+ let m = merge(Some(&stored), project("p", vec![img("a"), img("b")]), &known(&["a"]));
+ assert_eq!(m.extra.get("lab"), Some(&Value::from("doc")));
+ let mut cleared = project("p", vec![img("a")]);
+ cleared.extra.insert("lab".into(), Value::Null);
+ assert_eq!(merge(Some(&m), cleared, &known(&["a", "b"])).extra.get("lab"), Some(&Value::Null));
+ }
+
#[test]
fn the_stored_format_reads_what_the_app_saved_before() {
// A project as 0.10's UI kept it in browser storage.
diff --git a/crates/qls-core/src/store.rs b/crates/qls-core/src/store.rs
index c1f7715..2c3e0ce 100644
--- a/crates/qls-core/src/store.rs
+++ b/crates/qls-core/src/store.rs
@@ -91,6 +91,10 @@ pub struct GenerationRecord {
pub request: serde_json::Value,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pipeline: Option<PipelineRecord>,
+ /// Left out of the gallery list (a step of the layers lab: only the image the user saves from
+ /// it is shown). The project still opens it by id.
+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+ pub hidden: bool,
}
impl GenerationRecord {
@@ -240,6 +244,15 @@ impl Store {
);",
)
.map_err(db_err)?;
+ // 0.12: `hidden` (layers lab steps). Added when missing; an older app ignores the column
+ // and its inserts get the default.
+ let has_hidden = conn
+ .prepare("SELECT 1 FROM pragma_table_info('generations') WHERE name = 'hidden'")
+ .and_then(|mut st| st.exists([]))
+ .map_err(db_err)?;
+ if !has_hidden {
+ conn.execute_batch("ALTER TABLE generations ADD COLUMN hidden INTEGER NOT NULL DEFAULT 0;").map_err(db_err)?;
+ }
conn.pragma_update(None, "user_version", 1).map_err(db_err)?;
Ok(Self { conn: Mutex::new(conn), data_dir: data_dir.to_path_buf() })
}
@@ -274,8 +287,8 @@ impl Store {
.lock()
.unwrap()
.execute(
- "INSERT OR REPLACE INTO generations (id, created_at, prompt, mode, favorite, file_path, record) VALUES (?1,?2,?3,?4,?5,?6,?7)",
- params![r.id, r.created_at, r.prompt, r.mode, r.favorite as i32, r.file_path, json],
+ "INSERT OR REPLACE INTO generations (id, created_at, prompt, mode, favorite, file_path, record, hidden) VALUES (?1,?2,?3,?4,?5,?6,?7,?8)",
+ params![r.id, r.created_at, r.prompt, r.mode, r.favorite as i32, r.file_path, json, r.hidden as i32],
)
.map_err(db_err)?;
Ok(())
@@ -327,7 +340,7 @@ impl Store {
let esc = s.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_");
format!("%{esc}%")
});
- let where_ = "WHERE (?1 IS NULL OR prompt LIKE ?1 ESCAPE '\\' OR mode LIKE ?1 ESCAPE '\\') AND (?2 = 0 OR favorite = 1)";
+ let where_ = "WHERE (?1 IS NULL OR prompt LIKE ?1 ESCAPE '\\' OR mode LIKE ?1 ESCAPE '\\') AND (?2 = 0 OR favorite = 1) AND hidden = 0";
let conn = self.conn.lock().unwrap();
let total: i64 = conn
.query_row(&format!("SELECT COUNT(*) FROM generations {where_}"), params![pattern, q.favorites as i32], |r| r.get(0))
@@ -349,6 +362,15 @@ impl Store {
Ok((out, total.max(0) as u64))
}
+ /// Shows or hides result `id` in the gallery list (see [`GenerationRecord::hidden`]).
+ pub fn set_hidden(&self, id: &str, hidden: bool) -> AppResult<()> {
+ let mut rec = self.get_generation(id)?;
+ rec.hidden = hidden;
+ let json = serde_json::to_string(&rec).map_err(AppError::internal)?;
+ self.conn.lock().unwrap().execute("UPDATE generations SET hidden = ?2, record = ?3 WHERE id = ?1", params![id, hidden as i32, json]).map_err(db_err)?;
+ Ok(())
+ }
+
pub fn set_favorite(&self, id: &str, fav: bool) -> AppResult<()> {
let n = self.conn.lock().unwrap().execute("UPDATE generations SET favorite = ?2 WHERE id = ?1", params![id, fav as i32]).map_err(db_err)?;
if n == 0 {
@@ -541,6 +563,7 @@ mod tests {
client: Some("ui".into()),
request: serde_json::json!({"prompt": prompt}),
pipeline: None,
+ hidden: false,
}
}
@@ -619,6 +642,31 @@ mod tests {
assert_eq!(s.get_generation("c").unwrap().references[0].input_id, "abc");
}
+ #[test]
+ fn hidden_results_leave_the_list_only() {
+ let dir = tempfile::tempdir().unwrap();
+ // A gallery made before `hidden`: the column is added on open.
+ {
+ let c = Connection::open(dir.path().join("gallery.db")).unwrap();
+ c.execute_batch("CREATE TABLE generations (id TEXT PRIMARY KEY, created_at TEXT NOT NULL, prompt TEXT NOT NULL, mode TEXT NOT NULL, favorite INTEGER NOT NULL DEFAULT 0, file_path TEXT NOT NULL, record TEXT NOT NULL);").unwrap();
+ let r = rec("old", "kept", "2026-10-06T09:00:00.000Z");
+ c.execute("INSERT INTO generations (id, created_at, prompt, mode, favorite, file_path, record) VALUES (?1,?2,?3,?4,0,?5,?6)", params![r.id, r.created_at, r.prompt, r.mode, r.file_path, serde_json::to_string(&r).unwrap()]).unwrap();
+ }
+ let s = Store::open(dir.path()).unwrap();
+ s.insert_generation(&rec("a", "a step", "2026-10-06T10:00:00.000Z")).unwrap();
+ s.insert_generation(&GenerationRecord { hidden: true, ..rec("b", "a lab step", "2026-10-06T11:00:00.000Z") }).unwrap();
+ let ids = |s: &Store| s.list_generations(&ListQuery::default()).unwrap().0.into_iter().map(|r| r.id).collect::<Vec<_>>();
+ assert_eq!(ids(&s), vec!["a", "old"]);
+ assert!(s.get_generation("b").unwrap().hidden, "a project still opens it");
+ s.set_hidden("a", true).unwrap();
+ s.set_hidden("b", false).unwrap();
+ assert_eq!(ids(&s), vec!["b", "old"]);
+ assert_eq!(s.list_generations(&ListQuery::default()).unwrap().1, 2);
+ assert!(s.set_hidden("zzz", true).is_err());
+ // The record leaves `hidden` out unless set (older apps read the same JSON).
+ assert!(serde_json::to_value(rec("x", "p", "t")).unwrap().get("hidden").is_none());
+ }
+
#[test]
fn inputs_are_deduplicated() {
let dir = tempfile::tempdir().unwrap();
diff --git a/crates/qls-service/src/api.rs b/crates/qls-service/src/api.rs
index 733704c..b4ae30d 100644
--- a/crates/qls-service/src/api.rs
+++ b/crates/qls-service/src/api.rs
@@ -107,6 +107,7 @@ pub fn router(state: Arc<AppState>, lan: bool) -> Router {
.route("/inputs/{id}/thumb", get(input_thumb))
.route("/inputs/{id}/preview", get(input_preview))
.route("/gallery", get(gallery_list))
+ .route("/gallery/layers", post(gallery_save_layers))
.route("/gallery/{id}", get(gallery_get).patch(gallery_patch).delete(gallery_delete))
.route("/gallery/{id}/image", get(gallery_image))
.route("/gallery/{id}/thumb", get(gallery_thumb))
@@ -1121,6 +1122,9 @@ async fn gallery_get(State(s): St, AxPath(id): AxPath<String>) -> ApiResult<Json
#[derive(Deserialize)]
struct PatchBody {
favorite: Option<bool>,
+ /// Left out of the gallery list (layers lab steps).
+ #[serde(default)]
+ hidden: Option<bool>,
}
async fn gallery_patch(State(s): St, AxPath(id): AxPath<String>, Json(b): Json<PatchBody>) -> ApiResult<Json<GenerationRecord>> {
@@ -1129,9 +1133,31 @@ async fn gallery_patch(State(s): St, AxPath(id): AxPath<String>, Json(b): Json<P
store.set_favorite(&id, f)?;
let _ = s.engine.events.send(qls_core::engine::Event::Gallery { id: id.clone(), change: "updated" });
}
+ if let Some(h) = b.hidden {
+ store.set_hidden(&id, h)?;
+ let _ = s.engine.events.send(qls_core::engine::Event::Gallery { id: id.clone(), change: "updated" });
+ }
Ok(Json(store.get_generation(&id)?))
}
+#[derive(Deserialize)]
+struct LayersBody {
+ /// The flattened image ("input:<id>").
+ image: String,
+ /// The image the layers were split from.
+ #[serde(default)]
+ from: Option<String>,
+ #[serde(default)]
+ private: bool,
+}
+
+/// The layers lab's image, saved to the gallery (its steps are hidden there).
+async fn gallery_save_layers(State(s): St, Json(b): Json<LayersBody>) -> ApiResult<Json<GenerationRecord>> {
+ let engine = s.engine.clone();
+ let rec = tokio::task::spawn_blocking(move || engine.save_layers(&b.image, b.from.as_deref(), b.private)).await.map_err(AppError::internal)??;
+ Ok(Json(rec))
+}
+
async fn gallery_delete(State(s): St, AxPath(id): AxPath<String>) -> ApiResult<Json<Value>> {
let out = s.engine.output_root(&id);
s.engine.gen_store(&id).delete_generation(&id, &out)?;
diff --git a/package-lock.json b/package-lock.json
index 64160a5..513fc97 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -17,6 +17,7 @@
"@tauri-apps/plugin-process": "^2.4.0",
"@tauri-apps/plugin-updater": "^2.13.2",
"lucide-react": "^1.52.0",
+ "onnxruntime-web": "1.30.0",
"react": "^19.3.0",
"react-dom": "^19.3.0",
"uqr": "^0.1.3"
@@ -301,6 +302,63 @@
"url": "https://github.com/sponsors/oxc-project"
}
},
+ "node_modules/@protobufjs/aspromise": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+ "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
+ "license": "BSD-3-Clause"
+ },
+ "node_modules/@protobufjs/base64": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+ "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
+ "license": "BSD-3-Clause"
+ },
+ "node_modules/@protobufjs/codegen": {
+ "version": "2.0.5",
+ "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz",
+ "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==",
+ "license": "BSD-3-Clause"
+ },
+ "node_modules/@protobufjs/eventemitter": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz",
+ "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==",
+ "license": "BSD-3-Clause"
+ },
+ "node_modules/@protobufjs/fetch": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz",
+ "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==",
+ "license": "BSD-3-Clause",
+ "dependencies": {
+ "@protobufjs/aspromise": "^1.1.1"
+ }
+ },
+ "node_modules/@protobufjs/float": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+ "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
+ "license": "BSD-3-Clause"
+ },
+ "node_modules/@protobufjs/path": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+ "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
+ "license": "BSD-3-Clause"
+ },
+ "node_modules/@protobufjs/pool": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+ "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
+ "license": "BSD-3-Clause"
+ },
+ "node_modules/@protobufjs/utf8": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz",
+ "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==",
+ "license": "BSD-3-Clause"
+ },
"node_modules/@rolldown/binding-android-arm-eabi": {
"version": "1.2.12",
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.12.tgz",
@@ -860,6 +918,15 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/@types/node": {
+ "version": "26.6.6",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.6.tgz",
+ "integrity": "sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog==",
+ "license": "MIT",
+ "dependencies": {
+ "undici-types": "~8.9.0"
+ }
+ },
"node_modules/@types/react": {
"version": "19.3.0",
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz",
@@ -1428,6 +1495,12 @@
}
}
},
+ "node_modules/flatbuffers": {
+ "version": "25.9.23",
+ "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz",
+ "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==",
+ "license": "Apache-2.0"
+ },
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@@ -1443,6 +1516,12 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
+ "node_modules/guid-typescript": {
+ "version": "1.0.9",
+ "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
+ "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",
+ "license": "ISC"
+ },
"node_modules/html-encoding-sniffer": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz",
@@ -1765,6 +1844,12 @@
"url": "https://opencollective.com/parcel"
}
},
+ "node_modules/long": {
+ "version": "5.3.2",
+ "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
+ "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
+ "license": "Apache-2.0"
+ },
"node_modules/lru-cache": {
"version": "11.5.3",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz",
@@ -1840,6 +1925,26 @@
"node": ">=12.20.0"
}
},
+ "node_modules/onnxruntime-common": {
+ "version": "1.30.0",
+ "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz",
+ "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==",
+ "license": "MIT"
+ },
+ "node_modules/onnxruntime-web": {
+ "version": "1.30.0",
+ "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.30.0.tgz",
+ "integrity": "sha512-q0y+JrrtukXSzsBWEMccVfqX25LRmosXHF+CaRJmg8pZClzcV7svNc4rKY3jL02Vb7QmRMDs1SigqR4CXAfKYQ==",
+ "license": "MIT",
+ "dependencies": {
+ "flatbuffers": "^25.1.24",
+ "guid-typescript": "^1.0.9",
+ "long": "^5.2.3",
+ "onnxruntime-common": "1.30.0",
+ "platform": "^1.3.6",
+ "protobufjs": "^7.2.4"
+ }
+ },
"node_modules/parse5": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz",
@@ -1873,6 +1978,12 @@
"url": "https://github.com/sponsors/jonschlinkert"
}
},
+ "node_modules/platform": {
+ "version": "1.3.6",
+ "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+ "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
+ "license": "MIT"
+ },
"node_modules/postcss": {
"version": "8.5.29",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.29.tgz",
@@ -1902,6 +2013,29 @@
"node": "^10 || ^12 || >=14"
}
},
+ "node_modules/protobufjs": {
+ "version": "7.6.6",
+ "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz",
+ "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==",
+ "hasInstallScript": true,
+ "license": "BSD-3-Clause",
+ "dependencies": {
+ "@protobufjs/aspromise": "^1.1.2",
+ "@protobufjs/base64": "^1.1.2",
+ "@protobufjs/codegen": "^2.0.5",
+ "@protobufjs/eventemitter": "^1.1.1",
+ "@protobufjs/fetch": "^1.1.1",
+ "@protobufjs/float": "^1.0.2",
+ "@protobufjs/path": "^1.1.2",
+ "@protobufjs/pool": "^1.1.0",
+ "@protobufjs/utf8": "^1.1.1",
+ "@types/node": ">=13.7.0",
+ "long": "^5.3.2"
+ },
+ "engines": {
+ "node": ">=12.0.0"
+ }
+ },
"node_modules/punycode": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
@@ -2148,6 +2282,12 @@
"node": ">=20.18.1"
}
},
+ "node_modules/undici-types": {
+ "version": "8.9.0",
+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
+ "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
+ "license": "MIT"
+ },
"node_modules/uqr": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/uqr/-/uqr-0.1.3.tgz",
diff --git a/package.json b/package.json
index b308d67..d0292db 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,7 @@
"@tauri-apps/plugin-process": "^2.4.0",
"@tauri-apps/plugin-updater": "^2.13.2",
"lucide-react": "^1.52.0",
+ "onnxruntime-web": "1.30.0",
"react": "^19.3.0",
"react-dom": "^19.3.0",
"uqr": "^0.1.3"
diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json
index e21ba75..5384f51 100644
--- a/src-tauri/tauri.conf.json
+++ b/src-tauri/tauri.conf.json
@@ -27,7 +27,7 @@
}
],
"security": {
- "csp": "default-src 'self'; img-src 'self' data: blob: http://127.0.0.1:*; connect-src 'self' ipc: http://ipc.localhost http://127.0.0.1:*; style-src 'self' 'unsafe-inline'; font-src 'self' data:; script-src 'self'"
+ "csp": "default-src 'self'; img-src 'self' data: blob: http://127.0.0.1:*; connect-src 'self' ipc: http://ipc.localhost http://127.0.0.1:*; style-src 'self' 'unsafe-inline'; font-src 'self' data:; script-src 'self' 'wasm-unsafe-eval'"
}
},
"bundle": {
diff --git a/src/App.tsx b/src/App.tsx
index a43285a..348e310 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -13,6 +13,7 @@ import { Login } from "./components/Login";
import { ExpandEditor } from "./components/ExpandEditor";
import { Gallery } from "./components/Gallery";
import { InpaintEditor } from "./components/InpaintEditor";
+import { LayersEditor } from "./components/LayersEditor";
import { Settings as SettingsDialog } from "./components/Settings";
import { Setup } from "./components/Setup";
import { Sidebar } from "./components/Sidebar";
@@ -378,6 +379,7 @@ export default function App() {
{overlay === "gallery" && <Gallery />}
{overlay === "inpaint" && <InpaintEditor />}
{overlay === "expand" && <ExpandEditor />}
+ {overlay === "layers" && <LayersEditor />}
{dragging && <div className="drop-hint">{editing ? "Drop to add as a reference image" : "Drop an image to start editing it"}</div>}
</div>
{!needsSetup && overlay !== "inpaint" && overlay !== "expand" && overlay !== "gallery" && <Composer />}
diff --git a/src/components/Composer.tsx b/src/components/Composer.tsx
index 412222d..8202dd2 100644
--- a/src/components/Composer.tsx
+++ b/src/components/Composer.tsx
@@ -6,7 +6,7 @@ import { native } from "../lib/native";
import { haptic } from "../lib/haptics";
import { inBrowser, useNarrow } from "../lib/platform";
import {
- addAsReference, addInputs, cancelPromptReview, editShownImage, enhancerAvailable, getState, itemSource, openSettings, selectedItem, sessionLabels, setProjectLoras, setState, startNewImage, submitComposer, toastError, undoEnhancedPrompt, updateComposer, useStore, type SessionItem,
+ addAsReference, addInputs, openEditor, cancelPromptReview, editShownImage, enhancerAvailable, getState, itemSource, openSettings, selectedItem, sessionLabels, setProjectLoras, setState, startNewImage, submitComposer, toastError, undoEnhancedPrompt, updateComposer, useStore, type SessionItem,
} from "../lib/store";
import type { InpaintAction, Plan, Quality } from "../lib/types";
import { AdvancedPanel } from "./Advanced";
@@ -205,7 +205,7 @@ function TargetBar() {
const name = nameOf(first)?.toLowerCase() ?? (first.source.startsWith("gen:") ? "this result" : first.name);
const outPx = hasOutpaint(c.outpaint) ? expansionPixels(c.outpaint!, first.width, first.height) : null;
const what =
- c.refs.length > 1 ? `Combining ${c.refs.length} images` : c.mask ? `${INPAINT_VERB[c.inpaintAction]} the painted area of ${name}` : outPx ? `Expanding ${name}` : `Editing ${name}`;
+ c.refs.length > 1 ? `Combining ${c.refs.length} images` : c.mask?.marks ? `Adding at the coloured marks on ${name}` : c.mask ? `${INPAINT_VERB[c.inpaintAction]} the painted area of ${name}` : outPx ? `Expanding ${name}` : `Editing ${name}`;
const sub =
c.refs.length > 1
? "Refer to them as image 1, image 2… in the prompt"
@@ -223,8 +223,8 @@ function TargetBar() {
</span>
<span className="spacer" />
<span className="tool-group">
- <button className={`tool ${c.mask ? "on" : ""}`} data-tour="paint" onClick={() => setState({ overlay: "inpaint" })} title="Paint the area to change; the rest stays as it is">
- <Brush size={14} /> <span className="lbl">{c.mask ? "Area painted" : "Paint area"}</span>
+ <button className={`tool ${c.mask ? "on" : ""}`} data-tour="paint" onClick={() => openEditor("inpaint")} title="Paint where things go (one colour each), or the area to change; the rest stays as it is">
+ <Brush size={14} /> <span className="lbl">{c.mask?.marks ? "Colours painted" : c.mask ? "Area painted" : "Paint area"}</span>
</button>
{c.mask && (
<button className="tool-x" title="Remove the painted area" onClick={() => updateComposer({ mask: null })}>
@@ -233,7 +233,7 @@ function TargetBar() {
)}
</span>
<span className="tool-group">
- <button className={`tool ${outPx ? "on" : ""}`} onClick={() => setState({ overlay: "expand" })} title="Extend the image beyond its borders">
+ <button className={`tool ${outPx ? "on" : ""}`} onClick={() => openEditor("expand")} title="Extend the image beyond its borders">
<Expand size={14} /> <span className="lbl">{outPx ? "Expanded" : "Expand"}</span>
</button>
{outPx && (
@@ -242,6 +242,13 @@ function TargetBar() {
</button>
)}
</span>
+ {c.refs.length === 1 && (
+ <span className="tool-group">
+ <button className="tool" onClick={() => setState({ overlay: "layers" })} title="Experimental: put the objects into layers and save each one, and the background, as a new version">
+ <Layers size={14} /> <span className="lbl">Layers (lab)</span>
+ </button>
+ </span>
+ )}
</div>
);
}
@@ -568,6 +575,8 @@ export function Composer() {
: "Describe the new image with the reference: the scene, what it's doing…"
: c.refs.length > 1
? "Describe how to combine image 1, image 2…"
+ : c.mask?.marks
+ ? "Say what goes at each colour, e.g. “a leather armchair at the red, a lamp at the blue”…"
: c.mask
? INPAINT_ACTIONS.find((a) => a.value === c.inpaintAction)!.placeholder
: outPx
@@ -679,7 +688,7 @@ export function Composer() {
<div className="controls">
<AddImage />
- {c.mask && (
+ {c.mask && !c.mask.marks && (
<Seg value={c.inpaintAction} options={INPAINT_ACTIONS} onChange={(v) => updateComposer({ inpaintAction: v })} />
)}
diff --git a/src/components/ExpandEditor.tsx b/src/components/ExpandEditor.tsx
index 8b42583..3328338 100644
--- a/src/components/ExpandEditor.tsx
+++ b/src/components/ExpandEditor.tsx
@@ -2,7 +2,7 @@ import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Check, ChevronDown, Lock, Lo
import { useEffect, useLayoutEffect, useRef, useState } from "react";
import { urls } from "../lib/api";
import { LONG_SIDE_PRESETS, aspectLabel, expandToRatio, expandToSize, expansionPixels, expansionPreset, parseRatio, sizeForLongSide } from "../lib/logic";
-import { focusPrompt, setState, updateComposer, useStore } from "../lib/store";
+import { closeEditor, focusPrompt, updateComposer, useStore } from "../lib/store";
import type { OutpaintSpec, Units } from "../lib/types";
import { AspectGrid, RatioIcon } from "./SizeMenus";
import { IconButton, Popover, Seg } from "./ui";
@@ -104,10 +104,10 @@ export function ExpandEditor() {
}, []);
useEffect(() => {
- if (!ref0) setState({ overlay: null });
+ if (!ref0) closeEditor();
}, [ref0]);
useEffect(() => {
- const onKey = (e: KeyboardEvent) => e.key === "Escape" && setState({ overlay: null });
+ const onKey = (e: KeyboardEvent) => e.key === "Escape" && closeEditor();
window.addEventListener("keydown", onKey);
return () => window.removeEventListener("keydown", onKey);
}, []);
@@ -176,7 +176,7 @@ export function ExpandEditor() {
{W}×{H} ({aspectLabel(W, H)}) → <strong className="expand-final">{finalW}×{finalH} ({aspectLabel(finalW, finalH)})</strong>
</span>
<span className="spacer" />
- <IconButton title="Cancel (Esc)" onClick={() => setState({ overlay: null })}>
+ <IconButton title="Cancel (Esc)" onClick={() => closeEditor()}>
<X size={18} />
</IconButton>
</div>
@@ -280,7 +280,7 @@ export function ExpandEditor() {
title={tooBig ? `Max ${MAX_FINAL}px per side` : undefined}
onClick={() => {
updateComposer({ outpaint: spec, mask: null, mode: "auto" });
- setState({ overlay: null });
+ closeEditor();
focusPrompt();
}}
>
diff --git a/src/components/InpaintEditor.tsx b/src/components/InpaintEditor.tsx
index 78f19a3..c08ac4a 100644
--- a/src/components/InpaintEditor.tsx
+++ b/src/components/InpaintEditor.tsx
@@ -1,14 +1,27 @@
-import { Brush, Check, Contrast, Eraser, Hand, Redo2, Trash2, Undo2, X } from "lucide-react";
+import { Brush, Check, Contrast, Eraser, Hand, Loader2, MousePointerClick, Palette, Redo2, Trash2, Undo2, X } from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { api, urls } from "../lib/api";
+import { area, dilate, fromLogits, maskOrder, samSelection } from "../lib/layers";
import { inBrowser, shortcut } from "../lib/platform";
-import { focusPrompt, getState, setState, toastError, updateComposer, useStore } from "../lib/store";
+import { Segmenter, type SamResult } from "../lib/sam";
+import { MARK_COLOURS, marksPrompt, type MarkColour } from "../lib/sceneEdit";
+import { closeEditor, focusPrompt, getState, toastError, toastInfo, updateComposer, useStore } from "../lib/store";
import { IconButton } from "./ui";
import { isTyping } from "./Viewer";
import { ZoomPan } from "./ZoomPan";
+// Two ways to say where the change goes. Colours (the default): each thing gets a colour, painted
+// over the whole area it should fill, and a few words; the model gets the image with the marks
+// drawn on (one reference, no guide copy) and the result is pasted on the unmarked image.
+// Area: the classic mask (shown pink), with the prompt and an action (Edit, Remove, Replace, Add).
+// In both, Select (SAM) fills a clicked object instead of brushing it.
+
+type Kind = "colours" | "area";
+
type Op =
- | { kind: "stroke"; erase: boolean; size: number; points: [number, number][] }
+ | { kind: "stroke"; erase: boolean; size: number; points: [number, number][]; colour?: MarkColour }
+ /** A SAM selection (`w`×`h`, stretched over the mask). */
+ | { kind: "blob"; erase: boolean; mask: Uint8Array; w: number; h: number; colour?: MarkColour }
| { kind: "invert" }
| { kind: "clear" };
@@ -17,28 +30,63 @@ const MAX_MASK_SIDE = 2048;
/** Soft edge of the exported mask, in mask px. Brush strokes are never exactly binary, so the
* service keeps this softness for the blend instead of adding its own. */
const FEATHER = 8;
+/** Colours: the area pasted back reaches this far (share of the long side) past the marks, so a
+ * thing drawn a little larger than its marks is not cut at the edge. */
+const MARKS_MARGIN = 0.015;
+/** SAM runs at most at this size (its own masks are 256²; larger only costs memory). */
+const SAM_SIDE = 1024;
+
+const KIND_KEY = "kokon.inpaint.kind";
+/** One area is shown pink, as in the layers lab (the export is white on black either way). */
+const AREA_RGB = [255, 59, 107] as const;
+const AREA_CSS = "#ff3b6b";
+const cssOf = (c: MarkColour | undefined) => (c ? MARK_COLOURS.find((m) => m.name === c)!.css : AREA_CSS);
+/** What each colour asks for, per image (kept while the editor closes). */
+const lastWants = new Map<string, Partial<Record<MarkColour, string>>>();
+
+function storedKind(): Kind {
+ try {
+ return localStorage.getItem(KIND_KEY) === "area" ? "area" : "colours";
+ } catch {
+ return "colours";
+ }
+}
export function InpaintEditor() {
const ref0 = useStore((s) => s.composer.refs[0]);
const existingMask = useStore((s) => s.composer.mask);
+ const fromLab = useStore((s) => s.overlayBack === "layers");
const [ops, setOps] = useState<Op[]>([]);
const [redo, setRedo] = useState<Op[]>([]);
- const [tool, setTool] = useState<"brush" | "eraser" | "pan">("brush");
+ // A mask being edited again keeps its kind; the layers lab has its own pencil, so it opens on Area.
+ const [kind, setKindState] = useState<Kind>(() => (existingMask && existingMask.refKey === ref0?.key ? (existingMask.marks ? "colours" : "area") : fromLab ? "area" : storedKind()));
+ const [colour, setColour] = useState<MarkColour>("red");
+ const [wants, setWantsState] = useState<Partial<Record<MarkColour, string>>>(() => lastWants.get(ref0?.source ?? "") ?? {});
+ const [used, setUsed] = useState<MarkColour[]>([]);
+ const [tool, setTool] = useState<"brush" | "eraser" | "pan" | "select">("brush");
const [spacePressed, setSpacePressed] = useState(false);
// Right button held: the other tool is in use, and the toolbar shows it.
const [swapped, setSwapped] = useState(false);
const [brush, setBrush] = useState(48); // in screen px
const [busy, setBusy] = useState(false);
+ const [sam, setSam] = useState<"idle" | "loading" | "ready" | "missing" | "error">("idle");
+ const [selecting, setSelecting] = useState(false);
const [cursor, setCursor] = useState<{ x: number; y: number } | null>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
const imgRef = useRef<HTMLImageElement>(null);
const current = useRef<Op | null>(null);
+ const segmenter = useRef<Segmenter | null>(null);
+ // The last click of Select: the same spot again takes SAM's next output (part -> whole).
+ const lastClick = useRef<{ x: number; y: number; result: SamResult; order: number[]; pos: number; op: Op } | null>(null);
const srcW = ref0?.width ?? 1;
const srcH = ref0?.height ?? 1;
const scale = Math.min(1, MAX_MASK_SIDE / Math.max(srcW, srcH));
const mw = Math.max(1, Math.round(srcW * scale));
const mh = Math.max(1, Math.round(srcH * scale));
+ const samK = Math.min(1, SAM_SIDE / Math.max(mw, mh));
+ const sw = Math.max(1, Math.round(mw * samK));
+ const sh = Math.max(1, Math.round(mh * samK));
const src = ref0 ? urls.source(ref0.source) : "";
// In a browser the picture is drawn flat (ZoomPan `flat`: no image-sized layer, which cost an
// iPhone ~300 MB on a 4K image) and is the preview until the zoom needs the full image; the
@@ -47,18 +95,27 @@ export function InpaintEditor() {
const isPanningActive = spacePressed || tool === "pan";
const erase = tool === "eraser";
+ const colours = kind === "colours";
+ const paint = colours ? colour : undefined;
+
+ const setWants = (w: Partial<Record<MarkColour, string>>) => {
+ setWantsState(w);
+ if (ref0) lastWants.set(ref0.source, w);
+ };
- // Load an existing mask for re-editing (as a base raster op).
+ // Load an existing mask (or marks) for re-editing (as a base raster op).
const baseMask = useRef<HTMLImageElement | null>(null);
+ const baseMarks = useRef(false);
useEffect(() => {
if (!existingMask || !ref0 || existingMask.refKey !== ref0.key) return;
const img = new Image();
img.crossOrigin = "anonymous";
img.onload = () => {
baseMask.current = img;
+ baseMarks.current = !!existingMask.marks;
redraw(ops);
};
- img.src = urls.input(existingMask.inputId);
+ img.src = urls.input(existingMask.marks ?? existingMask.inputId);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
@@ -73,21 +130,24 @@ export function InpaintEditor() {
ctx.clearRect(0, 0, mw, mh);
const lastClear = list.map((o) => o.kind).lastIndexOf("clear");
if (baseMask.current && lastClear < 0) {
- // Stored masks are grayscale (white = edit): convert luminance to alpha.
- const tmp = document.createElement("canvas");
- tmp.width = mw;
- tmp.height = mh;
- const t = tmp.getContext("2d");
- if (t) {
- t.drawImage(baseMask.current, 0, 0, mw, mh);
- const d = t.getImageData(0, 0, mw, mh);
- for (let i = 0; i < d.data.length; i += 4) {
- const v = d.data[i];
- d.data[i] = d.data[i + 1] = d.data[i + 2] = 255;
- d.data[i + 3] = v;
+ if (baseMarks.current) ctx.drawImage(baseMask.current, 0, 0, mw, mh);
+ else {
+ // Stored masks are grayscale (white = edit): convert luminance to alpha.
+ const tmp = document.createElement("canvas");
+ tmp.width = mw;
+ tmp.height = mh;
+ const t = tmp.getContext("2d");
+ if (t) {
+ t.drawImage(baseMask.current, 0, 0, mw, mh);
+ const d = t.getImageData(0, 0, mw, mh);
+ for (let i = 0; i < d.data.length; i += 4) {
+ const v = d.data[i];
+ [d.data[i], d.data[i + 1], d.data[i + 2]] = AREA_RGB;
+ d.data[i + 3] = v;
+ }
+ t.putImageData(d, 0, 0);
+ ctx.drawImage(tmp, 0, 0);
}
- t.putImageData(d, 0, 0);
- ctx.drawImage(tmp, 0, 0);
}
}
for (const op of list.slice(lastClear + 1)) render2(ctx, op, mw, mh);
@@ -95,7 +155,60 @@ export function InpaintEditor() {
[mw, mh],
);
- useEffect(() => redraw(ops), [ops, redraw]);
+ useEffect(() => {
+ redraw(ops);
+ if (colours) setUsed(coloursOn(canvasRef.current, mw, mh));
+ }, [ops, redraw, colours, mw, mh]);
+
+ /** Colours <-> Area: the drawing starts again (one kind per request). */
+ const setKind = (k: Kind) => {
+ if (k === kind) return;
+ setKindState(k);
+ try {
+ localStorage.setItem(KIND_KEY, k);
+ } catch {
+ /* storage unavailable */
+ }
+ baseMask.current = null;
+ lastClick.current = null;
+ setOps([]);
+ setRedo([]);
+ };
+
+ // SAM loads the first time Select is picked (the image is encoded once), and keeps loading if
+ // another tool is picked meanwhile: only closing the editor drops it.
+ const samStarted = useRef(false);
+ const open = useRef(true);
+ useEffect(() => {
+ // Set again on mount: StrictMode mounts twice in development.
+ open.current = true;
+ return () => {
+ open.current = false;
+ };
+ }, []);
+ useEffect(() => {
+ if (tool !== "select" || samStarted.current || !src) return;
+ samStarted.current = true;
+ setSam("loading");
+ (async () => {
+ try {
+ const s = await Segmenter.load();
+ if (!open.current) return;
+ if (!s) return setSam("missing");
+ const img = new Image();
+ img.crossOrigin = "anonymous";
+ img.src = src;
+ await img.decode();
+ await s.encode(img, sw, sh);
+ if (!open.current) return;
+ segmenter.current = s;
+ setSam("ready");
+ } catch (e) {
+ console.error(e);
+ if (open.current) setSam("error");
+ }
+ })();
+ }, [tool, src, sw, sh]);
const toMask = (e: React.PointerEvent): [number, number] => {
const c = canvasRef.current;
@@ -113,12 +226,53 @@ export function InpaintEditor() {
return (brush / r.width) * mw;
};
+ /** Select: the object under the click fills (or, `remove`, leaves) the drawing. */
+ const select = async (x: number, y: number, remove: boolean) => {
+ const s = segmenter.current;
+ if (!s) {
+ toastInfo(sam === "missing" ? "The selection model is not in this build yet: paint with the brush." : sam === "error" ? "The selection model could not start: paint with the brush." : "The selection model is still getting ready.");
+ return;
+ }
+ const prev = lastClick.current;
+ const near = prev && Math.hypot(prev.x - x, prev.y - y) < Math.max(mw, mh) * 0.015;
+ if (!remove && prev && near && ops[ops.length - 1] === prev.op) {
+ const pos = (prev.pos + 1) % prev.order.length;
+ const op: Op = { kind: "blob", erase: false, mask: samSelection(prev.result.masks[prev.order[pos]], sw, sh), w: sw, h: sh, colour: paint };
+ lastClick.current = { ...prev, pos, op };
+ setOps((o) => [...o.slice(0, -1), op]);
+ setRedo([]);
+ return;
+ }
+ setSelecting(true);
+ try {
+ const result = await s.predict([{ x: x * samK, y: y * samK, label: 1 }], sw, sh);
+ const order = maskOrder(
+ result.iou,
+ result.masks.map((m) => area(fromLogits(m))),
+ );
+ const op: Op = { kind: "blob", erase: remove, mask: samSelection(result.masks[order[0]], sw, sh), w: sw, h: sh, colour: paint };
+ lastClick.current = remove ? null : { x, y, result, order, pos: 0, op };
+ setOps((o) => [...o, op]);
+ setRedo([]);
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setSelecting(false);
+ }
+ };
+
const onDown = (e: React.PointerEvent) => {
// If middle click or pan mode active, let event bubble to ZoomPan for panning
if (e.button === 1 || isPanningActive) return;
if (e.button !== 0 && e.button !== 2) return;
e.stopPropagation();
+ if (tool === "select") {
+ // Right click (or Alt): takes the object out.
+ const [x, y] = toMask(e);
+ if (!selecting) select(x, y, e.button === 2 || e.altKey);
+ return;
+ }
try {
(e.target as HTMLElement).setPointerCapture(e.pointerId);
} catch {}
@@ -126,13 +280,13 @@ export function InpaintEditor() {
// Right button: the other tool (erases with the brush, paints with the eraser), as in mask editors.
const isErasing = e.button === 2 ? !erase : erase;
setSwapped(e.button === 2);
- current.current = { kind: "stroke", erase: isErasing, size: maskBrush(), points: [toMask(e)] };
+ current.current = { kind: "stroke", erase: isErasing, size: maskBrush(), points: [toMask(e)], colour: paint };
const ctx = canvasRef.current?.getContext("2d");
if (ctx) render2(ctx, current.current, mw, mh);
};
const onMove = (e: React.PointerEvent, transformScale: number) => {
- if (isPanningActive) {
+ if (isPanningActive || tool === "select") {
setCursor(null);
return;
}
@@ -208,7 +362,7 @@ export function InpaintEditor() {
e.preventDefault();
redoOp();
} else if (e.key === "Escape") {
- setState({ overlay: null });
+ closeEditor();
} else if (e.key === "[") {
setBrush((b) => Math.max(4, b - 8));
} else if (e.key === "]") {
@@ -219,6 +373,10 @@ export function InpaintEditor() {
setTool((t) => (t === "eraser" ? "brush" : "eraser"));
} else if (e.key.toLowerCase() === "h") {
setTool((t) => (t === "pan" ? "brush" : "pan"));
+ } else if (e.key.toLowerCase() === "s") {
+ setTool((t) => (t === "select" ? "brush" : "select"));
+ } else if (e.key >= "1" && e.key <= String(MARK_COLOURS.length)) {
+ setColour(MARK_COLOURS[Number(e.key) - 1].name);
}
};
@@ -242,23 +400,14 @@ export function InpaintEditor() {
};
}, []);
+ const toBlob = (c: HTMLCanvasElement): Promise<Blob> => new Promise((res, rej) => c.toBlob((b) => (b ? res(b) : rej(new Error("export failed"))), "image/png"));
+
const done = async () => {
const c = canvasRef.current;
if (!c || !ref0) return;
- // Export: white = edit on black, feathered.
- const out = document.createElement("canvas");
- out.width = mw;
- out.height = mh;
- const ctx = out.getContext("2d")!;
- ctx.fillStyle = "#000";
- ctx.fillRect(0, 0, mw, mh);
- ctx.filter = `blur(${FEATHER / 2}px)`;
- ctx.drawImage(c, 0, 0);
- ctx.filter = "none";
-
- // Detect empty mask.
const data = c.getContext("2d")!.getImageData(0, 0, mw, mh).data;
+ // Detect empty mask.
let any = false;
for (let i = 3; i < data.length; i += 16) {
if (data[i] > 16) {
@@ -269,19 +418,49 @@ export function InpaintEditor() {
if (!any) {
updateComposer({ mask: null });
- setState({ overlay: null });
+ closeEditor();
return;
}
+ // Export: white = edit on black (from the drawing's alpha), feathered. Colours: the marks' area,
+ // a little larger.
+ const out = document.createElement("canvas");
+ out.width = mw;
+ out.height = mh;
+ const ctx = out.getContext("2d")!;
+ ctx.fillStyle = "#000";
+ ctx.fillRect(0, 0, mw, mh);
+ ctx.filter = `blur(${FEATHER / 2}px)`;
+ {
+ const a = new Uint8Array(mw * mh);
+ for (let i = 0; i < a.length; i++) a[i] = colours ? (data[i * 4 + 3] > 16 ? 255 : 0) : data[i * 4 + 3];
+ const grown = colours ? dilate(a, mw, mh, Math.max(4, Math.round(Math.max(mw, mh) * MARKS_MARGIN))) : a;
+ const white = document.createElement("canvas");
+ white.width = mw;
+ white.height = mh;
+ const w = white.getContext("2d")!;
+ const img = w.createImageData(mw, mh);
+ for (let i = 0; i < grown.length; i++) {
+ img.data[i * 4] = img.data[i * 4 + 1] = img.data[i * 4 + 2] = 255;
+ img.data[i * 4 + 3] = grown[i];
+ }
+ w.putImageData(img, 0, 0);
+ ctx.drawImage(white, 0, 0);
+ }
+ ctx.filter = "none";
+
setBusy(true);
try {
- const blob: Blob = await new Promise((res, rej) =>
- out.toBlob((b) => (b ? res(b) : rej(new Error("export failed"))), "image/png"),
- );
- const rec = await api.upload(blob, "mask.png");
+ const rec = await api.upload(await toBlob(out), "mask.png");
+ const marks = colours ? (await api.upload(await toBlob(c), "marks.png")).id : undefined;
const refKey = getState().composer.refs[0]?.key;
- if (refKey) updateComposer({ mask: { inputId: rec.id, refKey }, outpaint: null, mode: "auto" });
- setState({ overlay: null });
+ if (refKey) {
+ // Colours: the prompt names each colour's thing (Prompt+, when on, rewrites it seeing the marks).
+ const named = used.some((u) => wants[u]?.trim());
+ const prompt = marks && (named || !getState().composer.prompt.trim()) ? marksPrompt(used.map((u) => ({ colour: u, what: wants[u] ?? "" }))) : undefined;
+ updateComposer({ mask: { inputId: rec.id, refKey, ...(marks ? { marks } : {}) }, outpaint: null, mode: "auto", ...(prompt ? { prompt, enhancedFrom: null } : {}) });
+ }
+ closeEditor();
focusPrompt();
} catch (e) {
toastError(e);
@@ -291,19 +470,26 @@ export function InpaintEditor() {
};
useEffect(() => {
- if (!ref0) setState({ overlay: null });
+ if (!ref0) closeEditor();
}, [ref0]);
const contentSize = useMemo(() => ({ width: srcW, height: srcH }), [srcW, srcH]);
if (!ref0) return null;
+ const selectTitle =
+ sam === "missing"
+ ? "Select (S): the selection model is not in this build yet"
+ : sam === "loading"
+ ? "Select (S): getting the selection model ready…"
+ : `Select (S): click an object to ${colours ? "fill it with the colour" : "paint it"} · click again for a bigger part · right-click takes it out`;
+
return (
<div className="overlay">
<div className="overlay-head">
- <h2>Paint the area to change</h2>
+ <h2>{colours ? "Paint where things go" : "Paint the area to change"}</h2>
<span className="spacer" />
- <IconButton title="Cancel (Esc)" onClick={() => setState({ overlay: null })}>
+ <IconButton title="Cancel (Esc)" onClick={() => closeEditor()}>
<X size={18} />
</IconButton>
</div>
@@ -321,13 +507,14 @@ export function InpaintEditor() {
{({ transform, isPanning, flat, wantsFull }) => {
// Content px per screen px: the box is laid out at its drawn size when flat.
const k = flat ? 1 : transform.scale;
+ const pointer = isPanningActive ? (isPanning ? "grabbing" : "grab") : tool === "select" ? (selecting ? "progress" : "crosshair") : "none";
return (
<div
className="mask-wrap"
style={{
width: flat ? "100%" : srcW,
height: flat ? "100%" : srcH,
- cursor: isPanningActive ? (isPanning ? "grabbing" : "grab") : "none",
+ cursor: pointer,
}}
onContextMenu={(e) => e.preventDefault()}
>
@@ -342,12 +529,13 @@ export function InpaintEditor() {
ref={canvasRef}
width={mw}
height={mh}
+ className={colours ? "marks" : "area"}
style={{
position: "absolute",
inset: 0,
width: "100%",
height: "100%",
- cursor: isPanningActive ? (isPanning ? "grabbing" : "grab") : "none",
+ cursor: pointer,
}}
onPointerDown={onDown}
onPointerMove={(e) => onMove(e, k)}
@@ -355,7 +543,7 @@ export function InpaintEditor() {
onPointerCancel={onUp}
onPointerLeave={() => setCursor(null)}
/>
- {cursor && !isPanningActive && (
+ {cursor && !isPanningActive && tool !== "select" && (
<div
className="brush-cursor"
style={{
@@ -372,18 +560,69 @@ export function InpaintEditor() {
</ZoomPan>
</div>
+ {colours && (
+ <div className="marks-panel">
+ <p className="marks-help">
+ {used.length
+ ? "Say what goes at each colour. Works best when the colour fills the whole area the thing should take."
+ : "Fill the whole area where something should go, one colour per thing (Select fills an object in one click). The rest of the photo stays as it is."}
+ </p>
+ {used.map((c) => (
+ <label key={c} className="marks-want">
+ <span className="doc-swatch" style={{ background: cssOf(c) }} />
+ <input
+ value={wants[c] ?? ""}
+ placeholder={`What goes at the ${c} area, e.g. “a green velvet armchair”`}
+ onChange={(e) => setWants({ ...wants, [c]: e.target.value })}
+ spellCheck={false}
+ />
+ </label>
+ ))}
+ </div>
+ )}
+
<div className="editor-tools">
+ {/* How to paint, next to the brush: colours (one per thing) or one area. */}
+ <div className="group">
+ <span className="seg inpaint-kind" role="radiogroup" aria-label="How to paint">
+ <button className={colours ? "on" : ""} role="radio" aria-checked={colours} onClick={() => setKind("colours")} title="One colour per thing to add or change: fill its area, then say what goes there">
+ <Palette size={14} /> Colours
+ </button>
+ <button className={colours ? "" : "on"} role="radio" aria-checked={!colours} onClick={() => setKind("area")} title="One area and a prompt (edit, remove, replace or add)">
+ <Brush size={14} /> One area
+ </button>
+ </span>
+ </div>
+ {colours && (
+ <div className="group marks-colours" role="radiogroup" aria-label="Colour">
+ {MARK_COLOURS.map((m, i) => (
+ <button
+ key={m.name}
+ className={`doc-colour ${colour === m.name ? "on" : ""}`}
+ style={{ background: m.css }}
+ role="radio"
+ aria-checked={colour === m.name}
+ aria-label={m.name}
+ title={`${m.name[0].toUpperCase()}${m.name.slice(1)} (${i + 1})`}
+ onClick={() => setColour(m.name)}
+ />
+ ))}
+ </div>
+ )}
<div className="group">
<IconButton
- title="Brush (B) · right-click erases"
- active={(swapped ? tool === "eraser" : tool === "brush") && !spacePressed}
+ title={colours ? "Brush (B): fill the area · right-click erases" : "Brush (B) · right-click erases"}
+ active={tool !== "select" && (swapped ? tool === "eraser" : tool === "brush") && !spacePressed}
onClick={() => setTool("brush")}
>
<Brush size={17} />
</IconButton>
+ <IconButton title={selectTitle} active={tool === "select" && !spacePressed} onClick={() => setTool((t) => (t === "select" ? "brush" : "select"))} disabled={sam === "missing"}>
+ {sam === "loading" || selecting ? <Loader2 size={17} className="spin" /> : <MousePointerClick size={17} />}
+ </IconButton>
<IconButton
title="Eraser (E) · right-click paints"
- active={(swapped ? tool === "brush" : tool === "eraser") && !spacePressed}
+ active={tool !== "select" && (swapped ? tool === "brush" : tool === "eraser") && !spacePressed}
onClick={() => setTool("eraser")}
>
<Eraser size={17} />
@@ -411,10 +650,12 @@ export function InpaintEditor() {
<IconButton title={`Redo (${shortcut("Y")})`} onClick={redoOp} disabled={!redo.length}>
<Redo2 size={17} />
</IconButton>
- <IconButton title="Invert mask" onClick={() => push({ kind: "invert" })}>
- <Contrast size={17} />
- </IconButton>
- <IconButton title="Clear mask" onClick={() => push({ kind: "clear" })}>
+ {!colours && (
+ <IconButton title="Invert mask" onClick={() => push({ kind: "invert" })}>
+ <Contrast size={17} />
+ </IconButton>
+ )}
+ <IconButton title={colours ? "Clear the colours" : "Clear mask"} onClick={() => push({ kind: "clear" })}>
<Trash2 size={17} />
</IconButton>
</div>
@@ -427,6 +668,48 @@ export function InpaintEditor() {
);
}
+/** The mark colours on the canvas (sampled; an opaque pixel goes to the nearest colour). */
+function coloursOn(c: HTMLCanvasElement | null, w: number, h: number): MarkColour[] {
+ const ctx = c?.getContext("2d");
+ if (!ctx) return [];
+ const d = ctx.getImageData(0, 0, w, h).data;
+ const seen = new Set<MarkColour>();
+ for (let i = 0; i < d.length && seen.size < MARK_COLOURS.length; i += 4 * 7) {
+ if (d[i + 3] < 200) continue;
+ let best: (typeof MARK_COLOURS)[number] = MARK_COLOURS[0];
+ let bd = Infinity;
+ for (const m of MARK_COLOURS) {
+ const dd = (d[i] - m.rgb[0]) ** 2 + (d[i + 1] - m.rgb[1]) ** 2 + (d[i + 2] - m.rgb[2]) ** 2;
+ if (dd < bd) [best, bd] = [m, dd];
+ }
+ seen.add(best.name);
+ }
+ return MARK_COLOURS.map((m) => m.name).filter((n) => seen.has(n));
+}
+
+/** A selection as a canvas in its colour (made once per selection). */
+const blobCanvas = new WeakMap<Uint8Array, HTMLCanvasElement>();
+function blobOf(op: Extract<Op, { kind: "blob" }>): HTMLCanvasElement {
+ let c = blobCanvas.get(op.mask);
+ if (c) return c;
+ c = document.createElement("canvas");
+ c.width = op.w;
+ c.height = op.h;
+ const ctx = c.getContext("2d")!;
+ const img = ctx.createImageData(op.w, op.h);
+ const rgb = op.colour ? MARK_COLOURS.find((m) => m.name === op.colour)!.rgb : AREA_RGB;
+ for (let i = 0; i < op.mask.length; i++) {
+ if (!op.mask[i]) continue;
+ img.data[i * 4] = rgb[0];
+ img.data[i * 4 + 1] = rgb[1];
+ img.data[i * 4 + 2] = rgb[2];
+ img.data[i * 4 + 3] = 255;
+ }
+ ctx.putImageData(img, 0, 0);
+ blobCanvas.set(op.mask, c);
+ return c;
+}
+
/** Draws a single operation incrementally (no clearing). */
function render2(ctx: CanvasRenderingContext2D, op: Op, w: number, h: number) {
if (op.kind === "clear") {
@@ -435,14 +718,21 @@ function render2(ctx: CanvasRenderingContext2D, op: Op, w: number, h: number) {
}
if (op.kind === "invert") {
ctx.globalCompositeOperation = "xor";
- ctx.fillStyle = "#fff";
+ ctx.fillStyle = AREA_CSS;
ctx.fillRect(0, 0, w, h);
ctx.globalCompositeOperation = "source-over";
return;
}
ctx.globalCompositeOperation = op.erase ? "destination-out" : "source-over";
- ctx.strokeStyle = "#fff";
- ctx.fillStyle = "#fff";
+ if (op.kind === "blob") {
+ // Stretched smooth (a stepped edge otherwise); the colour stays solid, only the edge's alpha softens.
+ ctx.imageSmoothingQuality = "high";
+ ctx.drawImage(blobOf(op), 0, 0, w, h);
+ ctx.globalCompositeOperation = "source-over";
+ return;
+ }
+ ctx.strokeStyle = cssOf(op.colour);
+ ctx.fillStyle = cssOf(op.colour);
ctx.lineWidth = op.size;
ctx.lineCap = "round";
ctx.lineJoin = "round";
diff --git a/src/components/LabScene.ts b/src/components/LabScene.ts
new file mode 100644
index 0000000..9d02dde
--- /dev/null
+++ b/src/components/LabScene.ts
@@ -0,0 +1,335 @@
+// Layers lab: reading a whole-image edit back into layers (lib/sceneEdit.ts has the maths). The
+// edit of the flattened layers is put back over them (Qwen zooms plain edits a little), its change
+// of tone divided out, then: darkening near an object -> that object's shadow layer (it follows the
+// object); something new where the user drew coloured marks -> a new layer cut from the edit (SAM's
+// outline when the model is there); an object asked to take the edit's look -> its next version.
+
+import { api, urls } from "../lib/api";
+import { centre, type LayerDoc, type Place, type SceneEdit } from "../lib/layerDoc";
+import { dilate, fileSafe, fromLogits, refineEdges } from "../lib/layers";
+import { Segmenter } from "../lib/sam";
+import { alignEdit, blur, changed, darkness, IDENTITY, invertAffine, luminance, MARK_COLOURS, objectAt, scaleAffine, splitShadows, toneCurve, type Affine, type MarkColour } from "../lib/sceneEdit";
+import type { InputRecord } from "../lib/types";
+import { loadImage, pixels, sourceInfo, toPng } from "./LayerDoc";
+
+/** Long side of the analysis (shadows are soft: no need for every pixel). */
+const SIDE = 1536;
+
+/** What a result made: images to add to the document. Keys: object ids (shadows, looks) and
+ * mark colours (new objects, and their shadows). */
+export interface SceneResult {
+ objects: { colour: MarkColour; src: string; name: string }[];
+ shadows: { of: string; src: string }[];
+ looks: { id: string; src: string }[];
+ notes: string[];
+}
+
+let sam: Promise<Segmenter | null> | null = null;
+const segmenter = () => (sam ??= Segmenter.load().catch(() => null));
+
+const canvas = (w: number, h: number) => {
+ const c = document.createElement("canvas");
+ c.width = w;
+ c.height = h;
+ return c;
+};
+
+/** `img` (any size) drawn over a `w`×`h` frame through `t` (a frame point p shows the image's
+ * point t(p), both in frame px `W`×`H`): the edit put back over the layers. */
+function warp(img: CanvasImageSource, t: Affine, w: number, h: number, W: number, H: number): HTMLCanvasElement {
+ const c = canvas(w, h);
+ const ctx = c.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ const k = w / W;
+ // Frame point p -> image point t(p): the image is drawn through t's inverse, scaled to w×h.
+ const inv = invertAffine(t);
+ ctx.setTransform(k * inv.a, k * inv.d, k * inv.b, k * inv.e, k * inv.c, k * inv.f);
+ ctx.drawImage(img, 0, 0, W, H);
+ return c;
+}
+
+/** An object's pixels (alpha > 128) at `w`×`h`, drawn where `place` puts it. */
+async function footprint(source: string, place: Place, w: number, h: number, W: number, H: number): Promise<Uint8Array> {
+ const info = await sourceInfo(source, W, H);
+ const c = canvas(w, h);
+ const ctx = c.getContext("2d")!;
+ const cc = centre(info.box);
+ ctx.scale(w / W, h / H);
+ ctx.translate(cc.x + place.dx, cc.y + place.dy);
+ ctx.scale(place.s, place.s);
+ ctx.translate(-cc.x, -cc.y);
+ ctx.drawImage(info.img, 0, 0, W, H);
+ const px = ctx.getImageData(0, 0, w, h).data;
+ const m = new Uint8Array(w * h);
+ for (let i = 0; i < m.length; i++) m[i] = px[i * 4 + 3] > 128 ? 255 : 0;
+ return m;
+}
+
+/** One byte per pixel (`w`×`h`) -> a `W`×`H` image: black with that alpha (a shadow layer) or, with
+ * `from`, that image cut by it (a new object). */
+async function upload(alpha: Uint8Array, w: number, h: number, W: number, H: number, name: string, from?: HTMLCanvasElement): Promise<InputRecord> {
+ const small = canvas(w, h);
+ const sctx = small.getContext("2d")!;
+ const d = sctx.createImageData(w, h);
+ for (let i = 0; i < alpha.length; i++) d.data[i * 4 + 3] = alpha[i];
+ sctx.putImageData(d, 0, 0);
+ const c = canvas(W, H);
+ const ctx = c.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ if (from) {
+ ctx.drawImage(from, 0, 0, W, H);
+ ctx.globalCompositeOperation = "destination-in";
+ }
+ ctx.drawImage(small, 0, 0, W, H);
+ return api.upload(await toPng(c), `${fileSafe(name)}.png`);
+}
+
+const total = (m: ArrayLike<number>) => {
+ let s = 0;
+ for (let i = 0; i < m.length; i++) s += m[i];
+ return s;
+};
+
+/** "a green glass bottle" -> "Green glass bottle". */
+export function layerName(what: string, colour: MarkColour): string {
+ const t = what.trim().replace(/^(a|an|the|some|another)\s+/i, "");
+ return t ? t[0].toUpperCase() + t.slice(1, 40) : `New object (${colour})`;
+}
+
+interface Pair {
+ w: number;
+ h: number;
+ /** Before / the edit put back over it, RGBA at w×h. */
+ before: Uint8ClampedArray;
+ after: Uint8ClampedArray;
+ /** Blurred luminances. */
+ lb: Float32Array;
+ la: Float32Array;
+ /** Pixels the edit covers once put back (its border may not reach the frame's edge). */
+ valid: Uint8Array;
+ t: Affine;
+ img: HTMLImageElement;
+}
+
+/** The two images at the analysis size, the edit put back over the layers (`align`; `skip`:
+ * where it is meant to change, left out of the alignment). */
+async function pair(beforeSrc: string, afterSrc: string, W: number, H: number, align: boolean, skip?: (w: number, h: number) => Promise<Uint8Array>): Promise<Pair> {
+ const k = Math.min(1, SIDE / Math.max(W, H));
+ const w = Math.max(1, Math.round(W * k));
+ const h = Math.max(1, Math.round(H * k));
+ const [bi, ai] = await Promise.all([loadImage(urls.source(beforeSrc)), loadImage(urls.source(afterSrc))]);
+ const before = pixels(bi, w, h);
+ const n = w * h;
+ let t = IDENTITY;
+ if (align) {
+ const tk = alignEdit(luminance(before, n), luminance(pixels(ai, w, h), n), w, h, skip ? await skip(w, h) : undefined);
+ t = scaleAffine(tk, W / w);
+ }
+ const after = warp(ai, t, w, h, W, H).getContext("2d")!.getImageData(0, 0, w, h).data;
+ const valid = new Uint8Array(n);
+ // Both opaque there (a background with holes left has none to compare).
+ for (let i = 0; i < n; i++) valid[i] = after[i * 4 + 3] > 250 && before[i * 4 + 3] > 250 ? 1 : 0;
+ const r = Math.max(1, Math.round(Math.max(w, h) * 0.0015));
+ return { w, h, before, after, lb: blur(luminance(before, n), w, h, r), la: blur(luminance(after, n), w, h, r), valid, t, img: ai };
+}
+
+/** The darkening the edit added (tone change divided out), 0-1 per pixel; tone measured on the
+ * pixels farther than `reach` from every object and outside `busy` (the marks). */
+function addedDarkness(p: Pair, objects: Uint8Array[], reach: number, busy?: Uint8Array): Float32Array {
+ const near = new Uint8Array(p.w * p.h);
+ for (const m of objects) for (let i = 0; i < m.length; i++) if (m[i]) near[i] = 255;
+ // Grown at a quarter of the size (dilate is per pixel of radius).
+ const q = 4;
+ const qw = Math.ceil(p.w / q);
+ const qh = Math.ceil(p.h / q);
+ const small = new Uint8Array(qw * qh);
+ for (let i = 0; i < near.length; i++) if (near[i] || busy?.[i]) small[((((i / p.w) | 0) / q) | 0) * qw + (((i % p.w) / q) | 0)] = 255;
+ const grown = dilate(small, qw, qh, reach / q);
+ const far = new Uint8Array(p.w * p.h);
+ for (let i = 0; i < far.length; i++) far[i] = p.valid[i] && !grown[((((i / p.w) | 0) / q) | 0) * qw + (((i % p.w) / q) | 0)] ? 1 : 0;
+ const lut = toneCurve(p.lb, p.la, far);
+ const d = darkness(p.lb, p.la, lut);
+ for (let i = 0; i < d.length; i++) if (!p.valid[i]) d[i] = 0;
+ // A touch of smoothing: decode speckle left above the floor.
+ return blur(d, p.w, p.h, 1);
+}
+
+/** Visible objects of the document (not the background, not shadows). */
+const objectsOf = (doc: LayerDoc) => doc.layers.filter((l) => !l.base && !l.effect && !l.hidden);
+
+/** A whole-image edit's result -> what it made (see SceneResult). */
+export async function readScene(doc: LayerDoc, e: SceneEdit, result: string): Promise<SceneResult> {
+ const W = doc.width;
+ const H = doc.height;
+ const out: SceneResult = { objects: [], shadows: [], looks: [], notes: [] };
+ // The marks, one mask per colour (the pencil's colours, drawn opaque).
+ let marks: { colour: MarkColour; m: Uint8Array; what: string }[] = [];
+ let marked: Uint8Array | undefined;
+ const readMarks = async (w: number, h: number) => {
+ if (!e.marks || !e.wants?.length) return new Uint8Array(w * h);
+ const px = pixels(await loadImage(urls.input(e.marks)), w, h);
+ marks = e.wants.map((x) => ({ ...x, m: new Uint8Array(w * h) }));
+ const any = new Uint8Array(w * h);
+ for (let i = 0; i < any.length; i++) {
+ if (px[i * 4 + 3] < 128) continue;
+ let best = -1;
+ let bd = Infinity;
+ marks.forEach((x, k) => {
+ const c = MARK_COLOURS.find((mc) => mc.name === x.colour)!.rgb;
+ const d = Math.abs(px[i * 4] - c[0]) + Math.abs(px[i * 4 + 1] - c[1]) + Math.abs(px[i * 4 + 2] - c[2]);
+ if (d < bd) {
+ bd = d;
+ best = k;
+ }
+ });
+ if (best >= 0) marks[best].m[i] = 255;
+ any[i] = 255;
+ }
+ // Room around the marks: the object may be drawn a little bigger than them.
+ marked = dilate(any, w, h, Math.round(Math.max(w, h) * 0.03));
+ return marked;
+ };
+ const p = await pair(e.before, result, W, H, true, readMarks);
+ const { w, h } = p;
+ if (!marked) await readMarks(w, h);
+ const reach = (Math.max(w, h) * e.reach) / 100;
+ const ids = Object.keys(e.places).filter((id) => doc.layers.some((l) => l.id === id));
+ const feet = new Map<string, Uint8Array>();
+ for (const id of ids) feet.set(id, await footprint(e.places[id].source, e.places[id], w, h, W, H));
+
+ // New objects: what changed around each colour's marks.
+ const news: { colour: MarkColour; m: Uint8Array }[] = [];
+ if (marks.length) {
+ const lut = toneCurve(p.lb, p.la, Uint8Array.from(p.valid, (v, i) => (v && !marked![i] ? 1 : 0)));
+ const diff = changed(p.before, p.after, lut, w * h);
+ let full: HTMLCanvasElement | null = null;
+ const gray = Float32Array.from(luminance(p.after, w * h), (v) => v / 255);
+ for (const mk of marks) {
+ const region = dilate(mk.m, w, h, Math.round(Math.max(w, h) * 0.04));
+ const cand = Uint8Array.from(diff, (v, i) => (v && region[i] && p.valid[i] ? 255 : 0));
+ let obj = objectAt(cand, dilate(mk.m, w, h, Math.round(Math.max(w, h) * 0.01)), w, h);
+ const area = total(obj) / 255;
+ if (area < w * h * 0.0002) {
+ out.notes.push(`Nothing new was found at the ${mk.colour} marks.`);
+ continue;
+ }
+ obj = await samOutline(p, obj, w, h);
+ const soft = refineEdges(obj, gray, w, h, Math.max(2, Math.round(Math.max(w, h) * 0.003)));
+ full ??= warp(p.img, p.t, W, H, W, H);
+ const rec = await upload(soft, w, h, W, H, layerName(mk.what, mk.colour), full);
+ out.objects.push({ colour: mk.colour, src: `input:${rec.id}`, name: layerName(mk.what, mk.colour) });
+ news.push({ colour: mk.colour, m: obj });
+ }
+ }
+
+ // Shadows: only of the objects asked for. A new object comes alone (its shadow is made
+ // afterwards with Add shadow, the ring), whatever shadow Qwen drew for it is left out.
+ const targets = e.shadows.filter((id) => feet.has(id)).map((id) => ({ of: id, m: feet.get(id)! }));
+ if (targets.length) {
+ const others = new Uint8Array(w * h);
+ for (const [id, m] of feet) if (!e.shadows.includes(id)) for (let i = 0; i < m.length; i++) if (m[i]) others[i] = 255;
+ const dark = addedDarkness(p, [...feet.values(), ...news.map((n) => n.m)], reach, marked);
+ const split = splitShadows(dark, targets.map((t) => t.m), w, h, reach, Math.max(w, h) * 0.015, others, Math.round(Math.max(w, h) * 0.004));
+ for (let k = 0; k < targets.length; k++) {
+ // Less than a hint of shadow: nothing to add.
+ if (total(split[k]) < 255 * w * h * 0.0003) {
+ out.notes.push(`No shadow was found for ${nameOf(doc, targets[k].of, out)}.`);
+ continue;
+ }
+ const rec = await upload(split[k], w, h, W, H, `${nameOf(doc, targets[k].of, out)} shadow`);
+ out.shadows.push({ of: targets[k].of, src: `input:${rec.id}` });
+ }
+ }
+
+ // The edit's look of an object: its pixels in the edit, back in the object's own place.
+ if (e.look.length) {
+ const full = warp(p.img, p.t, W, H, W, H);
+ for (const id of e.look) {
+ const at = e.places[id];
+ const l = doc.layers.find((x) => x.id === id);
+ const foot = feet.get(id);
+ if (!at || !l || !foot) continue;
+ if (!differs(p, foot)) continue;
+ const info = await sourceInfo(at.source, W, H);
+ const cc = centre(info.box);
+ const c = canvas(W, H);
+ const ctx = c.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ // Frame -> the object's own pixels: the inverse of its move and scale.
+ ctx.setTransform(1 / at.s, 0, 0, 1 / at.s, cc.x - (cc.x + at.dx) / at.s, cc.y - (cc.y + at.dy) / at.s);
+ ctx.drawImage(full, 0, 0);
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.globalCompositeOperation = "destination-in";
+ ctx.drawImage(info.img, 0, 0, W, H);
+ const rec = await api.upload(await toPng(c), `${fileSafe(l.name)} (edited).png`);
+ out.looks.push({ id, src: `input:${rec.id}` });
+ }
+ }
+ return out;
+}
+
+const nameOf = (doc: LayerDoc, of: string, out: SceneResult) => doc.layers.find((l) => l.id === of)?.name ?? out.objects.find((o) => o.colour === of)?.name ?? of;
+
+/** Whether the edit changed an object beyond decode noise (mean change over its pixels). */
+function differs(p: Pair, foot: Uint8Array): boolean {
+ let n = 0;
+ let sum = 0;
+ for (let i = 0; i < foot.length; i++) {
+ if (!foot[i] || !p.valid[i]) continue;
+ n++;
+ for (let c = 0; c < 3; c++) sum += Math.abs(p.after[i * 4 + c] - p.before[i * 4 + c]);
+ }
+ return n > 0 && sum / (n * 3) > 6;
+}
+
+/** SAM's outline of a new object (box prompt around the changed pixels) when the model is there
+ * and agrees with them (IoU >= 0.5); else the changed pixels as they are. */
+async function samOutline(p: Pair, obj: Uint8Array, w: number, h: number): Promise<Uint8Array> {
+ const s = await segmenter();
+ if (!s) return obj;
+ let x0 = w;
+ let y0 = h;
+ let x1 = -1;
+ let y1 = -1;
+ for (let i = 0; i < obj.length; i++)
+ if (obj[i]) {
+ const x = i % w;
+ const y = (i / w) | 0;
+ x0 = Math.min(x0, x);
+ y0 = Math.min(y0, y);
+ x1 = Math.max(x1, x);
+ y1 = Math.max(y1, y);
+ }
+ const mx = (x1 - x0) * 0.04 + 2;
+ const my = (y1 - y0) * 0.04 + 2;
+ try {
+ const c = canvas(w, h);
+ c.getContext("2d")!.putImageData(new ImageData(new Uint8ClampedArray(p.after), w, h), 0, 0);
+ await s.encode(c, w, h);
+ const r = await s.predict([], w, h, { x0: Math.max(0, x0 - mx), y0: Math.max(0, y0 - my), x1: Math.min(w - 1, x1 + mx), y1: Math.min(h - 1, y1 + my) });
+ let best: Uint8Array | null = null;
+ let bestIou = 0.5;
+ for (const logits of r.masks) {
+ const m = fromLogits(logits);
+ let inter = 0;
+ let uni = 0;
+ for (let i = 0; i < m.length; i++) {
+ if (m[i] && obj[i]) inter++;
+ if (m[i] || obj[i]) uni++;
+ }
+ const iou = uni ? inter / uni : 0;
+ if (iou >= bestIou) {
+ bestIou = iou;
+ best = m;
+ }
+ }
+ return best ?? obj;
+ } catch {
+ return obj;
+ }
+}
+
+/** The objects the scene edits read, as they are now (their place and version). */
+export function placesOf(doc: LayerDoc): SceneEdit["places"] {
+ return Object.fromEntries(objectsOf(doc).map((l) => [l.id, { dx: l.dx, dy: l.dy, s: l.s, source: l.source }]));
+}
diff --git a/src/components/LabStage.tsx b/src/components/LabStage.tsx
new file mode 100644
index 0000000..0ab99c8
--- /dev/null
+++ b/src/components/LabStage.tsx
@@ -0,0 +1,183 @@
+// Layers lab in the project: once an image has been split, the project shows its layered image
+// (one frame per split image in the version strip, the composite on the Stage) instead of the lab's
+// own versions; clicking it opens that image's layers.
+
+import { Layers, Loader2, X } from "lucide-react";
+import { useEffect, useMemo, useRef, useState, useSyncExternalStore } from "react";
+import { docId, docsVersion, jobLayer, waitingMarks, labItems, leftovers, loadLab, saveLab, subscribeDocs, type LabStore, type LayerDoc } from "../lib/layerDoc";
+import { editSource, itemOrigin, itemSource, removeSessionItem, setState, useStore, type SessionItem } from "../lib/store";
+import type { JobView } from "../lib/types";
+import { urls } from "../lib/api";
+import { useJobPercent } from "./jobMotion";
+import { renderDoc } from "./LayerDoc";
+
+/** Opens `doc`'s layers: the prompt goes to its photo (or one of its layers), which picks the
+ * document (LayersEditor). */
+export function openLab(doc: LayerDoc) {
+ if (!(doc.photo && editSource(doc.photo, false))) doc.layers.some((l) => editSource(l.source, false));
+ setState({ overlay: "layers" });
+}
+
+const decode = (s: string) => {
+ try {
+ return decodeURIComponent(s);
+ } catch {
+ return s;
+ }
+};
+
+/** The open project's lab (its documents), and the versions that belong to a lab: key -> document
+ * id (null: an earlier lab's leftover, hidden too). */
+export function useLab(): Lab {
+ const projectId = useStore((s) => s.projectId);
+ const session = useStore((s) => s.session);
+ const v = useSyncExternalStore(subscribeDocs, docsVersion);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ const store = useMemo(() => loadLab(projectId), [projectId, v]);
+ // Once per project: leftovers of a lab whose document was replaced before documents were kept
+ // per image (found by the names the lab gives its images).
+ useEffect(() => {
+ if (store.swept || !session.some((i) => i.input || i.gen)) return;
+ const found = leftovers(session.map((i) => ({ source: itemSource(i), name: i.input?.original_name ? decode(i.input.original_name) : null, input: i.kind === "input" })));
+ saveLab(projectId, { ...store, made: [...new Set([...store.made, ...found])], swept: true });
+ }, [store, session, projectId]);
+ const hidden = useMemo(
+ () =>
+ labItems(
+ store,
+ session.map((i) => ({ key: i.key, source: itemSource(i), from: itemOrigin(i).from[0] ?? null, mask: i.gen?.mask_input_id ?? i.request?.mask?.replace(/^input:/, "") ?? null })),
+ ),
+ [store, session],
+ );
+ return { store, hidden };
+}
+
+export const docById = (lab: Lab, id: string | null | undefined) => (id ? lab.store.docs.find((d) => docId(d) === id) ?? null : null);
+
+/** The layered image as an object URL (redrawn when a layer changes; the old one stays until then). */
+export function useLabImage(doc: LayerDoc | null, maxSide: number): string | null {
+ const key = doc ? JSON.stringify([doc.width, doc.height, doc.layers.map((l) => [l.source, l.hidden, l.dx, l.dy, l.s, l.follows, l.at])]) : "";
+ const [url, setUrl] = useState<string | null>(null);
+ const shown = useRef<string | null>(null);
+ useEffect(() => {
+ if (!doc) return;
+ let gone = false;
+ renderDoc(doc, maxSide)
+ .then((c) => new Promise<Blob | null>((res) => c.toBlob(res, "image/png")))
+ .then((b) => {
+ if (gone || !b) return;
+ if (shown.current) URL.revokeObjectURL(shown.current);
+ shown.current = URL.createObjectURL(b);
+ setUrl(shown.current);
+ })
+ .catch(() => {});
+ return () => {
+ gone = true;
+ };
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [key, maxSide]);
+ useEffect(() => () => void (shown.current && URL.revokeObjectURL(shown.current)), []);
+ return url;
+}
+
+export type Lab = { store: LabStore; hidden: Map<string, string | null> };
+export interface LabJob {
+ item: SessionItem;
+ job: JobView;
+ doc: LayerDoc;
+ /** jobLayer's answer: a layer id, "shadows", or null. */
+ layer: string | null;
+}
+
+/** The lab's edits being made, or failed (the Stage's job card and the version strip do not show
+ * them: they are lab versions). */
+export function useLabJobs(lab: Lab): LabJob[] {
+ const session = useStore((s) => s.session);
+ const jobs = useStore((s) => s.jobs);
+ return useMemo(
+ () =>
+ session.flatMap((i) => {
+ const job = i.kind === "job" && i.jobId ? jobs[i.jobId] : undefined;
+ const doc = docById(lab, lab.hidden.get(i.key));
+ if (!job || !doc || job.status === "done" || job.status === "cancelled") return [];
+ return [{ item: i, job, doc, layer: jobLayer(doc, i.request?.images?.[0], i.request?.mask?.replace(/^input:/, "")) }];
+ }),
+ [lab, session, jobs],
+ );
+}
+
+const PHASE: Partial<Record<JobView["status"], string>> = { queued: "Waiting", starting_backend: "Starting the engine", enhancing_prompt: "Writing the prompt" };
+
+/** The edit's percentage, the same as its job card's. */
+export function JobPct({ job, item }: { job: JobView; item: SessionItem }) {
+ const { pct } = useJobPercent(job, item.request);
+ return <>{Math.floor(pct)}%</>;
+}
+
+function JobPill({ j }: { j: LabJob }) {
+ const layer = j.doc.layers.find((l) => l.id === j.layer);
+ const what = j.layer === "shadows" ? "the shadows" : j.layer === "scene" ? "the scene" : layer ? (layer.base ? "the background" : layer.name) : "";
+ if (j.job.status === "failed")
+ return (
+ <span className="lab-progress failed" title={j.job.error?.detail ?? undefined}>
+ {what ? `${what}: ` : ""}
+ {j.job.error?.message ?? "the edit failed"}
+ <button className="lab-dismiss" title="Dismiss" onClick={() => removeSessionItem(j.item.key)}>
+ <X size={13} />
+ </button>
+ </span>
+ );
+ return (
+ <span className="lab-progress">
+ <Loader2 className="spin" size={13} />
+ {PHASE[j.job.status] ? `${PHASE[j.job.status]}${what ? ` · ${what}` : ""}` : `Generating${what ? ` ${what}` : ""}`} · <JobPct job={j.job} item={j.item} />
+ </span>
+ );
+}
+
+/** "Generating the background · 38%": the lab edits in progress (or failed) of `doc`. */
+export function LabJobs({ lab, doc, className }: { lab: Lab; doc: LayerDoc; className?: string }) {
+ const list = useLabJobs(lab).filter((j) => docId(j.doc) === docId(doc));
+ if (!list.length) return null;
+ return (
+ <div className={`lab-jobs ${className ?? ""}`}>
+ {list.map((j) => (
+ <JobPill key={j.item.key} j={j} />
+ ))}
+ </div>
+ );
+}
+
+/** The Stage while a lab version is selected: the layered image to look at; its button opens the lab. */
+export function LabFrame({ doc, lab }: { doc: LayerDoc; lab: Lab }) {
+ const url = useLabImage(doc, 1600);
+ return (
+ <div className="stage-frame">
+ <LabJobs lab={lab} doc={doc} className="on-stage" />
+ <div className="lab-frame" style={{ aspectRatio: `${doc.width} / ${doc.height}` }}>
+ {url ? <img src={url} alt="The layered image" draggable={false} /> : <Loader2 className="spin" size={22} />}
+ {/* The pencil's marks sent for new objects, until the objects are there. */}
+ {url && waitingMarks(doc).map((m) => <img key={m} className="lab-marks" src={urls.input(m)} alt="" draggable={false} />)}
+ <button className="lab-open" onClick={() => openLab(doc)} title="Open this image's layers in the Layers lab">
+ <Layers size={14} /> Open Layers lab
+ </button>
+ </div>
+ </div>
+ );
+}
+
+/** The version strip's frame for the lab (with the percentage of an edit being made). */
+export function LabThumb({ doc, job }: { doc: LayerDoc; job?: LabJob }) {
+ const url = useLabImage(doc, 160);
+ const making = job && job.job.status !== "failed";
+ return (
+ <>
+ {url && <img className={making ? "dim" : ""} src={url} alt="" draggable={false} />}
+ {making && (
+ <span className="frame-pct">
+ <JobPct job={job.job} item={job.item} />
+ </span>
+ )}
+ </>
+ );
+}
diff --git a/src/components/LayerDoc.tsx b/src/components/LayerDoc.tsx
new file mode 100644
index 0000000..9054a90
--- /dev/null
+++ b/src/components/LayerDoc.tsx
@@ -0,0 +1,1865 @@
+// Layers lab, step 2: the layers a split made, over each other like in an image editor, only
+// simpler. Right bar: the layers top first (thumbnail, name, eye, drag to reorder) and what to do
+// with the selected one: go back to an older version, reset its place, delete it. The selected layer
+// is the image the prompt below edits: the result shows on the layer, here (the background: "Fill
+// the holes" sets up a brushed Remove on the photo). On the canvas: click a layer to select it, drag
+// to move it, drag the corner to scale it; Shift/Ctrl+click selects more (moved and scaled
+// together), Group keeps them together. Whole-image edits (LabScene.ts) give Qwen the flattened
+// layers: "Shadows" reads back each object's shadow as its own layer (it moves with the object),
+// "New objects" adds what the user drew with the coloured pencil as new layers, "Edit together"
+// gives the selected objects the edit's look. "Save as image" flattens the layers into a new version.
+
+import { Brush, Eraser, Eye, EyeOff, GripVertical, Group, Hand, History, ImageDown, Link2, Loader2, Lock, MousePointer2, Paintbrush, Pencil, Redo2, RotateCcw, Scissors, Sparkles, Sun, Trash2, Undo2, Ungroup, WandSparkles, X } from "lucide-react";
+import { useEffect, useMemo, useRef, useState } from "react";
+import { api, urls } from "../lib/api";
+import { dilate, erode, fileSafe, grow } from "../lib/layers";
+import { centre, CROP_MAX_RATIO, cropFor, docId, waitingMarks, shadowAlpha, mapBox, reorder, unmapPoint, withGroup, xfOf, type Box, type DocLayer, type Effect, type Feed, type LayerDoc, type SceneEdit, type Sticker, type Xf } from "../lib/layerDoc";
+import { buildRequest, refFromInput } from "../lib/logic";
+import { MARK_COLOURS, marksPrompt, type MarkColour } from "../lib/sceneEdit";
+import { placesOf } from "./LabScene";
+import { shortcut } from "../lib/platform";
+import { addResult, addVersions, cancelJob, editSource, focusPrompt, getState, itemSource, setState, submitRequest, toastError, toastInfo, updateComposer, useStore, type SessionItem } from "../lib/store";
+import type { InputRecord, JobView } from "../lib/types";
+import { useJobPercent } from "./jobMotion";
+import { JobPct, LabJobs, useLab, useLabJobs } from "./LabStage";
+import { IconButton } from "./ui";
+import { isTyping } from "./Viewer";
+import { ZoomPan } from "./ZoomPan";
+
+/** Clean edges' prompt, on the object in its scene (cropOf "clean"). No "sticker" and no "no outline"
+ * (11-01): a sticker has a border, and "no outline" took off a lettering's own outline; with the scene
+ * in view this kept only the bats / the lettering, clean (bench/20261101-stickers-fill). */
+const stickerPrompt = (name: string) =>
+ `Keep only the ${name.toLowerCase()}, exactly as it is with its own colours and details, in the same place and size. Remove the background and everything else around it, with no rim of background colour left along its edges, and empty any gap inside it (like the hole of a handle). Make the background fully transparent.`;
+
+/** Fill the holes' prompt, inside Kokon's Add wrapper (pipeline.rs inpaint_prompt). Naming what must not
+ * be there took the grey off on the seed that had kept it, and the contact shadows (11-01 bench). */
+const FILL_PROMPT =
+ "the empty background that continues the surrounding scene, with the same surfaces, textures, lighting and perspective; no objects, people, text or leftover pieces in it, and no shadows or reflections of the things that were there";
+
+/** The progress of an object's sticker: a ring filling up, the percentage inside. */
+function StickerRing({ job, item }: { job: JobView; item: SessionItem }) {
+ const { pct } = useJobPercent(job, item.request);
+ const r = 22;
+ const len = 2 * Math.PI * r;
+ return (
+ <svg width="56" height="56" viewBox="0 0 56 56" className="sticker-progress" aria-label={`${Math.floor(pct)} percent`}>
+ <circle cx="28" cy="28" r={r} className="track" />
+ <circle cx="28" cy="28" r={r} className="bar" strokeDasharray={len} strokeDashoffset={len * (1 - pct / 100)} transform="rotate(-90 28 28)" />
+ <text x="28" y="32" textAnchor="middle">
+ {Math.floor(pct)}%
+ </text>
+ </svg>
+ );
+}
+
+/** What the canvas needs of a layer's version: its alpha (for clicks), where its content is, a thumbnail. */
+export interface SourceInfo {
+ img: HTMLImageElement;
+ alpha: Uint8Array;
+ aw: number;
+ ah: number;
+ /** In frame px. */
+ box: Box;
+ thumb: string;
+}
+
+const ALPHA_SIDE = 512;
+export const toPng = (c: HTMLCanvasElement) => new Promise<Blob>((res, rej) => c.toBlob((b) => (b ? res(b) : rej(new Error("export failed"))), "image/png"));
+const cache = new Map<string, Promise<SourceInfo>>();
+
+export function sourceInfo(src: string, W: number, H: number): Promise<SourceInfo> {
+ const key = `${src}@${W}x${H}`;
+ let p = cache.get(key);
+ if (!p) {
+ p = new Promise<HTMLImageElement>((res, rej) => {
+ const img = new Image();
+ img.crossOrigin = "anonymous";
+ img.onload = () => res(img);
+ img.onerror = () => rej(new Error("A layer's image could not be loaded."));
+ img.src = urls.source(src);
+ }).then((img) => {
+ const k = Math.min(1, ALPHA_SIDE / Math.max(W, H));
+ const aw = Math.max(1, Math.round(W * k));
+ const ah = Math.max(1, Math.round(H * k));
+ const c = document.createElement("canvas");
+ c.width = aw;
+ c.height = ah;
+ const ctx = c.getContext("2d")!;
+ // Every version is drawn over the whole frame (an edit may come back at another size).
+ ctx.drawImage(img, 0, 0, aw, ah);
+ const px = ctx.getImageData(0, 0, aw, ah).data;
+ const alpha = new Uint8Array(aw * ah);
+ let x0 = aw;
+ let y0 = ah;
+ let x1 = -1;
+ let y1 = -1;
+ for (let i = 0; i < alpha.length; i++) {
+ const a = (alpha[i] = px[i * 4 + 3]);
+ if (a <= 16) continue;
+ const x = i % aw;
+ const y = (i / aw) | 0;
+ if (x < x0) x0 = x;
+ if (x > x1) x1 = x;
+ if (y < y0) y0 = y;
+ if (y > y1) y1 = y;
+ }
+ const box = x1 < 0 ? { x0: 0, y0: 0, x1: W, y1: H } : { x0: (x0 / aw) * W, y0: (y0 / ah) * H, x1: ((x1 + 1) / aw) * W, y1: ((y1 + 1) / ah) * H };
+ const t = document.createElement("canvas");
+ const tk = 64 / Math.max(W, H);
+ t.width = Math.max(1, Math.round(W * tk));
+ t.height = Math.max(1, Math.round(H * tk));
+ t.getContext("2d")!.drawImage(img, 0, 0, t.width, t.height);
+ return { img, alpha, aw, ah, box, thumb: t.toDataURL("image/png") };
+ });
+ cache.set(key, p);
+ p.catch(() => cache.delete(key));
+ }
+ return p;
+}
+
+export function loadImage(src: string): Promise<HTMLImageElement> {
+ return new Promise((res, rej) => {
+ const img = new Image();
+ img.crossOrigin = "anonymous";
+ img.onload = () => res(img);
+ img.onerror = () => rej(new Error("An image could not be loaded."));
+ img.src = src;
+ });
+}
+
+/** `img` drawn over a `W`×`H` frame, as RGBA. */
+export function pixels(img: CanvasImageSource, W: number, H: number): Uint8ClampedArray {
+ const c = document.createElement("canvas");
+ c.width = W;
+ c.height = H;
+ const ctx = c.getContext("2d")!;
+ ctx.drawImage(img, 0, 0, W, H);
+ return ctx.getImageData(0, 0, W, H).data;
+}
+
+/** The document's visible layers drawn into one image, at most `maxSide` px (the project's Stage
+ * and version strip show it). Layers whose version cannot be loaded are left out. */
+export async function renderDoc(doc: LayerDoc, maxSide: number): Promise<HTMLCanvasElement> {
+ const W = doc.width;
+ const H = doc.height;
+ const infos = new Map<string, SourceInfo>();
+ await Promise.all(doc.layers.map((l) => sourceInfo(l.source, W, H).then((i) => infos.set(l.source, i), () => {})));
+ const k = Math.min(1, maxSide / Math.max(W, H));
+ const c = document.createElement("canvas");
+ c.width = Math.max(1, Math.round(W * k));
+ c.height = Math.max(1, Math.round(H * k));
+ const ctx = c.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ ctx.scale(c.width / W, c.height / H);
+ for (const l of [...doc.layers].reverse()) {
+ const info = infos.get(l.source);
+ if (l.hidden || !info) continue;
+ const t = xfOf(doc.layers, l, (src) => infos.get(src)?.box, W, H);
+ ctx.save();
+ ctx.translate(t.c.x + t.dx, t.c.y + t.dy);
+ ctx.scale(t.s, t.s);
+ ctx.translate(-t.c.x, -t.c.y);
+ ctx.drawImage(info.img, 0, 0, W, H);
+ ctx.restore();
+ }
+ return c;
+}
+
+/** A shadow edit's result -> the shadow layer's image: black, as opaque as the edit darkened the
+ * flattened layers it was made from, inside the painted ring (lib/layerDoc.ts shadowAlpha). */
+export async function shadowImage(p: Effect, result: string, W: number, H: number, name: string): Promise<InputRecord> {
+ const [before, after, mask] = await Promise.all([loadImage(urls.source(p.source)), loadImage(urls.source(result)), loadImage(urls.input(p.mask))]);
+ const a = pixels(before, W, H);
+ const b = pixels(after, W, H);
+ const m = pixels(mask, W, H);
+ const area = new Uint8Array(W * H);
+ for (let i = 0; i < area.length; i++) area[i] = m[i * 4];
+ const alpha = shadowAlpha(a, b, area);
+ const c = document.createElement("canvas");
+ c.width = W;
+ c.height = H;
+ const ctx = c.getContext("2d")!;
+ const out = ctx.createImageData(W, H);
+ for (let i = 0; i < alpha.length; i++) out.data[i * 4 + 3] = alpha[i];
+ ctx.putImageData(out, 0, 0);
+ return api.upload(await toPng(c), `${name}.png`);
+}
+
+
+/** The painted area (a mask PNG: white = the edit) as a pink tint with a white outline, readable
+ * on any picture (the Stage's difference blend is hard to see on dark, warm photos). */
+export function useMaskTint(src: string | null): string | null {
+ const [url, setUrl] = useState<string | null>(null);
+ useEffect(() => {
+ setUrl(null);
+ if (!src) return;
+ let gone = false;
+ let made: string | null = null;
+ loadImage(src)
+ .then((img) => {
+ const w = img.naturalWidth;
+ const h = img.naturalHeight;
+ const m = pixels(img, w, h);
+ const c = document.createElement("canvas");
+ c.width = w;
+ c.height = h;
+ const ctx = c.getContext("2d")!;
+ const out = ctx.createImageData(w, h);
+ const on = (x: number, y: number) => x >= 0 && y >= 0 && x < w && y < h && m[(y * w + x) * 4] >= 128;
+ // The outline about 2 px wide at the mask's size (up to 2048), so it shows at fit.
+ const r = Math.max(1, Math.round(Math.max(w, h) / 1000));
+ for (let y = 0; y < h; y++)
+ for (let x = 0; x < w; x++) {
+ const i = (y * w + x) * 4;
+ const v = m[i];
+ const inside = v >= 128;
+ const edge = inside && (!on(x - r, y) || !on(x + r, y) || !on(x, y - r) || !on(x, y + r));
+ if (edge) out.data.set([255, 255, 255, 255], i);
+ else if (v) out.data.set([255, 59, 107, Math.round(v * 0.5)], i);
+ }
+ ctx.putImageData(out, 0, 0);
+ return toPng(c);
+ })
+ .then((b) => {
+ if (gone) return;
+ made = URL.createObjectURL(b);
+ setUrl(made);
+ })
+ .catch(() => {});
+ return () => {
+ gone = true;
+ if (made) URL.revokeObjectURL(made);
+ };
+ }, [src]);
+ return url;
+}
+
+/** A crop edit's result -> the layer's next image: shrunk back into the crop's place in the frame
+ * (the rest transparent: the crop holds the whole object), clipped to the grown selection for
+ * Clean edges (so the edit cannot bring a neighbour back). */
+export async function pasteCrop(f: Feed, result: string, W: number, H: number, name: string): Promise<InputRecord> {
+ const r = f.rect!;
+ let img: CanvasImageSource = await loadImage(urls.source(result));
+ const c = document.createElement("canvas");
+ c.width = W;
+ c.height = H;
+ const ctx = c.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ if (f.clip) {
+ const m = await loadImage(urls.input(f.clip));
+ const mw = m.naturalWidth;
+ const mh = m.naturalHeight;
+ const mc = document.createElement("canvas");
+ mc.width = mw;
+ mc.height = mh;
+ const mctx = mc.getContext("2d")!;
+ mctx.drawImage(m, 0, 0);
+ const d = mctx.getImageData(0, 0, mw, mh);
+ for (let i = 0; i < d.data.length; i += 4) d.data[i + 3] = d.data[i];
+ mctx.putImageData(d, 0, 0);
+ // In the ring the selection was grown by, faint pixels are the halo of what was around the
+ // object (Qwen leaves it half transparent): dropped. Inside the selection everything stays
+ // (a glass is see-through on purpose).
+ if (f.grow) {
+ const rc = document.createElement("canvas");
+ rc.width = mw;
+ rc.height = mh;
+ const rctx = rc.getContext("2d")!;
+ rctx.drawImage(img, 0, 0, mw, mh);
+ const px = rctx.getImageData(0, 0, mw, mh);
+ const sel = new Uint8Array(mw * mh);
+ for (let i = 0; i < sel.length; i++) sel[i] = d.data[i * 4] > 128 ? 255 : 0;
+ const inner = erode(sel, mw, mh, f.grow);
+ for (let i = 0; i < sel.length; i++) if (sel[i] && !inner[i] && px.data[i * 4 + 3] < 160) px.data[i * 4 + 3] = 0;
+ rctx.putImageData(px, 0, 0);
+ img = rc;
+ }
+ ctx.drawImage(img, r.x, r.y, r.w, r.h);
+ ctx.globalCompositeOperation = "destination-in";
+ ctx.drawImage(mc, r.x, r.y, r.w, r.h);
+ } else ctx.drawImage(img, r.x, r.y, r.w, r.h);
+ return api.upload(await toPng(c), `${name} (edited).png`);
+}
+
+/** One dab of the layer brushes, in the layer's own pixels: the eraser clears (solid core, soft
+ * rim); the shadow brush adds a little darkness each dab, so strokes build up softly. */
+function dab(ctx: CanvasRenderingContext2D, x: number, y: number, r: number, mode: "erase" | "shade") {
+ const g = ctx.createRadialGradient(x, y, 0, x, y, r);
+ if (mode === "erase") {
+ g.addColorStop(0, "rgba(0,0,0,1)");
+ g.addColorStop(0.6, "rgba(0,0,0,1)");
+ g.addColorStop(1, "rgba(0,0,0,0)");
+ } else {
+ g.addColorStop(0, "rgba(0,0,0,0.06)");
+ g.addColorStop(1, "rgba(0,0,0,0)");
+ }
+ ctx.globalCompositeOperation = mode === "erase" ? "destination-out" : "source-over";
+ ctx.fillStyle = g;
+ ctx.beginPath();
+ ctx.arc(x, y, r, 0, Math.PI * 2);
+ ctx.fill();
+ ctx.globalCompositeOperation = "source-over";
+}
+
+/** How far a shadow edit's painted area reaches below an object, in % of the image's long side
+ * (sideways half of it, upwards little: contact shadows fall under things). Kept while the app runs. */
+let lastReach = 4;
+
+/** How far the shadow area goes inside the objects, in frame px: the model redraws their bottom edge
+ * with the shadow, so the contact joins (as hand-painted areas did, 10-31). Kept while the app runs. */
+let lastInset = 8;
+
+/** How far a whole-image shadow edit's result is read around each object (% of the long side). */
+const SCENE_REACH = 12;
+
+/** The selected layers per document (the last one is the one the panel shows), kept while the
+ * view closes (Paint area, Expand, the Stage). */
+const lastSelected = new Map<string, string[]>();
+
+/** The pencil's marks per document, and what each colour asks for (kept while the view closes). */
+interface Mark {
+ colour: MarkColour;
+ /** Line width in frame px. */
+ r: number;
+ pts: { x: number; y: number }[];
+}
+const lastMarks = new Map<string, { marks: Mark[]; wants: Partial<Record<MarkColour, string>> }>();
+
+/** A pencil mark: an opaque line in its colour (LabScene tells the colours apart by nearest colour). */
+function drawMark(ctx: CanvasRenderingContext2D, m: Mark, from = 0) {
+ const css = MARK_COLOURS.find((c) => c.name === m.colour)!.css;
+ ctx.strokeStyle = css;
+ ctx.fillStyle = css;
+ ctx.lineWidth = m.r * 2;
+ ctx.lineCap = "round";
+ ctx.lineJoin = "round";
+ if (m.pts.length - from === 1) {
+ ctx.beginPath();
+ ctx.arc(m.pts[from].x, m.pts[from].y, m.r, 0, Math.PI * 2);
+ ctx.fill();
+ return;
+ }
+ ctx.beginPath();
+ ctx.moveTo(m.pts[from].x, m.pts[from].y);
+ for (let i = from + 1; i < m.pts.length; i++) ctx.lineTo(m.pts[i].x, m.pts[i].y);
+ ctx.stroke();
+}
+
+type Drag = { mode: "move" | "scale"; ids: string[]; x0: number; y0: number; l0: Record<string, DocLayer>; c: { x: number; y: number } };
+
+export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId: string; doc: LayerDoc; onChange: (d: LayerDoc) => void; onNewSplit: () => void }) {
+ const W = doc.width;
+ const H = doc.height;
+ const key = `${projectId}|${docId(doc)}`;
+ const [live, setLive] = useState(doc);
+ const [sels, setSelsState] = useState<string[]>(() => (lastSelected.get(key) ?? []).filter((id) => doc.layers.some((l) => l.id === id)));
+ const setSels = (ids: string[]) => {
+ lastSelected.set(key, ids);
+ setSelsState(ids);
+ };
+ /** The layer the panel shows and the prompt edits (the last one clicked). */
+ const selected: string | null = sels[sels.length - 1] ?? null;
+ /** Selects a layer with its group (it last, so the panel shows it). */
+ const setSelected = (id: string | null) => setSels(id ? [...withGroup(live.layers, id).filter((x) => x !== id), id] : []);
+ const multi = sels.length > 1;
+ const [infos, setInfos] = useState<Record<string, SourceInfo>>({});
+ const [tool, setTool] = useState<"move" | "pan" | "erase" | "shade" | "sketch" | "area">("move");
+ const [markColour, setMarkColour] = useState<MarkColour>("red");
+ const [sketch, setSketchState] = useState(() => lastMarks.get(key) ?? { marks: [], wants: {} });
+ const setSketch = (v: typeof sketch) => {
+ lastMarks.set(key, v);
+ setSketchState(v);
+ };
+ const marksRef = useRef<HTMLCanvasElement>(null);
+ const drawing = useRef<Mark | null>(null);
+ // Layer brushes: size in screen px, the pointer's place in frame px.
+ // The pencil has its own size: marks only need to be seen (thick marks hide the scene).
+ const [brushSize, setBrushSize] = useState(40);
+ const [pencilSize, setPencilSize] = useState(12);
+ const brush = tool === "sketch" ? pencilSize : brushSize;
+ const setBrush = (f: (b: number) => number) => (tool === "sketch" ? setPencilSize : setBrushSize)(f);
+ const [cursor, setCursor] = useState<{ x: number; y: number } | null>(null);
+ const paintRef = useRef<HTMLCanvasElement>(null);
+ const stroke = useRef<{ x: number; y: number } | null>(null);
+ const zoomK = useRef(1);
+ const [spacePressed, setSpacePressed] = useState(false);
+ const [busy, setBusy] = useState(false);
+ const [rowDrag, setRowDrag] = useState<{ from: number; over: number } | null>(null);
+ // The layer being renamed (double-click), so a click on a row leaves the arrows to nudging.
+ const [renaming, setRenaming] = useState<string | null>(null);
+ const [versionsOpen, setVersionsOpen] = useState(false);
+ const [inset, setInsetState] = useState(lastInset);
+ const setInset = (v: number) => {
+ lastInset = v;
+ setInsetState(v);
+ };
+ const [reach, setReachState] = useState(lastReach);
+ const setReach = (v: number) => {
+ lastReach = v;
+ setReachState(v);
+ };
+ useEffect(() => setVersionsOpen(false), [selected]);
+ const undoStack = useRef<LayerDoc[]>([]);
+ const redoStack = useRef<LayerDoc[]>([]);
+ const drag = useRef<Drag | null>(null);
+ const hitRef = useRef<HTMLDivElement>(null);
+ const rowsRef = useRef<(HTMLDivElement | null)[]>([]);
+ const lab = useLab();
+ const labJobs = useLabJobs(lab).filter((j) => docId(j.doc) === docId(doc));
+ // The area painted for the next edit, shown where it applies: on a layer's own image (moved with
+ // it), or on the whole frame (a fill, a shadow edit: images the lab made at the frame's size).
+ const mask = useStore((s) => s.composer.mask);
+ const target = useStore((s) => s.composer.refs[0]);
+ const composing = useStore((s) => s.composer.create);
+ const paintedSource = mask && target && mask.refKey === target.key && !composing ? target.source : null;
+ const paintedLayer = paintedSource ? doc.layers.find((l) => l.source === paintedSource) : undefined;
+ const paintedCrop = paintedSource ? doc.layers.flatMap((l) => (l.feeds ?? []).filter((f) => f.rect && f.source === paintedSource).map((f) => ({ l, f })))[0] : undefined;
+ const paintedFrame = !!paintedSource && !paintedLayer && !paintedCrop && (doc.layers.some((l) => l.feeds?.some((f) => f.source === paintedSource)) || !!doc.pending?.some((p) => p.source === paintedSource));
+ const maskTint = useMaskTint(mask && (paintedLayer || paintedFrame || paintedCrop) ? urls.input(mask.inputId) : null);
+ /** The edit being made for a row: its own, or (the background's row) a new "all objects" shadow. */
+ const rowJob = (l: DocLayer) => labJobs.find((j) => j.job.status !== "failed" && (j.layer === l.id || (l.base && j.layer === "shadows")));
+
+ useEffect(() => setLive(doc), [doc]);
+
+ useEffect(() => {
+ let gone = false;
+ for (const l of doc.layers) {
+ if (infos[l.source]) continue;
+ sourceInfo(l.source, W, H).then(
+ (info) => !gone && setInfos((m) => ({ ...m, [l.source]: info })),
+ (e) => !gone && toastError(e),
+ );
+ }
+ return () => {
+ gone = true;
+ };
+ }, [doc.layers, W, H, infos]);
+
+ const panning = spacePressed || tool === "pan";
+ const sel = live.layers.find((l) => l.id === selected) ?? null;
+ // A brush works on the selected layer: the eraser on any object or shadow, the shadow brush on
+ // shadow layers. Otherwise the pointer selects and moves.
+ const brushOn = (tool === "erase" || tool === "shade") && !panning && !!sel && !sel.base && !sel.hidden && (tool === "erase" || !!sel.effect);
+ // The painted area of the next edit (pink), brushed here: left adds, right (or Alt) takes out.
+ const areaOn = tool === "area" && !panning && !!maskTint;
+ const liveRef = useRef(live);
+ liveRef.current = live;
+ const xf = (l: DocLayer, d: LayerDoc = live): Xf => xfOf(d.layers, l, (src) => infos[src]?.box, W, H);
+
+ // The painted area at its own size (the image it applies to: the frame, a layer, or a layer's
+ // crop), changed by the Area brush and sent back to the prompt as a new mask when a stroke ends.
+ const areaCanvas = useRef<HTMLCanvasElement | null>(null);
+ const areaPreview = useRef<HTMLCanvasElement>(null);
+ const areaStroke = useRef<{ erase: boolean; last: { x: number; y: number } } | null>(null);
+ const areaUndo = useRef<string[]>([]);
+ const areaRedo = useRef<string[]>([]);
+ const maskId = maskTint ? mask?.inputId : undefined;
+ useEffect(() => {
+ areaCanvas.current = null;
+ if (!maskId) return;
+ let gone = false;
+ loadImage(urls.input(maskId))
+ .then((img) => {
+ if (gone) return;
+ const c = document.createElement("canvas");
+ c.width = img.naturalWidth;
+ c.height = img.naturalHeight;
+ c.getContext("2d")!.drawImage(img, 0, 0);
+ areaCanvas.current = c;
+ })
+ .catch(() => {});
+ return () => {
+ gone = true;
+ };
+ }, [maskId]);
+ // The stroke's preview goes once the new pink is drawn.
+ useEffect(() => {
+ const c = areaPreview.current;
+ c?.getContext("2d")?.clearRect(0, 0, c.width, c.height);
+ }, [maskTint]);
+ // A painted area opens on the Area brush (the area is what changes next).
+ useEffect(() => {
+ setTool((t) => (maskId && t === "move" ? "area" : !maskId && t === "area" ? "move" : t));
+ }, [maskId]);
+
+ /** A frame point in the area's px, and its px per frame px. */
+ const toArea = (p: { x: number; y: number }) => {
+ const c = areaCanvas.current;
+ if (!c) return null;
+ if (paintedCrop) {
+ const t = xf(paintedCrop.l);
+ const q = unmapPoint(t, p.x, p.y);
+ const r = paintedCrop.f.rect!;
+ const k = c.width / r.w;
+ return { x: (q.x - r.x) * k, y: (q.y - r.y) * k, k: k / t.s };
+ }
+ const k = c.width / W;
+ if (paintedLayer) {
+ const t = xf(paintedLayer);
+ const q = unmapPoint(t, p.x, p.y);
+ return { x: q.x * k, y: q.y * k, k: k / t.s };
+ }
+ return { x: p.x * k, y: p.y * k, k };
+ };
+
+ const areaTo = (p: { x: number; y: number }) => {
+ const st = areaStroke.current;
+ const c = areaCanvas.current;
+ if (!st || !c) return;
+ const r = brush / 2 / zoomK.current;
+ const a = toArea(st.last);
+ const b = toArea(p);
+ const line = (ctx: CanvasRenderingContext2D, x0: number, y0: number, x1: number, y1: number, w: number, colour: string) => {
+ ctx.strokeStyle = ctx.fillStyle = colour;
+ ctx.lineWidth = w;
+ ctx.lineCap = ctx.lineJoin = "round";
+ ctx.beginPath();
+ ctx.moveTo(x0, y0);
+ ctx.lineTo(x1, y1);
+ ctx.stroke();
+ };
+ if (a && b) line(c.getContext("2d")!, a.x, a.y, b.x, b.y, 2 * r * b.k, st.erase ? "#000" : "#fff");
+ const pv = areaPreview.current?.getContext("2d");
+ if (pv) line(pv, st.last.x, st.last.y, p.x, p.y, 2 * r, st.erase ? "rgba(20, 20, 20, 0.55)" : "rgba(255, 59, 107, 0.5)");
+ st.last = p;
+ };
+
+ const setArea = (inputId: string) => {
+ const m = getState().composer.mask;
+ if (m) updateComposer({ mask: { ...m, inputId } });
+ };
+ const endArea = async () => {
+ const c = areaCanvas.current;
+ areaStroke.current = null;
+ if (!c || !mask) return;
+ try {
+ const rec = await api.upload(await toPng(c), "mask.png");
+ areaUndo.current.push(mask.inputId);
+ areaRedo.current = [];
+ setArea(rec.id);
+ } catch (e) {
+ toastError(e);
+ }
+ };
+ /** Undo / Redo of the Area brush (its own steps, while it is the tool). */
+ const areaStep = (back: boolean) => {
+ const from = back ? areaUndo.current : areaRedo.current;
+ const id = from.pop();
+ if (!id || !mask) return false;
+ (back ? areaRedo.current : areaUndo.current).push(mask.inputId);
+ setArea(id);
+ return true;
+ };
+
+ /** An object-size crop of a layer for the model (cropFor): "edit" = the layer as it is with room
+ * around it (every prompt on an object layer); "clean" = the object with its selection grown ~3 px,
+ * filled from the photo there, so Qwen sees the real border and redraws it (Clean edges). Reused
+ * while the layer's version is the same. */
+ const cropOf = async (l: DocLayer, kind: "edit" | "clean"): Promise<Feed> => {
+ // An older crop of this version is reused, unless it is narrower than crops are now.
+ const have = l.feeds?.find((f) => f.rect && f.from === l.source && !f.clip && Math.max(f.rect.w / f.rect.h, f.rect.h / f.rect.w) <= CROP_MAX_RATIO + 0.01);
+ if (kind === "edit" && have) return have;
+ const info = await sourceInfo(l.source, W, H);
+ const { rect, scale } = cropFor(info.box, W, H, kind === "edit" ? 0.2 : 0.08);
+ const cw = Math.max(1, Math.round(rect.w * scale));
+ const ch = Math.max(1, Math.round(rect.h * scale));
+ // The image over the whole frame, moved and enlarged so the crop fills the canvas (the crop may
+ // reach past the frame: transparent there).
+ const draw = (img: HTMLImageElement, ctx: CanvasRenderingContext2D) => {
+ ctx.save();
+ ctx.imageSmoothingQuality = "high";
+ ctx.setTransform(cw / rect.w, 0, 0, ch / rect.h, (-rect.x * cw) / rect.w, (-rect.y * ch) / rect.h);
+ ctx.drawImage(img, 0, 0, W, H);
+ ctx.restore();
+ };
+ const c = document.createElement("canvas");
+ c.width = cw;
+ c.height = ch;
+ const ctx = c.getContext("2d")!;
+ let clip: string | undefined;
+ // ~3 px of the frame around the selection (in the enlarged crop: 3 x scale).
+ const g = Math.max(2, Math.round(3 * scale));
+ if (kind === "clean") {
+ const lc = document.createElement("canvas");
+ lc.width = cw;
+ lc.height = ch;
+ const lctx = lc.getContext("2d")!;
+ draw(info.img, lctx);
+ const px = lctx.getImageData(0, 0, cw, ch).data;
+ const sel0 = new Uint8Array(cw * ch);
+ for (let i = 0; i < sel0.length; i++) sel0[i] = px[i * 4 + 3] > 16 ? 255 : 0;
+ const grown = dilate(sel0, cw, ch, g);
+ const gc = document.createElement("canvas");
+ gc.width = cw;
+ gc.height = ch;
+ const gctx = gc.getContext("2d")!;
+ const ga = gctx.createImageData(cw, ch);
+ for (let i = 0; i < grown.length; i++) {
+ ga.data[i * 4] = ga.data[i * 4 + 1] = ga.data[i * 4 + 2] = 255;
+ ga.data[i * 4 + 3] = grown[i];
+ }
+ gctx.putImageData(ga, 0, 0);
+ // The whole photo around the object (when known), the layer over it. On transparency with a
+ // rim of photo, Qwen took the object for a sticker with a coloured border and kept the rim (bats,
+ // lettering, 11-01); seeing the scene, it knows what is background (the paste-back is clipped).
+ const photo = live.photo ? await loadImage(urls.source(live.photo)).catch(() => null) : null;
+ if (photo) draw(photo, ctx);
+ ctx.drawImage(lc, 0, 0);
+ // The clip for the paste-back: the grown selection, white on black.
+ const mk = document.createElement("canvas");
+ mk.width = cw;
+ mk.height = ch;
+ const mkx = mk.getContext("2d")!;
+ mkx.fillStyle = "#000";
+ mkx.fillRect(0, 0, cw, ch);
+ mkx.drawImage(gc, 0, 0);
+ clip = (await api.upload(await toPng(mk), "clip.png")).id;
+ } else draw(info.img, ctx);
+ const rec = await api.upload(await toPng(c), `${fileSafe(l.name)} (${kind === "edit" ? "object size" : "to clean"}).png`);
+ addVersions([rec], 0);
+ const f: Feed = { source: `input:${rec.id}`, mask: "", rect, scale, from: l.source, ...(clip ? { clip, grow: g } : {}) };
+ const d = liveRef.current;
+ commitRef.current({ ...d, layers: d.layers.map((x) => (x.id === l.id ? { ...x, feeds: [...(x.feeds ?? []), f] } : x)) });
+ return f;
+ };
+
+ // Selecting a layer (or a new version of it) makes it the image the prompt edits; its result comes
+ // back to the layer. An object layer is edited at its own size (an enlarged crop, cropOf), the
+ // background and shadows on the whole frame. Not when the view opens: the prompt keeps what it was
+ // set up for (a fill whose area was just painted again in Paint area).
+ // Several layers selected: the prompt stays as it is (Edit together sets it up for them).
+ const shownSel = useRef(`${selected}|${sel?.source}|${multi}`);
+ const selNow = useRef<string | null>(selected);
+ selNow.current = multi ? null : selected;
+ useEffect(() => {
+ const k = `${selected}|${sel?.source}|${multi}`;
+ if (k === shownSel.current) return;
+ shownSel.current = k;
+ if (!sel || multi) return;
+ if (sel.base || sel.effect) {
+ editSource(sel.source, false);
+ return;
+ }
+ const id = sel.id;
+ cropOf(sel, "edit").then(
+ (f) => selNow.current === id && editSource(f.source, false),
+ (e) => toastError(e),
+ );
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [selected, sel?.source, multi]);
+
+ /** Clean edges: the object with a little of what is around it, and a prompt to keep only the
+ * object, ready in the prompt box (the result is clipped to the grown selection when it comes back). */
+ const cleanEdges = async (l: DocLayer) => {
+ setBusy(true);
+ try {
+ const f = await cropOf(l, "clean");
+ if (!editSource(f.source, false)) throw new Error("The object could not be opened.");
+ updateComposer({ mask: null, outpaint: null, prompt: stickerPrompt(l.name) });
+ focusPrompt(true);
+ toastInfo("The object is ready to clean at its own size: check the prompt, then Generate. The result replaces the layer, inside its outline.");
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ // Stickers: after a split with "Clean the stickers automatically", every object goes to Clean edges
+ // on its own (a crop at object size and the sticker prompt, sent as lab steps); the service runs
+ // them one after the other and each result comes back to its layer like any Clean edges.
+ const stickersStarted = useRef(false);
+ const setSticker = (layer: string, p: Partial<Sticker>) => {
+ const d = liveRef.current;
+ const next = { ...d, stickers: (d.stickers ?? []).map((s) => (s.layer === layer ? { ...s, ...p } : s)) };
+ liveRef.current = next;
+ setLive(next);
+ onChange(next);
+ };
+ const makeStickers = async () => {
+ const d = liveRef.current;
+ const objects = d.layers.filter((l) => !l.base && !l.effect);
+ const base = d.layers.find((l) => l.base);
+ const start = { ...d, autoClean: undefined, stickers: [...objects.map((l): Sticker => ({ layer: l.id, job: null })), ...(base ? [{ layer: base.id, job: null, kind: "fill" as const }] : [])] };
+ liveRef.current = start;
+ setLive(start);
+ onChange(start);
+ // The crops become versions (and the prompt's image) on the way: put back what was there.
+ const before = getState();
+ const keep = { selected: before.selected, refs: before.composer.refs };
+ for (const l of objects) {
+ if (liveRef.current.stickers?.find((s) => s.layer === l.id && !s.kind)?.skipped) continue;
+ try {
+ const f = await cropOf(liveRef.current.layers.find((x) => x.id === l.id) ?? l, "clean");
+ const it = getState().session.find((i) => itemSource(i) === f.source);
+ if (!it?.input) throw new Error(`The crop of ${l.name} is not in the project.`);
+ // Skipped while its crop was made.
+ if (liveRef.current.stickers?.find((s) => s.layer === l.id)?.skipped) continue;
+ const c = getState().composer;
+ const req = buildRequest({ ...c, prompt: stickerPrompt(l.name), refs: [refFromInput(it.input)], mask: null, outpaint: null, create: false, mode: "auto", enhance: false, enhancedFrom: null, seed: null });
+ const job = await submitRequest({ ...req, hidden: true });
+ setSticker(l.id, job ? { job, from: l.source } : { failed: true });
+ } catch (e) {
+ console.error(e);
+ setSticker(l.id, { failed: true });
+ }
+ }
+ // Then the background: its holes filled, shadows and leftovers of the objects taken off.
+ const bg = liveRef.current.layers.find((x) => x.base);
+ if (bg && !liveRef.current.stickers?.find((s) => s.kind)?.skipped) {
+ try {
+ const { rec, mask } = await prepareFill(bg);
+ const job = await submitRequest(fillRequest(rec, mask));
+ setSticker(bg.id, job ? { job, from: bg.source, mask } : { failed: true });
+ } catch (e) {
+ console.error(e);
+ setSticker(bg.id, { failed: true });
+ }
+ }
+ setState((s) => ({ selected: s.session.some((i) => i.key === keep.selected) ? keep.selected : s.selected, composer: { ...s.composer, refs: keep.refs } }));
+ };
+ useEffect(() => {
+ if (!live.autoClean || stickersStarted.current) return;
+ stickersStarted.current = true;
+ void makeStickers();
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+ const jobs = useStore((s) => s.jobs);
+ const session = useStore((s) => s.session);
+ /** Where an object's sticker is. */
+ const stickerState = (s: Sticker): "waiting" | "making" | "done" | "skipped" | "failed" => {
+ if (s.skipped) return "skipped";
+ if (s.failed) return "failed";
+ const j = s.job ? jobs[s.job] : undefined;
+ if (!j) return s.job ? "done" : "waiting";
+ // Done once the result is pasted into the layer (a moment after the job).
+ if (j.status === "done") {
+ // The fill counts once its result was checked for grey patches (or it was the second try).
+ if (s.kind && !s.checked && !s.retried) return "making";
+ return s.from && live.layers.find((l) => l.id === s.layer)?.source === s.from ? "making" : "done";
+ }
+ if (j.status === "cancelled") return "skipped";
+ if (j.status === "failed") return "failed";
+ return j.status === "queued" ? "waiting" : "making";
+ };
+ // A fill that kept grey patches (Qwen copies the flat grey now and then, 11-01) is sent once more.
+ const checkedFill = useRef(new Set<string>());
+ useEffect(() => {
+ const s = live.stickers?.find((x) => x.kind && x.job && !x.retried && !x.skipped);
+ if (!s?.job || !s.mask || jobs[s.job]?.status !== "done" || checkedFill.current.has(s.job)) return;
+ const gen = session.find((i) => i.jobId === s.job)?.gen;
+ if (!gen) return;
+ checkedFill.current.add(s.job);
+ const mask = s.mask;
+ (async () => {
+ const [img, m] = await Promise.all([loadImage(urls.source(`gen:${gen.id}`)), loadImage(urls.input(mask))]);
+ const w = Math.min(512, img.naturalWidth);
+ const h = Math.max(1, Math.round((w * img.naturalHeight) / img.naturalWidth));
+ const a = pixels(img, w, h);
+ const b = pixels(m, w, h);
+ let inside = 0;
+ let grey = 0;
+ for (let i = 0; i < w * h; i++) {
+ if (b[i * 4] < 128) continue;
+ inside++;
+ if (Math.abs(a[i * 4] - 128) <= 12 && Math.abs(a[i * 4 + 1] - 128) <= 12 && Math.abs(a[i * 4 + 2] - 128) <= 12) grey++;
+ }
+ const rec = gen.request?.images?.[0] && session.find((i) => itemSource(i) === gen.request!.images[0])?.input;
+ if (!inside || grey / inside < 0.02 || !rec) return setSticker(s.layer, { checked: true });
+ toastInfo("The background fill kept grey patches: trying once more.");
+ const job = await submitRequest(fillRequest(rec, mask));
+ setSticker(s.layer, job ? { job, retried: true } : { checked: true });
+ })().catch((e) => {
+ console.error(e);
+ setSticker(s.layer, { checked: true });
+ });
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [jobs, session, live.stickers]);
+
+ const skipSticker = (s: Sticker) => {
+ if (s.job) void cancelJob(s.job);
+ setSticker(s.layer, { skipped: true });
+ };
+ const stickerRows = (live.stickers ?? []).filter((s) => live.layers.some((l) => l.id === s.layer));
+ const stickerStates = stickerRows.map(stickerState);
+ const stickersLeft = stickerStates.filter((x) => x === "waiting" || x === "making").length;
+ /** Stickers are being made: the Stage shows the objects on transparency, the one being made ringed. */
+ const stickering = stickersLeft > 0;
+ const stickerOf = (id: string) => {
+ const i = stickerRows.findIndex((s) => s.layer === id);
+ return i < 0 ? null : stickerStates[i];
+ };
+ const stickerNow = stickerRows[stickerStates.indexOf("making")] ?? stickerRows[stickerStates.indexOf("waiting")];
+ // All in: the list goes and the background fades back under them; the ones not cleaned are named.
+ useEffect(() => {
+ if (!stickerRows.length || stickersLeft) return;
+ const failed = stickerRows.filter((_, i) => stickerStates[i] === "failed").map((s) => live.layers.find((l) => l.id === s.layer)?.name ?? "");
+ toastInfo(failed.length ? `Stickers done; not cleaned: ${failed.join(", ")}. Select one and use Clean edges.` : "Stickers done. Not clean enough? Select the object and use Clean edges, or the Eraser.");
+ const d = { ...liveRef.current, stickers: undefined };
+ liveRef.current = d;
+ setLive(d);
+ onChange(d);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [stickersLeft, stickerRows.length]);
+
+ const commit = (next: LayerDoc) => {
+ undoStack.current = [...undoStack.current, doc].slice(-50);
+ redoStack.current = [];
+ setLive(next);
+ onChange(next);
+ };
+ // For work that ends after an upload: the latest render's commit (the document may have changed).
+ const commitRef = useRef(commit);
+ commitRef.current = commit;
+ const patchLayer = (id: string, p: Partial<DocLayer>) => commit({ ...live, layers: live.layers.map((l) => (l.id === id ? { ...l, ...p } : l)) });
+
+ const undo = () => {
+ const prev = undoStack.current.pop();
+ if (!prev) return;
+ redoStack.current.push(live);
+ setLive(prev);
+ onChange(prev);
+ };
+ const redo = () => {
+ const next = redoStack.current.pop();
+ if (!next) return;
+ undoStack.current.push(live);
+ setLive(next);
+ onChange(next);
+ };
+
+ /** Removes these layers (not the background), and the shadows that follow them. */
+ const removeLayers = (ids: string[]) => {
+ const gone = new Set(live.layers.filter((l) => ids.includes(l.id) && !l.base).map((l) => l.id));
+ if (!gone.size) return;
+ commit({ ...live, layers: live.layers.filter((x) => !gone.has(x.id) && !(x.follows && gone.has(x.follows))) });
+ setSels([]);
+ };
+
+ /** Selected layers that move (objects: the background and shadows stay where they are). */
+ const movable = (ids: string[]) =>
+ live.layers.filter(
+ (l) =>
+ ids.includes(l.id) &&
+ !l.base &&
+ infos[l.source] &&
+ // A shadow that follows an object moved with it already moves with it.
+ !(l.follows && ids.includes(l.follows)),
+ );
+
+ /** Moves the layers by (dx, dy) and scales them by `k` around `c` (frame px), from `l0`. */
+ const placeAll = (ids: string[], l0: Record<string, DocLayer>, dx: number, dy: number, k = 1, c = { x: 0, y: 0 }) => {
+ setLive((cur) => ({
+ ...cur,
+ layers: cur.layers.map((l) => {
+ const o = l0[l.id];
+ const info = o && ids.includes(l.id) ? infos[o.source] : undefined;
+ if (!o || !info) return l;
+ // Its content's centre where it is drawn, moved/scaled with the others around c.
+ const own = centre(info.box);
+ const at = { x: own.x + o.dx, y: own.y + o.dy };
+ return { ...l, s: Math.min(20, Math.max(0.05, o.s * k)), dx: c.x + k * (at.x - c.x) - own.x + dx, dy: c.y + k * (at.y - c.y) - own.y + dy };
+ }),
+ }));
+ };
+
+ const toFrame = (e: React.PointerEvent) => {
+ const r = hitRef.current!.getBoundingClientRect();
+ return { x: ((e.clientX - r.left) / r.width) * W, y: ((e.clientY - r.top) / r.height) * H };
+ };
+
+ /** The top visible layer with something under the point (shadow layers are picked in the list). */
+ const hit = (x: number, y: number): DocLayer | null => {
+ for (const l of live.layers) {
+ const info = infos[l.source];
+ if (l.hidden || l.effect || !info) continue;
+ const q = unmapPoint(xf(l), x, y);
+ const ix = Math.floor((q.x / W) * info.aw);
+ const iy = Math.floor((q.y / H) * info.ah);
+ if (ix < 0 || iy < 0 || ix >= info.aw || iy >= info.ah) continue;
+ if (info.alpha[iy * info.aw + ix] > 32) return l;
+ }
+ return null;
+ };
+
+ // While a brush is on, the selected layer is drawn on a canvas the brush paints into.
+ const paintInfo = brushOn && sel ? infos[sel.source] : undefined;
+ useEffect(() => {
+ const c = paintRef.current;
+ if (!c || !paintInfo || stroke.current) return;
+ const ctx = c.getContext("2d")!;
+ ctx.clearRect(0, 0, W, H);
+ ctx.drawImage(paintInfo.img, 0, 0, W, H);
+ }, [paintInfo, W, H]);
+
+ /** Brush dabs from the last point to frame point `p`, in the layer's own pixels. */
+ const paintTo = (p: { x: number; y: number }) => {
+ const c = paintRef.current;
+ if (!c || !sel) return;
+ const t = xf(sel);
+ const q = unmapPoint(t, p.x, p.y);
+ const r = Math.max(1, brush / 2 / zoomK.current / t.s);
+ const from = stroke.current ?? q;
+ const n = Math.max(1, Math.ceil(Math.hypot(q.x - from.x, q.y - from.y) / Math.max(1, r / 4)));
+ const ctx = c.getContext("2d")!;
+ for (let i = stroke.current ? 1 : 0; i <= n; i++) dab(ctx, from.x + ((q.x - from.x) * i) / n, from.y + ((q.y - from.y) * i) / n, r, tool === "erase" ? "erase" : "shade");
+ stroke.current = q;
+ };
+
+ /** A stroke ends: the layer as painted becomes its next version (Undo and Versions go back). */
+ const endStroke = async () => {
+ const c = paintRef.current;
+ const l = sel;
+ stroke.current = null;
+ if (!c || !l) return;
+ setBusy(true);
+ try {
+ const rec = await api.upload(await toPng(c), `${fileSafe(l.name)} (brushed).png`);
+ addVersions([rec], 0);
+ const src = `input:${rec.id}`;
+ const d = liveRef.current;
+ commitRef.current({
+ ...d,
+ layers: d.layers.map((x) => (x.id === l.id ? { ...x, source: src, history: [...x.history, src] } : x)),
+ known: [...d.known, src],
+ });
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ /** The drawn box around layers (union of their content where they are). */
+ const boxOfAll = (ls: DocLayer[]): Box | null => {
+ let b: Box | null = null;
+ for (const l of ls) {
+ const info = infos[l.source];
+ if (!info || l.hidden) continue;
+ const m = mapBox(info.box, xf(l));
+ b = b ? { x0: Math.min(b.x0, m.x0), y0: Math.min(b.y0, m.y0), x1: Math.max(b.x1, m.x1), y1: Math.max(b.y1, m.y1) } : m;
+ }
+ return b;
+ };
+
+ const startDrag = (e: React.PointerEvent, mode: Drag["mode"], ids: string[]) => {
+ const ls = movable(ids);
+ const box = boxOfAll(ls);
+ if (!ls.length || !box) return;
+ e.stopPropagation();
+ try {
+ hitRef.current?.setPointerCapture(e.pointerId);
+ } catch {}
+ const p = toFrame(e);
+ drag.current = { mode, ids: ls.map((l) => l.id), x0: p.x, y0: p.y, l0: Object.fromEntries(ls.map((l) => [l.id, l])), c: centre(box) };
+ };
+
+ // The pencil: marks in frame px, kept per document until they are sent with New objects.
+ useEffect(() => {
+ const c = marksRef.current;
+ if (!c) return;
+ const ctx = c.getContext("2d")!;
+ ctx.clearRect(0, 0, W, H);
+ for (const m of [...sketch.marks, ...(drawing.current ? [drawing.current] : [])]) drawMark(ctx, m);
+ });
+
+ const onDown = (e: React.PointerEvent) => {
+ if (areaOn && (e.button === 0 || e.button === 2)) {
+ e.stopPropagation();
+ try {
+ hitRef.current?.setPointerCapture(e.pointerId);
+ } catch {}
+ const p = toFrame(e);
+ areaStroke.current = { erase: e.button === 2 || e.altKey, last: p };
+ areaTo(p);
+ return;
+ }
+ if (e.button !== 0 || panning) return;
+ const p = toFrame(e);
+ if (brushOn) {
+ e.stopPropagation();
+ try {
+ hitRef.current?.setPointerCapture(e.pointerId);
+ } catch {}
+ stroke.current = null;
+ paintTo(p);
+ return;
+ }
+ if (tool === "sketch") {
+ e.stopPropagation();
+ try {
+ hitRef.current?.setPointerCapture(e.pointerId);
+ } catch {}
+ drawing.current = { colour: markColour, r: Math.max(2, brush / 2 / zoomK.current), pts: [p] };
+ const ctx = marksRef.current?.getContext("2d");
+ if (ctx) drawMark(ctx, drawing.current);
+ setCursor(p);
+ return;
+ }
+ const inSel = !e.shiftKey && !e.ctrlKey && !e.metaKey && !e.altKey && selLayers.some((x) => x.effect) ? boxOfAll(selLayers.filter((x) => !x.hidden)) : null;
+ if (inSel && p.x >= inSel.x0 && p.x <= inSel.x1 && p.y >= inSel.y0 && p.y <= inSel.y1) {
+ startDrag(e, "move", sels);
+ return;
+ }
+ const l = hit(p.x, p.y);
+ // Shift / Ctrl / Cmd: add or take out (with its group); Alt: this layer alone, even in a group.
+ if (l && (e.shiftKey || e.ctrlKey || e.metaKey)) {
+ const ids = withGroup(live.layers, l.id);
+ const on = sels.includes(l.id);
+ setSels(on ? sels.filter((x) => !ids.includes(x)) : [...sels.filter((x) => !ids.includes(x)), ...ids.filter((x) => x !== l.id), l.id]);
+ return;
+ }
+ if (!l) {
+ setSels([]);
+ return;
+ }
+ // A click on a layer already selected keeps the selection, so the drag moves all of them.
+ const ids = e.altKey ? [l.id] : sels.includes(l.id) ? [...sels.filter((x) => x !== l.id), l.id] : [...withGroup(live.layers, l.id).filter((x) => x !== l.id), l.id];
+ setSels(ids);
+ startDrag(e, "move", ids);
+ };
+
+ const onMove = (e: React.PointerEvent) => {
+ if (areaOn) {
+ const p = toFrame(e);
+ setCursor(p);
+ if (areaStroke.current) {
+ e.stopPropagation();
+ areaTo(p);
+ }
+ return;
+ }
+ if (brushOn || tool === "sketch") {
+ const p = toFrame(e);
+ setCursor(p);
+ if (brushOn && stroke.current) {
+ e.stopPropagation();
+ paintTo(p);
+ }
+ if (drawing.current) {
+ e.stopPropagation();
+ drawing.current.pts.push(p);
+ const ctx = marksRef.current?.getContext("2d");
+ if (ctx) drawMark(ctx, drawing.current, drawing.current.pts.length - 2);
+ }
+ return;
+ }
+ const d = drag.current;
+ if (!d) return;
+ e.stopPropagation();
+ const p = toFrame(e);
+ if (d.mode === "move") placeAll(d.ids, d.l0, p.x - d.x0, p.y - d.y0);
+ else {
+ const r0 = Math.hypot(d.x0 - d.c.x, d.y0 - d.c.y) || 1;
+ placeAll(d.ids, d.l0, 0, 0, Math.hypot(p.x - d.c.x, p.y - d.c.y) / r0, d.c);
+ }
+ };
+
+ const onUp = () => {
+ if (areaStroke.current) {
+ void endArea();
+ return;
+ }
+ if (stroke.current) {
+ void endStroke();
+ return;
+ }
+ if (drawing.current) {
+ const m = drawing.current;
+ drawing.current = null;
+ setSketch({ ...sketch, marks: [...sketch.marks, m] });
+ return;
+ }
+ const d = drag.current;
+ if (!d) return;
+ drag.current = null;
+ const moved = live.layers.some((l) => d.l0[l.id] && (l.dx !== d.l0[l.id].dx || l.dy !== d.l0[l.id].dy || l.s !== d.l0[l.id].s));
+ if (moved) commit(live);
+ };
+
+ // Reorder by dragging a row's grip (pointer events: the window's file drop takes HTML drag and drop).
+ const onGripDown = (e: React.PointerEvent, i: number) => {
+ e.stopPropagation();
+ (e.target as HTMLElement).setPointerCapture(e.pointerId);
+ setRowDrag({ from: i, over: i });
+ };
+ const onGripMove = (e: React.PointerEvent) => {
+ if (!rowDrag) return;
+ // The new index once the row is taken out: how many other rows are above the pointer.
+ let over = 0;
+ rowsRef.current.slice(0, live.layers.length).forEach((el, i) => {
+ if (!el || i === rowDrag.from) return;
+ const r = el.getBoundingClientRect();
+ if (e.clientY > r.top + r.height / 2) over++;
+ });
+ if (over !== rowDrag.over) setRowDrag({ ...rowDrag, over });
+ };
+ const onGripUp = () => {
+ if (!rowDrag) return;
+ const next = reorder(live, rowDrag.from, rowDrag.over);
+ setRowDrag(null);
+ if (next !== live) commit(next);
+ };
+
+ /** The background's holes as a brushed Add ready in the prompt, with a prompt to fine-tune: the
+ * background with its holes flat grey (opaque: an edit of the transparent background keeps the
+ * holes; no objects in them, as on the photo) and the holes as the painted area. Add keeps the
+ * image as it is under the area (Remove blurs it first and, empty, clears "everything that was
+ * there"). The result comes back as the background's next version. */
+ /** The background's holes ready to fill: the image (holes grey) as a version, and the area. */
+ const prepareFill = async (l: DocLayer) => {
+ const info = await sourceInfo(l.source, W, H);
+ const k = Math.min(1, 2048 / Math.max(W, H));
+ const mw = Math.max(1, Math.round(W * k));
+ const mh = Math.max(1, Math.round(H * k));
+ const c = document.createElement("canvas");
+ c.width = mw;
+ c.height = mh;
+ const ctx = c.getContext("2d")!;
+ ctx.drawImage(info.img, 0, 0, mw, mh);
+ const px = ctx.getImageData(0, 0, mw, mh).data;
+ let hole: Uint8Array = new Uint8Array(mw * mh);
+ for (let i = 0; i < hole.length; i++) hole[i] = px[i * 4 + 3] < 200 ? 255 : 0;
+ // Wider than the holes: the objects' fringe and the contact shadows at their feet (1.5 %: 0.6 %
+ // left the shadows, 11-01 bench; the Area brush widens it for longer ones).
+ hole = dilate(hole, mw, mh, Math.max(2, Math.round(Math.max(mw, mh) * 0.015)));
+ const m = ctx.createImageData(mw, mh);
+ for (let i = 0; i < hole.length; i++) {
+ m.data[i * 4] = m.data[i * 4 + 1] = m.data[i * 4 + 2] = hole[i];
+ m.data[i * 4 + 3] = 255;
+ }
+ ctx.putImageData(m, 0, 0);
+ const out = document.createElement("canvas");
+ out.width = mw;
+ out.height = mh;
+ const o = out.getContext("2d")!;
+ o.filter = "blur(4px)"; // the soft edge of a painted area (InpaintEditor FEATHER)
+ o.drawImage(c, 0, 0);
+ const mask = await api.upload(await toPng(out), "mask.png");
+
+ const g = document.createElement("canvas");
+ g.width = W;
+ g.height = H;
+ const gc = g.getContext("2d")!;
+ gc.fillStyle = "#808080";
+ gc.fillRect(0, 0, W, H);
+ gc.drawImage(info.img, 0, 0, W, H);
+ const rec = await api.upload(await toPng(g), "Background to fill.png");
+ addVersions([rec], 0);
+ const target = `input:${rec.id}`;
+ const d = liveRef.current;
+ commitRef.current({ ...d, layers: d.layers.map((x) => (x.id === l.id ? { ...x, feeds: [...(x.feeds ?? []), { source: target, mask: mask.id }] } : x)) });
+ return { rec, target, mask: mask.id };
+ };
+ /** The fill as a request (the automatic one after a split): an Add in the holes, a lab step. */
+ const fillRequest = (rec: InputRecord, mask: string) => {
+ const ref = refFromInput(rec);
+ const c = getState().composer;
+ return { ...buildRequest({ ...c, prompt: FILL_PROMPT, refs: [ref], mask: { inputId: mask, refKey: ref.key }, inpaintAction: "add", outpaint: null, create: false, mode: "auto", enhance: false, enhancedFrom: null, seed: null }), hidden: true };
+ };
+
+ const fillHoles = async (l: DocLayer) => {
+ setBusy(true);
+ try {
+ const { target, mask } = await prepareFill(l);
+ if (!editSource(target, false)) throw new Error("The background to fill could not be opened.");
+ const refKey = getState().composer.refs[0]?.key;
+ // Kokon wraps it: "In <image1>, add {this} inside the area marked …" (pipeline.rs inpaint_prompt).
+ if (refKey) updateComposer({ mask: { inputId: mask, refKey }, outpaint: null, mode: "auto", inpaintAction: "add", prompt: FILL_PROMPT });
+ focusPrompt(true);
+ toastInfo("The holes are painted as an Add with a prompt to fine-tune (e.g. name the surfaces: “the wooden table top and the tiled floor”), then Generate. The result becomes the background here.");
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ /** The visible layers drawn into one `w`×`h` image (`skip`: layers left out; `only`: just these). */
+ const render = async (w: number, h: number, skip?: Set<string>, only?: DocLayer[]) => {
+ const c = document.createElement("canvas");
+ c.width = w;
+ c.height = h;
+ const ctx = c.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ ctx.scale(w / W, h / H);
+ for (const l of [...(only ?? live.layers)].reverse()) {
+ if (l.hidden || skip?.has(l.id)) continue;
+ const info = await sourceInfo(l.source, W, H);
+ const t = xfOf(live.layers, l, (src) => (src === l.source ? info.box : infos[src]?.box), W, H);
+ ctx.save();
+ ctx.translate(t.c.x + t.dx, t.c.y + t.dy);
+ ctx.scale(t.s, t.s);
+ ctx.translate(-t.c.x, -t.c.y);
+ ctx.drawImage(info.img, 0, 0, W, H);
+ ctx.restore();
+ }
+ return c;
+ };
+
+ /** The background still has holes (grey in what is sent): Fill the holes first. */
+ const hasHoles = async () => {
+ const base = live.layers.find((l) => l.base);
+ const info = base && !base.hidden ? await sourceInfo(base.source, W, H) : undefined;
+ return !!info && info.alpha.some((a) => a < 200);
+ };
+
+ /** The colours the pencil has used, in the palette's order. */
+ const usedColours = MARK_COLOURS.map((c) => c.name).filter((c) => sketch.marks.some((m) => m.colour === c));
+ const wantsNow = () => usedColours.map((colour) => ({ colour, what: sketch.wants[colour] ?? "" }));
+
+ /** Sets up a whole-image edit in the prompt (LabScene reads its result): the flattened layers on
+ * grey (holes left would keep the image transparent), with the pencil's marks drawn on when there
+ * are any; `shadows`: objects whose shadows are read back (their shadow layers are left out of
+ * what is sent, so Qwen makes them again); `look`: objects that take the edit's look. */
+ const sendScene = async (o: { shadows: string[]; look: string[]; prompt: string; marks: boolean; hint: string }) => {
+ setBusy(true);
+ try {
+ const skip = new Set(live.layers.filter((l) => l.effect && (o.shadows.includes(l.follows ?? "") || (!l.follows && o.shadows.length > 0))).map((l) => l.id));
+ const flat = document.createElement("canvas");
+ flat.width = W;
+ flat.height = H;
+ const fctx = flat.getContext("2d")!;
+ fctx.fillStyle = "#808080";
+ fctx.fillRect(0, 0, W, H);
+ fctx.drawImage(await render(W, H, skip), 0, 0);
+ const before = await api.upload(await toPng(flat), "Layers for the scene.png");
+ let source = before;
+ let marksId: string | undefined;
+ if (o.marks && sketch.marks.length) {
+ const mc = document.createElement("canvas");
+ mc.width = W;
+ mc.height = H;
+ const mctx = mc.getContext("2d")!;
+ for (const m of sketch.marks) drawMark(mctx, m);
+ marksId = (await api.upload(await toPng(mc), "marks.png")).id;
+ fctx.drawImage(mc, 0, 0);
+ source = await api.upload(await toPng(flat), "Layers with marks.png");
+ }
+ addVersions([source], 0);
+ const target = `input:${source.id}`;
+ if (!editSource(target, false)) throw new Error("The flattened layers could not be opened.");
+ updateComposer({ mask: null, outpaint: null, mode: "auto", prompt: o.prompt });
+ const scene: SceneEdit = {
+ source: target,
+ before: `input:${before.id}`,
+ ...(marksId ? { marks: marksId, wants: wantsNow() } : {}),
+ shadows: o.shadows,
+ look: o.look,
+ places: placesOf(live),
+ reach: SCENE_REACH,
+ };
+ commit({ ...live, scenes: [...(live.scenes ?? []), scene] });
+ if (marksId) setSketch({ marks: [], wants: sketch.wants });
+ setSels([]);
+ focusPrompt(true);
+ toastInfo((await hasHoles()) ? "Ready in the prompt, but the background still has holes (grey in the image): Fill the holes first." : o.hint);
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ /** Sets up a shadow edit in the prompt (the way that works, 10-31): the flattened layers
+ * (without the shadow layer it replaces) as the image, a ring around the object (or every object)
+ * as the painted area, reaching `inset` px into the objects (their bottom edge is redrawn with the
+ * contact shadow); deeper the objects are left out. */
+ const addShadows = async (of: DocLayer | null) => {
+ setBusy(true);
+ try {
+ const objects = of ? [of] : live.layers.filter((l) => !l.base && !l.effect && !l.hidden);
+ if (!objects.length) throw new Error("No visible object to make shadows for.");
+ const existing = live.layers.find((l) => l.effect && (of ? l.follows === of.id : !l.follows));
+ // On grey: holes the background still has would make the image transparent, and an edit of a
+ // transparent image keeps its transparency.
+ const layersOnly = await render(W, H, existing ? new Set([existing.id]) : undefined);
+ const flat = document.createElement("canvas");
+ flat.width = W;
+ flat.height = H;
+ const fctx = flat.getContext("2d")!;
+ fctx.fillStyle = "#808080";
+ fctx.fillRect(0, 0, W, H);
+ fctx.drawImage(layersOnly, 0, 0);
+ const holes = await hasHoles();
+ const k = Math.min(1, 2048 / Math.max(W, H));
+ const mw = Math.max(1, Math.round(W * k));
+ const mh = Math.max(1, Math.round(H * k));
+ const alphaOf = async (ls: DocLayer[]) => {
+ const px = (await render(mw, mh, undefined, ls)).getContext("2d")!.getImageData(0, 0, mw, mh).data;
+ const a = new Uint8Array(mw * mh);
+ for (let i = 0; i < a.length; i++) a[i] = px[i * 4 + 3] > 128 ? 255 : 0;
+ return a;
+ };
+ const obj = await alphaOf(objects);
+ // Every visible object stays out of the painted area, not only the one getting its shadow.
+ const all = of ? await alphaOf(live.layers.filter((l) => !l.base && !l.effect && !l.hidden)) : obj;
+ // The area: mostly below the object (where contact shadows fall), half as far sideways, a little
+ // above; `reach` % of the side (the Shadow reach slider).
+ const r = Math.max(4, Math.round((Math.max(mw, mh) * reach) / 100));
+ const grown = grow(obj, mw, mh, { down: r, left: r * 0.5, right: r * 0.5, up: r * 0.15 });
+ // The objects' own outline `inset` px deep is in the area too (left out: everything deeper).
+ const kept = inset > 0 ? erode(all, mw, mh, Math.max(1, Math.round(inset * k))) : all;
+ const mc = document.createElement("canvas");
+ mc.width = mw;
+ mc.height = mh;
+ const mctx = mc.getContext("2d")!;
+ const m = mctx.createImageData(mw, mh);
+ for (let i = 0; i < obj.length; i++) {
+ const v = grown[i] && !kept[i] ? 255 : 0;
+ m.data[i * 4] = m.data[i * 4 + 1] = m.data[i * 4 + 2] = v;
+ m.data[i * 4 + 3] = 255;
+ }
+ mctx.putImageData(m, 0, 0);
+ const soft = document.createElement("canvas");
+ soft.width = mw;
+ soft.height = mh;
+ const sctx = soft.getContext("2d")!;
+ sctx.filter = "blur(3px)";
+ sctx.drawImage(mc, 0, 0);
+ const mask = await api.upload(await toPng(soft), "mask.png");
+ const rec = await api.upload(await toPng(flat), "Layers for shadows.png");
+ addVersions([rec], 0);
+ const target = `input:${rec.id}`;
+ if (!editSource(target, false)) throw new Error("The flattened layers could not be opened.");
+ const refKey = getState().composer.refs[0]?.key;
+ const what = of ? `the ${of.name.toLowerCase()}` : "the objects";
+ const prompt = `Add a natural contact shadow under ${what} and the soft shadow ${of ? "it casts" : "they cast"} on the surface, matching the light of the scene. Keep ${what} and everything else unchanged.`;
+ if (refKey) updateComposer({ mask: { inputId: mask.id, refKey }, outpaint: null, mode: "auto", inpaintAction: "edit", prompt });
+ const effect: Effect = { source: target, mask: mask.id, of: of?.id ?? null, at: of ? { dx: of.dx, dy: of.dy, s: of.s } : undefined, layer: existing?.id };
+ commit({ ...live, pending: [...(live.pending ?? []), effect] });
+ focusPrompt(true);
+ toastInfo(
+ holes
+ ? "Ready in the prompt, but the background still has holes (grey here): Fill the holes first for real shadows."
+ : "The shadow area is painted on the flattened layers and the prompt is ready: change it if you like, then Generate. The result becomes a shadow layer here.",
+ );
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ const newObjects = () =>
+ sendScene({
+ shadows: [],
+ look: [],
+ marks: true,
+ prompt: marksPrompt(wantsNow()),
+ hint: "The whole scene with your marks is in the prompt (Prompt+ sees the marks too), then Generate. Each new object comes back as its own layer at the marks' place and size (no shadow: Add shadow makes it).",
+ });
+
+ /** Several objects edited as one scene: the prompt is the user's; they take the edit's look. */
+ const editTogether = (objects: DocLayer[]) =>
+ sendScene({
+ shadows: [],
+ look: objects.map((l) => l.id),
+ marks: sketch.marks.length > 0,
+ prompt: sketch.marks.length ? marksPrompt(wantsNow()) : "",
+ hint: `Write what to change in the whole scene, then Generate: the ${objects.length} selected objects take the result's look, the other layers stay.`,
+ });
+
+ const setGroup = (ids: string[], group: string | undefined) => commit({ ...live, layers: live.layers.map((l) => (ids.includes(l.id) && !l.base ? { ...l, group } : l)) });
+
+ const flatten = async () => {
+ setBusy(true);
+ try {
+ // The one image of the lab that goes to the gallery (its steps are kept out of it).
+ const rec = await api.upload(await toPng(await render(W, H)), "Layers flattened.png");
+ addResult(await api.saveLayers(`input:${rec.id}`, live.photo ?? null, getState().privateMode));
+ setState({ overlay: null });
+ toastInfo("Saved the layers as one image: a new version, and in All images (the layers stay: open Layers lab again).");
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ useEffect(() => {
+ const onKeyDown = (e: KeyboardEvent) => {
+ if (isTyping(e.target)) return;
+ const k = e.key.toLowerCase();
+ if (e.code === "Space" && !e.repeat) {
+ e.preventDefault();
+ setSpacePressed(true);
+ } else if ((e.ctrlKey || e.metaKey) && k === "z") {
+ e.preventDefault();
+ if (!(tool === "area" && areaStep(!e.shiftKey))) e.shiftKey ? redo() : undo();
+ } else if ((e.ctrlKey || e.metaKey) && k === "y") {
+ e.preventDefault();
+ if (!(tool === "area" && areaStep(false))) redo();
+ } else if ((e.ctrlKey || e.metaKey) && k === "a") {
+ // Every visible object (not the background, not shadows).
+ e.preventDefault();
+ setSels(live.layers.filter((l) => !l.base && !l.effect && !l.hidden).map((l) => l.id));
+ } else if ((e.ctrlKey || e.metaKey) && k === "g") {
+ e.preventDefault();
+ if (e.shiftKey) setGroup(sels, undefined);
+ else if (movable(sels).length > 1) setGroup(sels, `g-${Date.now()}`);
+ } else if (e.key === "Escape") setState({ overlay: null });
+ else if ((e.key === "Delete" || e.key === "Backspace") && sels.length) removeLayers(sels);
+ else if (e.key.startsWith("Arrow") && movable(sels).length) {
+ e.preventDefault();
+ const step = e.shiftKey ? 10 : 1;
+ const dx = e.key === "ArrowLeft" ? -step : e.key === "ArrowRight" ? step : 0;
+ const dy = e.key === "ArrowUp" ? -step : e.key === "ArrowDown" ? step : 0;
+ const ids = movable(sels).map((l) => l.id);
+ commit({ ...live, layers: live.layers.map((l) => (ids.includes(l.id) ? { ...l, dx: l.dx + dx, dy: l.dy + dy } : l)) });
+ } else if (k === "v") setTool("move");
+ else if (k === "h") setTool((t) => (t === "pan" ? "move" : "pan"));
+ else if (k === "e") setTool("erase");
+ else if (k === "b") setTool("shade");
+ else if (k === "p") setTool("sketch");
+ else if (k === "a" && maskTint) setTool("area");
+ else if (e.key === "[") setBrush((b) => Math.max(4, b - 8));
+ else if (e.key === "]") setBrush((b) => Math.min(400, b + 8));
+ };
+ const onKeyUp = (e: KeyboardEvent) => e.code === "Space" && setSpacePressed(false);
+ const onBlur = () => setSpacePressed(false);
+ window.addEventListener("keydown", onKeyDown);
+ window.addEventListener("keyup", onKeyUp);
+ window.addEventListener("blur", onBlur);
+ return () => {
+ window.removeEventListener("keydown", onKeyDown);
+ window.removeEventListener("keyup", onKeyUp);
+ window.removeEventListener("blur", onBlur);
+ };
+ });
+
+ const contentSize = useMemo(() => ({ width: W, height: H }), [W, H]);
+ const base = live.layers.find((l) => l.base);
+ const selLayers = live.layers.filter((l) => sels.includes(l.id));
+ const selObjects = selLayers.filter((l) => !l.base && !l.effect);
+ const grouped = selLayers.some((l) => l.group);
+ const groupable = selLayers.filter((l) => !l.base).length > 1;
+
+ const status = areaOn
+ ? "Area brush: paint to add to the pink area · right-click takes out · [ ] size · then Generate"
+ : brushOn
+ ? `${tool === "erase" ? "Eraser" : "Shadow brush"} on “${sel!.name}”: paint · [ ] size · V to select and move`
+ : tool === "erase"
+ ? "Eraser: select an object or shadow layer in the list"
+ : tool === "shade"
+ ? "Shadow brush: select a shadow layer in the list"
+ : tool === "sketch"
+ ? "Pencil: draw where a new object goes, one colour per object · [ ] size · then say what it is on the right"
+ : "Click a layer to select it · Shift+click adds · drag to move · corner scales · Ctrl+A all · Ctrl+G group";
+
+ return (
+ <div className="overlay">
+ <div className="overlay-head">
+ <h2>Layers (lab)</h2>
+ {labJobs.length && !stickering ? <LabJobs lab={lab} doc={doc} /> : <span className="layers-status">{stickering ? "Making stickers: each object is redrawn on its own; the background comes back when they are in" : status}</span>}
+ <span className="spacer" />
+ <button className="btn" onClick={onNewSplit} title="Split this image again (replaces these layers; other images keep theirs)">
+ <Scissors size={14} /> New split
+ </button>
+ <IconButton title="Close (Esc)" onClick={() => setState({ overlay: null })}>
+ <X size={18} />
+ </IconButton>
+ </div>
+
+ <div className="layers-body">
+ <div className="editor-stage">
+ {stickering && (
+ <div className="sticker-strip">
+ <Loader2 size={14} className="spin" />
+ <span>
+ Making stickers · {stickerRows.length - stickersLeft} of {stickerRows.length}
+ {stickerNow ? ` · ${stickerNow.kind ? "filling the background" : (live.layers.find((l) => l.id === stickerNow.layer)?.name ?? "")}` : ""}
+ </span>
+ {stickerNow && (
+ <button className="btn" onClick={() => skipSticker(stickerNow)} title="Keep this object's rough cut-out">
+ Skip
+ </button>
+ )}
+ <button className="btn" onClick={() => stickerRows.forEach((s, i) => (stickerStates[i] === "waiting" || stickerStates[i] === "making") && skipSticker(s))} title="Keep the rough cut-outs of the objects not done yet">
+ Skip all
+ </button>
+ </div>
+ )}
+ {maskTint && !stickering && <span className="doc-mask-chip">Painted area: Generate redoes what is pink · brush it here to change it (A)</span>}
+ {!maskTint && waitingMarks(live).length > 0 && <span className="doc-mask-chip">Your marks: Generate adds the new objects there</span>}
+ <ZoomPan contentSize={contentSize} resetKey="layer-doc" enableToggleZoom={false} enablePan={panning} className="inpaint-zoom-viewport" minimapSrc={base ? urls.source(base.source) : undefined}>
+ {({ transform, isPanning }) => {
+ const k = transform.scale;
+ zoomK.current = k;
+ const cur = panning ? (isPanning ? "grabbing" : "grab") : brushOn || areaOn || tool === "sketch" ? "none" : drag.current ? "move" : "default";
+ const shown = selLayers.filter((l) => !l.hidden);
+ const box = boxOfAll(shown);
+ const canScale = movable(sels).length > 0;
+ return (
+ <div className={`mask-wrap doc-frame ${stickering ? "stickering" : ""}`} style={{ width: W, height: H }} onContextMenu={(e) => e.preventDefault()}>
+ {[...live.layers].reverse().map((l) => {
+ if (l.hidden) return null;
+ const t = xf(l);
+ // Stickers: background and shadows out of sight (they fade back), objects by their state.
+ const st = stickering ? (l.base || l.effect ? "hide" : (stickerOf(l.id) ?? "done")) : null;
+ if (brushOn && l.id === sel!.id)
+ return (
+ <canvas
+ key={`${l.id}:paint`}
+ ref={paintRef}
+ width={W}
+ height={H}
+ className="doc-layer"
+ style={{ transformOrigin: `${t.c.x}px ${t.c.y}px`, transform: `translate(${t.dx}px, ${t.dy}px) scale(${t.s})` }}
+ />
+ );
+ return (
+ <img
+ key={`${l.id}:${l.source}`}
+ src={urls.source(l.source)}
+ alt=""
+ draggable={false}
+ className={`doc-layer ${st ? `st-${st}` : ""}`}
+ style={{ transformOrigin: `${t.c.x}px ${t.c.y}px`, transform: `translate(${t.dx}px, ${t.dy}px) scale(${t.s})` }}
+ />
+ );
+ })}
+ {/* Stickers: a ring with the percentage on the object being made, a spinner on the ones waiting. */}
+ {stickering &&
+ stickerRows.map((s, i) => {
+ const st = stickerStates[i];
+ const l = live.layers.find((x) => x.id === s.layer)!;
+ const info = infos[l.source];
+ // The background's fill: a ring only while it is made (in the middle).
+ if ((st !== "waiting" && st !== "making") || !info || l.hidden || (s.kind && st !== "making")) return null;
+ const c = centre(mapBox(info.box, xf(l)));
+ const job = s.job ? jobs[s.job] : undefined;
+ const item = s.job ? session.find((x) => x.jobId === s.job) : undefined;
+ return (
+ <div key={`ring-${s.layer}`} className={`sticker-ring ${st}`} style={{ left: c.x, top: c.y, transform: `translate(-50%, -50%) scale(${1 / k})` }}>
+ {st === "making" && job && item && job.status !== "done" ? <StickerRing job={job} item={item} /> : <Loader2 size={st === "making" ? 26 : 16} className="spin" />}
+ </div>
+ );
+ })}
+ {/* Marks already sent: shown until the new objects come back. */}
+ {waitingMarks(live).map((m) => (
+ <img key={m} className="doc-layer doc-marks" src={urls.input(m)} alt="" draggable={false} />
+ ))}
+ <canvas ref={marksRef} width={W} height={H} className="doc-layer doc-marks" />
+ {maskTint && paintedCrop && (() => {
+ const t = xf(paintedCrop.l);
+ const r = paintedCrop.f.rect!;
+ return (
+ <div className="doc-layer" style={{ transformOrigin: `${t.c.x}px ${t.c.y}px`, transform: `translate(${t.dx}px, ${t.dy}px) scale(${t.s})` }}>
+ <img className="doc-crop-mask" src={maskTint} alt="" draggable={false} style={{ left: r.x, top: r.y, width: r.w, height: r.h }} />
+ </div>
+ );
+ })()}
+ {maskTint &&
+ !paintedCrop &&
+ (() => {
+ const t = paintedLayer ? xf(paintedLayer) : null;
+ return (
+ <img
+ className="doc-mask"
+ src={maskTint}
+ alt=""
+ draggable={false}
+ style={t ? { transformOrigin: `${t.c.x}px ${t.c.y}px`, transform: `translate(${t.dx}px, ${t.dy}px) scale(${t.s})` } : undefined}
+ />
+ );
+ })()}
+ <canvas ref={areaPreview} width={W} height={H} className="doc-layer doc-marks" />
+ <div ref={hitRef} className="doc-hit" style={{ cursor: cur }} onContextMenu={(e) => e.preventDefault()} onPointerDown={onDown} onPointerMove={onMove} onPointerUp={onUp} onPointerCancel={onUp} onPointerLeave={() => setCursor(null)} />
+ {(brushOn || areaOn || tool === "sketch") && cursor && !panning && (
+ <div
+ className={`brush-cursor ${tool === "sketch" ? "pencil" : ""}`}
+ style={{ left: cursor.x, top: cursor.y, width: brush / k, height: brush / k, borderWidth: 1.5 / k, ...(tool === "sketch" ? { borderColor: MARK_COLOURS.find((c) => c.name === markColour)!.css } : {}) }}
+ />
+ )}
+ {/* Several selected: a thin box on each, one box with the handle around all. */}
+ {shown.length > 1 &&
+ tool !== "sketch" &&
+ shown.map((l) => {
+ const b = boxOfAll([l]);
+ return b && <div key={l.id} className="doc-sel doc-sel-part" style={{ left: b.x0, top: b.y0, width: b.x1 - b.x0, height: b.y1 - b.y0, borderWidth: 1 / k }} />;
+ })}
+ {box && tool !== "sketch" && (
+ <div className="doc-sel" style={{ left: box.x0, top: box.y0, width: box.x1 - box.x0, height: box.y1 - box.y0, borderWidth: 1.5 / k }}>
+ {canScale && (
+ <div
+ className="doc-handle"
+ title="Drag to scale"
+ style={{ width: 12 / k, height: 12 / k, right: -6 / k, bottom: -6 / k, borderWidth: 1.5 / k }}
+ onPointerDown={(e) => startDrag(e, "scale", sels)}
+ onPointerMove={onMove}
+ onPointerUp={onUp}
+ />
+ )}
+ </div>
+ )}
+ </div>
+ );
+ }}
+ </ZoomPan>
+ </div>
+
+ <aside className="layers-panel">
+ <div className="layers-title">Layers</div>
+ {live.layers.map((l, i) => {
+ const info = infos[l.source];
+ const mark = rowDrag && rowDrag.from !== i && rowDrag.over === i ? (rowDrag.over > rowDrag.from ? "drop-below" : "drop-above") : "";
+ return (
+ <div
+ key={l.id}
+ ref={(el) => {
+ rowsRef.current[i] = el;
+ }}
+ className={`layer-row doc-row ${sels.includes(l.id) ? "on" : ""} ${mark} ${rowDrag?.from === i ? "dragging" : ""}`}
+ onClick={(e) => {
+ if (e.shiftKey || e.ctrlKey || e.metaKey) {
+ const ids = withGroup(live.layers, l.id);
+ setSels(sels.includes(l.id) ? sels.filter((x) => !ids.includes(x)) : [...sels.filter((x) => !ids.includes(x)), ...ids.filter((x) => x !== l.id), l.id]);
+ } else if (e.altKey) setSels([l.id]);
+ else setSelected(l.id);
+ }}
+ >
+ {l.base ? (
+ <span className="grip" title="The background stays at the bottom">
+ <Lock size={13} />
+ </span>
+ ) : (
+ <span className="grip" title="Drag to reorder" onPointerDown={(e) => onGripDown(e, i)} onPointerMove={onGripMove} onPointerUp={onGripUp}>
+ <GripVertical size={14} />
+ </span>
+ )}
+ <span className="doc-thumb">{info && <img src={info.thumb} alt="" />}</span>
+ {renaming === l.id ? (
+ <input
+ autoFocus
+ value={l.name}
+ onChange={(e) => onChange({ ...live, layers: live.layers.map((x) => (x.id === l.id ? { ...x, name: e.target.value } : x)) })}
+ onBlur={() => setRenaming(null)}
+ onKeyDown={(e) => (e.key === "Enter" || e.key === "Escape") && (e.target as HTMLInputElement).blur()}
+ spellCheck={false}
+ />
+ ) : (
+ <span className="doc-name" title="Double-click to rename" onDoubleClick={() => setRenaming(l.id)}>
+ {l.name}
+ </span>
+ )}
+ {l.group && (
+ <span className="doc-group" title="In a group: a click selects the whole group (Alt+click: this layer alone)">
+ <Link2 size={12} />
+ </span>
+ )}
+ {rowJob(l) ? (
+ <span className="layer-pct doc-running" title="An edit of this layer is being made">
+ <Loader2 className="spin" size={12} /> <JobPct job={rowJob(l)!.job} item={rowJob(l)!.item} />
+ </span>
+ ) : (
+ l.history.length > 1 && <span className="layer-pct" title="Versions">v{l.history.indexOf(l.source) + 1}</span>
+ )}
+ <IconButton title={l.hidden ? "Show" : "Hide"} onClick={() => patchLayer(l.id, { hidden: !l.hidden })}>
+ {l.hidden ? <EyeOff size={14} /> : <Eye size={14} />}
+ </IconButton>
+ </div>
+ );
+ })}
+
+ {(tool === "sketch" || sketch.marks.length > 0) && (
+ <div className="doc-actions doc-new">
+ <div className="doc-actions-title">
+ <Pencil size={13} /> New objects
+ </div>
+ <p className="layers-help">
+ {sketch.marks.length
+ ? "Say what goes at each colour. The whole scene goes to Qwen with your marks; each object comes back as its own layer at that place and size, without a shadow (Add shadow makes it). Prompt+ sees the marks too."
+ : "Draw where a new object goes (an outline or a scribble), one colour per object."}
+ </p>
+ {usedColours.map((c) => (
+ <label key={c} className="doc-want">
+ <span className="doc-swatch" style={{ background: MARK_COLOURS.find((m) => m.name === c)!.css }} />
+ <input
+ value={sketch.wants[c] ?? ""}
+ placeholder={`What goes at the ${c} marks, e.g. “a green glass bottle”`}
+ onChange={(e) => setSketch({ ...sketch, wants: { ...sketch.wants, [c]: e.target.value } })}
+ spellCheck={false}
+ />
+ </label>
+ ))}
+ {sketch.marks.length > 0 && (
+ <div className="doc-grid">
+ <button className="btn" onClick={() => setSketch({ ...sketch, marks: sketch.marks.slice(0, -1) })} title="Take back the last mark">
+ <Undo2 size={14} /> Undo mark
+ </button>
+ <button className="btn" onClick={() => setSketch({ marks: [], wants: sketch.wants })} title="Remove every mark">
+ <Trash2 size={14} /> Clear marks
+ </button>
+ <button className="btn primary doc-wide" onClick={newObjects} disabled={busy} title="The flattened layers with your marks as the image to edit, and a prompt naming each colour's object">
+ <WandSparkles size={14} /> Set up the prompt
+ </button>
+ </div>
+ )}
+ </div>
+ )}
+
+ {multi ? (
+ <div className="doc-actions">
+ <div className="doc-actions-title">{sels.length} layers</div>
+ <p className="layers-help">Moved and scaled together. Edit together gives Qwen the whole scene; the selected objects take the result's look.</p>
+ <div className="doc-grid">
+ {groupable &&
+ (grouped ? (
+ <button className="btn" onClick={() => setGroup(sels, undefined)} title={`Ungroup (${shortcut("Shift+G")})`}>
+ <Ungroup size={14} /> Ungroup
+ </button>
+ ) : (
+ <button className="btn" onClick={() => setGroup(sels, `g-${Date.now()}`)} title={`Keep them together: a click on one selects all (${shortcut("G")})`}>
+ <Group size={14} /> Group
+ </button>
+ ))}
+ {selObjects.length > 0 && (
+ <button className="btn" onClick={() => editTogether(selObjects)} disabled={busy} title="The whole scene as the image to edit; the selected objects take the result's look, the other layers stay">
+ <Sparkles size={14} /> Edit together
+ </button>
+ )}
+ {selObjects.length > 0 && (
+ <button className="btn" onClick={() => commit({ ...live, layers: live.layers.map((l) => (selObjects.some((o) => o.id === l.id) ? { ...l, dx: 0, dy: 0, s: 1 } : l)) })} title="Back to where they were in the photo">
+ <RotateCcw size={14} /> Reset place
+ </button>
+ )}
+ <button className="btn" onClick={() => removeLayers(sels)} title="Remove them from the layers (with their shadows; the versions stay in the project)">
+ <Trash2 size={14} /> Delete
+ </button>
+ </div>
+ </div>
+ ) : sel ? (
+ <div className="doc-actions">
+ <div className="doc-actions-title">{sel.name}</div>
+ <p className="layers-help">
+ {sel.base
+ ? "The prompt edits the background. Fill the holes sets up the fill; widen the area with Paint area (shadows too) before Generate."
+ : sel.effect
+ ? sel.follows
+ ? "Moves and scales with its object. Make it again after a big move."
+ : "Made where the objects are: make them again after moving objects."
+ : `The prompt edits this layer at its own size, e.g. “Complete the ${sel.name.toLowerCase()}, keep the transparent background”.`}
+ </p>
+ <div className="doc-grid">
+ {sel.base && (
+ <button className="btn" onClick={() => fillHoles(sel)} disabled={busy} title="Paints the holes as an Add with a prompt to fine-tune, ready to generate">
+ <Eraser size={14} /> Fill the holes
+ </button>
+ )}
+ {!sel.base && !sel.effect && (
+ <button className="btn" onClick={() => cleanEdges(sel)} disabled={busy} title="Redraw the object's border at its own size, as a die-cut sticker on transparency; the result stays inside its outline (grown ~3 px)">
+ <Sparkles size={14} /> Clean edges
+ </button>
+ )}
+ {!sel.base && !sel.effect && (
+ <button className="btn" onClick={() => addShadows(sel)} disabled={busy} title="A brushed edit of the flattened layers around this object; the result becomes its shadow layer">
+ <Sun size={14} /> {live.layers.some((l) => l.effect && l.follows === sel.id) ? "Shadow again" : "Add shadow"}
+ </button>
+ )}
+ {sel.history.length > 1 && (
+ <button className={`btn ${versionsOpen ? "on" : ""}`} onClick={() => setVersionsOpen((o) => !o)} aria-expanded={versionsOpen} title="Show an earlier version of this layer">
+ <History size={14} /> Versions · {sel.history.length}
+ </button>
+ )}
+ {!sel.base && !sel.effect && (
+ <button className="btn" onClick={() => patchLayer(sel.id, { dx: 0, dy: 0, s: 1 })} disabled={!sel.dx && !sel.dy && sel.s === 1} title="Back to where it was in the photo">
+ <RotateCcw size={14} /> Reset place
+ </button>
+ )}
+ {!sel.base && (
+ <button className="btn" onClick={() => removeLayers([sel.id])} title="Remove it from the layers (the version stays in the project)">
+ <Trash2 size={14} /> Delete
+ </button>
+ )}
+ </div>
+ {/* Inline under the buttons, the panel's full width (a popover from the right column ran off the panel). */}
+ {versionsOpen && sel.history.length > 1 && (
+ <div className="doc-versions">
+ {[...sel.history].reverse().map((v) => (
+ <button key={v} className={`doc-version ${v === sel.source ? "on" : ""}`} onClick={() => patchLayer(sel.id, { source: v })} title={v === sel.source ? "Shown now" : "Show this version"}>
+ <img src={urls.source(v)} alt="" />
+ <span>v{sel.history.indexOf(v) + 1}</span>
+ </button>
+ ))}
+ </div>
+ )}
+ </div>
+ ) : (
+ <p className="layers-help doc-none">Select a layer (here or on the image) to edit it with the prompt below. Shift+click selects more; the pencil (P) draws new objects.</p>
+ )}
+
+ <div className="doc-footer">
+ <label className="layers-opt" title="How far the area painted for shadows reaches below the objects (half as far sideways). Check the pink area before Generate.">
+ Shadow reach
+ <input type="range" min={1} max={12} value={reach} onChange={(e) => setReach(Number(e.target.value))} />
+ <span>{reach}%</span>
+ </label>
+ <label className="layers-opt" title="How many pixels the shadow area goes inside the objects, so the model redraws their bottom edge with the contact shadow (0: the area stops at the objects)">
+ Into objects
+ <input type="range" min={0} max={30} value={inset} onChange={(e) => setInset(Number(e.target.value))} />
+ <span>{inset} px</span>
+ </label>
+ <div className="doc-grid">
+ <button className="btn doc-wide" onClick={() => addShadows(null)} disabled={busy} title="One brushed edit of the flattened layers around every visible object; the result becomes a Shadows layer above the background">
+ <Sun size={14} /> Shadows for all objects
+ </button>
+ </div>
+ <button className="btn primary" onClick={flatten} disabled={busy}>
+ <ImageDown size={14} /> Save as image
+ </button>
+ </div>
+ </aside>
+ </div>
+
+ <div className="editor-tools">
+ <div className="group">
+ <IconButton title="Select and move (V)" active={tool === "move" && !spacePressed} onClick={() => setTool("move")}>
+ <MousePointer2 size={17} />
+ </IconButton>
+ <IconButton title="Pan (H) · or hold Space" active={tool === "pan" || spacePressed} onClick={() => setTool((t) => (t === "pan" ? "move" : "pan"))}>
+ <Hand size={17} />
+ </IconButton>
+ <IconButton title="Eraser (E): erase part of the selected layer (an object or a shadow)" active={tool === "erase" && !spacePressed} onClick={() => setTool("erase")}>
+ <Eraser size={17} />
+ </IconButton>
+ <IconButton title="Shadow brush (B): paint soft shadow on the selected shadow layer" active={tool === "shade" && !spacePressed} onClick={() => setTool("shade")}>
+ <Brush size={17} />
+ </IconButton>
+ <IconButton title="Pencil (P): draw where a new object goes, one colour per object" active={tool === "sketch" && !spacePressed} onClick={() => setTool("sketch")}>
+ <Pencil size={17} />
+ </IconButton>
+ {maskTint && (
+ <IconButton title="Area brush (A): change the pink area of the next edit · right-click takes out" active={tool === "area" && !spacePressed} onClick={() => setTool("area")}>
+ <Paintbrush size={17} />
+ </IconButton>
+ )}
+ </div>
+ {tool === "sketch" && (
+ <div className="group">
+ {MARK_COLOURS.map((c) => (
+ <button key={c.name} className={`doc-colour ${markColour === c.name ? "on" : ""}`} style={{ background: c.css }} title={`${c.name[0].toUpperCase()}${c.name.slice(1)}: one object`} aria-label={c.name} onClick={() => setMarkColour(c.name)} />
+ ))}
+ </div>
+ )}
+ {(tool === "erase" || tool === "shade" || tool === "sketch" || areaOn) && (
+ <div className="group">
+ <label className="slider" title="Brush size ( [ and ] )">
+ Size
+ <input type="range" min={4} max={400} value={brush} onChange={(e) => setBrush(() => Number(e.target.value))} />
+ </label>
+ </div>
+ )}
+ <div className="group">
+ <IconButton title={`Undo (${shortcut("Z")})`} onClick={() => (tool === "area" && areaStep(true)) || undo()} disabled={!undoStack.current.length && !(tool === "area" && areaUndo.current.length)}>
+ <Undo2 size={17} />
+ </IconButton>
+ <IconButton title={`Redo (${shortcut("Y")})`} onClick={() => (tool === "area" && areaStep(false)) || redo()} disabled={!redoStack.current.length && !(tool === "area" && areaRedo.current.length)}>
+ <Redo2 size={17} />
+ </IconButton>
+ </div>
+ </div>
+ </div>
+ );
+}
diff --git a/src/components/LayerSplit.tsx b/src/components/LayerSplit.tsx
new file mode 100644
index 0000000..8528947
--- /dev/null
+++ b/src/components/LayerSplit.tsx
@@ -0,0 +1,685 @@
+// Layers lab (experimental, branch feature/layers), step 1: the user puts the objects of an image
+// into layers by clicking them (MobileSAM in the webview, lib/sam.ts) or painting them, and "Split"
+// saves each layer cut from the photo (transparent elsewhere) and the background with transparent
+// holes where the layers were, as new versions of the project, and opens them as a layer document
+// (LayerDoc.tsx). Filling the holes or completing a cut object is done by hand with the usual
+// edits, to see what works before building the real feature.
+
+import { Brush, ChevronDown, ChevronUp, Eraser, Eye, EyeOff, Hand, Layers, MousePointerClick, Plus, Redo2, Trash2, Undo2, X } from "lucide-react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { api, urls } from "../lib/api";
+import { addTo, area, dilate, fileSafe, fromLogits, maskOrder, refineEdges, removeFrom, samSelection, union, type Mask } from "../lib/layers";
+import { shortcut } from "../lib/platform";
+import { Segmenter, type SamResult } from "../lib/sam";
+import type { DocLayer, LayerDoc } from "../lib/layerDoc";
+import { addVersions, setState, toastError, toastInfo, useStore } from "../lib/store";
+import type { InputRecord } from "../lib/types";
+import { IconButton } from "./ui";
+import { isTyping } from "./Viewer";
+import { ZoomPan } from "./ZoomPan";
+
+const STICKERS_KEY = "kokon.layers.stickers";
+
+/** Mask resolution cap. SAM's own masks are 256² for the whole image; the split refines the edges
+ * on the photo at this size (refineEdges), so it is the edge's resolution up to 2K. */
+const MAX_MASK_SIDE = 2048;
+const COLORS = ["#ff3b6b", "#2f9bff", "#ffb020", "#22c55e", "#a855f7", "#14b8a6", "#f97316", "#e11d48"];
+const MAX_UNDO = 30;
+
+interface LayerInfo {
+ id: string;
+ name: string;
+ color: string;
+ hidden: boolean;
+}
+
+interface Change {
+ before: Map<string, Mask>;
+ after: Map<string, Mask>;
+}
+
+/** The last click with the Select tool: clicking the same spot again cycles SAM's outputs. */
+interface LastClick {
+ x: number;
+ y: number;
+ layer: string;
+ result: SamResult;
+ order: number[];
+ pos: number;
+ change: Change;
+}
+
+type Tool = "select" | "brush" | "eraser" | "pan";
+type View = "layers" | "background" | "layer";
+type SamState = "loading" | "encoding" | "ready" | "missing" | "error";
+
+let nextId = 1;
+const newLayer = (n: number): LayerInfo => ({ id: `l${nextId++}`, name: `Layer ${n}`, color: COLORS[(n - 1) % COLORS.length], hidden: false });
+
+const rgb = (hex: string) => [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)];
+
+function loadImage(src: string): Promise<HTMLImageElement> {
+ return new Promise((res, rej) => {
+ const img = new Image();
+ img.crossOrigin = "anonymous";
+ img.onload = () => res(img);
+ img.onerror = () => rej(new Error("The image could not be loaded."));
+ img.src = src;
+ });
+}
+
+/** A mask as a white canvas with that alpha, to scale onto the full image. */
+function maskCanvas(m: Mask, w: number, h: number): HTMLCanvasElement {
+ const c = document.createElement("canvas");
+ c.width = w;
+ c.height = h;
+ const ctx = c.getContext("2d")!;
+ const d = ctx.createImageData(w, h);
+ for (let i = 0; i < m.length; i++) {
+ d.data[i * 4] = d.data[i * 4 + 1] = d.data[i * 4 + 2] = 255;
+ d.data[i * 4 + 3] = m[i];
+ }
+ ctx.putImageData(d, 0, 0);
+ return c;
+}
+
+const toBlob = (c: HTMLCanvasElement) => new Promise<Blob>((res, rej) => c.toBlob((b) => (b ? res(b) : rej(new Error("export failed"))), "image/png"));
+
+/** `onSplit`: the new layer document; `onBack`: back to the open document (none: close). */
+export function LayerSplit({ onSplit, onBack }: { onSplit: (doc: LayerDoc) => void; onBack?: () => void }) {
+ const ref0 = useStore((s) => s.composer.refs[0]);
+ const srcW = ref0?.width ?? 1;
+ const srcH = ref0?.height ?? 1;
+ const scale = Math.min(1, MAX_MASK_SIDE / Math.max(srcW, srcH));
+ const mw = Math.max(1, Math.round(srcW * scale));
+ const mh = Math.max(1, Math.round(srcH * scale));
+ const src = ref0 ? urls.source(ref0.source) : "";
+
+ const [layers, setLayers] = useState<LayerInfo[]>(() => [newLayer(1)]);
+ const [active, setActive] = useState(() => layers[0].id);
+ const [tool, setTool] = useState<Tool>("select");
+ const [view, setView] = useState<View>("layers");
+ const [brush, setBrush] = useState(40);
+ const [margin, setMargin] = useState(3);
+ const [grey, setGrey] = useState(false);
+ // Clean the stickers after the split (remembered; on unless turned off).
+ const [stickers, setStickersState] = useState(() => {
+ try {
+ return localStorage.getItem(STICKERS_KEY) !== "off";
+ } catch {
+ return true;
+ }
+ });
+ const setStickers = (on: boolean) => {
+ setStickersState(on);
+ try {
+ localStorage.setItem(STICKERS_KEY, on ? "on" : "off");
+ } catch {
+ /* storage unavailable */
+ }
+ };
+ const [spacePressed, setSpacePressed] = useState(false);
+ const [cursor, setCursor] = useState<{ x: number; y: number } | null>(null);
+ const [sam, setSam] = useState<SamState>("loading");
+ const [busy, setBusy] = useState<string | null>(null);
+ const [tick, setTick] = useState(0);
+ const [areas, setAreas] = useState<Record<string, number>>({});
+
+ const masks = useRef(new Map<string, Mask>());
+ const undoStack = useRef<Change[]>([]);
+ const redoStack = useRef<Change[]>([]);
+ const last = useRef<LastClick | null>(null);
+ const segmenter = useRef<Segmenter | null>(null);
+ const full = useRef<Promise<HTMLImageElement> | null>(null);
+ const canvasRef = useRef<HTMLCanvasElement>(null);
+ const stroke = useRef<{ erase: boolean; size: number; points: [number, number][] } | null>(null);
+ const layersRef = useRef(layers);
+ layersRef.current = layers;
+ const activeRef = useRef(active);
+ activeRef.current = active;
+
+ const maskOf = useCallback(
+ (id: string) => {
+ let m = masks.current.get(id);
+ if (!m) masks.current.set(id, (m = new Uint8Array(mw * mh)));
+ return m;
+ },
+ [mw, mh],
+ );
+
+ const panning = spacePressed || tool === "pan";
+ const bump = () => setTick((t) => t + 1);
+
+ // The full image (for SAM and for the split), and SAM's encoding of it.
+ useEffect(() => {
+ if (!src) return;
+ let gone = false;
+ full.current = loadImage(src);
+ (async () => {
+ try {
+ const s = await Segmenter.load();
+ if (gone) return;
+ if (!s) return setSam("missing");
+ setSam("encoding");
+ await s.encode(await full.current!, mw, mh);
+ if (gone) return;
+ segmenter.current = s;
+ setSam("ready");
+ } catch (e) {
+ console.error(e);
+ if (!gone) setSam("error");
+ }
+ })();
+ return () => {
+ gone = true;
+ };
+ }, [src, mw, mh]);
+
+ /** Runs `mutate` on the masks (top layer first) and records what changed for Undo. */
+ const commit = useCallback(
+ (mutate: (all: Mask[], ids: string[]) => void): Change | null => {
+ const ids = layersRef.current.map((l) => l.id);
+ const all = ids.map(maskOf);
+ const before = all.map((m) => m.slice());
+ mutate(all, ids);
+ const change: Change = { before: new Map(), after: new Map() };
+ all.forEach((m, k) => {
+ const b = before[k];
+ for (let i = 0; i < m.length; i++)
+ if (m[i] !== b[i]) {
+ change.before.set(ids[k], b);
+ change.after.set(ids[k], m.slice());
+ break;
+ }
+ });
+ if (!change.before.size) return null;
+ undoStack.current = [...undoStack.current, change].slice(-MAX_UNDO);
+ redoStack.current = [];
+ bump();
+ return change;
+ },
+ [maskOf],
+ );
+
+ const restore = (m: Map<string, Mask>) => {
+ for (const [id, v] of m) if (masks.current.has(id)) masks.current.set(id, v.slice());
+ };
+
+ const undo = useCallback(() => {
+ const c = undoStack.current.pop();
+ if (!c) return;
+ restore(c.before);
+ redoStack.current.push(c);
+ last.current = null;
+ bump();
+ }, []);
+
+ const redo = useCallback(() => {
+ const c = redoStack.current.pop();
+ if (!c) return;
+ restore(c.after);
+ undoStack.current.push(c);
+ bump();
+ }, []);
+
+ // The overlay: layer colours, or the background / the active layer alone on a checkerboard.
+ useEffect(() => {
+ const c = canvasRef.current;
+ const ctx = c?.getContext("2d");
+ if (!c || !ctx) return;
+ const n = mw * mh;
+ const img = ctx.createImageData(mw, mh);
+ const d = img.data;
+ const checker = (i: number) => ((((i % mw) >> 4) + (((i / mw) | 0) >> 4)) & 1 ? 255 : 200);
+ const nextAreas: Record<string, number> = {};
+ for (const l of layers) nextAreas[l.id] = area(maskOf(l.id)) / n;
+ if (view === "layers") {
+ const shown = layers.filter((l) => !l.hidden).map((l) => ({ m: maskOf(l.id), c: rgb(l.color), k: l.id === active ? 0.6 : 0.4 }));
+ for (let i = 0; i < n; i++) {
+ for (const s of shown) {
+ const a = s.m[i];
+ if (!a) continue;
+ d[i * 4] = s.c[0];
+ d[i * 4 + 1] = s.c[1];
+ d[i * 4 + 2] = s.c[2];
+ d[i * 4 + 3] = a * s.k;
+ break;
+ }
+ }
+ } else {
+ const hole =
+ view === "background" ? dilate(union(layers.map((l) => maskOf(l.id)), n), mw, mh, margin) : maskOf(active).map((a) => 255 - a);
+ for (let i = 0; i < n; i++) {
+ const a = hole[i];
+ if (!a) continue;
+ d[i * 4] = d[i * 4 + 1] = d[i * 4 + 2] = checker(i);
+ d[i * 4 + 3] = a;
+ }
+ }
+ ctx.putImageData(img, 0, 0);
+ setAreas(nextAreas);
+ }, [tick, layers, active, view, margin, mw, mh, maskOf]);
+
+ const toMask = (e: React.PointerEvent): [number, number] => {
+ const r = canvasRef.current?.getBoundingClientRect();
+ if (!r || r.width <= 0 || r.height <= 0) return [0, 0];
+ return [Math.min(mw - 1, Math.max(0, ((e.clientX - r.left) / r.width) * mw)), Math.min(mh - 1, Math.max(0, ((e.clientY - r.top) / r.height) * mh))];
+ };
+
+ const select = async (x: number, y: number, remove: boolean) => {
+ const s = segmenter.current;
+ if (!s) {
+ toastInfo(sam === "missing" ? "The selection model is not there: paint with the brush." : "The selection model is still getting ready.");
+ return;
+ }
+ const layer = activeRef.current;
+ const prev = last.current;
+ const near = prev && Math.hypot(prev.x - x, prev.y - y) < Math.max(mw, mh) * 0.015;
+ // The same spot again: undo that click and take SAM's next output (part -> whole).
+ if (!remove && prev && near && prev.layer === layer && undoStack.current[undoStack.current.length - 1] === prev.change) {
+ undo();
+ redoStack.current = [];
+ const pos = (prev.pos + 1) % prev.order.length;
+ const m = samSelection(prev.result.masks[prev.order[pos]], mw, mh);
+ const change = commit((all, ids) => addTo(all, ids.indexOf(layer), m, false));
+ last.current = change ? { ...prev, pos, change } : null;
+ return;
+ }
+ setBusy("Selecting…");
+ try {
+ const result = await s.predict([{ x, y, label: 1 }], mw, mh);
+ const order = maskOrder(
+ result.iou,
+ result.masks.map((m) => area(fromLogits(m))),
+ );
+ const m = samSelection(result.masks[order[0]], mw, mh);
+ const change = commit((all, ids) => (remove ? removeFrom(all[ids.indexOf(layer)], m) : addTo(all, ids.indexOf(layer), m, false)));
+ last.current = change && !remove ? { x, y, layer, result, order, pos: 0, change } : null;
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(null);
+ }
+ };
+
+ const strokeSize = () => {
+ const r = canvasRef.current?.getBoundingClientRect();
+ return r && r.width > 0 ? (brush / r.width) * mw : brush;
+ };
+
+ /** Draws the stroke's last segment on the overlay, in the layer's colour (or clears it). */
+ const drawSegment = (pts: [number, number][], erase: boolean, size: number) => {
+ const ctx = canvasRef.current?.getContext("2d");
+ if (!ctx) return;
+ const color = layers.find((l) => l.id === active)?.color ?? "#fff";
+ ctx.globalCompositeOperation = erase ? "destination-out" : "source-over";
+ ctx.strokeStyle = ctx.fillStyle = color + "99";
+ ctx.lineWidth = size;
+ ctx.lineCap = ctx.lineJoin = "round";
+ ctx.beginPath();
+ if (pts.length === 1) {
+ ctx.arc(pts[0][0], pts[0][1], size / 2, 0, Math.PI * 2);
+ ctx.fill();
+ } else {
+ ctx.moveTo(pts[0][0], pts[0][1]);
+ ctx.lineTo(pts[1][0], pts[1][1]);
+ ctx.stroke();
+ }
+ ctx.globalCompositeOperation = "source-over";
+ };
+
+ const onDown = (e: React.PointerEvent) => {
+ if (e.button === 1 || panning || busy) return;
+ if (e.button !== 0 && e.button !== 2) return;
+ e.stopPropagation();
+ const [x, y] = toMask(e);
+ if (tool === "select") {
+ void select(x, y, e.button === 2 || e.altKey);
+ return;
+ }
+ try {
+ (e.target as HTMLElement).setPointerCapture(e.pointerId);
+ } catch {}
+ const erase = (tool === "eraser") !== (e.button === 2);
+ stroke.current = { erase, size: strokeSize(), points: [[x, y]] };
+ drawSegment([[x, y]], erase, stroke.current.size);
+ };
+
+ const onMove = (e: React.PointerEvent, k: number) => {
+ const r = canvasRef.current?.getBoundingClientRect();
+ if (r && !panning) setCursor({ x: (e.clientX - r.left) / k, y: (e.clientY - r.top) / k });
+ const s = stroke.current;
+ if (!s) return;
+ e.stopPropagation();
+ const p = toMask(e);
+ drawSegment([s.points[s.points.length - 1], p], s.erase, s.size);
+ s.points.push(p);
+ };
+
+ const onUp = () => {
+ const s = stroke.current;
+ if (!s) return;
+ stroke.current = null;
+ // Rasterize the whole stroke once, then apply it to the masks.
+ const c = document.createElement("canvas");
+ c.width = mw;
+ c.height = mh;
+ const ctx = c.getContext("2d")!;
+ ctx.strokeStyle = ctx.fillStyle = "#fff";
+ ctx.lineWidth = s.size;
+ ctx.lineCap = ctx.lineJoin = "round";
+ ctx.beginPath();
+ if (s.points.length === 1) {
+ ctx.arc(s.points[0][0], s.points[0][1], s.size / 2, 0, Math.PI * 2);
+ ctx.fill();
+ } else {
+ ctx.moveTo(s.points[0][0], s.points[0][1]);
+ for (const p of s.points.slice(1)) ctx.lineTo(p[0], p[1]);
+ ctx.stroke();
+ }
+ const px = ctx.getImageData(0, 0, mw, mh).data;
+ const a = new Uint8Array(mw * mh);
+ for (let i = 0; i < a.length; i++) a[i] = px[i * 4 + 3];
+ const layer = activeRef.current;
+ last.current = null;
+ if (!commit((all, ids) => (s.erase ? removeFrom(all[ids.indexOf(layer)], a) : addTo(all, ids.indexOf(layer), a)))) bump();
+ };
+
+ const addLayer = () => {
+ const l = newLayer(layers.length + 1);
+ setLayers((ls) => [l, ...ls]);
+ setActive(l.id);
+ setView("layers");
+ };
+
+ const removeLayer = (id: string) => {
+ const rest = layers.filter((l) => l.id !== id);
+ if (!rest.length) return;
+ masks.current.delete(id);
+ // Undo steps that touched it can no longer be replayed in full.
+ undoStack.current = undoStack.current.filter((c) => !c.before.has(id));
+ redoStack.current = [];
+ last.current = null;
+ setLayers(rest);
+ if (active === id) setActive(rest[0].id);
+ };
+
+ const move = (id: string, by: number) =>
+ setLayers((ls) => {
+ const i = ls.findIndex((l) => l.id === id);
+ const j = i + by;
+ if (i < 0 || j < 0 || j >= ls.length) return ls;
+ const out = ls.slice();
+ [out[i], out[j]] = [out[j], out[i]];
+ return out;
+ });
+
+ const patch = (id: string, p: Partial<LayerInfo>) => setLayers((ls) => ls.map((l) => (l.id === id ? { ...l, ...p } : l)));
+
+ /** Saves the background and every non-empty layer at the image's full size as new versions. */
+ const split = async () => {
+ const used = layers.filter((l) => area(maskOf(l.id)) > 0);
+ if (!used.length) {
+ toastInfo("Put something in a layer first: click an object, or paint it.");
+ return;
+ }
+ setBusy("Splitting…");
+ try {
+ const img = await full.current!;
+ const W = img.naturalWidth;
+ const H = img.naturalHeight;
+ const out = document.createElement("canvas");
+ out.width = W;
+ out.height = H;
+ const ctx = out.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ const made: InputRecord[] = [];
+ const save = async (c: HTMLCanvasElement, name: string) => made.push(await api.upload(await toBlob(c), name));
+ const cut = (m: Mask, op: GlobalCompositeOperation) => {
+ ctx.globalCompositeOperation = "source-over";
+ ctx.clearRect(0, 0, W, H);
+ ctx.drawImage(img, 0, 0);
+ ctx.globalCompositeOperation = op;
+ ctx.drawImage(maskCanvas(m, mw, mh), 0, 0, W, H);
+ ctx.globalCompositeOperation = "source-over";
+ };
+
+ // Edges snapped to the photo's and anti-aliased (SAM's own are stepped and carry a sliver of
+ // what is behind): a band of ~0.4 % of the side (8 px at 2K) around each edge.
+ const g = document.createElement("canvas");
+ g.width = mw;
+ g.height = mh;
+ const gctx = g.getContext("2d")!;
+ gctx.drawImage(img, 0, 0, mw, mh);
+ const gpx = gctx.getImageData(0, 0, mw, mh).data;
+ const gray = new Float32Array(mw * mh);
+ for (let i = 0; i < gray.length; i++) gray[i] = (0.2126 * gpx[i * 4] + 0.7152 * gpx[i * 4 + 1] + 0.0722 * gpx[i * 4 + 2]) / 255;
+ const band = Math.max(2, Math.round(Math.max(mw, mh) * 0.004));
+ const refined = new Map(layers.map((l) => [l.id, area(maskOf(l.id)) ? refineEdges(maskOf(l.id), gray, mw, mh, band) : maskOf(l.id)]));
+ const holes = dilate(union([...refined.values()], mw * mh), mw, mh, margin);
+ cut(holes, "destination-out");
+ await save(out, "Background (holes).png");
+ const base = made[0];
+ const docLayers: DocLayer[] = [];
+
+ for (const l of used) {
+ cut(refined.get(l.id)!, "destination-in");
+ const name = fileSafe(l.name);
+ await save(out, `${name}.png`);
+ const src = `input:${made[made.length - 1].id}`;
+ docLayers.push({ id: l.id, name: l.name, source: src, history: [src], hidden: false, base: false, dx: 0, dy: 0, s: 1 });
+ if (grey) {
+ // On flat grey: the input that let a plain edit complete a cut object (s14, 10-31).
+ const g = document.createElement("canvas");
+ g.width = W;
+ g.height = H;
+ const gc = g.getContext("2d")!;
+ gc.fillStyle = "#808080";
+ gc.fillRect(0, 0, W, H);
+ gc.drawImage(out, 0, 0);
+ await save(g, `${name} (grey).png`);
+ }
+ }
+ const bg = `input:${base.id}`;
+ docLayers.push({ id: "background", name: "Background", source: bg, history: [bg], hidden: false, base: true, dx: 0, dy: 0, s: 1 });
+ addVersions(made, 0);
+ onSplit({ width: W, height: H, photo: ref0?.source, layers: docLayers, known: made.map((r) => `input:${r.id}`), ...(stickers && docLayers.some((l) => !l.base) ? { autoClean: true } : {}) });
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(null);
+ }
+ };
+
+ useEffect(() => {
+ const onKeyDown = (e: KeyboardEvent) => {
+ if (isTyping(e.target)) return;
+ const k = e.key.toLowerCase();
+ if (e.code === "Space" && !e.repeat) {
+ e.preventDefault();
+ setSpacePressed(true);
+ } else if ((e.ctrlKey || e.metaKey) && k === "z") {
+ e.preventDefault();
+ e.shiftKey ? redo() : undo();
+ } else if ((e.ctrlKey || e.metaKey) && k === "y") {
+ e.preventDefault();
+ redo();
+ } else if (e.key === "Escape") (onBack ?? (() => setState({ overlay: null })))();
+ else if (e.key === "[") setBrush((b) => Math.max(4, b - 8));
+ else if (e.key === "]") setBrush((b) => Math.min(400, b + 8));
+ else if (k === "s") setTool("select");
+ else if (k === "b") setTool("brush");
+ else if (k === "e") setTool("eraser");
+ else if (k === "h") setTool((t) => (t === "pan" ? "select" : "pan"));
+ };
+ const onKeyUp = (e: KeyboardEvent) => e.code === "Space" && setSpacePressed(false);
+ const onBlur = () => setSpacePressed(false);
+ window.addEventListener("keydown", onKeyDown);
+ window.addEventListener("keyup", onKeyUp);
+ window.addEventListener("blur", onBlur);
+ return () => {
+ window.removeEventListener("keydown", onKeyDown);
+ window.removeEventListener("keyup", onKeyUp);
+ window.removeEventListener("blur", onBlur);
+ };
+ }, [undo, redo, onBack]);
+
+ useEffect(() => {
+ if (!ref0) setState({ overlay: null });
+ }, [ref0]);
+
+ const contentSize = useMemo(() => ({ width: srcW, height: srcH }), [srcW, srcH]);
+ if (!ref0) return null;
+
+ const activeName = layers.find((l) => l.id === active)?.name ?? "";
+ const status =
+ busy ??
+ {
+ loading: "Loading the selection model…",
+ encoding: "Reading the image…",
+ ready: tool === "select" ? `Click an object to add it to “${activeName}” · again on the same spot: bigger part · right-click: take it out` : `Painting “${activeName}”`,
+ missing: "Selection model not found (lab-models/mobilesam): paint the layers with the brush",
+ error: "The selection model failed to start (see the console): paint the layers with the brush",
+ }[sam];
+ const brushCursor = tool === "brush" || tool === "eraser";
+
+ return (
+ <div className="overlay">
+ <div className="overlay-head">
+ <h2>Split into layers (lab)</h2>
+ <span className="layers-status">{status}</span>
+ <span className="spacer" />
+ {onBack && (
+ <button className="btn" onClick={onBack}>
+ Back to the layers
+ </button>
+ )}
+ <IconButton title="Close (Esc)" onClick={() => setState({ overlay: null })}>
+ <X size={18} />
+ </IconButton>
+ </div>
+
+ <div className="layers-body">
+ <div className="editor-stage">
+ <ZoomPan contentSize={contentSize} resetKey={src} enableToggleZoom={false} enablePan={panning} className="inpaint-zoom-viewport" minimapSrc={src}>
+ {({ transform, isPanning }) => {
+ const k = transform.scale;
+ const cur = panning ? (isPanning ? "grabbing" : "grab") : busy ? "progress" : brushCursor ? "none" : "crosshair";
+ return (
+ <div className="mask-wrap" style={{ width: srcW, height: srcH, cursor: cur }} onContextMenu={(e) => e.preventDefault()}>
+ <img src={src} alt="" draggable={false} style={{ width: "100%", height: "100%", display: "block" }} />
+ <canvas
+ ref={canvasRef}
+ width={mw}
+ height={mh}
+ style={{ position: "absolute", inset: 0, width: "100%", height: "100%", cursor: cur }}
+ onPointerDown={onDown}
+ onPointerMove={(e) => onMove(e, k)}
+ onPointerUp={onUp}
+ onPointerCancel={onUp}
+ onPointerLeave={() => setCursor(null)}
+ />
+ {cursor && brushCursor && !panning && <div className="brush-cursor" style={{ left: cursor.x, top: cursor.y, width: brush / k, height: brush / k }} />}
+ </div>
+ );
+ }}
+ </ZoomPan>
+ </div>
+
+ <aside className="layers-panel">
+ <div className="layers-title">
+ <Layers size={15} /> Layers
+ <span className="spacer" />
+ <button className="btn" onClick={addLayer} title="A new layer on top">
+ <Plus size={14} /> New layer
+ </button>
+ </div>
+ {layers.map((l, i) => (
+ <div key={l.id} className={`layer-row ${l.id === active ? "on" : ""}`} onClick={() => setActive(l.id)}>
+ <span className="swatch" style={{ background: l.color }} />
+ <input value={l.name} onChange={(e) => patch(l.id, { name: e.target.value })} spellCheck={false} />
+ <span className="layer-pct">{areas[l.id] ? `${Math.max(0.1, areas[l.id] * 100).toFixed(1)}%` : "empty"}</span>
+ <IconButton title={l.hidden ? "Show" : "Hide"} onClick={() => patch(l.id, { hidden: !l.hidden })}>
+ {l.hidden ? <EyeOff size={14} /> : <Eye size={14} />}
+ </IconButton>
+ <IconButton title="Up" disabled={i === 0} onClick={() => move(l.id, -1)}>
+ <ChevronUp size={14} />
+ </IconButton>
+ <IconButton title="Down" disabled={i === layers.length - 1} onClick={() => move(l.id, 1)}>
+ <ChevronDown size={14} />
+ </IconButton>
+ <IconButton title="Delete layer" disabled={layers.length < 2} onClick={() => removeLayer(l.id)}>
+ <Trash2 size={14} />
+ </IconButton>
+ </div>
+ ))}
+ <div className="layer-row bg">
+ <span className="swatch" />
+ <span>Background: everything else, transparent where the layers are</span>
+ </div>
+
+ <div className="layers-view">
+ <span>Show</span>
+ {(["layers", "background", "layer"] as const).map((v) => (
+ <button key={v} className={`btn ${view === v ? "primary" : ""}`} onClick={() => setView(v)}>
+ {{ layers: "Layers", background: "Background", layer: "Active" }[v]}
+ </button>
+ ))}
+ </div>
+ <label className="layers-opt" title="Grow the background's holes so no edge of the objects stays on it">
+ Hole margin
+ <input type="range" min={0} max={20} value={margin} onChange={(e) => setMargin(Number(e.target.value))} />
+ <span>{margin}</span>
+ </label>
+ <label className="layers-opt">
+ <input type="checkbox" checked={grey} onChange={(e) => setGrey(e.target.checked)} />
+ Also save each layer on grey, as extra versions (for "complete the object" edits)
+ </label>
+ <p className="layers-help">
+ A click adds the object to the active layer (never pixels another layer has: right-click it out of that layer first); the brush
+ (B) and eraser (E) fix edges and can move pixels between layers. Split saves the background and each layer as new versions and
+ opens them as layers: move them, hide them, edit them one by one.
+ </p>
+ <label className="layers-opt" title="After the split, each object is redrawn at its own size as a clean cut-out, then the background's holes are filled without the objects' shadows (one edit each; skip any while it runs). Clean edges and Fill the holes redo one later.">
+ <input type="checkbox" checked={stickers} onChange={(e) => setStickers(e.target.checked)} />
+ Clean the stickers and fill the background automatically (one edit each)
+ </label>
+ <button className="btn primary layers-split" onClick={split} disabled={!!busy}>
+ Split
+ </button>
+ </aside>
+ </div>
+
+ <div className="editor-tools">
+ <div className="group">
+ <IconButton title="Select objects (S) · right-click takes out" active={tool === "select" && !spacePressed} onClick={() => setTool("select")}>
+ <MousePointerClick size={17} />
+ </IconButton>
+ <IconButton title="Brush (B) · right-click erases" active={tool === "brush" && !spacePressed} onClick={() => setTool("brush")}>
+ <Brush size={17} />
+ </IconButton>
+ <IconButton title="Eraser (E) · right-click paints" active={tool === "eraser" && !spacePressed} onClick={() => setTool("eraser")}>
+ <Eraser size={17} />
+ </IconButton>
+ <IconButton title="Move (H) · or hold Space" active={tool === "pan" || spacePressed} onClick={() => setTool((t) => (t === "pan" ? "select" : "pan"))}>
+ <Hand size={17} />
+ </IconButton>
+ </div>
+ <div className="group">
+ <label className="slider" title="Brush size ( [ and ] )">
+ Size
+ <input type="range" min={4} max={400} value={brush} onChange={(e) => setBrush(Number(e.target.value))} />
+ </label>
+ </div>
+ <div className="group">
+ <IconButton title={`Undo (${shortcut("Z")})`} onClick={undo} disabled={!undoStack.current.length}>
+ <Undo2 size={17} />
+ </IconButton>
+ <IconButton title={`Redo (${shortcut("Y")})`} onClick={redo} disabled={!redoStack.current.length}>
+ <Redo2 size={17} />
+ </IconButton>
+ </div>
+ </div>
+ </div>
+ );
+}
diff --git a/src/components/LayersEditor.tsx b/src/components/LayersEditor.tsx
new file mode 100644
index 0000000..6b411cd
--- /dev/null
+++ b/src/components/LayersEditor.tsx
@@ -0,0 +1,194 @@
+// Layers lab (experimental): the layer document of the image being edited (LayerDoc.tsx; one per
+// image of the project), else the split that makes one for it (LayerSplit.tsx). The document follows
+// the edits made to its layers.
+
+import { useEffect, useRef, useState } from "react";
+import { docFor, docId, loadLab, putDoc, reconcile, saveLab, type DocLayer, type LayerDoc as Doc } from "../lib/layerDoc";
+import { editSource, getState, itemOrigin, itemSource, toastError, toastInfo, useStore } from "../lib/store";
+import { readScene } from "./LabScene";
+import { LayerDoc, pasteCrop, shadowImage } from "./LayerDoc";
+import { LayerSplit } from "./LayerSplit";
+
+export function LayersEditor() {
+ const projectId = useStore((s) => s.projectId);
+ return <LayersLab key={projectId} projectId={projectId} />;
+}
+
+function LayersLab({ projectId }: { projectId: string }) {
+ const session = useStore((s) => s.session);
+ // The image the prompt edits picks the document: the one split from it or the one that made it.
+ const [doc, setDoc] = useState<Doc | null>(() => docFor(loadLab(projectId), getState().composer.refs[0]?.source));
+ const [splitting, setSplitting] = useState(false);
+
+ const latest = useRef(doc);
+ const making = useRef(new Set<string>());
+ const update = (d: Doc) => {
+ latest.current = d;
+ // Saved into the project's lab as it is now (other documents, what earlier labs made).
+ saveLab(projectId, putDoc(loadLab(projectId), d));
+ setDoc(d);
+ };
+
+ // A shadow edit's result -> its shadow layer (a new one, or the next version of the one it redoes).
+ useEffect(() => {
+ if (!doc?.pending?.length) return;
+ for (const p of doc.pending) {
+ // Any painted edit of the image made for this shadow (the area may have been painted again).
+ const done = session.find((i) => i.gen?.mask_input_id && itemOrigin(i).from[0] === p.source && !doc.known.includes(`gen:${i.gen.id}`));
+ if (!done?.gen) continue;
+ const result = `gen:${done.gen.id}`;
+ if (making.current.has(result)) continue;
+ making.current.add(result);
+ const obj = p.of ? doc.layers.find((l) => l.id === p.of) : undefined;
+ const name = obj ? `${obj.name} shadow` : "Shadows";
+ shadowImage({ ...p, mask: done.gen.mask_input_id ?? p.mask }, result, doc.width, doc.height, name).then(
+ (rec) => {
+ const d = latest.current!;
+ const src = `input:${rec.id}`;
+ const known = [...d.known, result, src];
+ const pending = (d.pending ?? []).map((x) => (x.source === p.source ? { ...x, layer: x.layer ?? `fx-${result}` } : x));
+ const target = p.layer && d.layers.find((l) => l.id === p.layer);
+ let layers: DocLayer[];
+ if (target) layers = d.layers.map((l) => (l.id === target.id ? { ...l, source: src, history: [...l.history, src] } : l));
+ else {
+ const layer: DocLayer = { id: `fx-${result}`, name, source: src, history: [src], hidden: false, base: false, effect: true, follows: p.of ?? undefined, at: p.at, dx: 0, dy: 0, s: 1 };
+ const at = obj ? d.layers.findIndex((l) => l.id === obj.id) + 1 : d.layers.findIndex((l) => l.base);
+ layers = [...d.layers];
+ layers.splice(at < 0 ? layers.length : at, 0, layer);
+ }
+ update({ ...d, layers, pending, known });
+ },
+ (e) => toastError(e),
+ );
+ }
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [session, doc]);
+
+ // A whole-image edit's result (Shadows, New objects, Edit together) -> read back into layers: new
+ // objects at the top, each object's shadow just under it (following it), new looks as the
+ // objects' next versions. The newest result of each edit; a later one makes the next versions of
+ // the same layers.
+ useEffect(() => {
+ if (!doc?.scenes?.length) return;
+ for (const e of doc.scenes) {
+ const made = session.filter((i) => i.gen && itemOrigin(i).from[0] === e.source && !doc.known.includes(`gen:${i.gen.id}`));
+ const done = made[made.length - 1];
+ if (!done?.gen) continue;
+ const result = `gen:${done.gen.id}`;
+ if (making.current.has(result)) continue;
+ making.current.add(result);
+ readScene(doc, e, result).then(
+ (r) => {
+ const d = latest.current!;
+ const seen = made.map((i) => `gen:${i.gen!.id}`);
+ const madeBy = { ...(e.made ?? {}) };
+ let layers = d.layers;
+ const nextVersion = (id: string, src: string) => (layers = layers.map((l) => (l.id === id ? { ...l, source: src, history: [...l.history, src] } : l)));
+ for (const o of r.objects) {
+ const have = madeBy[o.colour] && layers.find((l) => l.id === madeBy[o.colour]);
+ if (have) nextVersion(have.id, o.src);
+ else {
+ const id = `new-${result}-${o.colour}`;
+ madeBy[o.colour] = id;
+ layers = [{ id, name: o.name, source: o.src, history: [o.src], hidden: false, base: false, dx: 0, dy: 0, s: 1 }, ...layers];
+ }
+ }
+ for (const s of r.shadows) {
+ // A new object's shadow follows the new layer; an object's, the object (from where it was).
+ const isNew = r.objects.some((o) => o.colour === s.of);
+ const obj = isNew ? madeBy[s.of] : s.of;
+ const key = isNew ? `${s.of}:shadow` : s.of;
+ const have = layers.find((l) => l.id === madeBy[key]) ?? layers.find((l) => l.effect && l.follows === obj);
+ if (have) {
+ nextVersion(have.id, s.src);
+ continue;
+ }
+ const owner = layers.find((l) => l.id === obj);
+ const id = `fx-${result}-${key}`;
+ madeBy[key] = id;
+ const place = isNew ? { dx: 0, dy: 0, s: 1 } : { dx: e.places[obj]?.dx ?? 0, dy: e.places[obj]?.dy ?? 0, s: e.places[obj]?.s ?? 1 };
+ const layer: DocLayer = { id, name: `${owner?.name ?? "Object"} shadow`, source: s.src, history: [s.src], hidden: false, base: false, effect: true, follows: obj, at: place, dx: 0, dy: 0, s: 1 };
+ const at = layers.findIndex((l) => l.id === obj);
+ layers = [...layers];
+ layers.splice(at < 0 ? layers.findIndex((l) => l.base) : at + 1, 0, layer);
+ }
+ for (const l of r.looks) nextVersion(l.id, l.src);
+ const srcs = [...r.objects.map((o) => o.src), ...r.shadows.map((s) => s.src), ...r.looks.map((l) => l.src)];
+ update({ ...d, layers, known: [...d.known, ...seen, ...srcs], scenes: (d.scenes ?? []).map((x) => (x.source === e.source ? { ...x, made: madeBy } : x)) });
+ if (r.notes.length) toastInfo(r.notes.join(" "));
+ },
+ (err) => toastError(err),
+ );
+ }
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [session, doc]);
+
+ // An edit of an object's crop -> shrunk back into place, the layer's next version (the newest
+ // result of each crop; the others are marked seen).
+ useEffect(() => {
+ if (!doc) return;
+ for (const l of doc.layers)
+ for (const f of l.feeds ?? []) {
+ if (!f.rect) continue;
+ const made = session.filter((i) => i.gen && itemOrigin(i).from[0] === f.source && !doc.known.includes(`gen:${i.gen.id}`));
+ const done = made[made.length - 1];
+ if (!done?.gen) continue;
+ const result = `gen:${done.gen.id}`;
+ if (making.current.has(result)) continue;
+ making.current.add(result);
+ pasteCrop(f, result, doc.width, doc.height, l.name).then(
+ (rec) => {
+ const d = latest.current!;
+ const src = `input:${rec.id}`;
+ const seen = made.map((i) => `gen:${i.gen!.id}`);
+ update({
+ ...d,
+ layers: d.layers.map((x) => (x.id === l.id ? { ...x, source: src, history: [...x.history, src] } : x)),
+ known: [...d.known, ...seen, src],
+ });
+ },
+ (e) => toastError(e),
+ );
+ }
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [session, doc]);
+
+ // Edits made to a layer (from the prompt under this view, or after closing it) become its version.
+ useEffect(() => {
+ if (!doc) return;
+ const items = session.flatMap((i) => {
+ const source = itemSource(i);
+ return source ? [{ source, from: itemOrigin(i).from[0] ?? null, mask: i.gen?.mask_input_id ?? null }] : [];
+ });
+ const next = reconcile(doc, items);
+ if (next !== doc) update(next);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [session, doc]);
+
+ if (!doc || splitting)
+ return (
+ <LayerSplit
+ onSplit={(d) => {
+ update(d);
+ setSplitting(false);
+ }}
+ onBack={doc ? () => setSplitting(false) : undefined}
+ />
+ );
+ return (
+ <LayerDoc
+ key={docId(doc)}
+ projectId={projectId}
+ doc={doc}
+ onChange={update}
+ onNewSplit={() => {
+ // Split this document's photo again (the prompt may be on one of its layers).
+ if (doc.photo && !editSource(doc.photo, false)) {
+ toastError(new Error("The photo these layers were split from is no longer in the project."));
+ return;
+ }
+ setSplitting(true);
+ }}
+ />
+ );
+}
diff --git a/src/components/PhoneStage.tsx b/src/components/PhoneStage.tsx
index 64447af..0edfa13 100644
--- a/src/components/PhoneStage.tsx
+++ b/src/components/PhoneStage.tsx
@@ -229,7 +229,7 @@ export function PhoneActions({ item, showOriginal, onLut }: { item: SessionItem;
};
return (
<div className="actions phone-actions">
- {g?.request && g.mode !== "upscale" && g.mode !== "lut" && (
+ {g?.request && g.mode !== "upscale" && g.mode !== "lut" && g.mode !== "layers" && (
<ActionButton
title="Again: same prompt and settings, new seed"
icon={<RefreshCw size={17} />}
@@ -299,7 +299,7 @@ function PhoneMore({ item, g, showOriginal }: { item: SessionItem; g?: Generatio
<span>Reuse prompt</span>
</button>
)}
- {g && g.mode !== "lut" && (
+ {g && g.mode !== "lut" && g.mode !== "layers" && (
<button
className="opt"
onClick={() => {
diff --git a/src/components/Stage.tsx b/src/components/Stage.tsx
index 98eea29..4de7e26 100644
--- a/src/components/Stage.tsx
+++ b/src/components/Stage.tsx
@@ -1,5 +1,5 @@
import {
- Brush, ClipboardPaste, Combine, Copy, Download, Ellipsis, Expand, FolderOpen, Heart, ImageUpscale, Info, Loader2, MousePointerClick, Palette, Pencil, RefreshCw, RotateCcw, SlidersHorizontal, Square, TextCursorInput, Trash2, Wand2, WandSparkles, X,
+ Brush, ClipboardPaste, Combine, Copy, Download, Ellipsis, Expand, FolderOpen, Heart, ImageUpscale, Info, Layers, Loader2, MousePointerClick, Palette, Pencil, RefreshCw, RotateCcw, SlidersHorizontal, Square, TextCursorInput, Trash2, Wand2, WandSparkles, X,
} from "lucide-react";
import { Fragment, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
import { api, urls } from "../lib/api";
@@ -16,13 +16,16 @@ import { Cocoon, Hatch } from "./Cocoon";
import { CompareFrame, CompareFullscreen } from "./Compare";
import { PROMPT_FADE_MS, cardMemory, usePromptLeave, useTypewriter, useJobPercent, wordStyle, type CardMemory } from "./jobMotion";
import { DevelopImg } from "./Develop";
+import { docById, LabFrame, LabThumb, useLab, useLabJobs, type Lab } from "./LabStage";
+import { useMaskTint } from "./LayerDoc";
+import { docId, type LayerDoc } from "../lib/layerDoc";
import { LutBar, LutButton, LutOverlay } from "./Lut";
import { IconButton, Popover, useFit } from "./ui";
import { Fullscreen, FullscreenButton, isTyping, ZoomImage, type ViewerNav } from "./Viewer";
import { fileManager, inBrowser, shortcut, useNarrow } from "../lib/platform";
import { PhoneActions, PhoneEmpty, Swipe } from "./PhoneStage";
-export const MODE_ICON: Record<Mode | "lut", React.ReactNode> = {
+export const MODE_ICON: Record<Mode | "lut" | "layers", React.ReactNode> = {
auto: <Wand2 size={11} />,
generate: <Wand2 size={11} />,
edit: <Pencil size={11} />,
@@ -32,6 +35,7 @@ export const MODE_ICON: Record<Mode | "lut", React.ReactNode> = {
outpaint: <Expand size={11} />,
upscale: <ImageUpscale size={11} />,
lut: <Palette size={11} />,
+ layers: <Layers size={11} />,
};
const MODE_DOING: Record<Mode, string> = {
@@ -382,7 +386,7 @@ function MoreMenu({ item, path }: { item: SessionItem; path: string | null }) {
>
{(close) => (
<>
- {g && g.mode !== "lut" && (
+ {g && g.mode !== "lut" && g.mode !== "layers" && (
<button
className="opt"
onClick={() => {
@@ -420,7 +424,7 @@ function MoreMenu({ item, path }: { item: SessionItem; path: string | null }) {
/** The size of the image an edit was made from (the upscale menu offers it back), or null. */
export function sourceSize(g: GenerationRecord): Size | null {
- return g.references[0] && g.mode !== "generate" && g.mode !== "compose" && g.mode !== "upscale" && g.mode !== "lut" ? { width: g.references[0].width, height: g.references[0].height } : null;
+ return g.references[0] && g.mode !== "generate" && g.mode !== "compose" && g.mode !== "upscale" && g.mode !== "lut" && g.mode !== "layers" ? { width: g.references[0].width, height: g.references[0].height } : null;
}
function GenActions({ item, g, showOriginal, onLut }: { item: SessionItem; g: GenerationRecord; showOriginal: boolean; onLut: (lut: LutEntry) => void }) {
@@ -438,7 +442,7 @@ function GenActions({ item, g, showOriginal, onLut }: { item: SessionItem; g: Ge
onClick={() => reusePrompt(g)}
/>
)}
- {g.request && g.mode !== "upscale" && g.mode !== "lut" && (
+ {g.request && g.mode !== "upscale" && g.mode !== "lut" && g.mode !== "layers" && (
<ActionButton
title="Regenerate: same prompt and settings, new seed"
icon={<RefreshCw size={16} />}
@@ -638,8 +642,28 @@ function linkText(link: TimelineLink | undefined, labels: Record<string, string>
/** The project's versions in order. A version continues the line of the one it was made from;
* a new image starts a new line; versions made from earlier ones (or from several) are joined
* to them by arcs. The image the prompt edits carries a pencil. */
-function Timeline({ labels }: { labels: Record<string, string> }) {
- const session = useStore((s) => s.session);
+function Timeline({ labels, lab }: { labels: Record<string, string>; lab: Lab }) {
+ const all = useStore((s) => s.session);
+ // The layers lab's versions are not shown: each split image is one frame, where the first of its
+ // versions was (LabStage.tsx); an earlier lab's leftovers are not shown at all.
+ const session = useMemo(() => (lab.hidden.size ? all.filter((i) => !lab.hidden.has(i.key)) : all), [all, lab]);
+ const labJobs = useLabJobs(lab);
+ const labSlots = useMemo(() => {
+ const out: { at: number; doc: LayerDoc; key: string }[] = [];
+ const seen = new Set<string>();
+ let shown = 0;
+ for (const i of all) {
+ if (!lab.hidden.has(i.key)) {
+ shown++;
+ continue;
+ }
+ const d = docById(lab, lab.hidden.get(i.key));
+ if (!d || seen.has(docId(d))) continue;
+ seen.add(docId(d));
+ out.push({ at: shown, doc: d, key: i.key });
+ }
+ return out;
+ }, [all, lab]);
const selected = useStore((s) => s.selected);
const jobs = useStore((s) => s.jobs);
const target = useStore((s) => s.composer.refs[0]?.source);
@@ -696,8 +720,36 @@ function Timeline({ labels }: { labels: Record<string, string> }) {
setCenters(out);
}, [session, labels]);
- if (!session.length) return null;
+ if (!session.length && !labSlots.length) return null;
const focus = hovered ?? selected;
+ const labFrame = (doc: LayerDoc, key: string, first: boolean) => {
+ const id = docId(doc);
+ const isSel = !!selected && lab.hidden.get(selected) === id;
+ const making = labJobs.find((j) => docId(j.doc) === id && j.job.status !== "failed");
+ return (
+ <Fragment key={`lab-${id}`}>
+ {!first && <Joint kind="next" hot={false} />}
+ <button data-key={`lab-${id}`} className={`frame ${isSel ? "sel" : ""}`} title="Layers: this image's layered version (open its layers from the Stage)" onClick={() => selectItem(key)}>
+ <span className="frame-box">
+ <span className="frame-img">
+ <LabThumb doc={doc} job={making} />
+ </span>
+ {(isSel || making) && (
+ <svg className={`frame-silk ${making ? "making" : ""}`} viewBox="0 0 60 60" aria-hidden>
+ <rect className="ring" x="1" y="1" width="58" height="58" rx="17" />
+ {making && <rect className="run" x="1" y="1" width="58" height="58" rx="17" pathLength={200} />}
+ </svg>
+ )}
+ </span>
+ <span className="frame-label">
+ <Layers size={10} />
+ Layers
+ </span>
+ </button>
+ </Fragment>
+ );
+ };
+ const labFirst = labSlots.filter((x) => x.at === 0);
const selParents = new Set(focus ? links[focus]?.parents : []);
const index = new Map(session.map((it, i) => [it.key, i]));
const arcs = session
@@ -730,6 +782,7 @@ function Timeline({ labels }: { labels: Record<string, string> }) {
))}
</svg>
)}
+ {labFirst.map((x, k) => labFrame(x.doc, x.key, k === 0))}
{session.map((it, i) => {
const job = it.jobId ? jobs[it.jobId] : undefined;
const src = itemSource(it);
@@ -737,7 +790,7 @@ function Timeline({ labels }: { labels: Record<string, string> }) {
const failed = pending && job && ["failed", "cancelled"].includes(job.status);
// Being made: a thread runs round its thumbnail (the shown version has a still ring).
const making = pending && !failed;
- const mode: Mode | "lut" | undefined = it.gen?.mode ?? job?.plan?.mode;
+ const mode: Mode | "lut" | "layers" | undefined = it.gen?.mode ?? job?.plan?.mode;
const link = links[it.key];
const pendingSrc = pending && it.request?.mode !== "generate" && it.request?.mode !== "compose" ? it.request?.images?.[0] : undefined;
const prompt = it.gen?.prompt || it.request?.prompt || "";
@@ -746,7 +799,7 @@ function Timeline({ labels }: { labels: Record<string, string> }) {
const hot = it.key === focus || link?.parents[0] === focus;
return (
<Fragment key={it.key}>
- {i > 0 && <Joint kind={link?.kind ?? "next"} hot={hot} />}
+ {(i > 0 || labFirst.length > 0) && <Joint kind={link?.kind ?? "next"} hot={hot} />}
<button
data-key={it.key}
className={`frame ${selected === it.key ? "sel" : ""} ${selParents.has(it.key) ? "src" : ""}`}
@@ -813,6 +866,7 @@ function Timeline({ labels }: { labels: Record<string, string> }) {
{labels[it.key]}
</span>
</button>
+ {labSlots.filter((x) => x.at === i + 1).map((x) => labFrame(x.doc, x.key, false))}
</Fragment>
);
})}
@@ -883,10 +937,14 @@ export function Stage() {
const phone = useNarrow();
const job = useStore((s) => (item?.jobId ? s.jobs[item.jobId] : undefined));
const labels = useMemo(() => sessionLabels(session), [session]);
+ const lab = useLab();
+ // A version of a layer document is selected: the Stage shows that layered image instead.
+ const labDoc = item ? docById(lab, lab.hidden.get(item.key)) : null;
+ const inLab = !!labDoc;
// The result whose pre-upscale original is on screen instead of it.
const [originalOf, setOriginalOf] = useState<string | null>(null);
const [full, setFull] = useState(false);
- const canFull = !!item && viewable(item);
+ const canFull = !!item && viewable(item) && !inLab;
// A LUT being previewed on the selected version (intensity 0-1), before saving it as a new one.
const [lut, setLut] = useState<{ itemKey: string; source: string; lut: LutEntry; intensity: number } | null>(null);
const [before, setBefore] = useState(false);
@@ -896,7 +954,10 @@ export function Stage() {
const mask = useStore((s) => s.composer.mask);
const ref0 = useStore((s) => s.composer.refs[0]);
const itemSource = item?.gen ? `gen:${item.gen.id}` : item?.input ? `input:${item.input.id}` : null;
- const maskSrc = mask && ref0 && mask.refKey === ref0.key && ref0.source === itemSource ? urls.input(mask.inputId) : undefined;
+ const masked = !!mask && !!ref0 && mask.refKey === ref0.key && ref0.source === itemSource;
+ // Pink with a light edge, as in the layers lab; coloured marks are shown as drawn.
+ const tint = useMaskTint(masked && !mask!.marks ? urls.input(mask!.inputId) : null);
+ const maskSrc = masked ? (mask!.marks ? urls.input(mask!.marks) : tint ?? undefined) : undefined;
// F opens full screen on the selected version (Esc or F again closes it).
useEffect(() => {
@@ -945,7 +1006,7 @@ export function Stage() {
return (
<div className={`stage ${phone ? "phone-stage" : ""}`}>
{phone ? <PhoneEmpty /> : <Empty />}
- <Timeline labels={labels} />
+ <Timeline labels={labels} lab={lab} />
</div>
);
}
@@ -953,7 +1014,9 @@ export function Stage() {
// The picture (or the job's card), and under it the bar with what to do with it.
let view: React.ReactNode;
let bar: React.ReactNode = null;
- if (item.kind === "job") {
+ if (inLab) {
+ view = <LabFrame doc={labDoc!} lab={lab} />;
+ } else if (item.kind === "job") {
view = !job ? (
<div className="stage-frame">
<Loader2 className="spin" size={22} />
@@ -1030,7 +1093,7 @@ export function Stage() {
{phone ? <Swipe onTap={canFull && !lutActive ? openFull : undefined}>{view}</Swipe> : view}
{/* A phone keeps the bar's room even without one (a job), so the timeline never moves. */}
{phone ? <div className="phone-bar">{bar}</div> : bar}
- <Timeline labels={labels} />
+ <Timeline labels={labels} lab={lab} />
{fullscreen}
</div>
);
diff --git a/src/lib/api.ts b/src/lib/api.ts
index bbb5068..3024b48 100644
--- a/src/lib/api.ts
+++ b/src/lib/api.ts
@@ -154,6 +154,11 @@ export const api = {
},
generation: (id: string) => call<GenerationRecord>(`/gallery/${id}`),
setFavorite: (id: string, favorite: boolean) => call<GenerationRecord>(`/gallery/${id}`, { method: "PATCH", body: json({ favorite }) }),
+ /** Leaves a result out of the gallery list (layers lab steps), or shows it again. */
+ setHidden: (id: string, hidden: boolean) => call<GenerationRecord>(`/gallery/${id}`, { method: "PATCH", body: json({ hidden }) }),
+ /** The layers lab's flattened image (an upload), saved to the gallery as made from `from`. */
+ saveLayers: (image: string, from: string | null, isPrivate: boolean) =>
+ call<GenerationRecord>("/gallery/layers", { method: "POST", body: json({ image, from, private: isPrivate }) }),
deleteGeneration: (id: string) => call<{ deleted: string }>(`/gallery/${id}`, { method: "DELETE" }),
/** Adds the bundled example projects to the gallery (idempotent) and lists them. */
installExamples: () => call<{ projects: { title: string; generations: string[] }[] }>("/examples", { method: "POST" }),
diff --git a/src/lib/layerDoc.test.ts b/src/lib/layerDoc.test.ts
new file mode 100644
index 0000000..cf1e7da
--- /dev/null
+++ b/src/lib/layerDoc.test.ts
@@ -0,0 +1,223 @@
+import { describe, expect, it } from "vitest";
+import { cropFor, docFor, keepLabsIn, labSources, loadLab, saveLab, waitingMarks, jobLayer, labItems, leftovers, putDoc, type LabStore, shadowAlpha, mapPoint, placed, reconcile, reorder, toLayer, xfOf, type DocLayer, type LayerDoc } from "./layerDoc";
+
+const layer = (id: string, source: string, base = false): DocLayer => ({ id, name: id, source, history: [source], hidden: false, base, dx: 0, dy: 0, s: 1 });
+const doc = (layers: DocLayer[]): LayerDoc => ({ width: 100, height: 100, layers, known: [] });
+
+describe("layer document", () => {
+ it("a layer follows the newest edit made from it, then the edits of that", () => {
+ const d = doc([layer("a", "input:a"), layer("bg", "input:bg", true)]);
+ const items = [
+ { source: "gen:1", from: "input:a" },
+ { source: "gen:2", from: "input:a" },
+ { source: "gen:3", from: "gen:2" },
+ { source: "gen:4", from: "input:x" },
+ ];
+ const r = reconcile(d, items);
+ expect(r.layers[0].source).toBe("gen:3");
+ expect(r.layers[0].history).toEqual(["input:a", "gen:2", "gen:3"]);
+ expect(r.layers[1]).toBe(d.layers[1]);
+ expect(r.known).toEqual(expect.arrayContaining(["gen:1", "gen:2", "gen:3"]));
+ });
+
+ it("an edit already seen is not adopted again (the user went back from it)", () => {
+ const d = { ...doc([layer("a", "input:a")]), known: ["gen:1"] };
+ expect(reconcile(d, [{ source: "gen:1", from: "input:a" }])).toBe(d);
+ });
+
+ it("an edit of the photo with the background's holes as the painted area fills the background", () => {
+ const bg = { ...layer("bg", "input:bg", true), feeds: [{ source: "input:photo", mask: "m1" }] };
+ const d0 = { ...doc([bg]), photo: "input:photo" };
+ const items = [
+ { source: "gen:other", from: "input:photo", mask: null },
+ { source: "gen:fill", from: "input:photo", mask: "m1" },
+ { source: "gen:more", from: "gen:fill", mask: null },
+ ];
+ const r = reconcile(d0, items);
+ expect(r.layers[0].history).toEqual(["input:bg", "gen:fill", "gen:more"]);
+ expect(r.known).not.toContain("gen:other");
+ });
+
+ it("a fill painted again in Paint area (another mask) still fills the background", () => {
+ const bg = { ...layer("bg", "input:bg", true), feeds: [{ source: "input:tofill", mask: "m1" }] };
+ const r = reconcile({ ...doc([bg]), photo: "input:photo" }, [{ source: "gen:wider", from: "input:tofill", mask: "m9" }]);
+ expect(r.layers[0].source).toBe("gen:wider");
+ });
+
+ it("reorder keeps the background last", () => {
+ const d = doc([layer("a", "x"), layer("b", "y"), layer("bg", "z", true)]);
+ expect(reorder(d, 0, 5).layers.map((l) => l.id)).toEqual(["b", "a", "bg"]);
+ expect(reorder(d, 2, 0)).toBe(d);
+ expect(reorder(d, 1, 0).layers.map((l) => l.id)).toEqual(["b", "a", "bg"]);
+ });
+
+ it("toLayer undoes the move and the scale around the content's centre", () => {
+ const l = { ...layer("a", "x"), dx: 10, dy: -5, s: 2 };
+ const b = { x0: 40, y0: 40, x1: 60, y1: 60 };
+ const p = placed(l, b);
+ expect(p).toEqual({ x0: 40, y0: 25, x1: 80, y1: 65 });
+ expect(toLayer(l, b, p.x0, p.y0)).toEqual({ x: 40, y: 40 });
+ expect(toLayer(l, b, p.x1, p.y1)).toEqual({ x: 60, y: 60 });
+ });
+
+ it("a shadow follows its object from where the object was when it was made", () => {
+ const box = { x0: 40, y0: 40, x1: 60, y1: 60 };
+ const obj = { ...layer("cup", "c"), dx: 30, dy: 0, s: 2 };
+ const shadow = { ...layer("sh", "s"), effect: true, follows: "cup", at: { dx: 10, dy: 0, s: 1 } };
+ const t = xfOf([obj, shadow], shadow, () => box, 100, 100);
+ // A shadow pixel at the object's old centre (50+10, 50) lands at its new centre (50+30, 50).
+ expect(mapPoint(t, 60, 50)).toEqual({ x: 80, y: 50 });
+ // 5 px right of the old centre: 10 px right of the new one (scale 1 -> 2).
+ expect(mapPoint(t, 65, 50)).toEqual({ x: 90, y: 50 });
+ // Nudged by hand (the model put it a little off): its own move adds to the object's.
+ const nudged = { ...shadow, dx: 3, dy: -2 };
+ expect(mapPoint(xfOf([obj, nudged], nudged, () => box, 100, 100), 60, 50)).toEqual({ x: 83, y: 48 });
+ // Without `at` (or the object gone) a layer uses its own place.
+ expect(xfOf([shadow], shadow, () => box, 100, 100)).toEqual({ c: { x: 50, y: 50 }, s: 1, dx: 0, dy: 0 });
+ });
+
+ it("shadowAlpha: how much darker, inside the painted area only; brighter pixels are left out", () => {
+ const px = (v: number) => [v, v, v, 255];
+ const before = [...px(200), ...px(200), ...px(200), ...px(200), ...px(100)];
+ const after = [...px(199), ...px(100), ...px(240), ...px(0), ...px(50)];
+ const a = [...shadowAlpha(before, after, [255, 255, 255, 255, 0])];
+ expect(a[0]).toBe(0); // noise
+ expect(a[1]).toBeGreaterThan(120); // half as bright: about half black
+ expect(a[1]).toBeLessThan(135);
+ expect(a[2]).toBe(0); // brighter
+ expect(a[3]).toBeGreaterThan(250); // black
+ expect(a[4]).toBe(0); // outside the area
+ });
+
+ it("labItems: the lab's versions and what was made from them; the photo and its own edits stay", () => {
+ const d: LayerDoc = {
+ ...doc([{ ...layer("bg", "input:bg", true), feeds: [{ source: "input:photo", mask: "m1" }] }, layer("cup", "gen:cup2")]),
+ photo: "input:photo",
+ known: ["input:bg", "input:cup"],
+ pending: [{ source: "input:flat", mask: "m2", of: null }],
+ };
+ const lab: LabStore = { v: 2, docs: [d], made: ["input:old"] };
+ const items = [
+ { key: "photo", source: "input:photo", from: null },
+ { key: "edit", source: "gen:e", from: "input:photo", mask: null },
+ { key: "bg", source: "input:bg", from: null },
+ { key: "cup", source: "input:cup", from: null },
+ { key: "cup2", source: "gen:cup2", from: "input:cup" },
+ { key: "cup3", source: "gen:cup3", from: "gen:cup2" },
+ { key: "fill", source: "gen:f", from: "input:photo", mask: "m1" },
+ { key: "flat", source: "input:flat", from: null },
+ { key: "shadow", source: "gen:s", from: "input:flat", mask: "m2" },
+ { key: "job", source: null, from: "gen:cup3" },
+ { key: "export", source: "input:flattened", from: null },
+ { key: "old", source: "input:old", from: null },
+ { key: "oldEdit", source: "gen:o", from: "input:old" },
+ ];
+ const got = labItems(lab, items);
+ expect([...got.keys()].sort()).toEqual(["bg", "cup", "cup2", "cup3", "fill", "flat", "job", "old", "oldEdit", "shadow"]);
+ expect(got.get("cup3")).toBe("input:photo");
+ expect(got.get("oldEdit")).toBeNull();
+ });
+
+ it("one document per image: found by its photo or by what it made; splitting again replaces it", () => {
+ const a = { ...doc([layer("bg", "input:bgA", true)]), photo: "gen:A" };
+ const b = { ...doc([layer("bg", "input:bgB", true)]), photo: "gen:B" };
+ let lab: LabStore = putDoc(putDoc({ v: 2, docs: [], made: [] }, a), b);
+ expect(lab.docs.length).toBe(2);
+ expect(docFor(lab, "gen:B")).toBe(b);
+ expect(docFor(lab, "input:bgA")).toBe(a);
+ expect(docFor(lab, "gen:other")).toBeNull();
+ const a2 = { ...doc([layer("bg", "input:bgA2", true)]), photo: "gen:A" };
+ lab = putDoc(lab, a2);
+ expect(lab.docs).toEqual([a2, b]);
+ });
+
+ it("leftovers: the lab's own images by name, and the cut-outs saved right after a split's background", () => {
+ const it = (source: string, name: string | null, input = true) => ({ source, name, input });
+ expect(
+ leftovers([
+ it("input:photo", "kitchen.png"),
+ it("gen:e", null, false),
+ it("input:bg", "Background (holes).png"),
+ it("input:cup", "Cup.png"),
+ it("input:glass", "Glass.png"),
+ it("gen:x", null, false),
+ it("input:crop", "Cup (object size).png"),
+ it("input:fill", "Background to fill.png"),
+ it("input:sh", "Cup shadow.png"),
+ it("input:export", "Layers flattened.png"),
+ it("input:mine", "holiday.png"),
+ ]),
+ ).toEqual(["input:bg", "input:cup", "input:glass", "input:crop", "input:fill", "input:sh"]);
+ });
+
+ it("jobLayer: an edit of a layer, of one of its fills, or a shadow being made", () => {
+ const d: LayerDoc = {
+ ...doc([layer("cup", "gen:cup2"), { ...layer("bg", "input:bg", true), feeds: [{ source: "input:fill", mask: "m1" }] }]),
+ pending: [
+ { source: "input:flat", mask: "m2", of: "cup" },
+ { source: "input:flat2", mask: "m3", of: null },
+ { source: "input:flat3", mask: "m4", of: null, layer: "fx-1" },
+ ],
+ };
+ expect(jobLayer(d, "gen:cup2", null)).toBe("cup");
+ expect(jobLayer(d, "input:fill", "m1")).toBe("bg");
+ expect(jobLayer(d, "input:fill", "other")).toBe("bg"); // painted again in Paint area
+ expect(jobLayer(d, "input:flat", "m2")).toBe("cup");
+ expect(jobLayer(d, "input:flat2", "m3")).toBe("shadows");
+ expect(jobLayer(d, "input:flat3", "m4")).toBe("fx-1");
+ expect(jobLayer(d, "input:photo", null)).toBeNull();
+ });
+
+ it("cropFor: the object's box with a margin, inside the frame, enlarged to about 1 MP", () => {
+ const { rect, scale } = cropFor({ x0: 100, y0: 200, x1: 300, y1: 400 }, 2048, 2048, 0.2);
+ expect(rect).toEqual({ x: 60, y: 160, w: 280, h: 280 });
+ expect(scale).toBeCloseTo(1024 / 280, 3);
+ // At the frame's edge, and never enlarged more than 4 times or shrunk.
+ expect(cropFor({ x0: 0, y0: 0, x1: 50, y1: 50 }, 100, 100, 0.2)).toEqual({ rect: { x: 0, y: 0, w: 60, h: 60 }, scale: 4 });
+ expect(cropFor({ x0: 0, y0: 0, x1: 2000, y1: 2000 }, 2048, 2048, 0).scale).toBe(1);
+ });
+
+ it("cropFor: a narrow object gets transparent room at its sides, up to 1:2 (past the frame if needed)", () => {
+ // A bottle 100 x 400 (with the margin 140 x 560): widened to 280, centred on it.
+ const { rect, scale } = cropFor({ x0: 1000, y0: 500, x1: 1100, y1: 900 }, 2752, 1536, 0.2);
+ expect(rect).toEqual({ x: 910, y: 420, w: 280, h: 560 });
+ expect(scale).toBeCloseTo(Math.sqrt((1024 * 1024) / (280 * 560)), 3);
+ // At the frame's left edge: the room goes past it.
+ expect(cropFor({ x0: 0, y0: 0, x1: 50, y1: 300 }, 1000, 1000, 0).rect).toEqual({ x: -50, y: 0, w: 150, h: 300 });
+ // A wide one gets height instead.
+ expect(cropFor({ x0: 0, y0: 400, x1: 600, y1: 500 }, 1000, 1000, 0).rect).toEqual({ x: 0, y: 300, w: 600, h: 300 });
+ });
+
+ it("marks sent with a scene edit show until its result made layers", () => {
+ const scene = { source: "input:s", before: "input:b", shadows: [], look: [], places: {}, reach: 12 };
+ const d = { ...doc([layer("bg", "input:bg", true)]), scenes: [{ ...scene, marks: "m1" }, { ...scene, marks: "m2", made: {} }, scene] };
+ expect(waitingMarks(d)).toEqual(["m1"]);
+ });
+
+ it("crop edits are not adopted as they are (they are pasted back first)", () => {
+ const cup = { ...layer("cup", "input:cup"), feeds: [{ source: "input:crop", mask: "", rect: { x: 0, y: 0, w: 10, h: 10 }, scale: 2, from: "input:cup" }] };
+ const d = doc([cup]);
+ expect(reconcile(d, [{ source: "gen:c", from: "input:crop", mask: null }])).toBe(d);
+ });
+});
+
+describe("where the lab is kept", () => {
+ it("reads the project's copy, else the one this browser kept before", () => {
+ const old = { v: 2, docs: [], made: ["gen:old"] };
+ localStorage.setItem("kokon.layersLab.p", JSON.stringify(old));
+ const kept = new Map<string, unknown>();
+ keepLabsIn({ get: (id) => kept.get(id), set: (id, lab) => kept.set(id, lab) });
+ try {
+ expect(loadLab("p").made).toEqual(["gen:old"]);
+ saveLab("p", { ...loadLab("p"), made: [...loadLab("p").made, "gen:new"] });
+ expect((kept.get("p") as { made: string[] }).made).toEqual(["gen:old", "gen:new"]);
+ expect(loadLab("p").made).toEqual(["gen:old", "gen:new"]);
+ expect(labSources(loadLab("p")).has("gen:new")).toBe(true);
+ // The browser's copy stays as it was (a backup).
+ expect(JSON.parse(localStorage.getItem("kokon.layersLab.p")!).made).toEqual(["gen:old"]);
+ } finally {
+ keepLabsIn(null);
+ localStorage.removeItem("kokon.layersLab.p");
+ }
+ });
+});
diff --git a/src/lib/layerDoc.ts b/src/lib/layerDoc.ts
new file mode 100644
index 0000000..3129950
--- /dev/null
+++ b/src/lib/layerDoc.ts
@@ -0,0 +1,458 @@
+// Layers lab: the layer document a split makes (components/LayerDoc.tsx). Each layer is a version
+// of the project ("gen:<id>" / "input:<id>") drawn over the whole frame, moved and scaled; editing a
+// layer with Kokon makes a new version that the layer then follows. Kept with the project in
+// kokon-service (store.ts, `keepLabsIn`), so every device and an update see it; labs made before
+// that were in this browser's storage and move there when first read.
+
+import type { MarkColour } from "./sceneEdit";
+
+/** An edit of another image that also belongs to a layer: "Fill the holes" runs a brushed Remove
+ * on the original photo with the background's holes as the painted area (an edit of the transparent
+ * background keeps its holes transparent), and that result is the background's next version. */
+export interface Feed {
+ source: string;
+ /** Input id of the painted area as set up (the user may paint it again in Paint area). */
+ mask: string;
+ /** An object-size crop of the layer (`rect` in the layer's own frame px, enlarged `scale` times
+ * for the model): its edits are shrunk back into `rect` (pasteCrop), not adopted as they are. */
+ rect?: Rect;
+ scale?: number;
+ /** The layer version the crop was cut from. */
+ from?: string;
+ /** Clean edges: input id of the grown selection (crop size) the result is clipped to, and how
+ * much it was grown (crop px): faint pixels in that ring are dropped (the halo of what was around). */
+ clip?: string;
+ grow?: number;
+}
+
+export interface Rect {
+ x: number;
+ y: number;
+ w: number;
+ h: number;
+}
+
+/** Below about 1 MP Qwen-Image does worse (planner.rs INPAINT_MIN_AREA): a crop smaller than that is
+ * enlarged for the model, at most 4 times, and its result shrunk back. */
+export const CROP_AREA = 1024 * 1024;
+
+/** A crop is never narrower than 1:2 (Qwen does worse on very narrow images: a bottle came back
+ * 544x1720, 10-31). */
+export const CROP_MAX_RATIO = 2;
+
+/** The crop for an object: its content box grown by `margin` of its size each side (room for a part
+ * the edit completes), inside the frame, in whole px; widened (or made taller) evenly to at most
+ * CROP_MAX_RATIO, past the frame's edge if needed: the layer is transparent there, so the model gets
+ * transparent room, not more scene. And how much to enlarge it for the model. */
+export function cropFor(box: Box, W: number, H: number, margin: number): { rect: Rect; scale: number } {
+ const mx = (box.x1 - box.x0) * margin;
+ const my = (box.y1 - box.y0) * margin;
+ let x = Math.max(0, Math.floor(box.x0 - mx));
+ let y = Math.max(0, Math.floor(box.y0 - my));
+ let w = Math.min(W, Math.ceil(box.x1 + mx)) - x;
+ let h = Math.min(H, Math.ceil(box.y1 + my)) - y;
+ if (h > w * CROP_MAX_RATIO) {
+ const nw = Math.ceil(h / CROP_MAX_RATIO);
+ x -= Math.floor((nw - w) / 2);
+ w = nw;
+ } else if (w > h * CROP_MAX_RATIO) {
+ const nh = Math.ceil(w / CROP_MAX_RATIO);
+ y -= Math.floor((nh - h) / 2);
+ h = nh;
+ }
+ const scale = Math.min(4, Math.max(1, Math.sqrt(CROP_AREA / Math.max(1, w * h))));
+ return { rect: { x, y, w, h }, scale };
+}
+
+/** An edit of `from` (painted area `mask`) belongs to the layer with feed `f`: any painted edit of an
+ * image the lab made for it ("Background to fill"); on the photo itself (an older document's
+ * Remove) only the area the lab painted, since the photo has edits of its own. */
+const fed = (doc: LayerDoc, f: Feed, from: string | null | undefined, mask: string | null | undefined) =>
+ f.source === from && (from !== doc.photo || f.mask === mask);
+
+export interface DocLayer {
+ id: string;
+ name: string;
+ /** The version shown. */
+ source: string;
+ /** Every version this layer has shown, oldest first (the Versions menu). */
+ history: string[];
+ hidden: boolean;
+ /** The background: not moved, always at the bottom. */
+ base: boolean;
+ feeds?: Feed[];
+ /** Shadows and light made by an edit (only the pixels it changed): not clicked on the canvas. */
+ effect?: boolean;
+ /** An object's shadow moves and scales with the object (`follows`), from where the object was
+ * when the shadow was made (`at`). */
+ follows?: string;
+ at?: Place;
+ /** Layers with the same group are selected, moved and scaled together. */
+ group?: string;
+ /** Offset in frame px, and scale around the centre of what the layer shows. */
+ dx: number;
+ dy: number;
+ s: number;
+}
+
+export interface Place {
+ dx: number;
+ dy: number;
+ s: number;
+}
+
+/** An older shadow edit (until 10-31; its result is still adopted when it comes): a brushed edit of
+ * the flattened layers (`source`, made for it alone) with a ring around one object (`of`) or all of
+ * them (null) as the painted area (`mask`, input id). Its results become an effect layer (`layer`
+ * once made; new results are its next versions). Now SceneEdit. */
+export interface Effect {
+ source: string;
+ mask: string;
+ of: string | null;
+ at?: Place;
+ layer?: string;
+}
+
+/** A whole-image edit set up in the lab ("Shadows", "New objects", "Edit together"): the
+ * flattened layers (`source`; with the coloured marks drawn on when there are marks, and `before`
+ * without them) sent as a plain edit, so Qwen sees the whole scene. Its results are put back over
+ * the layers and read (components/LabScene.ts): the shadows of the `shadows` objects, the new look
+ * of the `look` objects, one new layer per mark colour (`wants`). */
+export interface SceneEdit {
+ source: string;
+ before: string;
+ /** Input id of the marks alone (colours on transparency, frame size). */
+ marks?: string;
+ wants?: { colour: MarkColour; what: string }[];
+ shadows: string[];
+ look: string[];
+ /** Each visible object when the edit was made (it may move or change while the edit runs). */
+ places: Record<string, Place & { source: string }>;
+ /** How far shadows are looked for around an object, in % of the image's long side. */
+ reach: number;
+ /** Layers its results made, so the next result becomes their next version: object id -> its
+ * shadow layer, colour -> the new layer, "<colour>:shadow" -> the new layer's shadow. */
+ made?: Record<string, string>;
+}
+
+export interface LayerDoc {
+ /** The frame: the size of the photo that was split. */
+ width: number;
+ height: number;
+ /** The photo that was split (older documents: unknown). */
+ photo?: string;
+ /** Top first; the background last. */
+ layers: DocLayer[];
+ pending?: Effect[];
+ scenes?: SceneEdit[];
+ /** Versions already looked at for "follow the edit" (adopted, or left by going back). */
+ known: string[];
+ /** Split with "Clean the stickers automatically": LayerDoc sends every object to Clean edges once. */
+ autoClean?: boolean;
+ /** Those automatic Clean edges, one per object layer (shown while any is left). */
+ stickers?: Sticker[];
+}
+
+/** An object's automatic Clean edges: its job once sent (null: not yet, or it could not be sent). */
+export interface Sticker {
+ layer: string;
+ job: string | null;
+ /** The layer's version the crop was made from: the sticker is in once the layer moved on. */
+ from?: string;
+ /** "fill": the background's holes filled (its painted area: `mask`), after the objects. */
+ kind?: "fill";
+ mask?: string;
+ /** Sent again once already (the fill left grey patches). */
+ retried?: boolean;
+ /** The fill's result was looked at (no grey patches). */
+ checked?: boolean;
+ skipped?: boolean;
+ failed?: boolean;
+}
+
+export interface Box {
+ x0: number;
+ y0: number;
+ x1: number;
+ y1: number;
+}
+
+const KEY = "kokon.layersLab.";
+
+/** A project's lab: one layer document per image that was split (`photo`), and every version any
+ * lab of the project has made (`made`: kept when a document is split again, so its old versions stay
+ * out of the version strip). `swept`: older leftovers were looked for once (`leftovers`). */
+export interface LabStore {
+ v: 2;
+ docs: LayerDoc[];
+ made: string[];
+ swept?: boolean;
+}
+
+/** A document's id: the image it was split from (older documents: the background's first version). */
+export const docId = (d: LayerDoc) => d.photo ?? d.layers.find((l) => l.base)?.history[0] ?? d.layers[0]?.history[0] ?? "";
+
+/** Every version a document has made or shown (not the photo it was split from). */
+export function docSources(d: LayerDoc): Set<string> {
+ const s = new Set([...d.known, ...d.layers.flatMap((l) => [...l.history, ...(l.feeds ?? []).map((f) => f.source)]), ...(d.pending ?? []).map((p) => p.source), ...(d.scenes ?? []).flatMap((e) => [e.source, e.before])]);
+ if (d.photo) s.delete(d.photo);
+ return s;
+}
+
+/** The document an image belongs to: the one split from it, or the one that made it. */
+export function docFor(lab: LabStore, source: string | null | undefined): LayerDoc | null {
+ if (!source) return null;
+ return lab.docs.find((d) => d.photo === source) ?? lab.docs.find((d) => docSources(d).has(source)) ?? null;
+}
+
+/** A document added, or one split again (the same photo) replaced; `made` keeps the old versions. */
+export function putDoc(lab: LabStore, d: LayerDoc): LabStore {
+ const id = docId(d);
+ const at = lab.docs.findIndex((x) => docId(x) === id);
+ const docs = at < 0 ? [...lab.docs, d] : lab.docs.map((x, i) => (i === at ? d : x));
+ return { ...lab, docs };
+}
+
+// Saves are announced, so the Stage follows the documents (useSyncExternalStore).
+let version = 0;
+const listeners = new Set<() => void>();
+export function subscribeDocs(f: () => void) {
+ listeners.add(f);
+ return () => listeners.delete(f);
+}
+export const docsVersion = () => version;
+
+/** Where labs are kept (the project, see store.ts): `get` is undefined while the project has none. */
+export interface LabKeeper {
+ get(projectId: string): unknown;
+ set(projectId: string, lab: LabStore): void;
+}
+let keeper: LabKeeper | null = null;
+export function keepLabsIn(k: LabKeeper | null) {
+ keeper = k;
+}
+
+/** The project's lab changed elsewhere (another device, or the service's copy arrived). */
+export function labsChanged() {
+ version++;
+ for (const f of listeners) f();
+}
+
+function parseLab(data: unknown): LabStore | null {
+ if (!data || typeof data !== "object") return null;
+ // Before 10-31: one document per project, stored as it is.
+ if ((data as LabStore).v !== 2) return { v: 2, docs: [data as LayerDoc], made: [] };
+ return data as LabStore;
+}
+
+export function loadLab(projectId: string): LabStore {
+ const kept = parseLab(keeper?.get(projectId));
+ if (kept) return kept;
+ // Kept in this browser's storage before (the copy stays there as a backup).
+ try {
+ const raw = localStorage.getItem(KEY + projectId);
+ const old = raw ? parseLab(JSON.parse(raw)) : null;
+ if (old) return old;
+ } catch {
+ /* unreadable: no lab */
+ }
+ return { v: 2, docs: [], made: [] };
+}
+
+/** Saves the lab; everything its documents have made joins `made` for good. */
+export function saveLab(projectId: string, lab: LabStore) {
+ const made = new Set(lab.made);
+ for (const d of lab.docs) for (const s of docSources(d)) made.add(s);
+ const next = { ...lab, made: [...made] };
+ if (keeper) keeper.set(projectId, next);
+ else {
+ try {
+ localStorage.setItem(KEY + projectId, JSON.stringify(next));
+ } catch {
+ /* storage full or blocked: the documents live until the page closes */
+ }
+ }
+ labsChanged();
+}
+
+/** The project's images the lab made or uses (gallery steps to hide, requests to mark hidden). */
+export function labSources(lab: LabStore): Set<string> {
+ const out = new Set(lab.made);
+ for (const d of lab.docs) for (const s of docSources(d)) out.add(s);
+ return out;
+}
+
+export interface LabItem {
+ key: string;
+ source: string | null;
+ from: string | null;
+ mask?: string | null;
+}
+
+/** The project's versions that belong to a lab, so the project shows each layered image instead of
+ * them: per document, the split's images, every version a layer has shown, the images its crops,
+ * fills and shadows were made on, and everything made from those (edits not adopted yet too); and
+ * anything an earlier lab of the project made (`made`). The photos that were split and their other
+ * edits stay. Answer: version key -> the id of its document (null: an earlier lab's leftover). */
+export function labItems(lab: LabStore, items: LabItem[]): Map<string, string | null> {
+ const owner = new Map<string, string | null>();
+ for (const d of lab.docs) for (const s of docSources(d)) owner.set(s, docId(d));
+ for (const s of lab.made) if (!owner.has(s)) owner.set(s, null);
+ const photos = new Set(lab.docs.map((d) => d.photo).filter(Boolean) as string[]);
+ for (const p of photos) owner.delete(p);
+ const keys = new Map<string, string | null>();
+ for (const it of items) {
+ let doc: string | null | undefined = it.source ? owner.get(it.source) : undefined;
+ if (doc === undefined && it.from) {
+ if (owner.has(it.from)) doc = owner.get(it.from)!;
+ // From a photo itself: only a fill of an older document (a Remove on the photo).
+ else if (photos.has(it.from)) {
+ const d = lab.docs.find((x) => x.photo === it.from && x.layers.some((l) => l.feeds?.some((f) => !f.rect && fed(x, f, it.from, it.mask))));
+ if (d) doc = docId(d);
+ }
+ }
+ if (doc === undefined) continue;
+ keys.set(it.key, doc);
+ if (it.source) owner.set(it.source, doc);
+ }
+ return keys;
+}
+
+/** Leftovers of labs whose document is gone (a project split before documents were kept per image):
+ * images the lab saved, by their names, and the cut-outs saved right after a "Background (holes)". */
+export function leftovers(items: { source: string | null; name: string | null; input: boolean }[]): string[] {
+ const named = /^(Background \(holes\)|Background to fill|Background on grey|Layers for shadows|Layers for the scene|Layers with marks|Shadows)\.png$|\((grey|object size|to clean|brushed|edited)\)\.png$| shadow\.png$/;
+ const out: string[] = [];
+ let afterSplit = false;
+ for (const it of items) {
+ if (!it.input || !it.source || !it.name) {
+ afterSplit = false;
+ continue;
+ }
+ if (it.name === "Background (holes).png") {
+ out.push(it.source);
+ afterSplit = true;
+ } else if (named.test(it.name) || afterSplit) out.push(it.source);
+ else afterSplit = false;
+ }
+ return out;
+}
+
+/** Layers follow their edits: a version made from a layer's version (and not seen before) becomes
+ * the layer's version, then the same again from there (the newest edit wins at each step); so does
+ * an edit that matches one of the layer's feeds. `items`: the project's versions in order, with the
+ * version each was made from and the input id of its painted area. */
+export function reconcile(doc: LayerDoc, items: { source: string; from: string | null; mask?: string | null }[]): LayerDoc {
+ const known = new Set(doc.known);
+ let changed = false;
+ const layers = doc.layers.map((l) => {
+ let cur = l.source;
+ const history = l.history.slice();
+ // Crop feeds are not adopted as they are: their results are pasted back first (LayersEditor).
+ const mine = (i: (typeof items)[number]) => i.from === cur || !!l.feeds?.some((f) => !f.rect && fed(doc, f, i.from, i.mask));
+ for (;;) {
+ const next = [...items].reverse().find((i) => mine(i) && !known.has(i.source));
+ if (!next) break;
+ for (const i of items) if (mine(i)) known.add(i.source);
+ cur = next.source;
+ history.push(cur);
+ }
+ if (cur === l.source) return l;
+ changed = true;
+ return { ...l, source: cur, history };
+ });
+ return changed ? { ...doc, layers, known: [...known] } : doc;
+}
+
+/** Moves layer `from` to index `to`; the background stays last. */
+export function reorder(doc: LayerDoc, from: number, to: number): LayerDoc {
+ const ls = doc.layers.slice();
+ const last = ls.length - 1;
+ if (ls[from]?.base) return doc;
+ to = Math.max(0, Math.min(to, ls[last]?.base ? last - 1 : last));
+ if (from === to) return doc;
+ const [l] = ls.splice(from, 1);
+ ls.splice(to, 0, l);
+ return { ...doc, layers: ls };
+}
+
+/** A layer and every other layer of its group. */
+export const withGroup = (layers: DocLayer[], id: string): string[] => {
+ const l = layers.find((x) => x.id === id);
+ return l?.group ? layers.filter((x) => x.group === l.group).map((x) => x.id) : l ? [id] : [];
+};
+
+export const centre = (b: Box) => ({ x: (b.x0 + b.x1) / 2, y: (b.y0 + b.y1) / 2 });
+
+/** How a layer is drawn: scaled by `s` around `c`, then moved by `dx`, `dy` (frame px). */
+export interface Xf {
+ c: { x: number; y: number };
+ s: number;
+ dx: number;
+ dy: number;
+}
+
+/** A layer's own move and scale, around the centre of its content; a shadow that follows an
+ * object takes the object's move and scale since the shadow was made, plus its own move (the user
+ * nudging a shadow the model put a little off). `boxOf`: a version's content box (undefined while
+ * it loads). */
+export function xfOf(layers: DocLayer[], l: DocLayer, boxOf: (source: string) => Box | undefined, W: number, H: number): Xf {
+ const mid = (b: Box | undefined) => (b ? centre(b) : { x: W / 2, y: H / 2 });
+ const obj = l.follows ? layers.find((x) => x.id === l.follows) : undefined;
+ if (obj && l.at) {
+ const c = mid(boxOf(obj.source));
+ return { c: { x: c.x + l.at.dx, y: c.y + l.at.dy }, s: obj.s / l.at.s, dx: obj.dx - l.at.dx + l.dx, dy: obj.dy - l.at.dy + l.dy };
+ }
+ return { c: mid(boxOf(l.source)), s: l.s, dx: l.dx, dy: l.dy };
+}
+
+export const mapPoint = (t: Xf, x: number, y: number) => ({ x: (x - t.c.x) * t.s + t.c.x + t.dx, y: (y - t.c.y) * t.s + t.c.y + t.dy });
+export const unmapPoint = (t: Xf, x: number, y: number) => ({ x: (x - t.dx - t.c.x) / t.s + t.c.x, y: (y - t.dy - t.c.y) / t.s + t.c.y });
+
+export function mapBox(b: Box, t: Xf): Box {
+ const a = mapPoint(t, b.x0, b.y0);
+ const z = mapPoint(t, b.x1, b.y1);
+ return { x0: a.x, y0: a.y, x1: z.x, y1: z.y };
+}
+
+const ownXf = (l: DocLayer, b: Box): Xf => ({ c: centre(b), s: l.s, dx: l.dx, dy: l.dy });
+
+/** A point of the frame -> the point of the layer's own image under it (`b`: its content box). */
+export const toLayer = (l: DocLayer, b: Box, x: number, y: number) => unmapPoint(ownXf(l, b), x, y);
+
+/** The layer's content box where it is drawn in the frame. */
+export const placed = (l: DocLayer, b: Box): Box => mapBox(b, ownXf(l, b));
+
+/** A shadow edit's result as a shadow (black, this alpha): per pixel how much darker the edit made
+ * it (`after` luminance / `before` luminance), inside `area` (0-255 per pixel, the painted area).
+ * Darkness, not the edit's own pixels, so the shadow darkens whatever is under it when its object
+ * moves; brighter pixels are left out. `floor`: darkening ignored (decode noise, the colour match
+ * at the seam). */
+export function shadowAlpha(before: ArrayLike<number>, after: ArrayLike<number>, area: ArrayLike<number>, floor = 0.04): Uint8Array {
+ const out = new Uint8Array(area.length);
+ const lum = (d: ArrayLike<number>, j: number) => 0.2126 * d[j] + 0.7152 * d[j + 1] + 0.0722 * d[j + 2];
+ for (let i = 0; i < out.length; i++) {
+ if (!area[i]) continue;
+ const r = (lum(after, i * 4) + 1) / (lum(before, i * 4) + 1);
+ const v = Math.min(1, Math.max(0, (1 - r - floor) / (1 - floor)));
+ out[i] = Math.round(v * area[i]);
+ }
+ return out;
+}
+
+/** The pencil's marks sent with New objects / Edit together whose result has not come back yet
+ * (input ids): shown over the layers, here and on the Stage, until the objects are there. */
+export const waitingMarks = (doc: LayerDoc): string[] => (doc.scenes ?? []).filter((e) => e.marks && !e.made).map((e) => e.marks!);
+
+/** The layer an edit is for, from its image and painted area (`mask`: input id): the layer's own
+ * version, one of its fills, a whole-image edit ("scene"), an older ring shadow being made (the
+ * object's id; "all objects": "shadows"). Null: not a lab edit. */
+export function jobLayer(doc: LayerDoc, image: string | undefined, mask: string | null | undefined): string | null {
+ if (!image) return null;
+ const own = doc.layers.find((l) => l.source === image || l.feeds?.some((f) => fed(doc, f, image, mask)));
+ if (own) return own.id;
+ if (doc.scenes?.some((e) => e.source === image)) return "scene";
+ const p = doc.pending?.find((x) => x.source === image);
+ if (!p) return null;
+ return p.layer ?? p.of ?? "shadows";
+}
diff --git a/src/lib/layers.test.ts b/src/lib/layers.test.ts
new file mode 100644
index 0000000..2d20d9a
--- /dev/null
+++ b/src/lib/layers.test.ts
@@ -0,0 +1,110 @@
+import { describe, expect, it } from "vitest";
+import { addTo, dilate, erode, fileSafe, fromLogits, grow, maskOrder, refineEdges, removeFrom, samSelection, union } from "./layers";
+
+describe("layers lab masks", () => {
+ it("adding to a layer takes the pixels from the others", () => {
+ const masks = [new Uint8Array([255, 255, 0]), new Uint8Array([0, 0, 0])];
+ addTo(masks, 1, [0, 255, 255]);
+ expect([...masks[0]]).toEqual([255, 0, 0]);
+ expect([...masks[1]]).toEqual([0, 255, 255]);
+ });
+
+ it("a soft stroke leaves the rest of the other layer's alpha", () => {
+ const masks = [new Uint8Array([255]), new Uint8Array([0])];
+ addTo(masks, 1, [100]);
+ expect([...masks[0]]).toEqual([155]);
+ expect([...masks[1]]).toEqual([100]);
+ });
+
+ it("a click only takes what no other layer has", () => {
+ const masks = [new Uint8Array([255, 0, 100]), new Uint8Array([0, 0, 0])];
+ addTo(masks, 1, [255, 255, 255], false);
+ expect([...masks[0]]).toEqual([255, 0, 100]);
+ expect([...masks[1]]).toEqual([0, 255, 155]);
+ });
+
+ it("removing gives the pixels back to the background only", () => {
+ const m = new Uint8Array([255, 255]);
+ removeFrom(m, [255, 0]);
+ expect([...m]).toEqual([0, 255]);
+ });
+
+ it("logits above 0 are selected; union takes the largest alpha", () => {
+ expect([...fromLogits([-1, 0, 0.5])]).toEqual([0, 0, 255]);
+ expect([...union([new Uint8Array([10, 0]), new Uint8Array([5, 200])], 2)]).toEqual([10, 200]);
+ });
+
+ it("dilate grows by r in both directions and stays inside the image", () => {
+ const w = 5;
+ const h = 5;
+ const m = new Uint8Array(w * h);
+ m[2 * w + 2] = 255;
+ const d = dilate(m, w, h, 1);
+ const on = [...d].map((v, i) => (v ? i : -1)).filter((i) => i >= 0);
+ expect(on).toEqual([6, 7, 8, 11, 12, 13, 16, 17, 18]);
+ expect([...dilate(m, w, h, 0)]).toEqual([...m]);
+ });
+
+ it("mask order: best IoU first, then smallest to largest", () => {
+ expect(maskOrder([0.99, 0.89, 1.01, 0.82], [0.2, 0.17, 0.196, 0.275])).toEqual([2, 1, 3]);
+ expect(maskOrder([0.5], [0.1])).toEqual([0]);
+ });
+
+ it("file names lose the characters Windows refuses", () => {
+ expect(fileSafe('a/b: "c"')).toBe("a b c");
+ expect(fileSafe(" ")).toBe("layer");
+ });
+
+ it("erode shrinks what dilate grows", () => {
+ const w = 7;
+ const m = new Uint8Array(w * w);
+ m[3 * w + 3] = 255;
+ expect([...erode(dilate(m, w, w, 2), w, w, 2)]).toEqual([...m]);
+ });
+
+ it("grow reaches further down than up", () => {
+ const w = 1;
+ const h = 9;
+ const m = new Uint8Array(h);
+ m[4] = 255;
+ const g = grow(m, w, h, { left: 0, right: 0, up: 1, down: 3 });
+ expect([...g].map((v) => (v ? 1 : 0))).toEqual([0, 0, 0, 1, 1, 1, 1, 1, 0]);
+ });
+
+ it("refineEdges snaps a stepped mask to the photo's edge and keeps the inside solid", () => {
+ // The photo: a bright disc (r 10) on dark; the mask: the same disc in 4 px steps (SAM-like).
+ const w = 40;
+ const h = 40;
+ const gray = new Float32Array(w * h);
+ const coarse = new Uint8Array(w * h);
+ const truth = new Uint8Array(w * h);
+ for (let y = 0; y < h; y++)
+ for (let x = 0; x < w; x++) {
+ const inDisc = Math.hypot(x - 20, y - 20) <= 10;
+ gray[y * w + x] = inDisc ? 0.8 : 0.1;
+ truth[y * w + x] = inDisc ? 255 : 0;
+ coarse[y * w + x] = Math.hypot((x >> 2) * 4 + 2 - 20, (y >> 2) * 4 + 2 - 20) <= 10 ? 255 : 0;
+ }
+ const err = (m: Uint8Array) => m.reduce((s, v, i) => s + Math.abs(v - truth[i]), 0);
+ const refined = refineEdges(coarse, gray, w, h, 4);
+ expect(err(refined)).toBeLessThan(err(coarse) * 0.5);
+ expect(refined[20 * w + 20]).toBe(255);
+ expect(refined[0]).toBe(0);
+ });
+});
+
+describe("samSelection", () => {
+ it("takes SAM's object and a rim past it, two low-res cells wide", () => {
+ // 512 px: two cells of 256 = 4 px.
+ const w = 512;
+ const h = 512;
+ const logits = new Float32Array(w * h).fill(-5);
+ for (let y = 200; y < 300; y++) for (let x = 200; x < 300; x++) logits[y * w + x] = 3;
+ const m = samSelection(logits, w, h);
+ expect(m[250 * w + 250]).toBe(255);
+ expect(m[250 * w + 196]).toBe(255);
+ expect(m[250 * w + 303]).toBe(255);
+ expect(m[250 * w + 195]).toBe(0);
+ expect(m[195 * w + 250]).toBe(0);
+ });
+});
diff --git a/src/lib/layers.ts b/src/lib/layers.ts
new file mode 100644
index 0000000..2de58b8
--- /dev/null
+++ b/src/lib/layers.ts
@@ -0,0 +1,226 @@
+// Layers lab (components/LayersEditor.tsx): the masks behind "Split into layers". Each layer is an
+// alpha mask (0-255, one byte per mask pixel); a pixel belongs to one layer at most. The background
+// is everything no layer has.
+
+export type Mask = Uint8Array;
+
+/** Adds `a` (0-255 per pixel) to `layer`. `take`: the pixels leave every other layer (the brush);
+ * otherwise only what no other layer has is added (a SAM click: a loose mask, such as the whole
+ * scene when cycling to a bigger part, never eats a layer already made). */
+export function addTo(masks: Mask[], layer: number, a: ArrayLike<number>, take = true) {
+ for (let i = 0; i < a.length; i++) {
+ let v = a[i];
+ if (!v) continue;
+ if (!take) {
+ masks.forEach((m, k) => {
+ if (k !== layer) v = Math.min(v, 255 - m[i]);
+ });
+ if (v > 0) masks[layer][i] = Math.max(masks[layer][i], v);
+ continue;
+ }
+ masks.forEach((m, k) => {
+ if (k === layer) m[i] = Math.max(m[i], v);
+ else if (m[i]) m[i] = Math.min(m[i], 255 - v);
+ });
+ }
+}
+
+/** Removes `a` from `layer` only (those pixels go back to the background). */
+export function removeFrom(mask: Mask, a: ArrayLike<number>) {
+ for (let i = 0; i < a.length; i++) if (a[i]) mask[i] = Math.min(mask[i], 255 - a[i]);
+}
+
+/** SAM logits -> a hard 0/255 mask. */
+export function fromLogits(logits: ArrayLike<number>): Mask {
+ const out = new Uint8Array(logits.length);
+ for (let i = 0; i < logits.length; i++) out[i] = logits[i] > 0 ? 255 : 0;
+ return out;
+}
+
+/** How far a click's selection reaches past SAM's own edge, as a share of the long side: two cells
+ * of SAM's 256² low-res mask, about its uncertainty at an edge (its threshold at 0 keeps to the
+ * inside, which left a rim of the object out). The whole object is then inside: an inpaint repaints
+ * a little around it, the layers lab's split snaps the edge back to the photo (`refineEdges`). */
+export const SAM_GROW = 2 / 256;
+
+/** A SAM output (logits at `w`×`h`) as the selection a click makes: grown by `SAM_GROW`. */
+export function samSelection(logits: ArrayLike<number>, w: number, h: number): Mask {
+ return dilate(fromLogits(logits), w, h, Math.max(1, Math.round(Math.max(w, h) * SAM_GROW)));
+}
+
+/** Per pixel, the largest alpha of any layer: the background's holes. */
+export function union(masks: Mask[], n: number): Mask {
+ const out = new Uint8Array(n);
+ for (const m of masks) for (let i = 0; i < n; i++) if (m[i] > out[i]) out[i] = m[i];
+ return out;
+}
+
+/** Grows a mask by `r` px (square max filter, separable). */
+export function dilate(m: Mask, w: number, h: number, r: number): Mask {
+ r = Math.round(r);
+ if (r <= 0) return m.slice();
+ const tmp = new Uint8Array(m.length);
+ const out = new Uint8Array(m.length);
+ const run = (src: Mask, dst: Mask, len: number, count: number, step: number, stride: number) => {
+ for (let c = 0; c < count; c++) {
+ const base = c * stride;
+ for (let i = 0; i < len; i++) {
+ let v = 0;
+ const lo = Math.max(0, i - r);
+ const hi = Math.min(len - 1, i + r);
+ for (let j = lo; j <= hi && v < 255; j++) v = Math.max(v, src[base + j * step]);
+ dst[base + i * step] = v;
+ }
+ }
+ };
+ run(m, tmp, w, h, 1, w); // rows
+ run(tmp, out, h, w, w, 1); // columns
+ return out;
+}
+
+/** SAM's multimask outputs to try for one click, best first: the highest predicted IoU, then the
+ * others from the smallest to the largest. Clicking the same spot again moves to the next one
+ * (part -> whole); index 0, the single-mask output, is left out (S1: unreliable on one point). */
+export function maskOrder(iou: number[], areas: number[]): number[] {
+ const idx = [1, 2, 3].filter((i) => i < iou.length);
+ if (!idx.length) return [0];
+ const best = idx.reduce((a, b) => (iou[b] > iou[a] ? b : a));
+ return [best, ...idx.filter((i) => i !== best).sort((a, b) => areas[a] - areas[b])];
+}
+
+/** Count of pixels with any alpha. */
+export function area(m: ArrayLike<number>): number {
+ let n = 0;
+ for (let i = 0; i < m.length; i++) if (m[i]) n++;
+ return n;
+}
+
+/** A file name part from a layer name. */
+export function fileSafe(name: string): string {
+ return name.replace(/[\\/:*?"<>|]+/g, " ").replace(/\s+/g, " ").trim() || "layer";
+}
+
+/** Shrinks a mask by `r` px (the opposite of `dilate`). */
+export function erode(m: Mask, w: number, h: number, r: number): Mask {
+ const inv = new Uint8Array(m.length);
+ for (let i = 0; i < m.length; i++) inv[i] = 255 - m[i];
+ const g = dilate(inv, w, h, r);
+ for (let i = 0; i < g.length; i++) g[i] = 255 - g[i];
+ return g;
+}
+
+/** Grows a mask by a different amount each way (px): `down` = towards the bottom of the image. */
+export function grow(m: Mask, w: number, h: number, by: { left: number; right: number; up: number; down: number }): Mask {
+ const tmp = new Uint8Array(m.length);
+ const out = new Uint8Array(m.length);
+ // Output pixel x is on if an input pixel in [x - right, x + left] is (the mask moved right by up
+ // to `right`, left by up to `left`); the same with rows.
+ for (let y = 0; y < h; y++)
+ for (let x = 0; x < w; x++) {
+ let v = 0;
+ const lo = Math.max(0, x - Math.round(by.right));
+ const hi = Math.min(w - 1, x + Math.round(by.left));
+ for (let j = lo; j <= hi && v < 255; j++) v = Math.max(v, m[y * w + j]);
+ tmp[y * w + x] = v;
+ }
+ for (let x = 0; x < w; x++)
+ for (let y = 0; y < h; y++) {
+ let v = 0;
+ const lo = Math.max(0, y - Math.round(by.down));
+ const hi = Math.min(h - 1, y + Math.round(by.up));
+ for (let j = lo; j <= hi && v < 255; j++) v = Math.max(v, tmp[j * w + x]);
+ out[y * w + x] = v;
+ }
+ return out;
+}
+
+/** Mean of `a` over a (2r+1)² box, clamped at the borders (integral image). */
+function boxMean(a: Float32Array, w: number, h: number, r: number): Float32Array {
+ const ii = new Float64Array((w + 1) * (h + 1));
+ for (let y = 0; y < h; y++) {
+ let row = 0;
+ for (let x = 0; x < w; x++) {
+ row += a[y * w + x];
+ ii[(y + 1) * (w + 1) + x + 1] = ii[y * (w + 1) + x + 1] + row;
+ }
+ }
+ const out = new Float32Array(w * h);
+ for (let y = 0; y < h; y++) {
+ const y0 = Math.max(0, y - r);
+ const y1 = Math.min(h, y + r + 1);
+ for (let x = 0; x < w; x++) {
+ const x0 = Math.max(0, x - r);
+ const x1 = Math.min(w, x + r + 1);
+ const s = ii[y1 * (w + 1) + x1] - ii[y0 * (w + 1) + x1] - ii[y1 * (w + 1) + x0] + ii[y0 * (w + 1) + x0];
+ out[y * w + x] = s / ((y1 - y0) * (x1 - x0));
+ }
+ }
+ return out;
+}
+
+/** A cut-out's edge snapped to the photo's edges, soft (anti-aliased) instead of SAM's stepped
+ * yes/no: a guided filter (He et al.) of the mask with the photo's grey levels as the guide, kept to
+ * a band of `r` px around the edge (inside: solid, outside: empty, so textures inside an object
+ * never punch holes in it). `gray`: 0-1 per pixel. Works on the mask's bounding box only. */
+export function refineEdges(m: Mask, gray: ArrayLike<number>, w: number, h: number, r: number, eps = 1e-3): Mask {
+ let x0 = w;
+ let y0 = h;
+ let x1 = -1;
+ let y1 = -1;
+ for (let i = 0; i < m.length; i++)
+ if (m[i]) {
+ const x = i % w;
+ const y = (i / w) | 0;
+ if (x < x0) x0 = x;
+ if (x > x1) x1 = x;
+ if (y < y0) y0 = y;
+ if (y > y1) y1 = y;
+ }
+ if (x1 < 0 || r < 1) return m.slice();
+ const pad = 3 * r;
+ x0 = Math.max(0, x0 - pad);
+ y0 = Math.max(0, y0 - pad);
+ x1 = Math.min(w - 1, x1 + pad);
+ y1 = Math.min(h - 1, y1 + pad);
+ const cw = x1 - x0 + 1;
+ const ch = y1 - y0 + 1;
+ const I = new Float32Array(cw * ch);
+ const p = new Uint8Array(cw * ch);
+ for (let y = 0; y < ch; y++)
+ for (let x = 0; x < cw; x++) {
+ const k = (y + y0) * w + x + x0;
+ I[y * cw + x] = gray[k];
+ p[y * cw + x] = m[k];
+ }
+ const P = Float32Array.from(p, (v) => v / 255);
+ const mI = boxMean(I, cw, ch, r);
+ const mP = boxMean(P, cw, ch, r);
+ const II = boxMean(I.map((v) => v * v), cw, ch, r);
+ const IP = boxMean(I.map((v, i) => v * P[i]), cw, ch, r);
+ const A = new Float32Array(cw * ch);
+ const B = new Float32Array(cw * ch);
+ for (let i = 0; i < A.length; i++) {
+ const varI = II[i] - mI[i] * mI[i];
+ A[i] = (IP[i] - mI[i] * mP[i]) / (varI + eps);
+ B[i] = mP[i] - A[i] * mI[i];
+ }
+ const mA = boxMean(A, cw, ch, r);
+ const mB = boxMean(B, cw, ch, r);
+ const inner = erode(p, cw, ch, r);
+ const outer = dilate(p, cw, ch, r);
+ const out = m.slice();
+ for (let y = 0; y < ch; y++)
+ for (let x = 0; x < cw; x++) {
+ const i = y * cw + x;
+ const k = (y + y0) * w + x + x0;
+ if (inner[i]) out[k] = 255;
+ else if (!outer[i]) out[k] = 0;
+ else {
+ // The filter's ramp is as wide as the box: a contrast curve around 0.5 keeps the edge where
+ // it snapped and about 2 px soft (anti-aliased, not blurred).
+ const q = mA[i] * I[i] + mB[i];
+ out[k] = Math.round(Math.min(1, Math.max(0, (q - 0.5) * 2.5 + 0.5)) * 255);
+ }
+ }
+ return out;
+}
diff --git a/src/lib/logic.test.ts b/src/lib/logic.test.ts
index 54daecd..b8809f1 100644
--- a/src/lib/logic.test.ts
+++ b/src/lib/logic.test.ts
@@ -72,6 +72,18 @@ describe("buildRequest", () => {
expect(r.images).toEqual(["input:3", "input:1", "input:2"]);
expect(r.aspect_ratio).toBe("16:9");
});
+ it("coloured marks go with the mask, as an edit Prompt+ can write", () => {
+ const plain = { ...emptyComposer(), prompt: "x", refs: [ref(1)], mask: { inputId: "m", refKey: "k1" }, inpaintAction: "remove" as const, enhance: true };
+ expect(buildRequest(plain).marks).toBeUndefined();
+ expect(enhancerApplies(plain)).toBe(false);
+ expect(needsPrompt(plain)).toBe(false);
+ const marked = { ...plain, mask: { inputId: "m", refKey: "k1", marks: "k" } };
+ const r = buildRequest(marked);
+ expect([r.mask, r.marks, r.enhance]).toEqual(["input:m", "input:k", true]);
+ expect(needsPrompt(marked)).toBe(true);
+ // A mask of another image is not used, nor its marks.
+ expect(buildRequest({ ...marked, mask: { ...marked.mask, refKey: "other" } }).marks).toBeUndefined();
+ });
it("inpaint and expand ignore the aspect", () => {
const masked = { ...emptyComposer(), prompt: "x", refs: [ref(1)], mask: { inputId: "m", refKey: "k1" }, aspect: "16:9" };
expect(buildRequest(masked).aspect_ratio).toBeNull();
diff --git a/src/lib/logic.ts b/src/lib/logic.ts
index 603147f..d32d67d 100644
--- a/src/lib/logic.ts
+++ b/src/lib/logic.ts
@@ -16,8 +16,9 @@ export interface Ref {
export interface Composer {
prompt: string;
refs: Ref[];
- /** Mask painted on refs[0] (input id of the mask PNG). */
- mask: { inputId: string; refKey: string } | null;
+ /** Mask painted on refs[0] (input id of the mask PNG); `marks`: the coloured marks drawn there
+ * (input id of a transparent PNG), which the prompt names by colour. */
+ mask: { inputId: string; refKey: string; marks?: string } | null;
/** What to do in the painted area. */
inpaintAction: InpaintAction;
outpaint: OutpaintSpec | null;
@@ -148,6 +149,7 @@ export function buildRequest(c: Composer, parentId?: string | null): GenerationR
images: c.mode === "generate" && !composing(c) ? [] : c.refs.map((r) => r.source),
mask: maskValid ? `input:${c.mask!.inputId}` : null,
...(maskValid ? { inpaint_action: c.inpaintAction } : {}),
+ ...(maskValid && c.mask!.marks ? { marks: `input:${c.mask!.marks}` } : {}),
outpaint: outpaintInUse(c) ? c.outpaint : null,
quality: c.quality,
resolution: c.resolution,
@@ -174,15 +176,18 @@ export function reviewRequest(c: Composer): GenerationRequest {
return { ...buildRequest(c), enhance: true, enhance_only: true };
}
-/** The prompt enhancer works on new images, edits and combines; a painted area or an expansion
- * wraps the prompt in its own instruction. */
+/** The prompt enhancer works on new images, edits, combines and coloured marks; a painted area
+ * or an expansion wraps the prompt in its own instruction. */
export function enhancerApplies(c: Composer): boolean {
- return !maskInUse(c) && !outpaintInUse(c);
+ return (!maskInUse(c) || marksInUse(c)) && !outpaintInUse(c);
}
+/** The painted area in use is coloured marks (the prompt names them by colour). */
+export const marksInUse = (c: Composer) => maskInUse(c) && !!c.mask!.marks;
+
/** False only for a removal in a painted area, which needs no description. */
export function needsPrompt(c: Composer): boolean {
- return !(maskInUse(c) && c.inpaintAction === "remove");
+ return !(maskInUse(c) && !c.mask!.marks && c.inpaintAction === "remove");
}
/** Advanced fields that change how the model samples. They last for the session only: a value
@@ -231,7 +236,7 @@ export function composerFromRecord(g: GenerationRecord, base: Composer): Compose
inpaintAction: req?.inpaint_action ?? "edit",
outpaint: g.outpaint ? { ...g.outpaint, units: "pixels" } : null,
// Upscales are started from the image toolbar, not the composer.
- mode: req?.mode && req.mode !== "upscale" && req.mode !== "compose" && g.mode !== "lut" ? req.mode : "auto",
+ mode: req?.mode && req.mode !== "upscale" && req.mode !== "compose" && g.mode !== "lut" && g.mode !== "layers" ? req.mode : "auto",
quality: normalizeQuality(g.quality, base.quality),
resolution: req?.resolution ?? base.resolution,
aspect: req?.aspect_ratio ?? "auto",
@@ -254,7 +259,7 @@ export function variationRequest(g: GenerationRecord): GenerationRequest | null
return { ...g.request, seed: null, client: "ui", parent_id: g.id };
}
-export const MODE_LABEL: Record<Mode | "lut", string> = {
+export const MODE_LABEL: Record<Mode | "lut" | "layers", string> = {
auto: "Auto",
generate: "Create",
edit: "Edit",
@@ -264,6 +269,7 @@ export const MODE_LABEL: Record<Mode | "lut", string> = {
compose: "From references",
upscale: "Upscale",
lut: "LUT",
+ layers: "Layers",
};
/** How a LUT is shown: a preset ("kokon:<key>") by its title, a user's LUT by its file name. */
@@ -868,7 +874,7 @@ export interface CompareBase {
export function compareBase(g: GenerationRecord): CompareBase | null {
const r = g.references[0];
// A new image or a combination of several has no single "before".
- if (!r || !["edit", "inpaint", "outpaint", "upscale", "lut"].includes(g.mode)) return null;
+ if (!r || !["edit", "inpaint", "outpaint", "upscale", "lut", "layers"].includes(g.mode)) return null;
const o = g.outpaint;
if (g.mode === "outpaint" && o) {
// Expansion is in source pixels; the result is the source plus the expansion.
diff --git a/src/lib/sam.ts b/src/lib/sam.ts
new file mode 100644
index 0000000..6d5df86
--- /dev/null
+++ b/src/lib/sam.ts
@@ -0,0 +1,111 @@
+// Click-to-select for the layers lab (components/LayersEditor.tsx): MobileSAM (Apache-2.0, Acly's
+// ONNX export, MIT) run in the webview with onnxruntime-web. Experimental: the model files are not
+// shipped; the vite dev server serves them from `lab-models/mobilesam/` (gitignored). The planned
+// app version runs SAM in kokon-service instead.
+//
+// The encoder runs once per image (longest side 1024, 0-255 RGB, HWC); the decoder runs per click
+// on the cached embeddings and returns SAM's 4 masks at the size asked for: index 0 = the single-mask
+// output, 1-3 = sub-part / part / whole (their order and the predicted IoU are unreliable, S1).
+
+import type * as Ort from "onnxruntime-web";
+
+const MODEL_DIR = "/lab-models/mobilesam/";
+const SIDE = 1024;
+
+let ort: Promise<typeof Ort> | null = null;
+const loadOrt = () => (ort ??= import("onnxruntime-web"));
+
+export interface SamResult {
+ /** Logits (> 0 = selected) at the size passed to `predict`, one per output. */
+ masks: Float32Array[];
+ iou: number[];
+}
+
+export class Segmenter {
+ private emb: Ort.Tensor | null = null;
+ private scale = 1;
+
+ private constructor(
+ private o: typeof Ort,
+ private enc: Ort.InferenceSession,
+ private dec: Ort.InferenceSession,
+ /** The execution provider the encoder runs on. */
+ readonly device: string,
+ ) {}
+
+ /** Null when the model files are not there (a release build). */
+ static async load(webgpu = true): Promise<Segmenter | null> {
+ const head = await fetch(MODEL_DIR + "sam_mask_decoder_multi.onnx", { method: "HEAD" }).catch(() => null);
+ if (!head?.ok || head.headers.get("content-type")?.includes("text/html")) return null;
+ const o = await loadOrt();
+ o.env.logLevel = "error";
+ // WebGPU when the webview has it, else WASM (one thread: the page is not cross-origin isolated).
+ let enc: Ort.InferenceSession | null = null;
+ let device = "wasm";
+ if (webgpu && "gpu" in navigator) {
+ try {
+ enc = await o.InferenceSession.create(MODEL_DIR + "mobile_sam_image_encoder.onnx", { executionProviders: ["webgpu"] });
+ device = "webgpu";
+ } catch {
+ enc = null;
+ }
+ }
+ enc ??= await o.InferenceSession.create(MODEL_DIR + "mobile_sam_image_encoder.onnx", { executionProviders: ["wasm"] });
+ const dec = await o.InferenceSession.create(MODEL_DIR + "sam_mask_decoder_multi.onnx", { executionProviders: ["wasm"] });
+ return new Segmenter(o, enc, dec, device);
+ }
+
+ /** Encodes `img` (any size); clicks are then in the coordinates of a `w`×`h` mask of it. */
+ async encode(img: CanvasImageSource, w: number, h: number) {
+ this.scale = SIDE / Math.max(w, h);
+ const rw = Math.max(1, Math.round(w * this.scale));
+ const rh = Math.max(1, Math.round(h * this.scale));
+ const c = document.createElement("canvas");
+ c.width = rw;
+ c.height = rh;
+ const ctx = c.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ ctx.drawImage(img, 0, 0, rw, rh);
+ const px = ctx.getImageData(0, 0, rw, rh).data;
+ const x = new Float32Array(rw * rh * 3);
+ for (let i = 0, j = 0; i < px.length; i += 4, j += 3) {
+ x[j] = px[i];
+ x[j + 1] = px[i + 1];
+ x[j + 2] = px[i + 2];
+ }
+ const out = await this.enc.run({ input_image: new this.o.Tensor("float32", x, [rh, rw, 3]) });
+ this.emb = out.image_embeddings;
+ }
+
+ /** `points` in mask px (`w`×`h`, the size passed to `encode`); label 1 = object, 0 = not.
+ * `box`: the object is inside it (corners as labels 2 and 3, in place of the padding point). */
+ async predict(points: { x: number; y: number; label: 0 | 1 }[], w: number, h: number, box?: { x0: number; y0: number; x1: number; y1: number }): Promise<SamResult> {
+ if (!this.emb) throw new Error("encode first");
+ const all: { x: number; y: number; label: number }[] = [...points];
+ if (box) all.push({ x: box.x0, y: box.y0, label: 2 }, { x: box.x1, y: box.y1, label: 3 });
+ const n = all.length + (box ? 0 : 1);
+ const coords = new Float32Array(n * 2);
+ const labels = new Float32Array(n);
+ all.forEach((p, i) => {
+ coords[i * 2] = p.x * this.scale;
+ coords[i * 2 + 1] = p.y * this.scale;
+ labels[i] = p.label;
+ });
+ if (!box) labels[n - 1] = -1; // padding point: no box
+ const T = this.o.Tensor;
+ const out = await this.dec.run({
+ image_embeddings: this.emb,
+ point_coords: new T("float32", coords, [1, n, 2]),
+ point_labels: new T("float32", labels, [1, n]),
+ mask_input: new T("float32", new Float32Array(256 * 256), [1, 1, 256, 256]),
+ has_mask_input: new T("float32", new Float32Array([0]), [1]),
+ orig_im_size: new T("float32", new Float32Array([h, w]), [2]),
+ });
+ const m = out.masks;
+ const [, k, mh, mw] = m.dims as number[];
+ if (mw !== w || mh !== h) throw new Error(`mask size ${mw}x${mh}, expected ${w}x${h}`);
+ const data = m.data as Float32Array;
+ const masks = Array.from({ length: k }, (_, i) => data.subarray(i * w * h, (i + 1) * w * h));
+ return { masks, iou: Array.from(out.iou_predictions.data as Float32Array) };
+ }
+}
diff --git a/src/lib/sceneEdit.test.ts b/src/lib/sceneEdit.test.ts
new file mode 100644
index 0000000..c3a6906
--- /dev/null
+++ b/src/lib/sceneEdit.test.ts
@@ -0,0 +1,171 @@
+import { describe, expect, it } from "vitest";
+import { alignEdit, applyAffine, changed, darkness, fillHoles, IDENTITY, invertAffine, marksPrompt, objectAt, phaseShift, splitShadows, toneCurve } from "./sceneEdit";
+
+/** A textured test picture: smooth blobs of different sizes (edges everywhere, no repeats). */
+function picture(w: number, h: number, at: (x: number, y: number) => { x: number; y: number } = (x, y) => ({ x, y })): Float32Array {
+ const out = new Float32Array(w * h);
+ for (let y = 0; y < h; y++)
+ for (let x = 0; x < w; x++) {
+ const p = at(x, y);
+ let v = 120;
+ for (let k = 0; k < 40; k++) {
+ const cx = (k * 97.3) % w;
+ const cy = (k * 61.7 + k * k * 3.1) % h;
+ const r = 6 + ((k * 13) % 23);
+ const d = Math.hypot(p.x - cx, p.y - cy);
+ v += (k % 2 ? 60 : -50) * Math.max(0, 1 - d / r);
+ }
+ out[y * w + x] = Math.max(0, Math.min(255, v));
+ }
+ return out;
+}
+
+describe("scene edits", () => {
+ it("phase correlation finds a block's shift", () => {
+ const n = 64;
+ const a = picture(n, n);
+ const b = picture(n, n, (x, y) => ({ x: x - 5, y: y + 3 }));
+ const s = phaseShift(a, b, n);
+ // b(p + d) = a(p): b's content is 5 right and 3 up of a's.
+ expect(s.dx).toBeCloseTo(5, 0);
+ expect(s.dy).toBeCloseTo(-3, 0);
+ });
+
+ it("an edit that zoomed the picture is put back over the layers (the 10-31 kitchen case)", () => {
+ const w = 640;
+ const h = 384;
+ const before = picture(w, h);
+ // The edit's pixel p shows the layers' point at the centre + (p - centre) / zoom: 1.5 % wider,
+ // 2.5 % taller, as Qwen did.
+ const zx = 1.015;
+ const zy = 1.025;
+ const after = picture(w, h, (x, y) => ({ x: w / 2 + (x - w / 2) / zx, y: h / 2 + (y - h / 2) / zy }));
+ const t = alignEdit(before, after, w, h);
+ // The layers' corner (0, 0) is drawn at centre - centre * zoom in the edit.
+ const p = applyAffine(t, 0, 0);
+ expect(p.x).toBeCloseTo(w / 2 - (w / 2) * zx, 0);
+ expect(p.y).toBeCloseTo(h / 2 - (h / 2) * zy, 0);
+ const back = applyAffine(invertAffine(t), p.x, p.y);
+ expect(back.x).toBeCloseTo(0, 5);
+ });
+
+ it("an edit that moved nothing is the identity", () => {
+ const w = 400;
+ const h = 300;
+ const before = picture(w, h);
+ expect(alignEdit(before, before.slice(), w, h)).toEqual(IDENTITY);
+ });
+
+ it("the edit's change of tone is not a shadow; real darkening near an object is", () => {
+ const n = 100 * 100;
+ const before = new Float32Array(n).fill(150);
+ // The whole frame 10 % darker (the edit's grade), plus a shadow patch half as bright.
+ const after = before.map((v) => v * 0.9);
+ for (let y = 60; y < 70; y++) for (let x = 40; x < 60; x++) after[y * 100 + x] = 150 * 0.9 * 0.5;
+ const far = new Uint8Array(n).fill(1);
+ const lut = toneCurve(before, after, far);
+ expect(Math.abs(lut[150] - 135)).toBeLessThan(3);
+ const d = darkness(before, after, lut);
+ expect(d[10 * 100 + 10]).toBe(0);
+ expect(d[65 * 100 + 50]).toBeGreaterThan(0.4);
+ });
+
+ it("each object gets the darkening nearest to it, within its reach, and the same under its base", () => {
+ const w = 100;
+ const h = 40;
+ const left = new Uint8Array(w * h);
+ const right = new Uint8Array(w * h);
+ for (let y = 10; y < 20; y++)
+ for (let x = 10; x < 20; x++) {
+ left[y * w + x] = 255;
+ right[y * w + x + 60] = 255;
+ }
+ const dark = new Float32Array(w * h);
+ // A shadow right under the left object, touching it; another far from both.
+ for (let y = 20; y < 24; y++) for (let x = 10; x < 20; x++) dark[y * w + x] = 0.8;
+ dark[38 * w + 50] = 0.8;
+ const [a, b] = splitShadows(dark, [left, right], w, h, 12, 3);
+ expect(a[21 * w + 15]).toBeGreaterThan(150);
+ expect(b[21 * w + 15]).toBe(0);
+ // Under the object's bottom edge: carried in, so no seam.
+ expect(a[18 * w + 15]).toBeGreaterThan(150);
+ expect(a[11 * w + 15]).toBe(0);
+ // Too far from any object.
+ expect(a[38 * w + 50] + b[38 * w + 50]).toBe(0);
+ });
+
+ it("where two objects' shadows meet, their layers stack back to the whole darkness (no seam)", () => {
+ const w = 80;
+ const h = 30;
+ const a = new Uint8Array(w * h);
+ const b = new Uint8Array(w * h);
+ for (let y = 5; y < 10; y++)
+ for (let x = 0; x < 10; x++) {
+ a[y * w + x + 20] = 255;
+ b[y * w + x + 50] = 255;
+ }
+ const dark = new Float32Array(w * h);
+ for (let y = 12; y < 20; y++) for (let x = 15; x < 65; x++) dark[y * w + x] = 0.5;
+ const [sa, sb] = splitShadows(dark, [a, b], w, h, 40, 4);
+ for (const x of [38, 40, 42, 44]) {
+ const i = 15 * w + x;
+ const stacked = 1 - (1 - sa[i] / 255) * (1 - sb[i] / 255);
+ expect(Math.abs(stacked - 0.5)).toBeLessThan(0.02);
+ }
+ // Each is soft across the middle, not cut.
+ expect(sa[15 * w + 40]).toBeGreaterThan(20);
+ expect(sb[15 * w + 40]).toBeGreaterThan(20);
+ });
+
+ it("no light seam where the edit redrew an object's edge (no darkening in a thin ring around it)", () => {
+ const w = 60;
+ const h = 40;
+ const obj = new Uint8Array(w * h);
+ for (let y = 10; y < 20; y++) for (let x = 20; x < 30; x++) obj[y * w + x] = 255;
+ const dark = new Float32Array(w * h);
+ // Shadow below, but the 2 px right under the object's edge read nothing.
+ for (let y = 22; y < 30; y++) for (let x = 15; x < 35; x++) dark[y * w + x] = 0.6;
+ const [plain] = splitShadows(dark, [obj], w, h, 20, 4);
+ const [edged] = splitShadows(dark, [obj], w, h, 20, 4, undefined, 3);
+ expect(plain[20 * w + 25]).toBe(0);
+ expect(edged[20 * w + 25]).toBeGreaterThan(90);
+ });
+
+ it("a new object is what changed around its marks, without its shadow, holes filled", () => {
+ const w = 60;
+ const h = 40;
+ const before = new Uint8ClampedArray(w * h * 4);
+ const after = new Uint8ClampedArray(w * h * 4);
+ for (let i = 0; i < w * h; i++) {
+ before.set([180, 140, 100, 255], i * 4);
+ after.set([180, 140, 100, 255], i * 4);
+ }
+ // A green ring (the object, with a hole) and a darker strip of the same brown (its shadow).
+ for (let y = 10; y < 20; y++) for (let x = 20; x < 30; x++) if (y < 12 || y > 17 || x < 22 || x > 27) after.set([40, 160, 60, 255], (y * w + x) * 4);
+ for (let y = 20; y < 24; y++) for (let x = 20; x < 30; x++) after.set([90, 70, 50, 255], (y * w + x) * 4);
+ const lut = new Float32Array(256).map((_, i) => i);
+ const m = changed(before, after, lut, w * h);
+ expect(m[22 * w + 25]).toBe(0);
+ const seed = new Uint8Array(w * h);
+ seed[10 * w + 20] = 1;
+ const obj = objectAt(m, seed, w, h);
+ expect(obj[15 * w + 25]).toBe(255);
+ expect(obj[5 * w + 5]).toBe(0);
+ });
+
+ it("fills only enclosed holes", () => {
+ const w = 5;
+ const m = new Uint8Array([0, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 255, 0, 255, 0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0]);
+ const f = fillHoles(m, w, 5);
+ expect(f[12]).toBe(255);
+ expect(f[0]).toBe(0);
+ });
+
+ it("names each colour's object in the prompt", () => {
+ expect(marksPrompt([{ colour: "red", what: "a green glass bottle" }])).toMatch(/^Add a green glass bottle where the red marks are\. .*Remove all the coloured marks/);
+ expect(marksPrompt([
+ { colour: "red", what: "a bottle" },
+ { colour: "blue", what: "" },
+ ])).toMatch(/^Add a bottle where the red marks are and something that fits the scene where the blue marks are\./);
+ });
+});
diff --git a/src/lib/sceneEdit.ts b/src/lib/sceneEdit.ts
new file mode 100644
index 0000000..1ea6214
--- /dev/null
+++ b/src/lib/sceneEdit.ts
@@ -0,0 +1,509 @@
+// Layers lab: what a whole-image edit of the flattened layers changed (components/LabScene.ts).
+// Qwen's plain edits come back slightly zoomed (10-31 kitchen table: 1.4 % wider, 2.5 % taller,
+// up to ~25 px off at the corners), and with the whole picture's tone changed. So the result is
+// first put back over the layers (an affine fitted to block shifts of the two images' edges), its
+// tone change measured far from the objects and divided out, and only then read: darkening near
+// an object = its shadow; something new where the user drew coloured marks = a new object.
+//
+// Everything here works on plain arrays at an analysis size (one byte or float per pixel).
+
+import { dilate } from "./layers";
+
+/** x' = a x + b y + c, y' = d x + e y + f: a point of the layers -> the same point in the edit. */
+export interface Affine {
+ a: number;
+ b: number;
+ c: number;
+ d: number;
+ e: number;
+ f: number;
+}
+
+export const IDENTITY: Affine = { a: 1, b: 0, c: 0, d: 0, e: 1, f: 0 };
+
+export const applyAffine = (t: Affine, x: number, y: number) => ({ x: t.a * x + t.b * y + t.c, y: t.d * x + t.e * y + t.f });
+
+/** The same transform for images `k` times bigger. */
+export const scaleAffine = (t: Affine, k: number): Affine => ({ ...t, c: t.c * k, f: t.f * k });
+
+export function invertAffine(t: Affine): Affine {
+ const det = t.a * t.e - t.b * t.d || 1;
+ const a = t.e / det;
+ const b = -t.b / det;
+ const d = -t.d / det;
+ const e = t.a / det;
+ return { a, b, c: -(a * t.c + b * t.f), d, e, f: -(d * t.c + e * t.f) };
+}
+
+/** RGBA -> luminance (0-255). */
+export function luminance(px: ArrayLike<number>, n: number): Float32Array {
+ const out = new Float32Array(n);
+ for (let i = 0; i < n; i++) out[i] = 0.2126 * px[i * 4] + 0.7152 * px[i * 4 + 1] + 0.0722 * px[i * 4 + 2];
+ return out;
+}
+
+/** Box blur of radius `r`, three times (close to a Gaussian of sigma ~r). */
+export function blur(src: Float32Array, w: number, h: number, r: number): Float32Array {
+ r = Math.round(r);
+ if (r < 1) return src.slice();
+ let a = src.slice();
+ let b = new Float32Array(src.length);
+ const pass = (from: Float32Array, to: Float32Array, len: number, count: number, step: number, stride: number) => {
+ for (let c = 0; c < count; c++) {
+ const base = c * stride;
+ let sum = 0;
+ for (let j = -r; j <= r; j++) sum += from[base + Math.min(len - 1, Math.max(0, j)) * step];
+ for (let i = 0; i < len; i++) {
+ to[base + i * step] = sum / (2 * r + 1);
+ sum += from[base + Math.min(len - 1, i + r + 1) * step] - from[base + Math.max(0, i - r) * step];
+ }
+ }
+ };
+ for (let k = 0; k < 3; k++) {
+ pass(a, b, w, h, 1, w);
+ pass(b, a, h, w, w, 1);
+ }
+ return a;
+}
+
+/** Gradient magnitude: edges, which survive the edit's change of tone (the alignment looks at these). */
+export function edges(l: Float32Array, w: number, h: number): Float32Array {
+ const out = new Float32Array(l.length);
+ for (let y = 0; y < h - 1; y++)
+ for (let x = 0; x < w - 1; x++) {
+ const i = y * w + x;
+ out[i] = Math.hypot(l[i + 1] - l[i], l[i + w] - l[i]);
+ }
+ return out;
+}
+
+/** In-place radix-2 FFT of one row/column (`n` a power of 2). */
+function fft1(re: Float64Array, im: Float64Array, n: number, inverse: boolean) {
+ for (let i = 1, j = 0; i < n; i++) {
+ let bit = n >> 1;
+ for (; j & bit; bit >>= 1) j ^= bit;
+ j ^= bit;
+ if (i < j) {
+ [re[i], re[j]] = [re[j], re[i]];
+ [im[i], im[j]] = [im[j], im[i]];
+ }
+ }
+ for (let len = 2; len <= n; len <<= 1) {
+ const ang = ((inverse ? 2 : -2) * Math.PI) / len;
+ const wr = Math.cos(ang);
+ const wi = Math.sin(ang);
+ for (let i = 0; i < n; i += len) {
+ let cr = 1;
+ let ci = 0;
+ for (let k = 0; k < len / 2; k++) {
+ const ar = re[i + k + len / 2] * cr - im[i + k + len / 2] * ci;
+ const ai = re[i + k + len / 2] * ci + im[i + k + len / 2] * cr;
+ re[i + k + len / 2] = re[i + k] - ar;
+ im[i + k + len / 2] = im[i + k] - ai;
+ re[i + k] += ar;
+ im[i + k] += ai;
+ const t = cr * wr - ci * wi;
+ ci = cr * wi + ci * wr;
+ cr = t;
+ }
+ }
+ }
+}
+
+function fft2(re: Float64Array, im: Float64Array, n: number, inverse = false) {
+ const r = new Float64Array(n);
+ const i = new Float64Array(n);
+ for (let y = 0; y < n; y++) {
+ r.set(re.subarray(y * n, y * n + n));
+ i.set(im.subarray(y * n, y * n + n));
+ fft1(r, i, n, inverse);
+ re.set(r, y * n);
+ im.set(i, y * n);
+ }
+ for (let x = 0; x < n; x++) {
+ for (let y = 0; y < n; y++) {
+ r[y] = re[y * n + x];
+ i[y] = im[y * n + x];
+ }
+ fft1(r, i, n, inverse);
+ for (let y = 0; y < n; y++) {
+ re[y * n + x] = r[y];
+ im[y * n + x] = i[y];
+ }
+ }
+}
+
+/** Phase correlation of two `n`×`n` blocks: the shift `d` with b(p + d) ≈ a(p) (sub-pixel), and
+ * the peak's height (0-1: how sure). */
+export function phaseShift(a: Float32Array, b: Float32Array, n: number): { dx: number; dy: number; q: number } {
+ const ar = new Float64Array(n * n);
+ const ai = new Float64Array(n * n);
+ const br = new Float64Array(n * n);
+ const bi = new Float64Array(n * n);
+ let ma = 0;
+ let mb = 0;
+ for (let i = 0; i < n * n; i++) {
+ ma += a[i];
+ mb += b[i];
+ }
+ ma /= n * n;
+ mb /= n * n;
+ for (let y = 0; y < n; y++)
+ for (let x = 0; x < n; x++) {
+ const wgt = (0.5 - 0.5 * Math.cos((2 * Math.PI * (x + 0.5)) / n)) * (0.5 - 0.5 * Math.cos((2 * Math.PI * (y + 0.5)) / n));
+ ar[y * n + x] = (a[y * n + x] - ma) * wgt;
+ br[y * n + x] = (b[y * n + x] - mb) * wgt;
+ }
+ fft2(ar, ai, n);
+ fft2(br, bi, n);
+ // conj(A) * B, normalised.
+ for (let i = 0; i < n * n; i++) {
+ const re = ar[i] * br[i] + ai[i] * bi[i];
+ const im = ar[i] * bi[i] - ai[i] * br[i];
+ const m = Math.hypot(re, im) + 1e-9;
+ ar[i] = re / m;
+ ai[i] = im / m;
+ }
+ fft2(ar, ai, n, true);
+ let best = 0;
+ for (let i = 1; i < n * n; i++) if (ar[i] > ar[best]) best = i;
+ const at = (x: number, y: number) => ar[((y + n) % n) * n + ((x + n) % n)];
+ const px = best % n;
+ const py = (best / n) | 0;
+ const sub = (m: number, c: number, p: number) => (m - 2 * c + p !== 0 ? (0.5 * (m - p)) / (m - 2 * c + p) : 0);
+ const fx = sub(at(px - 1, py), at(px, py), at(px + 1, py));
+ const fy = sub(at(px, py - 1), at(px, py), at(px, py + 1));
+ const wrap = (v: number) => (v > n / 2 ? v - n : v);
+ return { dx: wrap(px) + fx, dy: wrap(py) + fy, q: ar[best] / (n * n) };
+}
+
+export interface BlockShift {
+ x: number;
+ y: number;
+ dx: number;
+ dy: number;
+}
+
+/** Shifts of `n`×`n` blocks every `step` px between two edge images; blocks with little texture
+ * (under a third of the median block's: a plain wall) or mostly inside `skip` (where the edit is
+ * meant to change things: the marks) are left out. */
+export function blockShifts(A: Float32Array, B: Float32Array, w: number, h: number, n = 128, step = 96, skip?: Uint8Array): BlockShift[] {
+ const blocks: { x0: number; y0: number; sd: number }[] = [];
+ for (let y0 = 0; y0 + n <= h; y0 += step)
+ for (let x0 = 0; x0 + n <= w; x0 += step) {
+ let sum = 0;
+ let sq = 0;
+ let skipped = 0;
+ for (let y = 0; y < n; y++)
+ for (let x = 0; x < n; x++) {
+ const i = (y0 + y) * w + x0 + x;
+ sum += A[i];
+ sq += A[i] * A[i];
+ if (skip?.[i]) skipped++;
+ }
+ const mean = sum / (n * n);
+ if (skipped <= (n * n) / 4) blocks.push({ x0, y0, sd: Math.sqrt(Math.max(0, sq / (n * n) - mean * mean)) });
+ }
+ const sds = blocks.map((b) => b.sd).sort((x, y) => x - y);
+ const min = Math.max(1e-3, (sds[sds.length >> 1] ?? 0) / 3);
+ const out: BlockShift[] = [];
+ const a = new Float32Array(n * n);
+ const b = new Float32Array(n * n);
+ for (const { x0, y0, sd } of blocks) {
+ if (sd < min) continue;
+ for (let y = 0; y < n; y++)
+ for (let x = 0; x < n; x++) {
+ a[y * n + x] = A[(y0 + y) * w + x0 + x];
+ b[y * n + x] = B[(y0 + y) * w + x0 + x];
+ }
+ const s = phaseShift(a, b, n);
+ if (s.q > 0.06) out.push({ x: x0 + n / 2, y: y0 + n / 2, dx: s.dx, dy: s.dy });
+ }
+ return out;
+}
+
+function solve3(M: number[][], v: number[]): number[] | null {
+ // Normal equations of a 3-parameter least squares (Cramer's rule).
+ const det = (m: number[][]) => m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]);
+ const D = det(M);
+ if (Math.abs(D) < 1e-9) return null;
+ return [0, 1, 2].map((k) => det(M.map((row, i) => row.map((x, j) => (j === k ? v[i] : x)))) / D);
+}
+
+/** The affine that best maps block centres to where they moved (least squares, then again
+ * without the blocks it misses by much: a block on a changed object is an outlier). Null when
+ * too few blocks agree. */
+export function fitAffine(pts: BlockShift[]): { t: Affine; inliers: number; err: number } | null {
+ let keep = pts.map(() => true);
+ let t: Affine | null = null;
+ let errs: number[] = [];
+ for (let it = 0; it < 4; it++) {
+ const P = pts.filter((_, i) => keep[i]);
+ if (P.length < 6) return null;
+ const M = [
+ [0, 0, 0],
+ [0, 0, 0],
+ [0, 0, 0],
+ ];
+ const vx = [0, 0, 0];
+ const vy = [0, 0, 0];
+ for (const p of P) {
+ const r = [p.x, p.y, 1];
+ for (let i = 0; i < 3; i++) {
+ for (let j = 0; j < 3; j++) M[i][j] += r[i] * r[j];
+ vx[i] += r[i] * (p.x + p.dx);
+ vy[i] += r[i] * (p.y + p.dy);
+ }
+ }
+ const sx = solve3(M, vx);
+ const sy = solve3(M, vy);
+ if (!sx || !sy) return null;
+ t = { a: sx[0], b: sx[1], c: sx[2], d: sy[0], e: sy[1], f: sy[2] };
+ errs = pts.map((p) => {
+ const q = applyAffine(t!, p.x, p.y);
+ return Math.hypot(q.x - p.x - p.dx, q.y - p.y - p.dy);
+ });
+ const sorted = errs.filter((_, i) => keep[i]).sort((x, y) => x - y);
+ const lim = Math.max(1.5, sorted[sorted.length >> 1] * 2.5);
+ keep = errs.map((e) => e < lim);
+ }
+ const inl = errs.filter((_, i) => keep[i]);
+ if (inl.length < 6 || inl.length < pts.length * 0.4) return null;
+ return { t: t!, inliers: inl.length, err: inl.sort((x, y) => x - y)[inl.length >> 1] };
+}
+
+/** Where the edit put the layers' content: identity when nothing moved or nothing could be told. */
+export function alignEdit(before: Float32Array, after: Float32Array, w: number, h: number, skip?: Uint8Array): Affine {
+ const A = edges(blur(before, w, h, 1), w, h);
+ const B = edges(blur(after, w, h, 1), w, h);
+ const fit = fitAffine(blockShifts(A, B, w, h, 128, 96, skip));
+ if (!fit) return IDENTITY;
+ // Less than half a pixel anywhere: nothing moved (a brushed edit is pasted back as it was).
+ const far = [applyAffine(fit.t, 0, 0), applyAffine(fit.t, w, 0), applyAffine(fit.t, 0, h), applyAffine(fit.t, w, h)];
+ const corners = [
+ [0, 0],
+ [w, 0],
+ [0, h],
+ [w, h],
+ ];
+ return far.every((p, i) => Math.hypot(p.x - corners[i][0], p.y - corners[i][1]) < 0.5) ? IDENTITY : fit.t;
+}
+
+/** The edit's change of tone, as a lookup: luminance before (0-255) -> the luminance the edit gives
+ * it where nothing else happened (`far`: pixels away from the objects and the marks). Median per
+ * bin, made non-decreasing; ranges with too few pixels follow their neighbours. */
+export function toneCurve(before: Float32Array, after: Float32Array, far: Uint8Array): Float32Array {
+ const BINS = 32;
+ const hist = Array.from({ length: BINS }, () => new Uint32Array(256));
+ const count = new Uint32Array(BINS);
+ for (let i = 0; i < before.length; i++) {
+ if (!far[i]) continue;
+ const k = Math.min(BINS - 1, Math.floor(before[i] / (256 / BINS)));
+ hist[k][Math.min(255, Math.max(0, Math.round(after[i])))]++;
+ count[k]++;
+ }
+ const mids: number[] = [];
+ const vals: number[] = [];
+ for (let k = 0; k < BINS; k++) {
+ if (count[k] < 50) continue;
+ let acc = 0;
+ let v = 0;
+ for (; v < 255; v++) if ((acc += hist[k][v]) * 2 >= count[k]) break;
+ mids.push((k + 0.5) * (256 / BINS));
+ vals.push(v);
+ }
+ const lut = new Float32Array(256);
+ if (!mids.length) {
+ for (let i = 0; i < 256; i++) lut[i] = i;
+ return lut;
+ }
+ for (let i = 0; i < 256; i++) {
+ let v: number;
+ if (i <= mids[0]) v = (vals[0] * (i + 2)) / (mids[0] + 2);
+ else if (i >= mids[mids.length - 1]) v = (vals[vals.length - 1] * (i + 2)) / (mids[mids.length - 1] + 2);
+ else {
+ let k = 0;
+ while (mids[k + 1] < i) k++;
+ const t = (i - mids[k]) / (mids[k + 1] - mids[k]);
+ v = vals[k] + t * (vals[k + 1] - vals[k]);
+ }
+ lut[i] = i ? Math.max(v, lut[i - 1]) : v;
+ }
+ return lut;
+}
+
+/** Per pixel, how much darker the edit made it than its tone change alone would (0-1), above
+ * `floor` (decode noise, texture redrawn). Both luminances blurred already. */
+export function darkness(before: Float32Array, after: Float32Array, lut: Float32Array, floor = 0.06): Float32Array {
+ const out = new Float32Array(before.length);
+ for (let i = 0; i < out.length; i++) {
+ const want = lut[Math.min(255, Math.max(0, Math.round(before[i])))];
+ const d = 1 - (after[i] + 2) / (want + 2);
+ out[i] = d > floor ? Math.min(1, (d - floor) / (1 - floor)) : 0;
+ }
+ return out;
+}
+
+/** Nearest object and the distance to it (px, chamfer 3-4) for every pixel: `labels` = the
+ * object's index on its pixels, -1 elsewhere. */
+export function nearest(labels: Int16Array, w: number, h: number): { label: Int16Array; dist: Float32Array } {
+ const label = labels.slice();
+ const dist = new Float32Array(w * h);
+ for (let i = 0; i < dist.length; i++) dist[i] = label[i] >= 0 ? 0 : Infinity;
+ const look = (i: number, j: number, c: number) => {
+ if (dist[j] + c < dist[i]) {
+ dist[i] = dist[j] + c;
+ label[i] = label[j];
+ }
+ };
+ for (let y = 0; y < h; y++)
+ for (let x = 0; x < w; x++) {
+ const i = y * w + x;
+ if (x > 0) look(i, i - 1, 1);
+ if (y > 0) {
+ look(i, i - w, 1);
+ if (x > 0) look(i, i - w - 1, 1.4142);
+ if (x < w - 1) look(i, i - w + 1, 1.4142);
+ }
+ }
+ for (let y = h - 1; y >= 0; y--)
+ for (let x = w - 1; x >= 0; x--) {
+ const i = y * w + x;
+ if (x < w - 1) look(i, i + 1, 1);
+ if (y < h - 1) {
+ look(i, i + w, 1);
+ if (x < w - 1) look(i, i + w + 1, 1.4142);
+ if (x > 0) look(i, i + w - 1, 1.4142);
+ }
+ }
+ return { label, dist };
+}
+
+/** Each object's shadow (0-255 per pixel), from the darkening the edit added:
+ * - one darkness map first: the objects' own pixels and a few px around their outlines (`edge`:
+ * the edit redraws an object's edge, so nothing is read there) are filled from the readings
+ * around them (a soft average over ~`under` px), so there is no light seam between an object
+ * and its shadow, and the shadow drawn under an object stays whole when it moves;
+ * - kept within `reach` px of an object (fading out over the last quarter), not on `others`
+ * (objects that get no shadow here);
+ * - shared between the objects softly by how near each one is, as alphas that stack back to the
+ * map exactly (1 - (1 - a)^share each): no hard cut where two shadows meet, which any resampling
+ * would turn into a light line.
+ * `objects[k]`: object k's pixels (alpha > 128). */
+export function splitShadows(dark: Float32Array, objects: Uint8Array[], w: number, h: number, reach: number, under: number, others?: Uint8Array, edge = 0): Uint8Array[] {
+ const n = w * h;
+ const covered = new Uint8Array(n);
+ for (const m of edge >= 1 ? objects.map((o) => dilate(o, w, h, edge)) : objects) for (let i = 0; i < n; i++) if (m[i]) covered[i] = 1;
+ const known = Float32Array.from(covered, (c) => (c ? 0 : 1));
+ const r = Math.max(1, Math.round(under / 2));
+ const bv = blur(Float32Array.from(dark, (v, i) => v * known[i]), w, h, r);
+ const bk = blur(known, w, h, r);
+ const total = Float32Array.from(dark, (v, i) => (known[i] ? v : bk[i] > 0.02 ? bv[i] / bk[i] : 0));
+ const dists = objects.map((m) => nearest(Int16Array.from(m, (v) => (v ? 0 : -1)), w, h).dist);
+ const tau = Math.max(1.5, under / 2);
+ const out = objects.map(() => new Uint8Array(n));
+ const wk = new Float64Array(objects.length);
+ for (let i = 0; i < n; i++) {
+ if (!total[i] || others?.[i]) continue;
+ let dmin = Infinity;
+ for (const d of dists) dmin = Math.min(dmin, d[i]);
+ if (dmin > reach) continue;
+ const a = Math.min(0.999, total[i] * Math.min(1, (reach - dmin) / (reach * 0.25)));
+ let sum = 0;
+ for (let k = 0; k < dists.length; k++) sum += wk[k] = Math.exp(-(dists[k][i] - dmin) / tau);
+ for (let k = 0; k < dists.length; k++) out[k][i] = Math.round((1 - Math.pow(1 - a, wk[k] / sum)) * 255);
+ }
+ return out;
+}
+
+/** Pixels the edit changed beyond its tone change, not counting shadows (darker, same colour):
+ * where a new object was drawn. RGBA in, both at the analysis size; `lut` from toneCurve. */
+export function changed(before: ArrayLike<number>, after: ArrayLike<number>, lut: Float32Array, n: number, threshold = 28): Uint8Array {
+ const out = new Uint8Array(n);
+ for (let i = 0; i < n; i++) {
+ const j = i * 4;
+ const lb = 0.2126 * before[j] + 0.7152 * before[j + 1] + 0.0722 * before[j + 2];
+ const la = 0.2126 * after[j] + 0.7152 * after[j + 1] + 0.0722 * after[j + 2];
+ const g = (lut[Math.min(255, Math.round(lb))] + 2) / (lb + 2);
+ const diff = (Math.abs(after[j] - before[j] * g) + Math.abs(after[j + 1] - before[j + 1] * g) + Math.abs(after[j + 2] - before[j + 2] * g)) / 3;
+ if (diff < threshold) continue;
+ // A shadow: darker with the same hue (chromaticity within a few %).
+ const sb = before[j] + before[j + 1] + before[j + 2] + 3;
+ const sa = after[j] + after[j + 1] + after[j + 2] + 3;
+ const hue = Math.abs((after[j] + 1) / sa - (before[j] + 1) / sb) + Math.abs((after[j + 1] + 1) / sa - (before[j + 1] + 1) / sb);
+ if (la < lb * g && hue < 0.04) continue;
+ out[i] = 255;
+ }
+ return out;
+}
+
+/** The pieces of `m` that touch `seed`, with their holes filled: one new object from the changed
+ * pixels around its marks. */
+export function objectAt(m: Uint8Array, seed: Uint8Array, w: number, h: number): Uint8Array {
+ const out = new Uint8Array(w * h);
+ const stack: number[] = [];
+ for (let i = 0; i < m.length; i++)
+ if (m[i] && seed[i] && !out[i]) {
+ out[i] = 255;
+ stack.push(i);
+ }
+ while (stack.length) {
+ const i = stack.pop()!;
+ const x = i % w;
+ for (const j of [x > 0 ? i - 1 : -1, x < w - 1 ? i + 1 : -1, i - w, i + w])
+ if (j >= 0 && j < m.length && m[j] && !out[j]) {
+ out[j] = 255;
+ stack.push(j);
+ }
+ }
+ return fillHoles(out, w, h);
+}
+
+/** Fills what the mask encloses (not reachable from the border without crossing it). */
+export function fillHoles(m: Uint8Array, w: number, h: number): Uint8Array {
+ const outside = new Uint8Array(w * h);
+ const stack: number[] = [];
+ const push = (i: number) => {
+ if (!m[i] && !outside[i]) {
+ outside[i] = 1;
+ stack.push(i);
+ }
+ };
+ for (let x = 0; x < w; x++) {
+ push(x);
+ push((h - 1) * w + x);
+ }
+ for (let y = 0; y < h; y++) {
+ push(y * w);
+ push(y * w + w - 1);
+ }
+ while (stack.length) {
+ const i = stack.pop()!;
+ const x = i % w;
+ if (x > 0) push(i - 1);
+ if (x < w - 1) push(i + 1);
+ if (i >= w) push(i - w);
+ if (i < m.length - w) push(i + w);
+ }
+ const out = new Uint8Array(w * h);
+ for (let i = 0; i < out.length; i++) out[i] = outside[i] ? 0 : 255;
+ return out;
+}
+
+/** The colours of the sketch pencil, as Qwen reads them (named in the prompt). */
+export const MARK_COLOURS = [
+ { name: "red", css: "#ff2020", rgb: [255, 32, 32] },
+ { name: "blue", css: "#1f5bff", rgb: [31, 91, 255] },
+ { name: "green", css: "#14c814", rgb: [20, 200, 20] },
+ { name: "yellow", css: "#ffd400", rgb: [255, 212, 0] },
+] as const;
+
+export type MarkColour = (typeof MARK_COLOURS)[number]["name"];
+
+/** The prompt for new objects drawn as coloured marks: one clause per colour (`what` empty: Qwen
+ * picks something that fits). Prompt+ may rewrite it; it sees the marks too. */
+export function marksPrompt(marks: { colour: MarkColour; what: string }[]): string {
+ const clauses = marks.map((m) => `${m.what.trim() || "something that fits the scene"} where the ${m.colour} marks are`);
+ const list = clauses.length > 1 ? `${clauses.slice(0, -1).join(", ")} and ${clauses[clauses.length - 1]}` : clauses[0];
+ return `Add ${list}. Draw each one at the size and place of its marks, sitting naturally in the scene with matching perspective and light. Remove all the coloured marks completely. Keep everything else in the image unchanged.`;
+}
diff --git a/src/lib/store.ts b/src/lib/store.ts
index 63ed9a1..ba362fb 100644
--- a/src/lib/store.ts
+++ b/src/lib/store.ts
@@ -10,9 +10,10 @@ import { useSyncExternalStore } from "react";
import { markFresh } from "../components/Cocoon";
import { api, ApiError, setPrivateInputs, subscribe, urls } from "./api";
import { activeLoras, buildRequest, composing, emptyComposer, goneVersions, macEngineOfferDue, needsPrompt, newKey, pageOutdated, persistentAdvanced, refFromGen, refFromInput, reviewRequest, stageFitsPreview, starPromptDue, timelineLinks, undoTarget, versionLabels, wantsPromptReview, type CompareMode, type Composer, type Ref } from "./logic";
+import { keepLabsIn, labsChanged, labSources, loadLab, type LabStore } from "./layerDoc";
import type { AppErrorBody, GenerationRecord, GenerationRequest, InputRecord, JobView, LoraItem, MacBenchState, ServerState, StatusResponse } from "./types";
-export type Overlay = null | "gallery" | "settings" | "inpaint" | "expand" | "welcome" | "whats-new" | "mac-engine" | "prompt-tips" | "shortcuts";
+export type Overlay = null | "gallery" | "settings" | "inpaint" | "expand" | "layers" | "welcome" | "whats-new" | "mac-engine" | "prompt-tips" | "shortcuts";
export interface SessionItem {
key: string;
@@ -70,6 +71,8 @@ export interface State {
/** Last submitted prompt (Up arrow in an empty prompt brings it back). */
lastPrompt: string;
overlay: Overlay;
+ /** Where Paint area / Expand go back to when they close (the layers lab), else nothing. */
+ overlayBack: Overlay;
/** The upscaler comparison, shown over any overlay (Settings, the Advanced menu). */
compareUpscalers: boolean;
toasts: Toast[];
@@ -144,6 +147,8 @@ export interface ProjectData {
loras?: string[];
/** The service's revision of it (see crates/qls-core/src/projects.rs). */
rev?: number;
+ /** Its layers lab (lib/layerDoc.ts `LabStore`); null once removed. */
+ lab?: unknown;
}
// The project list lives in kokon-service, shared by every device (0.11). Before, each app
@@ -165,7 +170,50 @@ const edits = new Map<string, number>();
let saveTimer: number | undefined;
/** What other devices need to hear about (the selection alone is this device's own). */
-const content = (p: ProjectData | undefined) => (p ? JSON.stringify([p.items.map((i) => i.key), p.meta.title, p.meta.named, p.loras]) : "");
+const content = (p: ProjectData | undefined) => (p ? JSON.stringify([p.items.map((i) => i.key), p.meta.title, p.meta.named, p.loras, p.lab ?? null]) : "");
+
+// The layers lab lives in its project (saved and merged with it). A lab made before its project
+// reached the service waits here; private projects keep theirs here only.
+const labsHere = new Map<string, LabStore>();
+/** Gallery results already hidden as lab steps (this session). */
+const hiddenSteps = new Set<string>();
+
+/** Leaves the lab's steps out of the gallery (they stay in the project): the ones made before
+ * requests were marked hidden, and any that slipped through. Only the image saved from the lab shows. */
+function hideLabSteps(lab: LabStore) {
+ for (const s of labSources(lab)) {
+ if (!s.startsWith("gen:") || hiddenSteps.has(s)) continue;
+ hiddenSteps.add(s);
+ const g = genCache.get(s.slice(4));
+ if (g?.hidden) continue;
+ api.setHidden(s.slice(4), true).catch(() => hiddenSteps.delete(s));
+ }
+}
+
+keepLabsIn({
+ get(id) {
+ const lab = projects[id]?.lab ?? labsHere.get(id);
+ if (lab && !state.privateMode) hideLabSteps(lab as LabStore);
+ return lab;
+ },
+ set(id, lab) {
+ if (state.privateMode || !projects[id]) {
+ labsHere.set(id, lab);
+ return;
+ }
+ labsHere.delete(id);
+ projects[id] = { ...projects[id], lab };
+ hideLabSteps(lab);
+ persistProjects([id]);
+ },
+});
+
+/** The lab's own images: a request editing one is a lab step (kept out of the gallery). */
+function isLabStep(source: string | undefined): boolean {
+ if (!source) return false;
+ const lab = loadLab(state.projectId);
+ return labSources(lab).has(source);
+}
function persistProjects(ids: string[]) {
for (const id of ids) {
@@ -201,6 +249,7 @@ async function applyProject(p: ProjectData) {
if (dirty.has(id)) return;
const old = projects[id];
projects[id] = { ...p, selected: old?.selected && p.items.some((i) => i.key === old.selected) ? old.selected : p.selected };
+ if (JSON.stringify(p.lab ?? null) !== JSON.stringify(old?.lab ?? null)) labsChanged();
if (id !== state.projectId || state.privateMode) {
setState({ projects: projectList() });
return;
@@ -332,6 +381,7 @@ function syncProject(prev: State) {
selected: state.selected,
loras: state.composer.loras ?? undefined,
rev: old?.rev,
+ lab: old?.lab ?? labsHere.get(id),
};
// Saved (and heard by the other devices) only when something they show changed.
if (content(projects[id]) !== content(old)) persistProjects([id]);
@@ -405,6 +455,7 @@ let state: State = {
composer: emptyComposer(loadPrefs()),
lastPrompt: "",
overlay: null,
+ overlayBack: null,
compareUpscalers: false,
toasts: [],
galleryVersion: 0,
@@ -995,7 +1046,7 @@ export function startNewImage() {
/** "Edit image": the prompt edits the version shown (or the latest image). When it is one of a
* new image's references it becomes image 1 and the others stay; a version that is not (such as
* the new image just made) is edited on its own. */
-export function editShownImage() {
+export function editShownImage(focus = true) {
setState((s) => {
const shown = s.session.find((i) => i.key === s.selected);
const ref = refOf(shown) ?? refOf([...s.session].reverse().find((i) => i.gen || i.input));
@@ -1008,7 +1059,16 @@ export function editShownImage() {
}
return { composer: { ...retarget(c, ref), create: false } };
});
- focusPrompt();
+ if (focus) focusPrompt();
+}
+
+/** Opens Paint area / Expand; from the layers lab they return there when they close. */
+export function openEditor(o: "inpaint" | "expand") {
+ setState((s) => ({ overlay: o, overlayBack: s.overlay === "layers" ? "layers" : null }));
+}
+
+export function closeEditor() {
+ setState((s) => ({ overlay: s.overlayBack, overlayBack: null }));
}
/** The sources ("gen:<id>"/"input:<id>") a version was made from; `variant` when it is the
@@ -1123,18 +1183,38 @@ export function addInputs(records: InputRecord[]) {
if (getState().composer.refs.length >= 10) toastInfo("Up to 10 reference images are supported.");
}
-export async function submitRequest(req: GenerationRequest) {
+/** Images made in the app (the layers lab's split): new versions at the end of the project, not
+ * references; `select` (an index in `records`) is shown and becomes the image to edit. */
+export function addVersions(records: InputRecord[], select = 0) {
+ if (!records.length) return;
+ const items: SessionItem[] = records.map((r) => ({ key: newKey(), kind: "input", input: r }));
+ setState((s) => ({ session: [...s.session, ...items], redo: [] }));
+ selectItem(items[Math.min(select, items.length - 1)].key);
+}
+
+/** Makes a version the image to edit (the layers lab's selected layer); false when the version
+ * is no longer in the project. `focus`: the prompt box takes the keyboard. */
+export function editSource(source: string, focus = true): boolean {
+ const it = getState().session.find((i) => itemSource(i) === source);
+ if (!it) return false;
+ selectItem(it.key);
+ editShownImage(focus);
+ return true;
+}
+
+/** Sends a request; its job joins the project as a version. The job's id, or null when it failed. */
+export async function submitRequest(req: GenerationRequest): Promise<string | null> {
try {
if (getState().privateMode) req = { ...req, private: true };
const job = await api.submit(req);
const item: SessionItem = { key: newKey(), kind: "job", jobId: job.id, request: req };
setState((s) => ({ jobs: { ...s.jobs, [job.id]: job }, session: [...s.session, item], selected: item.key, redo: [] }));
- return true;
+ return job.id;
} catch (e) {
// The prompt and images stay in the composer: once the files are back, Generate again.
if (isMissingModel(e)) openModelSetup();
toastError(e);
- return false;
+ return null;
}
}
@@ -1156,6 +1236,8 @@ export async function submitComposer() {
const target = composing(s.composer) ? undefined : s.composer.refs[0]?.source;
const parent = target?.startsWith("gen:") ? target.slice(4) : null;
const req: GenerationRequest = { ...buildRequest(s.composer, parent), loras: activeLoras(s.composer, s.loras) };
+ // Made from the layers lab: a step, out of the gallery (only the image saved from the lab shows).
+ if (s.overlay === "layers" || isLabStep(target)) req.hidden = true;
if (!req.prompt && needsPrompt(s.composer)) {
toastInfo("Describe what to create or change.");
return;
diff --git a/src/lib/types.ts b/src/lib/types.ts
index dd9c394..5008e5b 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -3,7 +3,8 @@
export type Mode = "auto" | "generate" | "edit" | "inpaint" | "outpaint" | "multi_reference" | "compose" | "upscale";
export type ResolvedMode = Exclude<Mode, "auto">;
/** What made a saved result: a generation mode, or a colour LUT applied to an image. */
-export type RecordMode = ResolvedMode | "lut";
+/** "layers": the layers lab's image saved to the gallery. */
+export type RecordMode = ResolvedMode | "lut" | "layers";
/** What an inpaint does in the painted area. */
export type InpaintAction = "edit" | "remove" | "replace" | "add";
/** Presets of the Turbo model (8 steps, no CFG): ~1 MP, ~1.5K, native 2K. */
@@ -67,6 +68,10 @@ export interface GenerationRequest {
enhanced_from?: string | null;
/** LoRA ids to use (the app always sends its list; omitted = none, as for MCP). */
loras?: string[] | null;
+ /** Inpaint: coloured marks (a transparent image the mask's size) that the prompt names by colour. */
+ marks?: string | null;
+ /** The result is left out of the gallery list (a layers lab step). */
+ hidden?: boolean;
}
export interface WorkingCanvas { source: Size; content: Size; canvas: Size; content_offset: [number, number]; scale: number }
@@ -157,6 +162,8 @@ export interface GenerationRecord {
notes: string[];
request: GenerationRequest | null;
pipeline?: PipelineRecord | null;
+ /** Left out of the gallery list (a layers lab step). */
+ hidden?: boolean;
}
/** A LoRA as an image used it. */
diff --git a/src/styles.css b/src/styles.css
index 02cd91d..4c09861 100644
--- a/src/styles.css
+++ b/src/styles.css
@@ -528,6 +528,114 @@ pre.diag { font-family: ui-monospace, "Cascadia Mono", Consolas, monospace; font
.mask-wrap canvas { position: absolute; inset: 0; width: 100%; height: 100%; mix-blend-mode: difference; opacity: 0.75; cursor: none; }
.brush-cursor { position: absolute; pointer-events: none; border: 1.5px solid #fff; mix-blend-mode: difference; border-radius: 50%; transform: translate(-50%, -50%); }
+/* layers lab (LayersEditor.tsx) */
+.layers-status { font-size: 12px; color: var(--text-2); min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+/* The zoom viewport fills its stage edge to edge (inset 0): here the stage is a framed area with a
+ gap on both sides, so a zoomed image never runs into the window edge or the right bar. */
+.layers-body { flex: 1; min-height: 0; display: flex; gap: 16px; padding: 0 16px; }
+.layers-body .editor-stage { flex: 1; min-width: 0; padding: 0; border-radius: var(--radius); background: var(--surface); }
+.layers-body .mask-wrap canvas { mix-blend-mode: normal; opacity: 1; }
+/* Inpaint with colours: the marks as they go to the model (opaque), a little see-through here. */
+.mask-wrap canvas.marks { mix-blend-mode: normal; opacity: 0.85; }
+/* One area: pink with a light edge, as the painted area in the layers lab. */
+.mask-wrap canvas.area { mix-blend-mode: normal; opacity: 0.5; filter: drop-shadow(0 0 1px #fff) drop-shadow(0 0 1px #fff); }
+.inpaint-kind { flex: none; background: transparent; padding: 0; }
+.inpaint-kind button { display: inline-flex; align-items: center; gap: 5px; padding: 0 10px; }
+.marks-panel { flex: none; display: flex; flex-direction: column; align-items: center; gap: 6px; padding: 8px 16px 0; }
+.marks-help { margin: 0; font-size: 12px; color: var(--text-2); text-align: center; max-width: 560px; }
+.marks-want { display: flex; align-items: center; gap: 8px; width: min(480px, 100%); }
+.marks-want input { flex: 1; min-width: 0; height: 30px; padding: 0 8px; border: 1px solid var(--border); border-radius: 8px; background: var(--bg); color: var(--text); font: inherit; }
+.marks-want input:focus { outline: none; border-color: var(--border-2); }
+.marks-colours { gap: 6px; padding: 5px 8px; }
+.layers-panel { width: 300px; flex: none; display: flex; flex-direction: column; gap: 6px; padding: 0 0 4px; overflow-y: auto; font-size: 12px; }
+.layers-title { display: flex; align-items: center; gap: 6px; font-weight: 600; font-size: 13px; }
+.layers-title .spacer { flex: 1; }
+.layer-row { display: flex; align-items: center; gap: 4px; padding: 4px 4px 4px 8px; border-radius: 10px; background: var(--bg); box-shadow: var(--lift); cursor: pointer; border: 1.5px solid transparent; }
+.layer-row.on { border-color: var(--text); }
+.layer-row.bg { cursor: default; color: var(--text-2); padding: 8px; }
+.layer-row .swatch { width: 12px; height: 12px; border-radius: 4px; flex: none; background: repeating-conic-gradient(var(--checker) 0 25%, var(--bg) 0 50%) 0 0 / 6px 6px; }
+.layer-row input { flex: 1; min-width: 0; border: none; background: transparent; color: var(--text); font: inherit; padding: 2px 4px; border-radius: 6px; }
+.layer-row input:focus { outline: 1px solid var(--border-2); }
+.doc-name { flex: 1; min-width: 0; padding: 2px 4px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.layer-pct { color: var(--text-3); font-variant-numeric: tabular-nums; }
+.layers-view { display: flex; align-items: center; gap: 4px; margin-top: 8px; }
+.layers-view span { color: var(--text-2); margin-right: 4px; }
+.layers-opt { display: flex; align-items: center; gap: 8px; color: var(--text-2); }
+.layers-opt input[type="range"] { flex: 1; }
+.layers-help { color: var(--text-3); line-height: 1.45; margin: 4px 0; }
+.layers-split { margin-top: auto; }
+.doc-frame { background: repeating-conic-gradient(var(--checker) 0 25%, var(--bg) 0 50%) 0 0 / 24px 24px; overflow: hidden; }
+.mask-wrap img.doc-layer { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; }
+.doc-hit { position: absolute; inset: 0; }
+.mask-wrap img.doc-mask { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; }
+.mask-wrap div.doc-layer { position: absolute; inset: 0; pointer-events: none; }
+.mask-wrap img.doc-crop-mask { position: absolute; pointer-events: none; }
+.doc-mask-chip { position: absolute; top: 12px; left: 50%; transform: translateX(-50%); z-index: 3; padding: 6px 12px; border-radius: 999px; background: #ff3b6b; color: #fff; font-size: 12px; font-weight: 500; white-space: nowrap; pointer-events: none; box-shadow: var(--lift); }
+.doc-sel { position: absolute; border: 1.5px dashed #fff; outline: 1px solid rgba(0, 0, 0, 0.45); pointer-events: none; box-sizing: border-box; }
+.doc-handle { position: absolute; background: #fff; border: 1.5px solid #000; border-radius: 2px; pointer-events: auto; cursor: nwse-resize; box-sizing: border-box; }
+.brush-cursor.pencil { mix-blend-mode: normal; }
+.doc-sel-part { border-style: solid; outline: none; opacity: 0.6; }
+.layers-body .mask-wrap canvas.doc-marks { pointer-events: none; opacity: 0.9; }
+.doc-group { display: flex; color: var(--text-3); }
+.doc-actions-title { display: flex; align-items: center; gap: 6px; }
+.doc-want { display: flex; align-items: center; gap: 8px; }
+.doc-want input { flex: 1; min-width: 0; height: 30px; padding: 0 8px; border: 1px solid var(--border); border-radius: 8px; background: var(--bg); color: var(--text); font: inherit; }
+.doc-want input:focus { outline: none; border-color: var(--border-2); }
+.doc-swatch { width: 14px; height: 14px; flex: none; border-radius: 50%; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.25); }
+.doc-colour { width: 22px; height: 22px; border-radius: 50%; border: 2px solid transparent; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.25); cursor: pointer; padding: 0; }
+.doc-colour.on { border-color: var(--bg); box-shadow: 0 0 0 2px var(--text); }
+.doc-grid .doc-wide { grid-column: 1 / -1; }
+.doc-row { position: relative; }
+.lab-frame { position: relative; max-width: 100%; max-height: 100%; height: 100%; padding: 0; border: none; background: repeating-conic-gradient(var(--checker) 0 25%, var(--bg) 0 50%) 0 0 / 24px 24px; border-radius: var(--radius-img); overflow: hidden; box-shadow: var(--print); display: grid; place-items: center; }
+.lab-frame img { width: 100%; height: 100%; object-fit: contain; display: block; }
+.lab-frame img.lab-marks { position: absolute; inset: 0; opacity: 0.9; pointer-events: none; }
+.mask-wrap img.doc-marks { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; opacity: 0.9; }
+.lab-open { position: absolute; left: 50%; bottom: 14px; transform: translateX(-50%); display: flex; align-items: center; gap: 6px; padding: 7px 12px; border: none; border-radius: 999px; background: var(--invert-bg); color: var(--invert-text); font: inherit; font-size: 12px; font-weight: 500; cursor: pointer; box-shadow: var(--lift); opacity: 0.92; transition: opacity 0.15s; }
+.lab-open:hover, .lab-open:focus-visible { opacity: 1; }
+.lab-jobs { display: flex; flex-wrap: wrap; gap: 6px; min-width: 0; }
+.lab-jobs.on-stage { position: absolute; top: 12px; left: 50%; transform: translateX(-50%); z-index: 2; justify-content: center; }
+.lab-progress { display: inline-flex; align-items: center; gap: 6px; padding: 6px 12px; border-radius: 999px; background: var(--invert-bg); color: var(--invert-text); font-size: 12px; font-weight: 500; font-variant-numeric: tabular-nums; white-space: nowrap; box-shadow: var(--lift); }
+.lab-progress.failed { background: #b42318; color: #fff; white-space: normal; }
+.lab-dismiss { display: grid; place-items: center; border: none; background: transparent; color: inherit; padding: 0; margin-left: 2px; cursor: pointer; opacity: 0.8; }
+.lab-dismiss:hover { opacity: 1; }
+.doc-running { display: inline-flex; align-items: center; gap: 4px; }
+.doc-row.dragging { opacity: 0.5; }
+.doc-row.drop-above { box-shadow: 0 -2px 0 0 var(--text), var(--lift); }
+.doc-row.drop-below { box-shadow: 0 2px 0 0 var(--text), var(--lift); }
+.doc-running { color: var(--text); font-weight: 600; }
+.doc-row .grip { display: flex; color: var(--text-3); cursor: grab; touch-action: none; padding: 2px 0; }
+.doc-thumb { width: 40px; height: 30px; flex: none; border-radius: 4px; overflow: hidden; display: flex; align-items: center; justify-content: center; background: repeating-conic-gradient(var(--checker) 0 25%, var(--bg) 0 50%) 0 0 / 8px 8px; }
+.doc-thumb img { max-width: 100%; max-height: 100%; display: block; }
+/* Stickers being made (LayerDoc): objects on transparency, the background fading back at the end. */
+.mask-wrap img.doc-layer { transition: opacity 0.6s; }
+.doc-frame.stickering { background: repeating-conic-gradient(var(--checker) 0 25%, var(--bg) 0 50%) 0 0 / 24px 24px; }
+.mask-wrap img.doc-layer.st-hide { opacity: 0; }
+.mask-wrap img.doc-layer.st-waiting { opacity: 0.4; }
+.mask-wrap img.doc-layer.st-making { filter: drop-shadow(0 0 2px #ff3b6b) drop-shadow(0 0 8px rgba(255, 59, 107, 0.6)); }
+.mask-wrap img.doc-layer.st-done { animation: sticker-in 0.6s ease-out; }
+@keyframes sticker-in { from { opacity: 0; filter: blur(6px); } to { opacity: 1; filter: none; } }
+.sticker-ring { position: absolute; pointer-events: none; display: grid; place-items: center; color: #fff; filter: drop-shadow(0 1px 3px rgba(0, 0, 0, 0.5)); }
+.sticker-ring.waiting { opacity: 0.8; }
+.sticker-progress .track { fill: rgba(0, 0, 0, 0.45); stroke: rgba(255, 255, 255, 0.25); stroke-width: 4; }
+.sticker-progress .bar { fill: none; stroke: #ff3b6b; stroke-width: 4; stroke-linecap: round; transition: stroke-dashoffset 0.4s; }
+.sticker-progress text { fill: #fff; font-size: 12px; font-weight: 600; font-variant-numeric: tabular-nums; }
+.sticker-strip { position: absolute; top: 12px; left: 50%; transform: translateX(-50%); z-index: 3; display: flex; align-items: center; gap: 8px; padding: 6px 6px 6px 12px; border-radius: 999px; background: var(--invert-bg); color: var(--invert-text); font-size: 12px; font-weight: 500; white-space: nowrap; box-shadow: var(--lift); font-variant-numeric: tabular-nums; }
+/* The strip is inverted (dark on light themes, light on dark): its buttons follow it, not the page. */
+.sticker-strip .btn { height: 26px; padding: 0 10px; font-size: 12px; border-radius: 999px; background: color-mix(in srgb, var(--invert-text) 12%, transparent); color: var(--invert-text); }
+.sticker-strip .btn:hover { background: color-mix(in srgb, var(--invert-text) 22%, transparent); }
+.doc-actions { display: flex; flex-direction: column; gap: 8px; margin-top: 6px; padding-top: 12px; border-top: 1px solid var(--border); }
+.doc-actions-title { font-weight: 600; font-size: 12px; }
+.doc-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 6px; }
+.doc-grid .btn, .doc-footer .btn { justify-content: center; height: 34px; min-width: 0; white-space: nowrap; }
+.doc-grid > :only-child { grid-column: 1 / -1; }
+.doc-footer { margin-top: auto; display: flex; flex-direction: column; gap: 6px; padding-top: 12px; }
+.doc-none { margin-top: 10px; }
+.layers-panel > * { flex-shrink: 0; }
+.doc-versions { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 6px; padding: 8px; border-radius: 12px; background: var(--bg); box-shadow: var(--lift); }
+.doc-version { min-width: 0; border: 1.5px solid transparent; border-radius: 8px; padding: 2px; background: repeating-conic-gradient(var(--checker) 0 25%, var(--bg) 0 50%) 0 0 / 8px 8px; cursor: pointer; display: flex; flex-direction: column; align-items: center; font-size: 11px; color: var(--text-2); }
+.doc-version.on { border-color: var(--text); }
+.doc-version img { width: 100%; aspect-ratio: 3 / 2; object-fit: contain; }
+
.expand-area { position: relative; }
.expand-canvas { position: absolute; border: 1.5px dashed var(--text-2); border-radius: 4px; background-image: conic-gradient(var(--checker) 25%, transparent 0 50%, var(--checker) 0 75%, transparent 0); background-size: 16px 16px; }
.expand-src { position: absolute; }
@@ -616,7 +724,8 @@ pre.diag { font-family: ui-monospace, "Cascadia Mono", Consolas, monospace; font
.compare-img { background-image: conic-gradient(var(--checker) 25%, transparent 0 50%, var(--checker) 0 75%, transparent 0); background-size: 16px 16px; }
.compare-box.slider { cursor: ew-resize; }
/* the area painted for the next edit, drawn like in the inpaint editor */
-.stage-mask { position: absolute; inset: 0; width: 100%; height: 100%; border-radius: var(--radius-img); mix-blend-mode: difference; opacity: 0.75; pointer-events: none; }
+/* The painted area (pink, see useMaskTint) or the coloured marks of the next edit. */
+.stage-mask { position: absolute; inset: 0; width: 100%; height: 100%; border-radius: var(--radius-img); opacity: 0.9; pointer-events: none; }
.compare-box.slider.zoomed { cursor: inherit; }
.compare-img, .compare-diff { position: absolute; inset: 0; width: 100%; height: 100%; display: block; }
/* Areas the original did not cover (expanded edges) show the checkerboard. */
diff --git a/vite.config.ts b/vite.config.ts
index 31cf5a7..9e5b6e7 100644
--- a/vite.config.ts
+++ b/vite.config.ts
@@ -10,6 +10,8 @@ export default defineConfig({
clearScreen: false,
// warmup: compile the app as soon as the dev server starts, not when the window asks for it.
server: { port: 1420, strictPort: true, host: "127.0.0.1", warmup: { clientFiles: ["./src/main.tsx"] } },
+ // onnxruntime-web (layers lab, lib/sam.ts) finds its .wasm next to its own module: no pre-bundling.
+ optimizeDeps: { exclude: ["onnxruntime-web"] },
build: { target: "es2022", sourcemap: false, chunkSizeWarningLimit: 1500 },
test: { include: ["src/**/*.test.{ts,tsx}"], environment: "jsdom" },
});