Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Layers lab: fast stickers, Fast/Quality choice on split, Expand on objects

Gabriele Lozupone1ac62b4ade02
diff --git a/Cargo.lock b/Cargo.lock
index 83ce9a1..189e0e2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -885,6 +885,16 @@ dependencies = [
"thiserror 2.0.21",
]
+[[package]]
+name = "der"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004"
+dependencies = [
+ "pem-rfc7468",
+ "zeroize",
+]
+
[[package]]
name = "deranged"
version = "0.5.8"
@@ -1790,6 +1800,12 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+[[package]]
+name = "hmac-sha256"
+version = "1.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad320b3b96fb2a455a0726d16efe0a5afdbd34b71dea5bc53b05ea057714d4e"
+
[[package]]
name = "html5ever"
version = "0.39.0"
@@ -2484,6 +2500,12 @@ version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
+[[package]]
+name = "lzma-rust2"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619"
+
[[package]]
name = "markup5ever"
version = "0.39.0"
@@ -2510,6 +2532,16 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
+[[package]]
+name = "matrixmultiply"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
[[package]]
name = "memchr"
version = "2.8.3"
@@ -2626,6 +2658,21 @@ dependencies = [
"tempfile",
]
+[[package]]
+name = "ndarray"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
[[package]]
name = "ndk"
version = "0.9.0"
@@ -2689,12 +2736,30 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "num-conv"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
+[[package]]
+name = "num-integer"
+version = "0.1.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -3033,6 +3098,30 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "ort"
+version = "2.0.0-rc.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08ca9287c05aace7e95250104a00d54b7da0fc9ac5f25efa424b80827d2e25eb"
+dependencies = [
+ "ndarray",
+ "ort-sys",
+ "smallvec",
+ "tracing",
+ "ureq",
+]
+
+[[package]]
+name = "ort-sys"
+version = "2.0.0-rc.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4af4763e553f916650b0bfe4b8d217d61c28607f6117e48b8ab5584d1138f22f"
+dependencies = [
+ "hmac-sha256",
+ "lzma-rust2",
+ "ureq",
+]
+
[[package]]
name = "os_pipe"
version = "1.2.3"
@@ -3121,6 +3210,15 @@ dependencies = [
"phc",
]
+[[package]]
+name = "pem-rfc7468"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
+dependencies = [
+ "base64ct",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -3394,6 +3492,7 @@ dependencies = [
"futures-util",
"hex",
"image",
+ "ort",
"png 0.18.1",
"regex",
"reqwest",
@@ -3486,6 +3585,12 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539"
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -5422,6 +5527,35 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+[[package]]
+name = "ureq"
+version = "3.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7be76686bdf7198002062423ca0d8cf9b75a27c8e1de804513b97e97b555d9d"
+dependencies = [
+ "base64 0.23.1",
+ "der",
+ "log",
+ "native-tls",
+ "percent-encoding",
+ "rustls-pki-types",
+ "ureq-proto",
+ "utf8-zero",
+ "webpki-root-certs",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622dce33c252115466d83f63e9744a18f12b4406aae51a01c15ad8fcf1185eb3"
+dependencies = [
+ "base64 0.23.1",
+ "http",
+ "httparse",
+ "log",
+]
+
[[package]]
name = "url"
version = "2.5.8"
@@ -5447,6 +5581,12 @@ dependencies = [
"url",
]
+[[package]]
+name = "utf8-zero"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
+
[[package]]
name = "utf8_iter"
version = "1.0.4"
@@ -5760,9 +5900,9 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
-version = "1.0.9"
+version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
+checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a"
dependencies = [
"rustls-pki-types",
]
diff --git a/crates/qls-core/Cargo.toml b/crates/qls-core/Cargo.toml
index 431c2c3..03c7e38 100644
--- a/crates/qls-core/Cargo.toml
+++ b/crates/qls-core/Cargo.toml
@@ -14,6 +14,8 @@ futures-util = "0.3.34"
hex = "0.4.3"
image = { version = "0.25.10", default-features = false, features = ["png", "jpeg", "webp", "bmp", "gif", "tiff"] }
png = "0.18.1"
+# Fast stickers (matting.rs): ONNX Runtime on the CPU, linked in from pyke's prebuilt binaries.
+ort = { version = "=2.0.0-rc.14", default-features = false, features = ["std", "download-binaries", "tls-native", "api-31"] }
regex = "1.13.1"
reqwest = { version = "0.13.5", default-features = false, features = ["json", "native-tls", "stream"] }
diff --git a/crates/qls-core/src/engine.rs b/crates/qls-core/src/engine.rs
index ded5ccd..cc856bc 100644
--- a/crates/qls-core/src/engine.rs
+++ b/crates/qls-core/src/engine.rs
@@ -904,6 +904,19 @@ impl Engine {
}
}
+ /// The fast-sticker matte of an image already in Kokon ("input:"/"gen:"), as a grey PNG at
+ /// its size (white = the main thing in it; the layers lab combines it with the selection).
+ pub fn matte(&self, src: &str) -> AppResult<Vec<u8>> {
+ if !src.starts_with("input:") && !src.starts_with("gen:") {
+ return Err(AppError::invalid("Stickers are made from images already in Kokon."));
+ }
+ let (bytes, _, _) = self.read_source(src, None)?;
+ let img = imaging::decode_image(&bytes)?;
+ let m = crate::matting::matte(&self.config().sdcpp_dir, &img.pixels)?;
+ let rgba = image::DynamicImage::ImageLuma8(m).to_rgba8();
+ imaging::encode_png(&rgba, &[], None, true)
+ }
+
/// Applies a library LUT to a "gen:"/"input:" source at `intensity` (0-1) and saves the
/// result as a new version made from it (mode "lut"; the source stays as it is). Private
/// when asked or when the source itself is private. Same file format as a source result.
diff --git a/crates/qls-core/src/lib.rs b/crates/qls-core/src/lib.rs
index c28cc41..6f4a918 100644
--- a/crates/qls-core/src/lib.rs
+++ b/crates/qls-core/src/lib.rs
@@ -13,6 +13,7 @@ pub mod imaging;
pub mod installer;
pub mod lora;
pub mod lut;
+pub mod matting;
pub mod mac_bench;
pub mod modelfile;
pub mod paths;
diff --git a/crates/qls-core/src/matting.rs b/crates/qls-core/src/matting.rs
new file mode 100644
index 0000000..f98c283
--- /dev/null
+++ b/crates/qls-core/src/matting.rs
@@ -0,0 +1,159 @@
+//! Fast stickers for the layers lab: BiRefNet lite (MIT; onnx-community's ONNX export, pinned) run
+//! on the CPU with ONNX Runtime. It returns the model's matte of an image; the app turns it into a
+//! sticker with the user's selection (`src/lib/matte.ts`: the selection decides what, the matte the
+//! edge). Spike s17 (2026-11-02): cleaner edges than SAM, deterministic, ~4 s per object on a
+//! 16-thread CPU, no VRAM taken from the image engine. The webview could not run it (WebGPU kernels
+//! rejected, WASM out of memory), hence here.
+
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+
+use image::{GrayImage, RgbaImage, imageops::FilterType};
+use ort::session::Session;
+use ort::session::builder::GraphOptimizationLevel;
+use ort::value::Tensor;
+
+use crate::catalog::{Download, FileKind, join_rel};
+use crate::error::{AppError, AppResult, ErrorCode};
+
+/// The model file, downloaded on first use (Settings never lists it: the lab asks for it).
+pub const BIREFNET_LITE: Download = Download {
+ id: "birefnet_lite",
+ label: "BiRefNet lite (fast stickers)",
+ kind: FileKind::Model,
+ url: "https://huggingface.co/onnx-community/BiRefNet_lite-ONNX/resolve/de15b22ba131738a16dff04aab8bdf8dc32e3ac1/onnx/model.onnx",
+ fallback_urls: &[],
+ page: "https://huggingface.co/onnx-community/BiRefNet_lite-ONNX",
+ size: 224_005_088,
+ sha256: "5600024376f572a557870a5eb0afb1e5961636bef4e1e22132025467d0f03333",
+ dest: "models/matting/birefnet_lite.onnx",
+ provides: &[],
+ engine_tag: None,
+};
+
+/// The model's input side (the export is fixed at 1024²).
+const SIDE: u32 = 1024;
+const MEAN: [f32; 3] = [0.485, 0.456, 0.406];
+const STD: [f32; 3] = [0.229, 0.224, 0.225];
+
+pub fn model_path(sdcpp_dir: &Path) -> PathBuf {
+ join_rel(sdcpp_dir, BIREFNET_LITE.dest)
+}
+
+/// Loaded once and kept (~0.5 GB of RAM while the lab is used): loading takes ~4 s. The
+/// environment is kept with it (sessions need it alive).
+static SESSION: Mutex<Option<(PathBuf, ort::environment::Environment, Session)>> = Mutex::new(None);
+
+fn ort_err(e: impl std::fmt::Display) -> AppError {
+ AppError::new(ErrorCode::Internal, format!("The sticker model failed: {e}"))
+}
+
+/// The model's input: RGB at 1024², normalised as in training, CHW.
+pub fn input_tensor(img: &RgbaImage) -> Vec<f32> {
+ let small = image::imageops::resize(img, SIDE, SIDE, FilterType::Triangle);
+ let n = (SIDE * SIDE) as usize;
+ let mut x = vec![0f32; 3 * n];
+ for (i, p) in small.pixels().enumerate() {
+ for k in 0..3 {
+ x[k * n + i] = (p.0[k] as f32 / 255.0 - MEAN[k]) / STD[k];
+ }
+ }
+ x
+}
+
+/// The matte (logits) -> 0-255 at `w`×`h`.
+pub fn matte_image(logits: &[f32], w: u32, h: u32) -> GrayImage {
+ let m = GrayImage::from_fn(SIDE, SIDE, |x, y| {
+ let v = logits[(y * SIDE + x) as usize];
+ image::Luma([(255.0 / (1.0 + (-v).exp())).round() as u8])
+ });
+ image::imageops::resize(&m, w, h, FilterType::Triangle)
+}
+
+/// The matte of `img` at its own size (white = the main thing in it).
+pub fn matte(sdcpp_dir: &Path, img: &RgbaImage) -> AppResult<GrayImage> {
+ let path = model_path(sdcpp_dir);
+ if !path.is_file() {
+ return Err(AppError::new(ErrorCode::MissingModel, "The fast sticker model is not downloaded yet."));
+ }
+ let x = input_tensor(img);
+ let mut guard = SESSION.lock().unwrap_or_else(|p| p.into_inner());
+ if guard.as_ref().is_none_or(|(p, _, _)| p != &path) {
+ let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4);
+ let env = ort::init().with_name("kokon-stickers").build().map_err(ort_err)?;
+ let s = Session::builder(&env)
+ .map_err(ort_err)?
+ .with_optimization_level(GraphOptimizationLevel::Level3)
+ .map_err(ort_err)?
+ .with_intra_threads(threads)
+ .map_err(ort_err)?
+ .commit_from_file(&path)
+ .map_err(ort_err)?;
+ *guard = Some((path.clone(), env, s));
+ }
+ let (_, _, session) = guard.as_mut().unwrap();
+ let input = Tensor::from_array(([1usize, 3, SIDE as usize, SIDE as usize], x.into_boxed_slice())).map_err(ort_err)?;
+ let outputs = session.run(ort::inputs![input]).map_err(ort_err)?;
+ let (_, logits) = outputs[outputs.len() - 1].try_extract_tensor::<f32>().map_err(ort_err)?;
+ if logits.len() != (SIDE * SIDE) as usize {
+ return Err(AppError::new(ErrorCode::Internal, "The sticker model returned an unexpected size."));
+ }
+ Ok(matte_image(logits, img.width(), img.height()))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn input_is_normalised_chw() {
+ let img = RgbaImage::from_pixel(8, 8, image::Rgba([255, 0, 128, 255]));
+ let x = input_tensor(&img);
+ let n = (SIDE * SIDE) as usize;
+ assert_eq!(x.len(), 3 * n);
+ assert!((x[0] - (1.0 - MEAN[0]) / STD[0]).abs() < 1e-4);
+ assert!((x[n] - (0.0 - MEAN[1]) / STD[1]).abs() < 1e-4);
+ assert!((x[2 * n] - (128.0 / 255.0 - MEAN[2]) / STD[2]).abs() < 1e-3);
+ }
+
+ #[test]
+ fn logits_become_a_matte_at_the_image_size() {
+ let mut logits = vec![-20f32; (SIDE * SIDE) as usize];
+ // The left half is the object.
+ for y in 0..SIDE {
+ for x in 0..SIDE / 2 {
+ logits[(y * SIDE + x) as usize] = 20.0;
+ }
+ }
+ let m = matte_image(&logits, 300, 200);
+ assert_eq!(m.dimensions(), (300, 200));
+ assert_eq!(m.get_pixel(10, 100).0[0], 255);
+ assert_eq!(m.get_pixel(290, 100).0[0], 0);
+ }
+
+ #[test]
+ fn missing_model_is_said() {
+ let dir = tempfile::tempdir().unwrap();
+ let e = matte(dir.path(), &RgbaImage::new(4, 4)).unwrap_err();
+ assert!(e.to_string().contains("not downloaded"));
+ }
+}
+
+#[cfg(test)]
+mod real_model {
+ use super::*;
+
+ /// `KOKON_MATTE_DIR` = a folder with `models/matting/birefnet_lite.onnx`, `KOKON_MATTE_IMAGE` = a photo.
+ #[test]
+ #[ignore = "needs the model file"]
+ fn mattes_a_photo() {
+ let dir = PathBuf::from(std::env::var("KOKON_MATTE_DIR").unwrap());
+ let img = image::open(std::env::var("KOKON_MATTE_IMAGE").unwrap()).unwrap().to_rgba8();
+ for _ in 0..2 {
+ let t = std::time::Instant::now();
+ let m = matte(&dir, &img).unwrap();
+ eprintln!("matte {:?} in {:.2} s", m.dimensions(), t.elapsed().as_secs_f32());
+ m.save(std::env::temp_dir().join("kokon-matte.png")).unwrap();
+ }
+ }
+}
diff --git a/crates/qls-core/src/remote.rs b/crates/qls-core/src/remote.rs
index 1b3ef8b..c483021 100644
--- a/crates/qls-core/src/remote.rs
+++ b/crates/qls-core/src/remote.rs
@@ -270,6 +270,7 @@ pub fn browser_allowed(method: &str, path: &str) -> bool {
("GET", ["gallery", _] | ["gallery", _, "image" | "thumb" | "preview" | "original"]) => true,
("GET", ["projects", _, "records"]) => true,
("GET", ["luts", _, "preview"]) => true,
+ ("GET" | "POST", ["matte"]) | ("POST", ["matte", "download"]) => true,
("GET" | "PUT" | "DELETE", ["projects", _]) => true,
("PUT", ["config"]) => true,
("POST", ["setup", "install" | "gpu"] | ["setup", "install", "cancel"]) => true,
diff --git a/crates/qls-service/src/api.rs b/crates/qls-service/src/api.rs
index b4ae30d..ebf86c0 100644
--- a/crates/qls-service/src/api.rs
+++ b/crates/qls-service/src/api.rs
@@ -80,6 +80,8 @@ pub fn router(state: Arc<AppState>, lan: bool) -> Router {
.route("/models/dit", post(models_new_dit))
.route("/upscalers", get(list_upscalers))
.route("/upscalers/{name}/download", post(download_upscaler))
+ .route("/matte", get(matte_status).post(matte_image))
+ .route("/matte/download", post(matte_download))
.route("/config", get(get_config).put(put_config))
.route("/enhancer/defaults", get(enhancer_defaults))
.route("/mac-engine/bench", get(mac_bench_status).post(mac_bench_start).delete(mac_bench_clear))
@@ -467,6 +469,31 @@ async fn download_upscaler(State(s): St, AxPath(name): AxPath<String>) -> ApiRes
Ok(Json(serde_json::to_value(s.engine.installer.status()).unwrap()))
}
+/// The fast-sticker model (layers lab): there or not, and its download size.
+async fn matte_status(State(s): St) -> Json<Value> {
+ let path = qls_core::matting::model_path(&s.engine.config().sdcpp_dir);
+ Json(json!({ "installed": path.is_file(), "bytes": qls_core::matting::BIREFNET_LITE.size }))
+}
+
+async fn matte_download(State(s): St) -> ApiResult<Json<Value>> {
+ let dir = s.engine.config().sdcpp_dir;
+ s.engine.installer.start(dir, "matting", vec![qls_core::matting::BIREFNET_LITE], Box::new(|_| {}))?;
+ Ok(Json(serde_json::to_value(s.engine.installer.status()).unwrap()))
+}
+
+#[derive(Deserialize)]
+struct MatteBody {
+ /// "input:<id>" / "gen:<id>".
+ image: String,
+}
+
+/// The matte of an image (grey PNG at its size) for a fast sticker.
+async fn matte_image(State(s): St, Json(b): Json<MatteBody>) -> ApiResult<Response> {
+ let engine = s.engine.clone();
+ let png = tokio::task::spawn_blocking(move || engine.matte(&b.image)).await.map_err(AppError::internal)??;
+ Ok(([(header::CONTENT_TYPE, "image/png")], png).into_response())
+}
+
/// Upscaler models of earlier versions in the app's own folder (Settings offers to delete
/// them; files elsewhere are never touched).
#[derive(Deserialize)]
diff --git a/crates/qls-service/src/security_tests.rs b/crates/qls-service/src/security_tests.rs
index 0c4669e..894848a 100644
--- a/crates/qls-service/src/security_tests.rs
+++ b/crates/qls-service/src/security_tests.rs
@@ -353,3 +353,17 @@ async fn a_browser_opens_a_project_with_one_request_and_previews() {
// Signed out: nothing.
assert_eq!(send(&a, "GET", "/api/v1/projects/p/records", &[], None).await.status, StatusCode::UNAUTHORIZED);
}
+
+#[tokio::test]
+async fn fast_stickers_take_only_images_already_in_kokon() {
+ let f = fixture(true);
+ let a = app(&f, true, "192.168.1.50");
+ let cookie = sign_in(&a).await;
+ let c = ("cookie", cookie.as_str());
+ let s = send(&a, "GET", "/api/v1/matte", &[c], None).await;
+ assert_eq!((s.status, s.body["installed"].clone()), (StatusCode::OK, json!(false)));
+ for image in ["/etc/passwd", r"C:\Users\me\x.png", "file:///etc/passwd"] {
+ let r = send(&a, "POST", "/api/v1/matte", &[c, CSRF], Some(json!({ "image": image }))).await;
+ assert_eq!((r.status, r.code()), (StatusCode::BAD_REQUEST, "invalid_input"), "{image}");
+ }
+}
diff --git a/src/components/LabScene.ts b/src/components/LabScene.ts
index 14ef0a5..f486b9e 100644
--- a/src/components/LabScene.ts
+++ b/src/components/LabScene.ts
@@ -10,7 +10,7 @@ import { dilate, fileSafe, fromLogits, refineEdges } from "../lib/layers";
import { Segmenter } from "../lib/sam";
import { alignEdit, attached, blur, changed, darkness, IDENTITY, invertAffine, luminance, MARK_COLOURS, objectAt, scaleAffine, splitShadows, toneCurve, type Affine, type MarkColour } from "../lib/sceneEdit";
import type { InputRecord } from "../lib/types";
-import { loadImage, pixels, sourceInfo, toPng } from "./LayerDoc";
+import { loadImage, pixels, drawSource, sourceInfo, toPng } from "./LayerDoc";
/** Long side of the analysis (shadows are soft: no need for every pixel). */
const SIDE = 1536;
@@ -61,7 +61,7 @@ async function footprint(source: string, place: Place, w: number, h: number, W:
ctx.translate(cc.x + place.dx, cc.y + place.dy);
ctx.scale(place.s, place.s);
ctx.translate(-cc.x, -cc.y);
- ctx.drawImage(info.img, 0, 0, W, H);
+ drawSource(ctx, info);
const px = ctx.getImageData(0, 0, w, h).data;
const m = new Uint8Array(w * h);
for (let i = 0; i < m.length; i++) m[i] = px[i * 4 + 3] > 128 ? 255 : 0;
@@ -335,7 +335,7 @@ export async function readScene(doc: LayerDoc, e: SceneEdit, result: string): Pr
ctx.drawImage(full, 0, 0);
ctx.setTransform(1, 0, 0, 1, 0, 0);
ctx.globalCompositeOperation = "destination-in";
- ctx.drawImage(info.img, 0, 0, W, H);
+ drawSource(ctx, info);
const rec = await api.upload(await toPng(c), `${fileSafe(l.name)} (edited).png`);
out.looks.push({ id, src: `input:${rec.id}` });
}
diff --git a/src/components/LayerDoc.tsx b/src/components/LayerDoc.tsx
index 4bc3265..13321ca 100644
--- a/src/components/LayerDoc.tsx
+++ b/src/components/LayerDoc.tsx
@@ -12,8 +12,9 @@
import { Brush, Eraser, Eye, EyeOff, FileDown, GripVertical, Group, Hand, History, ImageDown, Link2, Loader2, Lock, MousePointer2, Paintbrush, Pencil, Redo2, RotateCcw, Scissors, Sparkles, Sun, Trash2, Undo2, Ungroup, WandSparkles, X } from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { api, urls } from "../lib/api";
+import { focusPixels, foreground, modelMatte, stickerAlpha } from "../lib/matte";
import { dilate, erode, fileSafe } from "../lib/layers";
-import { centre, copiedShare, CROP_MAX_RATIO, cropFor, docId, waitingMarks, shadowAlpha, mapBox, reorder, unmapPoint, withGroup, xfOf, type Box, type DocLayer, type Effect, type Feed, type LayerDoc, type SceneEdit, type Sticker, type Xf } from "../lib/layerDoc";
+import { centre, copiedShare, CROP_MAX_RATIO, cropFor, docId, extOf, unionRect, type Rect, waitingMarks, shadowAlpha, mapBox, reorder, unmapPoint, withGroup, xfOf, type Box, type DocLayer, type Effect, type Feed, type LayerDoc, type SceneEdit, type Sticker, type Xf } from "../lib/layerDoc";
import { buildRequest, refFromInput } from "../lib/logic";
import { native } from "../lib/native";
import { writePsd } from "../lib/psd";
@@ -73,6 +74,20 @@ export interface SourceInfo {
/** In frame px. */
box: Box;
thumb: string;
+ /** Where the image lies in the frame (frame px; past the edge for an expanded object). */
+ ext: Rect;
+}
+
+/** Versions whose image is not the whole frame (`LayerDoc.ext`), noted from every document shown:
+ * ids are unique, so `sourceInfo` finds them without the document. */
+const extents = new Map<string, Rect>();
+export function noteExtents(doc: { ext?: Record<string, Rect> }) {
+ for (const [k, r] of Object.entries(doc.ext ?? {})) extents.set(k, r);
+}
+
+/** A layer's version drawn where it lies in the frame (on a context set up in frame px). */
+export function drawSource(ctx: CanvasRenderingContext2D, info: SourceInfo) {
+ ctx.drawImage(info.img, info.ext.x, info.ext.y, info.ext.w, info.ext.h);
}
const ALPHA_SIDE = 512;
@@ -80,7 +95,8 @@ export const toPng = (c: HTMLCanvasElement) => new Promise<Blob>((res, rej) => c
const cache = new Map<string, Promise<SourceInfo>>();
export function sourceInfo(src: string, W: number, H: number): Promise<SourceInfo> {
- const key = `${src}@${W}x${H}`;
+ const ext = extents.get(src) ?? { x: 0, y: 0, w: W, h: H };
+ const key = `${src}@${W}x${H}@${ext.x},${ext.y},${ext.w},${ext.h}`;
let p = cache.get(key);
if (!p) {
p = new Promise<HTMLImageElement>((res, rej) => {
@@ -90,9 +106,10 @@ export function sourceInfo(src: string, W: number, H: number): Promise<SourceInf
img.onerror = () => rej(new Error("A layer's image could not be loaded."));
img.src = urls.source(src);
}).then((img) => {
- const k = Math.min(1, ALPHA_SIDE / Math.max(W, H));
- const aw = Math.max(1, Math.round(W * k));
- const ah = Math.max(1, Math.round(H * k));
+ // The alpha map covers the image's own rect (the frame, or more for an expanded object).
+ const k = Math.min(1, ALPHA_SIDE / Math.max(ext.w, ext.h));
+ const aw = Math.max(1, Math.round(ext.w * k));
+ const ah = Math.max(1, Math.round(ext.h * k));
const c = document.createElement("canvas");
c.width = aw;
c.height = ah;
@@ -115,13 +132,16 @@ export function sourceInfo(src: string, W: number, H: number): Promise<SourceInf
if (y < y0) y0 = y;
if (y > y1) y1 = y;
}
- const box = x1 < 0 ? { x0: 0, y0: 0, x1: W, y1: H } : { x0: (x0 / aw) * W, y0: (y0 / ah) * H, x1: ((x1 + 1) / aw) * W, y1: ((y1 + 1) / ah) * H };
+ const box =
+ x1 < 0
+ ? { x0: 0, y0: 0, x1: W, y1: H }
+ : { x0: ext.x + (x0 / aw) * ext.w, y0: ext.y + (y0 / ah) * ext.h, x1: ext.x + ((x1 + 1) / aw) * ext.w, y1: ext.y + ((y1 + 1) / ah) * ext.h };
const t = document.createElement("canvas");
const tk = 64 / Math.max(W, H);
t.width = Math.max(1, Math.round(W * tk));
t.height = Math.max(1, Math.round(H * tk));
- t.getContext("2d")!.drawImage(img, 0, 0, t.width, t.height);
- return { img, alpha, aw, ah, box, thumb: t.toDataURL("image/png") };
+ t.getContext("2d")!.drawImage(img, ext.x * tk, ext.y * tk, ext.w * tk, ext.h * tk);
+ return { img, alpha, aw, ah, box, thumb: t.toDataURL("image/png"), ext };
});
cache.set(key, p);
p.catch(() => cache.delete(key));
@@ -154,6 +174,7 @@ export function pixels(img: CanvasImageSource, W: number, H: number): Uint8Clamp
export async function renderDoc(doc: LayerDoc, maxSide: number): Promise<HTMLCanvasElement> {
const W = doc.width;
const H = doc.height;
+ noteExtents(doc);
const infos = new Map<string, SourceInfo>();
await Promise.all(doc.layers.map((l) => sourceInfo(l.source, W, H).then((i) => infos.set(l.source, i), () => {})));
const k = Math.min(1, maxSide / Math.max(W, H));
@@ -171,7 +192,7 @@ export async function renderDoc(doc: LayerDoc, maxSide: number): Promise<HTMLCan
ctx.translate(t.c.x + t.dx, t.c.y + t.dy);
ctx.scale(t.s, t.s);
ctx.translate(-t.c.x, -t.c.y);
- ctx.drawImage(info.img, 0, 0, W, H);
+ drawSource(ctx, info);
ctx.restore();
}
return c;
@@ -248,16 +269,24 @@ export function useMaskTint(src: string | null): string | null {
/** A crop edit's result -> the layer's next image: shrunk back into the crop's place in the frame
* (the rest transparent: the crop holds the whole object), clipped to the grown selection for
- * Clean edges (so the edit cannot bring a neighbour back). */
-export async function pasteCrop(f: Feed, result: string, W: number, H: number, name: string): Promise<InputRecord> {
- const r = f.rect!;
+ * Clean edges (so the edit cannot bring a neighbour back). An Expand of the crop (`grown`: where its
+ * result lies, expandedRect) is put there at the same scale, not squeezed into the crop. The image
+ * covers the frame, or more when the object reaches past its edge (`ext`, for LayerDoc.ext). */
+export async function pasteCrop(f: Feed, result: string, W: number, H: number, name: string, grown?: Rect): Promise<{ rec: InputRecord; ext?: Rect }> {
+ const r = grown ?? f.rect!;
let img: CanvasImageSource = await loadImage(urls.source(result));
+ const frame = { x: 0, y: 0, w: W, h: H };
+ const u = unionRect(frame, r);
+ const out = { x: Math.floor(u.x), y: Math.floor(u.y), w: Math.ceil(u.x + u.w) - Math.floor(u.x), h: Math.ceil(u.y + u.h) - Math.floor(u.y) };
+ const past = out.x < 0 || out.y < 0 || out.w > W || out.h > H;
const c = document.createElement("canvas");
- c.width = W;
- c.height = H;
+ c.width = out.w;
+ c.height = out.h;
const ctx = c.getContext("2d")!;
ctx.imageSmoothingQuality = "high";
- if (f.clip) {
+ ctx.translate(-out.x, -out.y);
+ // The clip is the crop's own selection: an expanded result reaches past it on purpose.
+ if (f.clip && !grown) {
const m = await loadImage(urls.input(f.clip));
const mw = m.naturalWidth;
const mh = m.naturalHeight;
@@ -290,7 +319,8 @@ export async function pasteCrop(f: Feed, result: string, W: number, H: number, n
ctx.globalCompositeOperation = "destination-in";
ctx.drawImage(mc, r.x, r.y, r.w, r.h);
} else ctx.drawImage(img, r.x, r.y, r.w, r.h);
- return api.upload(await toPng(c), `${name} (edited).png`);
+ const rec = await api.upload(await toPng(c), `${name} (${grown ? "expanded" : "edited"}).png`);
+ return past ? { rec, ext: out } : { rec };
}
/** One dab of the layer brushes, in the layer's own pixels: the eraser clears (solid core, soft
@@ -421,6 +451,7 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
useEffect(() => {
let gone = false;
+ noteExtents(doc);
for (const l of doc.layers) {
if (infos[l.source]) continue;
sourceInfo(l.source, W, H).then(
@@ -567,11 +598,11 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
const ch = Math.max(1, Math.round(rect.h * scale));
// The image over the whole frame, moved and enlarged so the crop fills the canvas (the crop may
// reach past the frame: transparent there).
- const draw = (img: HTMLImageElement, ctx: CanvasRenderingContext2D) => {
+ const draw = (img: HTMLImageElement, ctx: CanvasRenderingContext2D, at: Rect = { x: 0, y: 0, w: W, h: H }) => {
ctx.save();
ctx.imageSmoothingQuality = "high";
ctx.setTransform(cw / rect.w, 0, 0, ch / rect.h, (-rect.x * cw) / rect.w, (-rect.y * ch) / rect.h);
- ctx.drawImage(img, 0, 0, W, H);
+ ctx.drawImage(img, at.x, at.y, at.w, at.h);
ctx.restore();
};
const c = document.createElement("canvas");
@@ -586,7 +617,7 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
lc.width = cw;
lc.height = ch;
const lctx = lc.getContext("2d")!;
- draw(info.img, lctx);
+ draw(info.img, lctx, info.ext);
const px = lctx.getImageData(0, 0, cw, ch).data;
const sel0 = new Uint8Array(cw * ch);
for (let i = 0; i < sel0.length; i++) sel0[i] = px[i * 4 + 3] > 16 ? 255 : 0;
@@ -616,7 +647,7 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
mkx.fillRect(0, 0, cw, ch);
mkx.drawImage(gc, 0, 0);
clip = (await api.upload(await toPng(mk), "clip.png")).id;
- } else draw(info.img, ctx);
+ } else draw(info.img, ctx, info.ext);
const rec = await api.upload(await toPng(c), `${fileSafe(l.name)} (${kind === "edit" ? "object size" : "to clean"}).png`);
addVersions([rec], 0);
const f: Feed = { source: `input:${rec.id}`, mask: "", rect, scale, from: l.source, ...(clip ? { clip, grow: g } : {}) };
@@ -667,9 +698,105 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
}
};
- // Stickers: after a split with "Clean the stickers automatically", every object goes to Clean edges
- // on its own (a crop at object size and the sticker prompt, sent as lab steps); the service runs
- // them one after the other and each result comes back to its layer like any Clean edges.
+ /** A fast sticker, as the layer's next version: the object over the photo around it, matted by the
+ * service (lib/matte.ts: the selection decides what, the matte only the edge; edge colours freed of
+ * the background). Deterministic, a few seconds; Clean edges is the AI redraw. */
+ const fastSticker = async (l: DocLayer) => {
+ const e = extOf(liveRef.current, l.source);
+ if (e.x < 0 || e.y < 0 || e.w > W || e.h > H) throw new Error(`${l.name} reaches past the photo: use Clean edges.`);
+ const info = await sourceInfo(l.source, W, H);
+ const b = info.box;
+ const m = Math.round(Math.max(b.x1 - b.x0, b.y1 - b.y0) * 0.1) + 8;
+ const x0 = Math.max(0, Math.floor(b.x0 - m));
+ const y0 = Math.max(0, Math.floor(b.y0 - m));
+ const cw = Math.min(W, Math.ceil(b.x1 + m)) - x0;
+ const ch = Math.min(H, Math.ceil(b.y1 + m)) - y0;
+ if (cw < 8 || ch < 8) throw new Error(`${l.name} is too small for a sticker.`);
+ // Worked on at most 1600 px (the model sees 1024 anyway); solid parts keep full-size pixels.
+ const k = Math.min(1, 1600 / Math.max(cw, ch));
+ const w = Math.max(1, Math.round(cw * k));
+ const h = Math.max(1, Math.round(ch * k));
+ const photo = live.photo ? await loadImage(urls.source(live.photo)).catch(() => null) : null;
+ const scene = (sw: number, sh: number, withPhoto: boolean) => {
+ const c = document.createElement("canvas");
+ c.width = sw;
+ c.height = sh;
+ const ctx = c.getContext("2d")!;
+ ctx.imageSmoothingQuality = "high";
+ ctx.setTransform(sw / cw, 0, 0, sh / ch, (-x0 * sw) / cw, (-y0 * sh) / ch);
+ if (withPhoto && photo) ctx.drawImage(photo, 0, 0, W, H);
+ drawSource(ctx, info);
+ return c;
+ };
+ const own = scene(w, h, false).getContext("2d")!.getImageData(0, 0, w, h).data;
+ const sel = new Uint8Array(w * h);
+ for (let i = 0; i < sel.length; i++) sel[i] = own[i * 4 + 3] > 127 ? 255 : 0;
+ // The object in its scene (a new object drawn by the lab is over the photo too).
+ const rgba = scene(w, h, true).getContext("2d")!.getImageData(0, 0, w, h).data;
+ const matte = await modelMatte(focusPixels(rgba, sel, w, h), w, h);
+ const { alpha } = stickerAlpha(matte, sel, w, h);
+ const small = document.createElement("canvas");
+ small.width = w;
+ small.height = h;
+ small.getContext("2d")!.putImageData(new ImageData(foreground(rgba, alpha, w, h) as Uint8ClampedArray<ArrayBuffer>, w, h), 0, 0);
+ const out = document.createElement("canvas");
+ out.width = W;
+ out.height = H;
+ const octx = out.getContext("2d")!;
+ octx.imageSmoothingQuality = "high";
+ octx.drawImage(small, x0, y0, cw, ch);
+ if (k < 1) {
+ const full = scene(cw, ch, true).getContext("2d")!.getImageData(0, 0, cw, ch).data;
+ const d = octx.getImageData(x0, y0, cw, ch);
+ for (let i = 0; i < cw * ch; i++) if (d.data[i * 4 + 3] > 250) for (let c = 0; c < 3; c++) d.data[i * 4 + c] = full[i * 4 + c];
+ octx.putImageData(d, x0, y0);
+ }
+ const rec = await api.upload(await toPng(out), `${fileSafe(l.name)} (sticker).png`);
+ addVersions([rec], 0);
+ const src = `input:${rec.id}`;
+ const now = liveRef.current;
+ commitRef.current({ ...now, layers: now.layers.map((x) => (x.id === l.id ? { ...x, source: src, history: [...x.history, src] } : x)), known: [...now.known, src] });
+ };
+
+ /** The fast sticker model is downloaded once (224 MB), the first time Fast is used; the % shows. */
+ const [matteDl, setMatteDl] = useState<number | null>(null);
+ const ensureMatteModel = async () => {
+ if ((await api.matteStatus()).installed) return;
+ setMatteDl(0);
+ try {
+ await api.downloadMatte();
+ for (;;) {
+ await new Promise((r) => window.setTimeout(r, 1000));
+ const st = await api.installStatus();
+ const it = st.items.find((i) => i.id === "birefnet_lite");
+ if (it?.state === "failed" || it?.state === "cancelled") throw new Error(it.error ?? "The sticker model could not be downloaded.");
+ if (it?.state === "done" || it?.state === "present") return;
+ if (!st.running) {
+ if ((await api.matteStatus()).installed) return;
+ throw new Error("The sticker model could not be downloaded.");
+ }
+ if (it?.size) setMatteDl(Math.floor((100 * it.received) / it.size));
+ }
+ } finally {
+ setMatteDl(null);
+ }
+ };
+ const quickClean = async (l: DocLayer) => {
+ setBusy(true);
+ try {
+ await ensureMatteModel();
+ await fastSticker(l);
+ } catch (e) {
+ toastError(e);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ // Stickers: after a split with stickers on, every object is cleaned once. Quality: each goes to
+ // Clean edges on its own (a crop at object size and the sticker prompt, sent as lab steps; the
+ // service runs them one after the other and each result comes back like any Clean edges). Fast:
+ // the background's fill is sent first, then each object is matted here (fastSticker).
const stickersStarted = useRef(false);
const setSticker = (layer: string, p: Partial<Sticker>) => {
const d = liveRef.current;
@@ -680,6 +807,7 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
};
const makeStickers = async () => {
const d = liveRef.current;
+ const fast = d.autoClean === "fast";
const objects = d.layers.filter((l) => !l.base && !l.effect);
const base = d.layers.find((l) => l.base);
const start = { ...d, autoClean: undefined, stickers: [...objects.map((l): Sticker => ({ layer: l.id, job: null })), ...(base ? [{ layer: base.id, job: null, kind: "fill" as const }] : [])] };
@@ -689,6 +817,47 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
// The crops become versions (and the prompt's image) on the way: put back what was there.
const before = getState();
const keep = { selected: before.selected, refs: before.composer.refs };
+ const putBack = () => setState((s) => ({ selected: s.session.some((i) => i.key === keep.selected) ? keep.selected : s.selected, composer: { ...s.composer, refs: keep.refs } }));
+ const sendFill = async () => {
+ const bg = liveRef.current.layers.find((x) => x.base);
+ if (!bg || liveRef.current.stickers?.find((s) => s.kind)?.skipped) return;
+ try {
+ const { rec, mask } = await prepareFill(bg);
+ const job = await submitRequest(fillRequest(rec, mask));
+ setSticker(bg.id, job ? { job, from: bg.source, mask } : { failed: true });
+ } catch (e) {
+ console.error(e);
+ setSticker(bg.id, { failed: true });
+ }
+ };
+ if (fast) {
+ // The fill runs on the image engine meanwhile; the stickers on the CPU, one after the other.
+ await sendFill();
+ let ready = true;
+ try {
+ await ensureMatteModel();
+ } catch (e) {
+ toastError(e);
+ ready = false;
+ }
+ for (const l of objects) {
+ if (liveRef.current.stickers?.find((s) => s.layer === l.id && !s.kind)?.skipped) continue;
+ if (!ready) {
+ setSticker(l.id, { failed: true });
+ continue;
+ }
+ setSticker(l.id, { local: "making" });
+ try {
+ await fastSticker(liveRef.current.layers.find((x) => x.id === l.id) ?? l);
+ setSticker(l.id, { local: "done" });
+ } catch (e) {
+ console.error(e);
+ setSticker(l.id, { failed: true, local: undefined });
+ }
+ }
+ putBack();
+ return;
+ }
for (const l of objects) {
if (liveRef.current.stickers?.find((s) => s.layer === l.id && !s.kind)?.skipped) continue;
try {
@@ -707,18 +876,8 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
}
}
// Then the background: its holes filled, shadows and leftovers of the objects taken off.
- const bg = liveRef.current.layers.find((x) => x.base);
- if (bg && !liveRef.current.stickers?.find((s) => s.kind)?.skipped) {
- try {
- const { rec, mask } = await prepareFill(bg);
- const job = await submitRequest(fillRequest(rec, mask));
- setSticker(bg.id, job ? { job, from: bg.source, mask } : { failed: true });
- } catch (e) {
- console.error(e);
- setSticker(bg.id, { failed: true });
- }
- }
- setState((s) => ({ selected: s.session.some((i) => i.key === keep.selected) ? keep.selected : s.selected, composer: { ...s.composer, refs: keep.refs } }));
+ await sendFill();
+ putBack();
};
useEffect(() => {
if (!live.autoClean || stickersStarted.current) return;
@@ -732,6 +891,7 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
const stickerState = (s: Sticker): "waiting" | "making" | "done" | "skipped" | "failed" => {
if (s.skipped) return "skipped";
if (s.failed) return "failed";
+ if (s.local) return s.local;
const j = s.job ? jobs[s.job] : undefined;
if (!j) return s.job ? "done" : "waiting";
// Done once the result is pasted into the layer (a moment after the job).
@@ -911,8 +1071,8 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
const info = infos[l.source];
if (l.hidden || l.effect || !info) continue;
const q = unmapPoint(xf(l), x, y);
- const ix = Math.floor((q.x / W) * info.aw);
- const iy = Math.floor((q.y / H) * info.ah);
+ const ix = Math.floor(((q.x - info.ext.x) / info.ext.w) * info.aw);
+ const iy = Math.floor(((q.y - info.ext.y) / info.ext.h) * info.ah);
if (ix < 0 || iy < 0 || ix >= info.aw || iy >= info.ah) continue;
if (info.alpha[iy * info.aw + ix] > 32) return l;
}
@@ -925,8 +1085,8 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
const c = paintRef.current;
if (!c || !paintInfo || stroke.current) return;
const ctx = c.getContext("2d")!;
- ctx.clearRect(0, 0, W, H);
- ctx.drawImage(paintInfo.img, 0, 0, W, H);
+ ctx.clearRect(0, 0, c.width, c.height);
+ ctx.drawImage(paintInfo.img, 0, 0, c.width, c.height);
}, [paintInfo, W, H]);
/** Brush dabs from the last point to frame point `p`, in the layer's own pixels. */
@@ -939,7 +1099,9 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
const from = stroke.current ?? q;
const n = Math.max(1, Math.ceil(Math.hypot(q.x - from.x, q.y - from.y) / Math.max(1, r / 4)));
const ctx = c.getContext("2d")!;
- for (let i = stroke.current ? 1 : 0; i <= n; i++) dab(ctx, from.x + ((q.x - from.x) * i) / n, from.y + ((q.y - from.y) * i) / n, r, tool === "erase" ? "erase" : "shade");
+ // The canvas covers the layer's own rect (past the frame for an expanded object).
+ const e = extOf(live, sel.source);
+ for (let i = stroke.current ? 1 : 0; i <= n; i++) dab(ctx, from.x + ((q.x - from.x) * i) / n - e.x, from.y + ((q.y - from.y) * i) / n - e.y, r, tool === "erase" ? "erase" : "shade");
stroke.current = q;
};
@@ -955,8 +1117,10 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
addVersions([rec], 0);
const src = `input:${rec.id}`;
const d = liveRef.current;
+ const e = d.ext?.[l.source];
commitRef.current({
...d,
+ ...(e ? { ext: { ...d.ext, [src]: e } } : {}),
layers: d.layers.map((x) => (x.id === l.id ? { ...x, source: src, history: [...x.history, src] } : x)),
known: [...d.known, src],
});
@@ -1235,7 +1399,7 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
ctx.translate(t.c.x + t.dx, t.c.y + t.dy);
ctx.scale(t.s, t.s);
ctx.translate(-t.c.x, -t.c.y);
- ctx.drawImage(info.img, 0, 0, W, H);
+ drawSource(ctx, info);
ctx.restore();
}
return c;
@@ -1487,8 +1651,14 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
<div className="sticker-strip">
<Loader2 size={14} className="spin" />
<span>
- Making stickers · {stickerRows.length - stickersLeft} of {stickerRows.length}
- {stickerNow ? ` · ${stickerNow.kind ? "filling the background" : (live.layers.find((l) => l.id === stickerNow.layer)?.name ?? "")}` : ""}
+ {matteDl !== null ? (
+ <>Getting the sticker model (once, 224 MB) · {matteDl}%</>
+ ) : (
+ <>
+ Making stickers · {stickerRows.length - stickersLeft} of {stickerRows.length}
+ {stickerNow ? ` · ${stickerNow.kind ? "filling the background" : (live.layers.find((l) => l.id === stickerNow.layer)?.name ?? "")}` : ""}
+ </>
+ )}
</span>
{stickerNow && (
<button className="btn" onClick={() => skipSticker(stickerNow)} title="Keep this object's rough cut-out">
@@ -1528,17 +1698,11 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
const t = xf(l);
// Stickers: background and shadows out of sight (they fade back), objects by their state.
const st = stickering ? (l.base || l.effect ? "hide" : (stickerOf(l.id) ?? "done")) : null;
+ // An expanded object's image reaches past the frame: placed on its own rect.
+ const e = extOf(live, l.source);
+ const at = { left: e.x, top: e.y, width: e.w, height: e.h, transformOrigin: `${t.c.x - e.x}px ${t.c.y - e.y}px`, transform: `translate(${t.dx}px, ${t.dy}px) scale(${t.s})` };
if (brushOn && l.id === sel!.id)
- return (
- <canvas
- key={`${l.id}:paint`}
- ref={paintRef}
- width={W}
- height={H}
- className="doc-layer"
- style={{ transformOrigin: `${t.c.x}px ${t.c.y}px`, transform: `translate(${t.dx}px, ${t.dy}px) scale(${t.s})` }}
- />
- );
+ return <canvas key={`${l.id}:paint`} ref={paintRef} width={Math.round(e.w)} height={Math.round(e.h)} className="doc-layer" style={at} />;
return (
<img
key={`${l.id}:${l.source}`}
@@ -1546,7 +1710,7 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
alt=""
draggable={false}
className={`doc-layer ${st ? `st-${st}` : ""}`}
- style={{ transformOrigin: `${t.c.x}px ${t.c.y}px`, transform: `translate(${t.dx}px, ${t.dy}px) scale(${t.s})` }}
+ style={at}
/>
);
})}
@@ -1779,10 +1943,15 @@ export function LayerDoc({ projectId, doc, onChange, onNewSplit }: { projectId:
</button>
)}
{!sel.base && !sel.effect && (
- <button className="btn" onClick={() => cleanEdges(sel)} disabled={busy} title="Redraw the object's border at its own size, as a die-cut sticker on transparency; the result stays inside its outline (grown ~3 px)">
+ <button className="btn" onClick={() => cleanEdges(sel)} disabled={busy} title="Quality: the AI redraws the object's border at its own size, as a cut-out on transparency; the result stays inside its outline (grown ~3 px)">
<Sparkles size={14} /> Clean edges
</button>
)}
+ {!sel.base && !sel.effect && (
+ <button className="btn" onClick={() => quickClean(sel)} disabled={busy} title="Fast: a quick cut-out from the photo, a few seconds (the first time, the sticker model is downloaded: 224 MB)">
+ <Scissors size={14} /> {matteDl !== null ? `Getting the model · ${matteDl}%` : "Quick cut-out"}
+ </button>
+ )}
{sel.history.length > 1 && (
<button className={`btn ${versionsOpen ? "on" : ""}`} onClick={() => setVersionsOpen((o) => !o)} aria-expanded={versionsOpen} title="Show an earlier version of this layer">
<History size={14} /> Versions · {sel.history.length}
diff --git a/src/components/LayerSplit.tsx b/src/components/LayerSplit.tsx
index 8528947..caa9b11 100644
--- a/src/components/LayerSplit.tsx
+++ b/src/components/LayerSplit.tsx
@@ -14,6 +14,7 @@ import { Segmenter, type SamResult } from "../lib/sam";
import type { DocLayer, LayerDoc } from "../lib/layerDoc";
import { addVersions, setState, toastError, toastInfo, useStore } from "../lib/store";
import type { InputRecord } from "../lib/types";
+import { SplitChoice, type StickerMode } from "./SplitChoice";
import { IconButton } from "./ui";
import { isTyping } from "./Viewer";
import { ZoomPan } from "./ZoomPan";
@@ -102,18 +103,24 @@ export function LayerSplit({ onSplit, onBack }: { onSplit: (doc: LayerDoc) => vo
const [brush, setBrush] = useState(40);
const [margin, setMargin] = useState(3);
const [grey, setGrey] = useState(false);
- // Clean the stickers after the split (remembered; on unless turned off).
- const [stickers, setStickersState] = useState(() => {
+ // How the stickers are cleaned after the split, asked when Split is pressed (SplitChoice; the last
+ // answer is remembered and offered first): Quality = a Qwen edit per object (the way it was, "on"
+ // before 11-02), Fast = a matte from the service, Off = the rough cut-outs.
+ const [asking, setAsking] = useState(false);
+ const askingRef = useRef(false);
+ askingRef.current = asking;
+ const [stickers, setStickersState] = useState<StickerMode>(() => {
try {
- return localStorage.getItem(STICKERS_KEY) !== "off";
+ const v = localStorage.getItem(STICKERS_KEY);
+ return v === "off" || v === "fast" ? v : "quality";
} catch {
- return true;
+ return "quality";
}
});
- const setStickers = (on: boolean) => {
- setStickersState(on);
+ const setStickers = (m: StickerMode) => {
+ setStickersState(m);
try {
- localStorage.setItem(STICKERS_KEY, on ? "on" : "off");
+ localStorage.setItem(STICKERS_KEY, m);
} catch {
/* storage unavailable */
}
@@ -124,6 +131,8 @@ export function LayerSplit({ onSplit, onBack }: { onSplit: (doc: LayerDoc) => vo
const [busy, setBusy] = useState<string | null>(null);
const [tick, setTick] = useState(0);
const [areas, setAreas] = useState<Record<string, number>>({});
+ /** Layers with something in them (the Split button counts them). */
+ const usedCount = layers.filter((l) => (areas[l.id] ?? 0) > 0).length;
const masks = useRef(new Map<string, Mask>());
const undoStack = useRef<Change[]>([]);
@@ -416,13 +425,22 @@ export function LayerSplit({ onSplit, onBack }: { onSplit: (doc: LayerDoc) => vo
const patch = (id: string, p: Partial<LayerInfo>) => setLayers((ls) => ls.map((l) => (l.id === id ? { ...l, ...p } : l)));
- /** Saves the background and every non-empty layer at the image's full size as new versions. */
- const split = async () => {
- const used = layers.filter((l) => area(maskOf(l.id)) > 0);
- if (!used.length) {
+ /** Split pressed: how to cut the objects out is asked first (SplitChoice). */
+ const askSplit = () => {
+ if (!layers.some((l) => area(maskOf(l.id)) > 0)) {
toastInfo("Put something in a layer first: click an object, or paint it.");
return;
}
+ setAsking(true);
+ };
+
+ /** Saves the background and every non-empty layer at the image's full size as new versions; the
+ * objects are then cleaned as `mode` says (LayerDoc's sticker queue). */
+ const split = async (mode: StickerMode) => {
+ setAsking(false);
+ setStickers(mode);
+ const used = layers.filter((l) => area(maskOf(l.id)) > 0);
+ if (!used.length) return;
setBusy("Splitting…");
try {
const img = await full.current!;
@@ -483,7 +501,7 @@ export function LayerSplit({ onSplit, onBack }: { onSplit: (doc: LayerDoc) => vo
const bg = `input:${base.id}`;
docLayers.push({ id: "background", name: "Background", source: bg, history: [bg], hidden: false, base: true, dx: 0, dy: 0, s: 1 });
addVersions(made, 0);
- onSplit({ width: W, height: H, photo: ref0?.source, layers: docLayers, known: made.map((r) => `input:${r.id}`), ...(stickers && docLayers.some((l) => !l.base) ? { autoClean: true } : {}) });
+ onSplit({ width: W, height: H, photo: ref0?.source, layers: docLayers, known: made.map((r) => `input:${r.id}`), ...(mode !== "off" && docLayers.some((l) => !l.base) ? { autoClean: mode } : {}) });
} catch (e) {
toastError(e);
} finally {
@@ -493,7 +511,7 @@ export function LayerSplit({ onSplit, onBack }: { onSplit: (doc: LayerDoc) => vo
useEffect(() => {
const onKeyDown = (e: KeyboardEvent) => {
- if (isTyping(e.target)) return;
+ if (isTyping(e.target) || askingRef.current) return;
const k = e.key.toLowerCase();
if (e.code === "Space" && !e.repeat) {
e.preventDefault();
@@ -640,13 +658,6 @@ export function LayerSplit({ onSplit, onBack }: { onSplit: (doc: LayerDoc) => vo
(B) and eraser (E) fix edges and can move pixels between layers. Split saves the background and each layer as new versions and
opens them as layers: move them, hide them, edit them one by one.
</p>
- <label className="layers-opt" title="After the split, each object is redrawn at its own size as a clean cut-out, then the background's holes are filled without the objects' shadows (one edit each; skip any while it runs). Clean edges and Fill the holes redo one later.">
- <input type="checkbox" checked={stickers} onChange={(e) => setStickers(e.target.checked)} />
- Clean the stickers and fill the background automatically (one edit each)
- </label>
- <button className="btn primary layers-split" onClick={split} disabled={!!busy}>
- Split
- </button>
</aside>
</div>
@@ -679,7 +690,11 @@ export function LayerSplit({ onSplit, onBack }: { onSplit: (doc: LayerDoc) => vo
<Redo2 size={17} />
</IconButton>
</div>
+ <button className="btn primary layers-split-go" onClick={askSplit} disabled={!!busy || !usedCount} title={usedCount ? "Make each layer its own image, then clean the objects' edges" : "Click an object first: it goes into the active layer"}>
+ <Layers size={15} /> Split into layers{usedCount ? ` · ${usedCount}` : ""}
+ </button>
</div>
+ {asking && <SplitChoice count={usedCount} last={stickers} onPick={(m) => void split(m)} onCancel={() => setAsking(false)} />}
</div>
);
}
diff --git a/src/components/LayersEditor.tsx b/src/components/LayersEditor.tsx
index 6b89568..a3edd74 100644
--- a/src/components/LayersEditor.tsx
+++ b/src/components/LayersEditor.tsx
@@ -3,7 +3,7 @@
// the edits made to its layers.
import { useEffect, useRef, useState } from "react";
-import { docFor, docId, loadLab, putDoc, reconcile, saveLab, type DocLayer, type LayerDoc as Doc } from "../lib/layerDoc";
+import { docFor, docId, expandedRect, loadLab, putDoc, reconcile, saveLab, type DocLayer, type LayerDoc as Doc } from "../lib/layerDoc";
import { editSource, getState, itemOrigin, itemSource, toastError, toastInfo, useStore } from "../lib/store";
import { readScene } from "./LabScene";
import { LayerDoc, pasteCrop, shadowImage } from "./LayerDoc";
@@ -154,13 +154,18 @@ function LayersLab({ projectId }: { projectId: string }) {
const result = `gen:${done.gen.id}`;
if (making.current.has(result)) continue;
making.current.add(result);
- pasteCrop(f, result, doc.width, doc.height, l.name).then(
- (rec) => {
+ // Expand of the crop: its result is the crop plus the expansion, put around the crop.
+ const o = done.gen.mode === "outpaint" ? done.gen.outpaint : null;
+ const ref = done.gen.references[0];
+ const grown = o && ref ? expandedRect(f.rect, ref.width, ref.height, o) : undefined;
+ pasteCrop(f, result, doc.width, doc.height, l.name, grown).then(
+ ({ rec, ext }) => {
const d = latest.current!;
const src = `input:${rec.id}`;
const seen = made.map((i) => `gen:${i.gen!.id}`);
update({
...d,
+ ...(ext ? { ext: { ...d.ext, [src]: ext } } : {}),
layers: d.layers.map((x) => (x.id === l.id ? { ...x, source: src, history: [...x.history, src] } : x)),
known: [...d.known, ...seen, src],
});
diff --git a/src/components/SplitChoice.tsx b/src/components/SplitChoice.tsx
new file mode 100644
index 0000000..259d2ca
--- /dev/null
+++ b/src/components/SplitChoice.tsx
@@ -0,0 +1,93 @@
+// The question asked when the user splits an image into layers (LayerSplit): how the objects are cut
+// out. Fast = the sticker model in kokon-service (crates/qls-core/src/matting.rs, a few seconds each);
+// Quality = Qwen redraws each object's edge (about half a minute each). Either way the background's
+// holes are then filled by the image model.
+
+import { Check, Scissors, Sparkles, X } from "lucide-react";
+import { useEffect, useRef, useState } from "react";
+import { api } from "../lib/api";
+import { IconButton } from "./ui";
+
+export type StickerMode = "quality" | "fast" | "off";
+
+/** Seconds per object, as measured (11-02, RTX 5070 Ti + 16-thread CPU): for the time shown. */
+const SECONDS = { fast: 4, quality: 30 };
+
+const total = (n: number, mode: "fast" | "quality") => {
+ const s = n * SECONDS[mode];
+ return s < 60 ? `about ${Math.max(5, Math.round(s / 5) * 5)} s` : `about ${Math.round(s / 60)} min`;
+};
+
+export function SplitChoice({ count, last, onPick, onCancel }: { count: number; last: StickerMode; onPick: (m: StickerMode) => void; onCancel: () => void }) {
+ // The fast model is downloaded the first time it is used: said on its card.
+ const [missing, setMissing] = useState(false);
+ useEffect(() => {
+ api.matteStatus().then((s) => setMissing(!s.installed), () => {});
+ }, []);
+ const preferred = last === "off" ? "fast" : last;
+ const first = useRef<HTMLButtonElement>(null);
+ useEffect(() => first.current?.focus(), []);
+ useEffect(() => {
+ const onKey = (e: KeyboardEvent) => {
+ if (e.key !== "Escape") return;
+ e.preventDefault();
+ e.stopImmediatePropagation();
+ onCancel();
+ };
+ window.addEventListener("keydown", onKey, true);
+ return () => window.removeEventListener("keydown", onKey, true);
+ }, [onCancel]);
+ const objects = `${count} object${count === 1 ? "" : "s"}`;
+
+ const card = (mode: "fast" | "quality") => {
+ const fast = mode === "fast";
+ return (
+ <button key={mode} ref={mode === preferred ? first : undefined} className={`split-card ${mode === preferred ? "last" : ""}`} onClick={() => onPick(mode)}>
+ <span className="split-card-icon">{fast ? <Scissors size={20} /> : <Sparkles size={20} />}</span>
+ <span className="split-card-title">
+ {fast ? "Fast" : "Quality"}
+ {mode === last && <span className="split-card-tag">Last used</span>}
+ </span>
+ <span className="split-card-text">{fast ? "A quick, precise cut-out of each object, the same every time." : "The AI redraws each object's edge as a clean cut-out."}</span>
+ <ul className="split-card-points">
+ {(fast
+ ? ["Solid objects with clear edges", "Objects that stand out from what is behind them"]
+ : ["Hair, fur, fine or see-through parts", "Objects that blend into a busy background"]
+ ).map((p) => (
+ <li key={p}>
+ <Check size={13} /> {p}
+ </li>
+ ))}
+ </ul>
+ <span className="split-card-time">
+ {total(count, mode)} for {objects}
+ {fast && missing ? " · first time: downloads 224 MB" : ""}
+ </span>
+ </button>
+ );
+ };
+
+ return (
+ <div className="modal-scrim" onPointerDown={(e) => e.target === e.currentTarget && onCancel()}>
+ <div className="split-choice" role="dialog" aria-modal="true" aria-labelledby="split-choice-title">
+ <span className="split-choice-close">
+ <IconButton title="Back to selecting (Esc)" onClick={onCancel}>
+ <X size={18} />
+ </IconButton>
+ </span>
+ <h2 id="split-choice-title">How should the objects be cut out?</h2>
+ <p className="split-choice-lead">
+ Each object becomes its own layer. Pick by how hard its edges are to cut out; you can redo any object later with Clean edges (Quality) or Quick
+ cut-out (Fast).
+ </p>
+ <div className="split-cards">{(["fast", "quality"] as const).map(card)}</div>
+ <div className="split-choice-foot">
+ <span>Then the background behind them is filled in by the AI.</span>
+ <button className="btn ghost" onClick={() => onPick("off")}>
+ Skip: keep the rough cut-outs
+ </button>
+ </div>
+ </div>
+ </div>
+ );
+}
diff --git a/src/lib/api.ts b/src/lib/api.ts
index 3024b48..f4acd13 100644
--- a/src/lib/api.ts
+++ b/src/lib/api.ts
@@ -113,6 +113,24 @@ export const api = {
upscalers: () => call<{ selected: string; active: string | null; items: UpscalerItem[] }>("/upscalers"),
/** Progress in `installStatus` (preset "upscaler:<name>"). */
downloadUpscaler: (name: string) => call<InstallStatus>(`/upscalers/${encodeURIComponent(name)}/download`, { method: "POST" }),
+ /** The fast-sticker model of the layers lab: there or not, and its download size. */
+ matteStatus: () => call<{ installed: boolean; bytes: number }>("/matte"),
+ /** Progress in `installStatus` (preset "matting"). */
+ downloadMatte: () => call<InstallStatus>("/matte/download", { method: "POST" }),
+ /** The fast-sticker matte of an image in Kokon ("input:"/"gen:"): a grey PNG at its size. */
+ matte: async (image: string): Promise<Blob> => {
+ let res: Response;
+ try {
+ res = await fetch(base + "/matte", { method: "POST", headers: { ...authHeaders("POST"), "Content-Type": "application/json" }, body: json({ image }) });
+ } catch (e) {
+ throw new ApiError({ code: "server_unavailable", message: inBrowser ? "Kokon is not reachable." : "The local service is not reachable.", detail: String(e) }, 0);
+ }
+ if (!res.ok) {
+ const data = await res.json().catch(() => null);
+ throw new ApiError(data?.error ?? { code: "internal", message: `HTTP ${res.status}` }, res.status);
+ }
+ return res.blob();
+ },
/** ESRGAN models of earlier versions in the app's own folder. */
legacyUpscalers: (keep: string) => call<{ files: string[]; bytes: number }>(`/setup/legacy-upscalers?keep=${encodeURIComponent(keep)}`),
/** `keep`: the model chosen in Settings, even if not saved yet. */
diff --git a/src/lib/layerDoc.test.ts b/src/lib/layerDoc.test.ts
index f4cdad8..b999c56 100644
--- a/src/lib/layerDoc.test.ts
+++ b/src/lib/layerDoc.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { copiedShare, cropFor, docFor, keepLabsIn, labSources, loadLab, saveLab, waitingMarks, jobLayer, labItems, leftovers, putDoc, type LabStore, shadowAlpha, mapPoint, placed, reconcile, reorder, toLayer, xfOf, type DocLayer, type LayerDoc } from "./layerDoc";
+import { copiedShare, cropFor, expandedRect, extOf, unionRect, docFor, keepLabsIn, labSources, loadLab, saveLab, waitingMarks, jobLayer, labItems, leftovers, putDoc, type LabStore, shadowAlpha, mapPoint, placed, reconcile, reorder, toLayer, xfOf, type DocLayer, type LayerDoc } from "./layerDoc";
const layer = (id: string, source: string, base = false): DocLayer => ({ id, name: id, source, history: [source], hidden: false, base, dx: 0, dy: 0, s: 1 });
const doc = (layers: DocLayer[]): LayerDoc => ({ width: 100, height: 100, layers, known: [] });
@@ -230,3 +230,19 @@ describe("copiedShare", () => {
expect(copiedShare(img([[30, 60, 160], [40, 70, 170], [90, 50, 120], [128, 128, 128]]), mask, 4)).toBe(0);
});
});
+
+describe("expanded objects", () => {
+ it("puts an Expand result around the crop, not squeezed into it", () => {
+ // A 200x400 crop sent enlarged 2x (400x800), expanded 200 px right and 100 px at the bottom.
+ const r = expandedRect({ x: 1000, y: 300, w: 200, h: 400 }, 400, 800, { left: 0, right: 200, top: 0, bottom: 100 });
+ expect(r).toEqual({ x: 1000, y: 300, w: 300, h: 450 });
+ const l = expandedRect({ x: 10, y: 20, w: 100, h: 100 }, 100, 100, { left: 50, right: 0, top: 25, bottom: 0 });
+ expect(l).toEqual({ x: -40, y: -5, w: 150, h: 125 });
+ });
+ it("knows where a version lies", () => {
+ const doc = { width: 800, height: 600, ext: { "input:b": { x: -40, y: 0, w: 900, h: 600 } } };
+ expect(extOf(doc, "input:a")).toEqual({ x: 0, y: 0, w: 800, h: 600 });
+ expect(extOf(doc, "input:b")).toEqual({ x: -40, y: 0, w: 900, h: 600 });
+ expect(unionRect({ x: 0, y: 0, w: 800, h: 600 }, { x: 700, y: -50, w: 300, h: 100 })).toEqual({ x: 0, y: -50, w: 1000, h: 650 });
+ });
+});
diff --git a/src/lib/layerDoc.ts b/src/lib/layerDoc.ts
index 0315493..3cf5cd2 100644
--- a/src/lib/layerDoc.ts
+++ b/src/lib/layerDoc.ts
@@ -32,6 +32,25 @@ export interface Rect {
h: number;
}
+/** Where a version's image lies in the frame (frame px): the whole frame unless `doc.ext` says. */
+export const extOf = (doc: { width: number; height: number; ext?: Record<string, Rect> }, source: string): Rect =>
+ doc.ext?.[source] ?? { x: 0, y: 0, w: doc.width, h: doc.height };
+
+export const unionRect = (a: Rect, b: Rect): Rect => {
+ const x = Math.min(a.x, b.x);
+ const y = Math.min(a.y, b.y);
+ return { x, y, w: Math.max(a.x + a.w, b.x + b.w) - x, h: Math.max(a.y + a.h, b.y + b.h) - y };
+};
+
+/** An Expand of an object's crop (`rect` in the frame, sent as a `refW`×`refH` image): where its
+ * result lies in the frame. The result is the crop plus the expansion, given in the sent image's px
+ * (GenerationRecord.outpaint), so it is wider or taller than the crop, never squeezed into it. */
+export function expandedRect(rect: Rect, refW: number, refH: number, o: { left: number; right: number; top: number; bottom: number }): Rect {
+ const kx = rect.w / refW;
+ const ky = rect.h / refH;
+ return { x: rect.x - o.left * kx, y: rect.y - o.top * ky, w: (refW + o.left + o.right) * kx, h: (refH + o.top + o.bottom) * ky };
+}
+
/** Below about 1 MP Qwen-Image does worse (planner.rs INPAINT_MIN_AREA): a crop smaller than that is
* enlarged for the model, at most 4 times, and its result shrunk back. */
export const CROP_AREA = 1024 * 1024;
@@ -154,10 +173,14 @@ export interface LayerDoc {
scenes?: SceneEdit[];
/** Versions already looked at for "follow the edit" (adopted, or left by going back). */
known: string[];
- /** Split with "Clean the stickers automatically": LayerDoc sends every object to Clean edges once. */
- autoClean?: boolean;
+ /** Split with the stickers cleaned: LayerDoc cleans every object once, with a Qwen edit each
+ * ("quality"; `true` from before 11-02) or the service's matte ("fast"). */
+ autoClean?: boolean | "quality" | "fast";
/** Those automatic Clean edges, one per object layer (shown while any is left). */
stickers?: Sticker[];
+ /** Versions whose image is not the whole frame (`extOf`): an Expand of an object took it past the
+ * frame's edge, so its image covers this rect (frame px) and the part outside shows once moved in. */
+ ext?: Record<string, Rect>;
}
/** An object's automatic Clean edges: its job once sent (null: not yet, or it could not be sent). */
@@ -175,6 +198,8 @@ export interface Sticker {
checked?: boolean;
skipped?: boolean;
failed?: boolean;
+ /** A fast sticker, made here (no job): being made, or in. */
+ local?: "making" | "done";
}
export interface Box {
diff --git a/src/lib/matte.test.ts b/src/lib/matte.test.ts
new file mode 100644
index 0000000..1f3ca07
--- /dev/null
+++ b/src/lib/matte.test.ts
@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+import { foreground, stickerAlpha } from "./matte";
+
+const W = 64;
+const H = 64;
+/** A disc selection (255 inside), radius r around the centre. */
+const disc = (r: number, cx = 32, cy = 32) => Uint8Array.from({ length: W * H }, (_, i) => (Math.hypot((i % W) - cx, Math.floor(i / W) - cy) < r ? 255 : 0));
+
+describe("fast sticker alpha", () => {
+ it("lets the matte empty a hole inside the selection, adds nothing far outside", () => {
+ const sel = disc(20);
+ // The matte: the disc with a hole in the middle (a handle's gap), plus a blob far away.
+ const matte = Float32Array.from(sel, (v, i) => {
+ const x = i % W;
+ const y = Math.floor(i / W);
+ if (Math.hypot(x - 32, y - 32) < 5) return 0;
+ if (x < 4 && y < 4) return 1;
+ return v ? 1 : 0;
+ });
+ const { alpha, fellBack } = stickerAlpha(matte, sel, W, H);
+ expect(fellBack).toBe(false);
+ expect(alpha[32 * W + 32]).toBe(0);
+ expect(alpha[32 * W + 45]).toBeGreaterThan(0.9);
+ expect(alpha[1 * W + 1]).toBe(0);
+ });
+
+ it("falls back to the selection's edge band when the matte drops a whole piece", () => {
+ // Two discs selected; the matte keeps only one (the bats it lost on the poster).
+ const a = disc(10, 18, 32);
+ const b = disc(10, 46, 32);
+ const sel = Uint8Array.from(a, (v, i) => Math.max(v, b[i]));
+ const matte = Float32Array.from(a, (v) => (v ? 1 : 0));
+ const { alpha, fellBack } = stickerAlpha(matte, sel, W, H);
+ expect(fellBack).toBe(true);
+ expect(alpha[32 * W + 46]).toBe(1);
+ });
+});
+
+describe("foreground colours", () => {
+ it("takes the background's tint out of a soft edge", () => {
+ // White object on red: the edge pixels are half white, half red; the estimate gets nearer white.
+ const alpha = Float32Array.from({ length: W * H }, (_, i) => Math.min(1, Math.max(0, (32 - (i % W)) / 4 + 0.5)));
+ const rgba = new Uint8ClampedArray(W * H * 4);
+ for (let i = 0; i < W * H; i++) {
+ const a = alpha[i];
+ rgba.set([255, Math.round(255 * a), Math.round(255 * a), 255], i * 4);
+ }
+ const out = foreground(rgba, alpha, W, H);
+ const i = 10 * W + 32; // alpha 0.5: the photo is (255, 128, 128)
+ expect(rgba[i * 4 + 1]).toBeLessThan(140);
+ expect(out[i * 4 + 1]).toBeGreaterThan(200);
+ expect(out[i * 4 + 3]).toBe(Math.round(255 * alpha[i]));
+ });
+});
diff --git a/src/lib/matte.ts b/src/lib/matte.ts
new file mode 100644
index 0000000..8e99d45
--- /dev/null
+++ b/src/lib/matte.ts
@@ -0,0 +1,209 @@
+// Fast stickers for the layers lab: BiRefNet lite (MIT) gives the matte, run by kokon-service on the
+// CPU (crates/qls-core/src/matting.rs: the webview could not run it, WebGPU kernels rejected and WASM out
+// of memory, 11-02).
+//
+// Spike s17 (11-02, wiki/layers.md): a salient-object model cuts out the main thing of a crop, not the
+// object the user chose, so SAM decides what and the matte decides the edge: the model sees the object
+// with the rest faded to grey (`focusPixels`), its matte may empty holes inside the selection (spokes,
+// a handle, a halo SAM took) but adds nothing beyond a thin band around it, and when it drops a whole
+// piece of the selection the selection's own edge band is used instead (`stickerAlpha`). The edge
+// colours are then freed of the background bleeding into them (`foreground`, Germer et al. 2020).
+
+import { api } from "./api";
+import { dilate, erode, type Mask } from "./layers";
+
+/** The model's matte (0-1) of RGBA pixels `w`×`h`, at that size (uploaded, matted by the service). */
+export async function modelMatte(rgba: Uint8ClampedArray, w: number, h: number): Promise<Float32Array> {
+ const c = document.createElement("canvas");
+ c.width = w;
+ c.height = h;
+ const ctx = c.getContext("2d")!;
+ ctx.putImageData(new ImageData(rgba as Uint8ClampedArray<ArrayBuffer>, w, h), 0, 0);
+ const png = await new Promise<Blob>((res, rej) => c.toBlob((b) => (b ? res(b) : rej(new Error("export failed"))), "image/png"));
+ const rec = await api.upload(png, "sticker input.png");
+ const m = await createImageBitmap(await api.matte(`input:${rec.id}`));
+ ctx.clearRect(0, 0, w, h);
+ ctx.drawImage(m, 0, 0, w, h);
+ const px = ctx.getImageData(0, 0, w, h).data;
+ const out = new Float32Array(w * h);
+ for (let i = 0; i < out.length; i++) out[i] = px[i * 4] / 255;
+ return out;
+}
+
+/** Box blur (radius r, 3 passes ~ Gaussian) of a 0-1 image. */
+function blur(a: Float32Array, w: number, h: number, r: number): Float32Array {
+ r = Math.max(1, Math.round(r));
+ let cur = a.slice();
+ const tmp = new Float32Array(a.length);
+ for (let pass = 0; pass < 3; pass++) {
+ for (let y = 0; y < h; y++) {
+ let s = 0;
+ for (let x = -r; x <= r; x++) s += cur[y * w + Math.min(w - 1, Math.max(0, x))];
+ for (let x = 0; x < w; x++) {
+ tmp[y * w + x] = s / (2 * r + 1);
+ s += cur[y * w + Math.min(w - 1, x + r + 1)] - cur[y * w + Math.max(0, x - r)];
+ }
+ }
+ for (let x = 0; x < w; x++) {
+ let s = 0;
+ for (let y = -r; y <= r; y++) s += tmp[Math.min(h - 1, Math.max(0, y)) * w + x];
+ for (let y = 0; y < h; y++) {
+ cur[y * w + x] = s / (2 * r + 1);
+ s += tmp[Math.min(h - 1, y + r + 1) * w + x] - tmp[Math.max(0, y - r) * w + x];
+ }
+ }
+ }
+ return cur;
+}
+
+const toUnit = (m: Mask) => Float32Array.from(m, (v) => (v > 127 ? 1 : 0));
+
+/** The model's input: the selected object, the rest faded to grey 4 % of the side away from it. */
+export function focusPixels(rgba: Uint8ClampedArray, sel: Mask, w: number, h: number): Uint8ClampedArray {
+ const r = Math.max(6, 0.04 * Math.max(w, h));
+ const g = blur(toUnit(dilate(sel, w, h, r)), w, h, r / 3);
+ const out = new Uint8ClampedArray(rgba.length);
+ for (let i = 0; i < w * h; i++) {
+ for (let k = 0; k < 3; k++) out[i * 4 + k] = rgba[i * 4 + k] * g[i] + 128 * (1 - g[i]);
+ out[i * 4 + 3] = 255;
+ }
+ return out;
+}
+
+/** The band around the selection's edge where the matte decides (px). */
+export const bandOf = (w: number, h: number) => Math.max(4, 0.015 * Math.max(w, h));
+
+/** SAM decides what, the matte only the edge: 1 deep inside the selection, 0 beyond the band. */
+export function bandAlpha(matte: Float32Array, sel: Mask, w: number, h: number): Float32Array {
+ const r = bandOf(w, h);
+ const inner = erode(sel, w, h, r);
+ const outer = dilate(sel, w, h, r);
+ const a = new Float32Array(w * h);
+ for (let i = 0; i < a.length; i++) a[i] = inner[i] > 127 ? 1 : outer[i] > 127 ? Math.min(1, Math.max(0, matte[i])) : 0;
+ return a;
+}
+
+/** A piece of the selection whose core (eroded by the band) is bigger than 2 % of the selection and
+ * keeps less than 30 % in the matte: the matte dropped it (the bats of the poster, s17). */
+export function dropsPiece(a: Float32Array, sel: Mask, w: number, h: number): boolean {
+ const core = erode(sel, w, h, bandOf(w, h));
+ let total = 0;
+ for (const v of sel) if (v > 127) total++;
+ const seen = new Uint8Array(w * h);
+ const stack: number[] = [];
+ for (let s = 0; s < core.length; s++) {
+ if (core[s] < 128 || seen[s]) continue;
+ let n = 0;
+ let kept = 0;
+ seen[s] = 1;
+ stack.push(s);
+ while (stack.length) {
+ const i = stack.pop()!;
+ n++;
+ if (a[i] > 0.5) kept++;
+ const x = i % w;
+ const nb = [x > 0 ? i - 1 : -1, x < w - 1 ? i + 1 : -1, i - w, i + w];
+ for (const j of nb)
+ if (j >= 0 && j < core.length && !seen[j] && core[j] > 127) {
+ seen[j] = 1;
+ stack.push(j);
+ }
+ }
+ if (n > 0.02 * total && kept / n < 0.3) return true;
+ }
+ return false;
+}
+
+/** The sticker's alpha from the model's matte and the selection (both `w`×`h`): the matte inside the
+ * selection grown by the band (it may empty holes, it adds nothing farther out), or the band alone
+ * when the matte dropped a whole piece of the selection (`fellBack`). */
+export function stickerAlpha(matte: Float32Array, sel: Mask, w: number, h: number): { alpha: Float32Array; fellBack: boolean } {
+ const outer = blur(toUnit(dilate(sel, w, h, bandOf(w, h))), w, h, 1);
+ const a = new Float32Array(w * h);
+ for (let i = 0; i < a.length; i++) a[i] = Math.min(1, Math.max(0, matte[i])) * outer[i];
+ return dropsPiece(a, sel, w, h) ? { alpha: bandAlpha(matte, sel, w, h), fellBack: true } : { alpha: a, fellBack: false };
+}
+
+/** Foreground colours of `rgba` under `alpha` (0-1): what the edge pixels would be without the
+ * background mixed in (multilevel estimation of Germer, Uelwer, Conrad, Harmeling 2020, as in
+ * pymatting's estimate_foreground_ml: coarse to fine, F and B solved per pixel from the neighbours). */
+export function foreground(rgba: Uint8ClampedArray, alpha: Float32Array, w: number, h: number): Uint8ClampedArray {
+ const n0 = w * h;
+ const img = new Float32Array(n0 * 3);
+ for (let i = 0; i < n0; i++) for (let k = 0; k < 3; k++) img[i * 3 + k] = rgba[i * 4 + k] / 255;
+ const levels = Math.ceil(Math.log2(Math.max(w, h)));
+ let F = new Float32Array(3);
+ let B = new Float32Array(3);
+ let pw = 1;
+ let ph = 1;
+ for (let k = 0; k < 3; k++) F[k] = B[k] = img[k];
+ for (let lv = 0; lv <= levels; lv++) {
+ const lw = Math.max(1, Math.round(w ** (lv / levels)));
+ const lh = Math.max(1, Math.round(h ** (lv / levels)));
+ const n = lw * lh;
+ const I = new Float32Array(n * 3);
+ const A = new Float32Array(n);
+ const Fn = new Float32Array(n * 3);
+ const Bn = new Float32Array(n * 3);
+ for (let y = 0; y < lh; y++)
+ for (let x = 0; x < lw; x++) {
+ const sx = Math.min(w - 1, Math.floor((x * w) / lw));
+ const sy = Math.min(h - 1, Math.floor((y * h) / lh));
+ const px = Math.min(pw - 1, Math.floor((x * pw) / lw));
+ const py = Math.min(ph - 1, Math.floor((y * ph) / lh));
+ const i = y * lw + x;
+ A[i] = alpha[sy * w + sx];
+ for (let k = 0; k < 3; k++) {
+ I[i * 3 + k] = img[(sy * w + sx) * 3 + k];
+ Fn[i * 3 + k] = F[(py * pw + px) * 3 + k];
+ Bn[i * 3 + k] = B[(py * pw + px) * 3 + k];
+ }
+ }
+ const iters = n <= 32 * 32 ? 10 : 2;
+ for (let it = 0; it < iters; it++)
+ for (let y = 0; y < lh; y++)
+ for (let x = 0; x < lw; x++) {
+ const i = y * lw + x;
+ const a = A[i];
+ // A tiny regularisation: with no neighbours (the 1×1 level) the system is singular.
+ let a00 = a * a + 1e-5;
+ let a11 = (1 - a) * (1 - a) + 1e-5;
+ const a01 = a * (1 - a);
+ let f0 = 0, f1 = 0, f2 = 0, b0 = 0, b1 = 0, b2 = 0;
+ for (let q = 0; q < 4; q++) {
+ const nx = q === 0 ? x - 1 : q === 1 ? x + 1 : x;
+ const ny = q === 2 ? y - 1 : q === 3 ? y + 1 : y;
+ if (nx < 0 || ny < 0 || nx >= lw || ny >= lh) continue;
+ const j = ny * lw + nx;
+ const wgt = 5e-3 + 0.1 * Math.abs(a - A[j]);
+ a00 += wgt;
+ a11 += wgt;
+ f0 += wgt * Fn[j * 3];
+ f1 += wgt * Fn[j * 3 + 1];
+ f2 += wgt * Fn[j * 3 + 2];
+ b0 += wgt * Bn[j * 3];
+ b1 += wgt * Bn[j * 3 + 1];
+ b2 += wgt * Bn[j * 3 + 2];
+ }
+ const det = a00 * a11 - a01 * a01;
+ for (let k = 0; k < 3; k++) {
+ const r0 = (k === 0 ? f0 : k === 1 ? f1 : f2) + a * I[i * 3 + k];
+ const r1 = (k === 0 ? b0 : k === 1 ? b1 : b2) + (1 - a) * I[i * 3 + k];
+ Fn[i * 3 + k] = Math.min(1, Math.max(0, (a11 * r0 - a01 * r1) / det));
+ Bn[i * 3 + k] = Math.min(1, Math.max(0, (a00 * r1 - a01 * r0) / det));
+ }
+ }
+ F = Fn;
+ B = Bn;
+ pw = lw;
+ ph = lh;
+ }
+ const out = new Uint8ClampedArray(n0 * 4);
+ for (let i = 0; i < n0; i++) {
+ // Solid object: the photo's own pixels (full detail); estimated colours only where alpha is soft.
+ const solid = alpha[i] > 0.98;
+ for (let k = 0; k < 3; k++) out[i * 4 + k] = Math.round(255 * (solid ? img[i * 3 + k] : F[i * 3 + k]));
+ out[i * 4 + 3] = Math.round(255 * alpha[i]);
+ }
+ return out;
+}
diff --git a/src/styles.css b/src/styles.css
index dae9942..78678d6 100644
--- a/src/styles.css
+++ b/src/styles.css
@@ -1227,3 +1227,25 @@ kbd { min-width: 22px; height: 22px; padding: 0 6px; border-radius: 6px; display
.gallery-bar { padding: 0 10px 10px; flex-wrap: wrap; }
.grid { grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); gap: 8px; padding: 4px 10px calc(20px + env(safe-area-inset-bottom)); }
}
+
+/* Split into layers: how the objects are cut out (SplitChoice) */
+.split-choice { position: relative; width: min(680px, calc(100vw - 32px)); max-height: 90vh; overflow-y: auto; background: var(--bg); border-radius: 22px; box-shadow: var(--shadow); padding: 30px 30px 22px; display: flex; flex-direction: column; gap: 10px; animation: welcome-in 0.3s ease-out; }
+.split-choice h2 { margin: 0; font-size: 20px; font-weight: 600; letter-spacing: -0.01em; }
+.split-choice-lead { margin: 0 0 8px; color: var(--text-2); line-height: 1.5; max-width: 56ch; }
+.split-choice-close { position: absolute; top: 14px; right: 14px; }
+.split-cards { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
+.split-card { display: flex; flex-direction: column; align-items: flex-start; gap: 8px; text-align: left; padding: 18px; border-radius: 16px; background: var(--ground); border: 1.5px solid transparent; color: var(--text); cursor: pointer; transition: border-color 0.15s, background 0.15s, transform 0.15s, box-shadow 0.2s; }
+.split-card:hover { background: var(--bg); border-color: var(--border-2); box-shadow: var(--lift); transform: translateY(-1px); }
+.split-card:focus-visible { outline: none; border-color: var(--text); }
+.split-card.last { border-color: var(--border-2); }
+.split-card-icon { width: 38px; height: 38px; border-radius: 11px; display: grid; place-items: center; background: var(--bg); box-shadow: var(--lift); }
+.split-card-title { display: flex; align-items: center; gap: 8px; font-size: 16px; font-weight: 600; }
+.split-card-tag { font-size: 11px; font-weight: 500; color: var(--text-2); background: var(--surface); padding: 2px 8px; border-radius: 999px; }
+.split-card-text { color: var(--text-2); line-height: 1.45; }
+.split-card-points { list-style: none; margin: 2px 0 0; padding: 0; display: flex; flex-direction: column; gap: 4px; color: var(--text); font-size: 12.5px; }
+.split-card-points li { display: flex; gap: 6px; align-items: flex-start; line-height: 1.4; }
+.split-card-points svg { flex: none; margin-top: 2px; color: var(--text-3); }
+.split-card-time { margin-top: auto; padding-top: 8px; font-size: 12px; color: var(--text-3); font-variant-numeric: tabular-nums; }
+.split-choice-foot { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; margin-top: 6px; color: var(--text-3); font-size: 12.5px; }
+@media (max-width: 560px) { .split-cards { grid-template-columns: 1fr; } .split-choice { padding: 24px 18px 18px; } }
+@media (prefers-reduced-motion: reduce) { .split-choice, .split-card { animation: none; transition: none; } }