LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
M12: SAM 3 Object and Describe masks (pure Rust, Metal)
Guinther Kovalski6468bfc06bf9
diff --git a/Cargo.lock b/Cargo.lockindex 1952f5c4f..668739305 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -594,6 +594,12 @@ dependencies = ["no_std_io2",]+[[package]]+name = "block"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"+[[package]]name = "block2"version = "0.5.1"@@ -678,6 +684,12 @@ dependencies = ["syn 3.0.6",]+[[package]]+name = "byteorder"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"+[[package]]name = "byteorder-lite"version = "0.1.0"@@ -741,6 +753,76 @@ dependencies = ["wayland-client",]+[[package]]+name = "candle-core"+version = "0.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c15b675b80d994b2eadb20a4bbe434eabeb454eac3ee5e2b4cf6f147ee9be091"+dependencies = [+ "byteorder",+ "candle-metal-kernels",+ "candle-ug",+ "float8",+ "gemm 0.19.0",+ "half",+ "libm",+ "memmap2",+ "num-traits",+ "num_cpus",+ "objc2-foundation 0.3.2",+ "objc2-metal 0.3.2",+ "rand",+ "rand_distr",+ "rayon",+ "safetensors 0.7.0",+ "thiserror 2.0.21",+ "yoke 0.8.3",+ "zip",+]++[[package]]+name = "candle-metal-kernels"+version = "0.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2fdfe9d06de16ce49961e49084e5b79a75a9bdf157246e7c7b6328e87a7aa25d"+dependencies = [+ "half",+ "objc2 0.6.4",+ "objc2-foundation 0.3.2",+ "objc2-metal 0.3.2",+ "once_cell",+ "thiserror 2.0.21",+ "tracing",+]++[[package]]+name = "candle-nn"+version = "0.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3045fa9e7aef8567d209a27d56b692f60b96f4d0569f4c3011f8ca6715c65e03"+dependencies = [+ "candle-core",+ "candle-metal-kernels",+ "half",+ "libc",+ "num-traits",+ "objc2-metal 0.3.2",+ "rayon",+ "safetensors 0.7.0",+ "serde",+ "thiserror 2.0.21",+]++[[package]]+name = "candle-ug"+version = "0.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c22d62be69068bf58987a45f690612739d8d2ea1bf508c1b87dc6815a019575d"+dependencies = [+ "ug",+ "ug-metal",+]+[[package]]name = "cc"version = "1.5.1"@@ -987,6 +1069,22 @@ version = "0.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"+[[package]]+name = "dyn-stack"+version = "0.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8"+dependencies = [+ "bytemuck",+ "dyn-stack-macros",+]++[[package]]+name = "dyn-stack-macros"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9"+[[package]]name = "ecolor"version = "0.36.2"@@ -1129,6 +1227,18 @@ version = "1.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099"+[[package]]+name = "enum-as-inner"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc"+dependencies = [+ "heck",+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]+[[package]]name = "enumflags2"version = "0.7.12"@@ -1296,6 +1406,18 @@ dependencies = ["zlib-rs",]+[[package]]+name = "float8"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5"+dependencies = [+ "half",+ "num-traits",+ "rand",+ "rand_distr",+]+[[package]]name = "fnv"version = "1.0.7"@@ -1399,6 +1521,244 @@ dependencies = ["slab",]+[[package]]+name = "gemm"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451"+dependencies = [+ "dyn-stack",+ "gemm-c32 0.18.2",+ "gemm-c64 0.18.2",+ "gemm-common 0.18.2",+ "gemm-f16 0.18.2",+ "gemm-f32 0.18.2",+ "gemm-f64 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb"+dependencies = [+ "dyn-stack",+ "gemm-c32 0.19.0",+ "gemm-c64 0.19.0",+ "gemm-common 0.19.0",+ "gemm-f16 0.19.0",+ "gemm-f32 0.19.0",+ "gemm-f64 0.19.0",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-c32"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847"+dependencies = [+ "dyn-stack",+ "gemm-common 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-c32"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c"+dependencies = [+ "dyn-stack",+ "gemm-common 0.19.0",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-c64"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf"+dependencies = [+ "dyn-stack",+ "gemm-common 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-c64"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f"+dependencies = [+ "dyn-stack",+ "gemm-common 0.19.0",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-common"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3"+dependencies = [+ "bytemuck",+ "dyn-stack",+ "half",+ "libm",+ "num-complex",+ "num-traits",+ "once_cell",+ "paste",+ "pulp 0.21.5",+ "raw-cpuid",+ "rayon",+ "seq-macro",+ "sysctl",+]++[[package]]+name = "gemm-common"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e"+dependencies = [+ "bytemuck",+ "dyn-stack",+ "half",+ "libm",+ "num-complex",+ "num-traits",+ "once_cell",+ "paste",+ "pulp 0.22.3",+ "raw-cpuid",+ "rayon",+ "seq-macro",+ "sysctl",+]++[[package]]+name = "gemm-f16"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109"+dependencies = [+ "dyn-stack",+ "gemm-common 0.18.2",+ "gemm-f32 0.18.2",+ "half",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "rayon",+ "seq-macro",+]++[[package]]+name = "gemm-f16"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e"+dependencies = [+ "dyn-stack",+ "gemm-common 0.19.0",+ "gemm-f32 0.19.0",+ "half",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "rayon",+ "seq-macro",+]++[[package]]+name = "gemm-f32"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864"+dependencies = [+ "dyn-stack",+ "gemm-common 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-f32"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c"+dependencies = [+ "dyn-stack",+ "gemm-common 0.19.0",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-f64"+version = "0.18.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd"+dependencies = [+ "dyn-stack",+ "gemm-common 0.18.2",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]++[[package]]+name = "gemm-f64"+version = "0.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a"+dependencies = [+ "dyn-stack",+ "gemm-common 0.19.0",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid",+ "seq-macro",+]+[[package]]name = "gethostname"version = "1.1.0"@@ -1582,9 +1942,12 @@ version = "2.7.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"dependencies = [+ "bytemuck","cfg-if","crunchy","num-traits",+ "rand",+ "rand_distr","zerocopy",]@@ -1609,6 +1972,8 @@ dependencies = ["allocator-api2","equivalent","foldhash",+ "serde",+ "serde_core",][[package]]@@ -1620,6 +1985,12 @@ dependencies = ["foldhash",]+[[package]]+name = "heck"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"+[[package]]name = "hermit-abi"version = "0.5.3"@@ -2158,6 +2529,7 @@ version = "0.2.1"dependencies = ["lightcraft-color","lightcraft-geom",+ "miniz_oxide 0.8.9","proptest","serde","serde_json",@@ -2182,6 +2554,7 @@ dependencies = ["lightcraft-raster","lightcraft-raw","lightcraft-scenes",+ "lightcraft-segment","log","miniz_oxide 0.8.9","rayon",@@ -2324,6 +2697,18 @@ dependencies = ["serde",]+[[package]]+name = "lightcraft-segment"+version = "0.2.1"+dependencies = [+ "candle-core",+ "candle-nn",+ "log",+ "serde",+ "serde_json",+ "thiserror 2.0.21",+]+[[package]]name = "lightcraft-sysmem"version = "0.2.1"@@ -2426,6 +2811,15 @@ dependencies = ["imgref",]+[[package]]+name = "malloc_buf"+version = "0.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"+dependencies = [+ "libc",+]+[[package]]name = "maybe-rayon"version = "0.1.1"@@ -2449,6 +2843,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"dependencies = ["libc",+ "stable_deref_trait",][[package]]@@ -2460,6 +2855,21 @@ dependencies = ["autocfg",]+[[package]]+name = "metal"+version = "0.29.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ecfd3296f8c56b7c1f6fbac3c71cefa9d78ce009850c45000015f206dc7fa21"+dependencies = [+ "bitflags 2.13.2",+ "block",+ "core-graphics-types",+ "foreign-types",+ "log",+ "objc",+ "paste",+]+[[package]]name = "miniz_oxide"version = "0.8.9"@@ -2629,6 +3039,20 @@ version = "0.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"+[[package]]+name = "num"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"+dependencies = [+ "num-bigint",+ "num-complex",+ "num-integer",+ "num-iter",+ "num-rational",+ "num-traits",+]+[[package]]name = "num-bigint"version = "0.4.8"@@ -2639,6 +3063,16 @@ dependencies = ["num-traits",]+[[package]]+name = "num-complex"+version = "0.4.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"+dependencies = [+ "bytemuck",+ "num-traits",+]+[[package]]name = "num-derive"version = "0.4.2"@@ -2659,6 +3093,16 @@ dependencies = ["num-traits",]+[[package]]+name = "num-iter"+version = "0.1.46"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b"+dependencies = [+ "num-integer",+ "num-traits",+]+[[package]]name = "num-rational"version = "0.4.2"@@ -2680,6 +3124,16 @@ dependencies = ["libm",]+[[package]]+name = "num_cpus"+version = "1.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"+dependencies = [+ "hermit-abi",+ "libc",+]+[[package]]name = "num_enum"version = "0.7.6"@@ -2702,6 +3156,15 @@ dependencies = ["syn 2.0.119",]+[[package]]+name = "objc"+version = "0.2.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1"+dependencies = [+ "malloc_buf",+]+[[package]]name = "objc-sys"version = "0.3.5"@@ -2868,6 +3331,7 @@ checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"dependencies = ["bitflags 2.13.2","block2 0.6.2",+ "libc","objc2 0.6.4","objc2-core-foundation",]@@ -2915,7 +3379,9 @@ checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794"dependencies = ["bitflags 2.13.2","block2 0.6.2",+ "dispatch2","objc2 0.6.4",+ "objc2-core-foundation","objc2-foundation 0.3.2",]@@ -3352,6 +3818,43 @@ dependencies = ["unarray",]+[[package]]+name = "pulp"+version = "0.21.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "96b86df24f0a7ddd5e4b95c94fc9ed8a98f1ca94d3b01bdce2824097e7835907"+dependencies = [+ "bytemuck",+ "cfg-if",+ "libm",+ "num-complex",+ "reborrow",+ "version_check",+]++[[package]]+name = "pulp"+version = "0.22.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a"+dependencies = [+ "bytemuck",+ "cfg-if",+ "libm",+ "num-complex",+ "paste",+ "pulp-wasm-simd-flag",+ "raw-cpuid",+ "reborrow",+ "version_check",+]++[[package]]+name = "pulp-wasm-simd-flag"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740"+[[package]]name = "pxfm"version = "0.1.30"@@ -3438,6 +3941,16 @@ dependencies = ["getrandom 0.3.4",]+[[package]]+name = "rand_distr"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"+dependencies = [+ "num-traits",+ "rand",+]+[[package]]name = "rand_xorshift"version = "0.4.0"@@ -3503,6 +4016,15 @@ dependencies = ["rgb",]+[[package]]+name = "raw-cpuid"+version = "11.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"+dependencies = [+ "bitflags 2.13.2",+]+[[package]]name = "raw-window-handle"version = "0.6.2"@@ -3552,6 +4074,12 @@ dependencies = ["once_cell",]+[[package]]+name = "reborrow"+version = "0.5.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"+[[package]]name = "redox_syscall"version = "0.4.1"@@ -3695,6 +4223,27 @@ dependencies = ["wait-timeout",]+[[package]]+name = "safetensors"+version = "0.4.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6"+dependencies = [+ "serde",+ "serde_json",+]++[[package]]+name = "safetensors"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5"+dependencies = [+ "hashbrown 0.16.1",+ "serde",+ "serde_json",+]+[[package]]name = "same-file"version = "1.0.6"@@ -3728,6 +4277,12 @@ version = "1.0.28"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"+[[package]]+name = "seq-macro"+version = "0.3.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"+[[package]]name = "serde"version = "1.0.229"@@ -3990,6 +4545,42 @@ dependencies = ["unicode-ident",]+[[package]]+name = "synstructure"+version = "0.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "synstructure"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.6",+]++[[package]]+name = "sysctl"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"+dependencies = [+ "bitflags 2.13.2",+ "byteorder",+ "enum-as-inner",+ "libc",+ "thiserror 1.0.69",+ "walkdir",+]+[[package]]name = "tempfile"version = "3.27.0"@@ -4179,6 +4770,12 @@ dependencies = ["rustc-hash 2.1.3",]+[[package]]+name = "typed-path"+version = "0.12.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"+[[package]]name = "uds_windows"version = "1.2.1"@@ -4190,6 +4787,41 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "ug"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "76b761acf8af3494640d826a8609e2265e19778fb43306c7f15379c78c9b05b0"+dependencies = [+ "gemm 0.18.2",+ "half",+ "libloading",+ "memmap2",+ "num",+ "num-traits",+ "num_cpus",+ "rayon",+ "safetensors 0.4.5",+ "serde",+ "thiserror 1.0.69",+ "tracing",+ "yoke 0.7.5",+]++[[package]]+name = "ug-metal"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9f7adf545a99a086d362efc739e7cf4317c18cbeda22706000fd434d70ea3d95"+dependencies = [+ "half",+ "metal",+ "objc",+ "serde",+ "thiserror 1.0.69",+ "ug",+]+[[package]]name = "unarray"version = "0.1.4"@@ -5141,6 +5773,53 @@ version = "0.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448"+[[package]]+name = "yoke"+version = "0.7.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"+dependencies = [+ "serde",+ "stable_deref_trait",+ "yoke-derive 0.7.5",+ "zerofrom",+]++[[package]]+name = "yoke"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"+dependencies = [+ "stable_deref_trait",+ "yoke-derive 0.8.4",+ "zerofrom",+]++[[package]]+name = "yoke-derive"+version = "0.7.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+ "synstructure 0.13.2",+]++[[package]]+name = "yoke-derive"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.6",+ "synstructure 0.14.0",+]+[[package]]name = "zbus"version = "5.19.0"@@ -5267,6 +5946,39 @@ dependencies = ["syn 2.0.119",]+[[package]]+name = "zerofrom"+version = "0.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"+dependencies = [+ "zerofrom-derive",+]++[[package]]+name = "zerofrom-derive"+version = "0.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.6",+ "synstructure 0.14.0",+]++[[package]]+name = "zip"+version = "7.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0"+dependencies = [+ "crc32fast",+ "indexmap",+ "memchr",+ "typed-path",+]+[[package]]name = "zlib-rs"version = "0.6.8"diff --git a/Cargo.toml b/Cargo.tomlindex b1882f9bf..d59f8f45e 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -29,6 +29,7 @@ lightcraft-merge = { path = "crates/merge" }lightcraft-engine = { path = "crates/engine" }lightcraft-ui-egui = { path = "crates/ui-egui" }lightcraft-mcp = { path = "crates/mcp" }+lightcraft-segment = { path = "crates/segment" }# foundationthiserror = "2"@@ -62,6 +63,10 @@ jxl-oxide = { version = "0.12", default-features = false }ravif = { version = "0.13", default-features = false, features = ["threading"] } # no "asm": it needs nasm (non-Rust)# gpu (crates/gpu; native backends only, no GL: everything stays pure Rust)+# SAM 3 inference (lightcraft-segment). 0.9.2: later releases pull `tokenizers` with the+# Oniguruma C library into candle-core, and the product is pure Rust.+candle-core = { version = "=0.9.2", default-features = false }+candle-nn = { version = "=0.9.2", default-features = false }wgpu = { version = "30", default-features = false, features = ["std", "parking_lot", "metal", "vulkan", "dx12", "wgsl"] }pollster = "1"diff --git a/NOTICE b/NOTICEindex e6f95a8a9..1fcf11e74 100644--- a/NOTICE+++ b/NOTICE@@ -15,6 +15,12 @@ list (author, source, license) is in assets/ATTRIBUTION.md.Inter (assets/fonts/Inter-*.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-Inter.txt.Showcase photographs by Ansel Adams, Dorothea Lange and NASA (docs/images/) — public domain. See assets/ATTRIBUTION.md.+crates/segment ports the SAM 3 model code of Hugging Face Transformers+(https://github.com/huggingface/transformers, src/transformers/models/sam3 and sam3_tracker),+Copyright The HuggingFace Team and Meta Platforms, Inc., licensed under the Apache License 2.0.+The SAM 3 weights are not part of LightCraft: users download the facebook/sam3 checkpoint+themselves, under Meta's SAM License.+Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases)also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md(the Japanese fonts BIZ UDPGothic, BIZ UDMincho and Shippori Mincho; the web build embeds only BIZdiff --git a/apps/lightcraft/Cargo.toml b/apps/lightcraft/Cargo.tomlindex 46acbb73c..a29299da0 100644--- a/apps/lightcraft/Cargo.toml+++ b/apps/lightcraft/Cargo.toml@@ -10,7 +10,7 @@ default-run = "lightcraft"workspace = true[dependencies]-lightcraft-engine = { workspace = true }+lightcraft-engine = { workspace = true, features = ["sam"] }lightcraft-sysmem = { workspace = true }lightcraft-ui-egui = { workspace = true }lightcraft-codecs = { workspace = true }diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rsindex 2bdfa16ff..26a4fa1fe 100644--- a/apps/lightcraft/src/main.rs+++ b/apps/lightcraft/src/main.rs@@ -415,10 +415,40 @@ OPTIONS:ENVIRONMENT:LIGHTCRAFT_GPU_BACKEND=dx12|vulkan|metal|auto|off graphics backend (default: DX12 on Windows, Metal on macOS,Vulkan on Linux; off = render on the CPU); else WGPU_BACKEND. LIGHTCRAFT_GPU=0: CPU rendering.+ LIGHTCRAFT_SAM3_DIR=DIR the SAM 3 model for Object / Describe masks (default: <settings folder>/models/sam3)";+/// Warnings and errors (failed commands, AI mask analysis) on stderr; `LIGHTCRAFT_LOG=info`+/// (or `debug`) shows more.+struct StderrLog(log::LevelFilter);++impl log::Log for StderrLog {+ fn enabled(&self, m: &log::Metadata) -> bool {+ m.level() <= self.0 && (m.level() <= log::Level::Warn || m.target().starts_with("lightcraft"))+ }+ fn log(&self, r: &log::Record) {+ if self.enabled(r.metadata()) {+ eprintln!("lightcraft: {} {}: {}", r.level(), r.target(), r.args());+ }+ }+ fn flush(&self) {}+}++fn install_log() {+ let level = match std::env::var("LIGHTCRAFT_LOG").unwrap_or_default().as_str() {+ "debug" => log::LevelFilter::Debug,+ "info" => log::LevelFilter::Info,+ _ => log::LevelFilter::Warn,+ };+ static LOGGER: std::sync::OnceLock<StderrLog> = std::sync::OnceLock::new();+ if log::set_logger(LOGGER.get_or_init(|| StderrLog(level))).is_ok() {+ log::set_max_level(level);+ }+}+fn main() -> eframe::Result {lightcraft_engine::guard::install_hook(std::env::temp_dir().join("lightcraft-panics.log"));+ install_log();alloc_release::install();let mut control_port: Option<u16> = std::env::var("LIGHTCRAFT_CONTROL_PORT").ok().and_then(|p| p.parse().ok());let mut files = Vec::new();@@ -481,7 +511,10 @@ fn main() -> eframe::Result {"LightCraft",options,Box::new(move |cc| {- let (session, problem) = open_session(in_memory, library_dir, seed_demo && files.is_empty());+ let (mut session, problem) = open_session(in_memory, library_dir, seed_demo && files.is_empty());+ // AI masks: the SAM 3 checkpoint (facebook/sam3) in <config>/models/sam3, or LIGHTCRAFT_SAM3_DIR+ session.segmenter.dir =+ std::env::var_os("LIGHTCRAFT_SAM3_DIR").map(std::path::PathBuf::from).or_else(|| config_dir().map(|d| d.join("models").join("sam3")));let mut app = LightcraftApp::new(session, services());if let Some(ui) = prefs {app.ui = ui;diff --git a/crates/develop/Cargo.toml b/crates/develop/Cargo.tomlindex c42aaaa3a..6d606148a 100644--- a/crates/develop/Cargo.toml+++ b/crates/develop/Cargo.toml@@ -13,6 +13,7 @@ lightcraft-geom = { workspace = true }lightcraft-color = { workspace = true }serde = { workspace = true }serde_json = { workspace = true }+miniz_oxide = { workspace = true }[dev-dependencies]proptest = { workspace = true }diff --git a/crates/develop/src/lib.rs b/crates/develop/src/lib.rsindex 72660afd9..ff4daedd9 100644--- a/crates/develop/src/lib.rs+++ b/crates/develop/src/lib.rs@@ -8,10 +8,12 @@pub mod controls;pub mod presets;+pub mod segmask;pub mod settings;pub use controls::{CONTROLS, ControlSpec, Section, Track};pub use presets::{Preset, SettingsGroup, apply_partial, extract_groups};+pub use segmask::SegMask;pub use settings::*;use serde_json::Value;diff --git a/crates/develop/src/segmask.rs b/crates/develop/src/segmask.rsnew file mode 100644index 000000000..724ebdf56--- /dev/null+++ b/crates/develop/src/segmask.rs@@ -0,0 +1,131 @@+//! Model segmentations stored with a mask ([`SegMask`]), so rendering never needs the model.++use serde::{Deserialize, Serialize};++/// Logits are stored in steps of 1/`SCALE`, clamped to ±127/`SCALE` (≈ ±15.9: a sigmoid of+/// 1 − 1.2e-7, i.e. fully in or out).+const SCALE: f32 = 8.0;+/// Largest grid a stored segmentation may claim (hostile input caps the allocation).+pub const MAX_SIDE: usize = 1024;++/// A segmentation computed by a model (SAM 3 in `lightcraft-segment`): `side × side` logits+/// over the uncropped, oriented image stretched to a square (row `y`, column `x` covers+/// normalized `((x + 0.5) / side, (y + 0.5) / side)`), quantized to `i8`, deflated and base64+/// encoded. Coverage at a point is the sigmoid of the bilinearly sampled logit, so edges stay+/// smooth at any output size.+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+pub struct SegMask {+ pub side: u32,+ pub data: String,+ /// The part of the image the grid covers, normalized `[x0, y0, x1, y1]` (`None` = all of+ /// it). A zoomed-in pass over one object covers just its surroundings, at a higher+ /// resolution than a pass over the whole photo.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub rect: Option<[f64; 4]>,+}++impl SegMask {+ /// From `side × side` logits over the whole image.+ pub fn from_logits(side: usize, logits: &[f32]) -> SegMask {+ let q: Vec<u8> = logits.iter().take(side * side).map(|l| ((l * SCALE).round().clamp(-127.0, 127.0) as i8) as u8).collect();+ let z = miniz_oxide::deflate::compress_to_vec(&q, 8);+ SegMask { side: side as u32, data: base64_encode(&z), rect: None }+ }++ /// From `side × side` logits over the part `rect` of the image.+ pub fn from_logits_in(side: usize, logits: &[f32], rect: [f64; 4]) -> SegMask {+ SegMask { rect: Some(rect), ..SegMask::from_logits(side, logits) }+ }++ /// The covered part, normalized (the whole image without a `rect`); `None` when damaged.+ pub fn bounds(&self) -> Option<[f64; 4]> {+ let r = self.rect.unwrap_or([0.0, 0.0, 1.0, 1.0]);+ (r.iter().all(|v| v.is_finite()) && r[2] > r[0] && r[3] > r[1]).then_some(r)+ }++ /// The logits, or `None` when the data is damaged or claims an absurd size.+ pub fn logits(&self) -> Option<Vec<f32>> {+ let side = self.side as usize;+ if side == 0 || side > MAX_SIDE {+ return None;+ }+ let z = base64_decode(&self.data)?;+ let q = miniz_oxide::inflate::decompress_to_vec_with_limit(&z, side * side).ok()?;+ (q.len() == side * side).then(|| q.iter().map(|b| f32::from(*b as i8) / SCALE).collect())+ }+}++const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";++fn base64_encode(bytes: &[u8]) -> String {+ let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);+ for chunk in bytes.chunks(3) {+ let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)];+ let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);+ for i in 0..4 {+ if i <= chunk.len() {+ out.push(char::from(B64[((n >> (18 - 6 * i)) & 63) as usize]));+ } else {+ out.push('=');+ }+ }+ }+ out+}++fn base64_decode(s: &str) -> Option<Vec<u8>> {+ let mut out = Vec::with_capacity(s.len() / 4 * 3);+ let (mut acc, mut bits) = (0u32, 0u32);+ for c in s.bytes().filter(|c| !c.is_ascii_whitespace()) {+ if c == b'=' {+ break;+ }+ let v = B64.iter().position(|b| *b == c)? as u32;+ acc = (acc << 6) | v;+ bits += 6;+ if bits >= 8 {+ bits -= 8;+ out.push((acc >> bits) as u8);+ acc &= (1 << bits) - 1;+ }+ }+ Some(out)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn base64_round_trips() {+ for n in 0..10 {+ let v: Vec<u8> = (0..n).map(|i| (i * 37 + 11) as u8).collect();+ assert_eq!(base64_decode(&base64_encode(&v)).unwrap(), v);+ }+ assert_eq!(base64_encode(b"Man"), "TWFu");+ assert_eq!(base64_encode(b"Ma"), "TWE=");+ assert!(base64_decode("TW!u").is_none());+ }++ #[test]+ fn logits_round_trip_quantized() {+ let side = 16;+ let l: Vec<f32> = (0..side * side).map(|i| (i as f32 - 128.0) / 4.0).collect();+ let m = SegMask::from_logits(side, &l);+ let back = m.logits().unwrap();+ for (a, b) in l.iter().zip(&back) {+ assert!((a.clamp(-127.0 / SCALE, 127.0 / SCALE) - b).abs() <= 0.5 / SCALE + 1e-6);+ }+ }++ #[test]+ fn damaged_data_is_none() {+ let mut m = SegMask::from_logits(8, &[1.0; 64]);+ m.side = 9;+ assert!(m.logits().is_none(), "size mismatch");+ m.side = 100_000;+ assert!(m.logits().is_none(), "absurd size");+ assert!(SegMask { side: 8, data: "not base64 !".into(), rect: None }.logits().is_none());+ assert!(SegMask { rect: Some([0.5, 0.0, 0.2, 1.0]), ..SegMask::from_logits(2, &[0.0; 4]) }.bounds().is_none());+ }+}diff --git a/crates/develop/src/settings.rs b/crates/develop/src/settings.rsindex eeb6ff048..736586fe9 100644--- a/crates/develop/src/settings.rs+++ b/crates/develop/src/settings.rs@@ -772,8 +772,33 @@ pub enum MaskShape {Subject,Sky,Background,+ /// One object picked by clicks (SAM 3 point prompts): `hint` holds the clicks that include,+ /// `exclude` the ones that exclude; `seg` the segmentation computed from them.Object {hint: Vec<Point>,+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ exclude: Vec<Point>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ seg: Option<crate::SegMask>,+ /// Zoomed-in passes over parts of the image (higher resolution than `seg`), used+ /// inside their rectangles.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ detail: Vec<crate::SegMask>,+ /// Edge −100..100: below 0 harder (a steeper transition), above 0 softer (feathered).+ #[serde(default, skip_serializing_if = "is_zero")]+ edge: f64,+ },+ /// Everything a description names ("sky", "the red car": SAM 3 concept prompts); `seg` is+ /// the segmentation computed for it.+ Prompt {+ text: String,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ seg: Option<crate::SegMask>,+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ detail: Vec<crate::SegMask>,+ /// Edge −100..100, as for `Object`.+ #[serde(default, skip_serializing_if = "is_zero")]+ edge: f64,},People {person: u32,diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.tomlindex 5c8e3e462..c2d4e8a80 100644--- a/crates/engine/Cargo.toml+++ b/crates/engine/Cargo.toml@@ -30,6 +30,11 @@ serde_json = { workspace = true }thiserror = { workspace = true }log = { workspace = true }ab_glyph = { workspace = true }+lightcraft-segment = { workspace = true, optional = true }++[features]+# AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds.+sam = ["dep:lightcraft-segment"][dev-dependencies]libc = { workspace = true }diff --git a/crates/engine/src/cmd/masks.rs b/crates/engine/src/cmd/masks.rsindex ec3ffac07..da96f967b 100644--- a/crates/engine/src/cmd/masks.rs+++ b/crates/engine/src/cmd/masks.rs@@ -5,6 +5,7 @@ use lightcraft_geom::Point;use serde_json::{Value, json};use super::{CommandSpec, bad, bool_or, cmd, f64_or, has_active, ok, point, str_param};+use crate::segment::Click;use crate::{Result, Session};fn shape_from(kind: &str, p: &Value, c: &str) -> Result<MaskShape> {@@ -38,10 +39,57 @@ fn shape_from(kind: &str, p: &Value, c: &str) -> Result<MaskShape> {.unwrap_or_default();MaskShape::ColorRange { samples, refine: f64_or(p, "refine", 50.0) }}- other => return Err(bad(c, format!("unknown mask kind `{other}` (brush|linear|radial|sky|subject|background|luminanceRange|colorRange)"))),+ "object" => MaskShape::Object { hint: points(p, "points"), exclude: points(p, "exclude"), seg: None, detail: vec![], edge: 0.0 },+ "prompt" => MaskShape::Prompt { text: str_param(p, "text").unwrap_or_default().trim().to_string(), seg: None, detail: vec![], edge: 0.0 },+ other => {+ return Err(bad(+ c,+ format!("unknown mask kind `{other}` (brush|linear|radial|sky|subject|background|luminanceRange|colorRange|object|prompt)"),+ ));+ }})}+/// `[[x, y], …]` normalized points under `key`.+fn points(p: &Value, key: &str) -> Vec<Point> {+ p.get(key)+ .and_then(Value::as_array)+ .map(|a| a.iter().filter_map(|q| Some(Point::new(q.get(0)?.as_f64()?, q.get(1)?.as_f64()?))).collect())+ .unwrap_or_default()+}++/// Compute the segmentation of a new AI shape (Object with clicks, Prompt): an Object+/// without clicks stays empty until the photo is clicked; a Prompt that matches nothing is+/// an error (nothing to select).+fn resolve_ai(s: &mut Session, shape: &mut MaskShape, c: &str) -> Result<()> {+ let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;+ match shape {+ MaskShape::Object { hint, exclude, seg, .. } if !hint.is_empty() => {+ *seg = Some(s.segment_clicks(id, &clicks_of(hint, exclude)).map_err(ai_err)?);+ }+ MaskShape::Object { .. } => {+ // the clicks come later: start analyzing the photo now, and refuse an Object mask that+ // could never be computed (no model, missing file) rather than leave an empty one+ s.segment_prepare(id).map_err(ai_err)?;+ }+ MaskShape::Prompt { text, seg, .. } => {+ let found = s.segment_text(id, text).map_err(ai_err)?;+ *seg = Some(found.ok_or_else(|| ai_err(format!("nothing matching “{text}” was found in this photo")))?);+ }+ _ => {}+ }+ Ok(())+}++/// AI mask failures are shown as they are (not as "invalid parameters").+fn ai_err(msg: impl Into<String>) -> crate::EngineError {+ crate::EngineError::Other(msg.into())+}++fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec<Click> {+ include.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect()+}+fn masks_edit(s: &mut Session, c: &str, label: &str, f: impl FnOnce(&mut Vec<Mask>, &mut Option<u32>) -> Result<()>) -> Result<Value> {let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;let mut d = (*s.develop_of(id).unwrap_or_default()).clone();@@ -67,11 +115,12 @@ pub fn specs() -> Vec<CommandSpec> {"Create New Mask",[],None,- "{kind: brush|linear|radial|sky|subject|background|luminanceRange|colorRange, ...shape params (start/end, center/rx/ry/angle/feather, lo/hi…), name?}",+ "{kind: brush|linear|radial|sky|subject|background|luminanceRange|colorRange|object|prompt, ...shape params (start/end, center/rx/ry/angle/feather, lo/hi…; object: points/exclude [[x,y],…]; prompt: text), name?}",has_active,|s, p| {let kind = str_param(p, "kind").unwrap_or("radial").to_string();- let shape = shape_from(&kind, p, "mask.add")?;+ let mut shape = shape_from(&kind, p, "mask.add")?;+ resolve_ai(s, &mut shape, "mask.add")?;let name = str_param(p, "name").map(str::to_string);let next = s.active().and_then(|id| s.develop_of(id)).map(|d| d.next_mask_id()).unwrap_or(1);masks_edit(s, "mask.add", "Add Mask", |masks, active| {@@ -95,7 +144,8 @@ pub fn specs() -> Vec<CommandSpec> {has_active,|s, p| {let kind = str_param(p, "kind").unwrap_or("brush").to_string();- let shape = shape_from(&kind, p, "mask.addComponent")?;+ let mut shape = shape_from(&kind, p, "mask.addComponent")?;+ resolve_ai(s, &mut shape, "mask.addComponent")?;let op: MaskOp =serde_json::from_value(p.get("op").cloned().unwrap_or(json!("add"))).map_err(|e| bad("mask.addComponent", e.to_string()))?;let active = s.active_mask;@@ -213,6 +263,89 @@ pub fn specs() -> Vec<CommandSpec> {Ok(json!({"samples": out}))}),+ cmd!(+ "mask.objectPoint",+ "Add Object Click",+ [],+ None,+ "{x, y: normalized image point, exclude?: bool (⌥-click: leave this part out), id?: maskId} — a click on the selected mask's Object selection (its last Object component); SAM 3 re-segments the object → {include, exclude}",+ has_active,+ |s, p| {+ let c = "mask.objectPoint";+ let at = Point::new(super::f64_req(p, "x", c)?, super::f64_req(p, "y", c)?);+ if !(0.0..=1.0).contains(&at.x) || !(0.0..=1.0).contains(&at.y) {+ return Err(bad(c, "the point is outside the photo"));+ }+ let exclude = bool_or(p, "exclude", false);+ let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;+ let mid = mask_id(p, s.active_mask, c)?;+ let d = s.develop_of(id).unwrap_or_default();+ let m = d.masks.iter().find(|m| m.id == mid).ok_or_else(|| bad(c, format!("no mask {mid}")))?;+ let k = m+ .components+ .iter()+ .rposition(|x| matches!(x.shape, MaskShape::Object { .. }))+ .ok_or_else(|| bad(c, "the mask has no Object selection (add one first)"))?;+ let Some(MaskShape::Object { hint, exclude: ex, edge, .. }) = m.components.get(k).map(|x| &x.shape) else {+ return Err(bad(c, "not an Object component"));+ };+ let (mut hint, mut ex, edge) = (hint.clone(), ex.clone(), *edge);+ if exclude {+ ex.push(at)+ } else {+ hint.push(at)+ }+ let seg = if hint.is_empty() { None } else { Some(s.segment_clicks(id, &clicks_of(&hint, &ex)).map_err(ai_err)?) };+ let out = json!({"include": hint.len(), "exclude": ex.len()});+ masks_edit(s, c, "Object Mask", |masks, _| {+ let i = find(masks, mid, c)?;+ if let Some(comp) = masks[i].components.get_mut(k) {+ comp.shape = MaskShape::Object { hint, exclude: ex, seg, detail: vec![], edge };+ }+ Ok(())+ })?;+ Ok(out)+ }+ ),+ cmd!(+ "mask.refineDetail",+ "Refine AI Mask Detail",+ [],+ None,+ "{id?: maskId, component?} — a zoomed-in SAM 3 pass over an Object/Describe selection (default: the mask's last one): the photo around it is analyzed again at a higher resolution, in the background; the mask updates when it's done → {started}",+ has_active,+ |s, p| {+ let c = "mask.refineDetail";+ let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;+ let mid = mask_id(p, s.active_mask, c)?;+ let d = s.develop_of(id).unwrap_or_default();+ let m = d.masks.iter().find(|m| m.id == mid).ok_or_else(|| bad(c, format!("no mask {mid}")))?;+ let k = match p.get("component").and_then(Value::as_u64) {+ Some(k) => k as usize,+ None => m+ .components+ .iter()+ .rposition(|x| matches!(x.shape, MaskShape::Object { .. } | MaskShape::Prompt { .. }))+ .ok_or_else(|| bad(c, "the mask has no Object or Describe selection"))?,+ };+ let started = s.segment_detail(id, mid, k).map_err(ai_err)?;+ Ok(json!({"started": started}))+ }+ ),+ cmd!(+ query "segment.prepare",+ "Prepare AI Masks",+ [],+ None,+ "{} — load SAM 3 and analyze the active photo in the background, so Object clicks are instant → {busy}",+ has_active,+ |s, _| {+ let c = "segment.prepare";+ let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;+ s.segment_prepare(id).map_err(ai_err)?;+ Ok(json!({"busy": s.segmenter.busy()}))+ }+ ),cmd!("mask.refine","Refine Mask Edges",diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rsindex 68200a356..40305c5ff 100644--- a/crates/engine/src/lib.rs+++ b/crates/engine/src/lib.rs@@ -32,6 +32,7 @@ pub mod preset_import;pub mod preset_luminar;pub mod presets;pub mod rename;+pub mod segment;pub mod sidecar;pub mod smart;mod view;@@ -150,6 +151,8 @@ pub struct Session {depth: u32,/// Selected mask (Masking panel), by mask id.pub active_mask: Option<u32>,+ /// AI masks (SAM 3): the model and the last photo prepared for it.+ pub segmenter: segment::Segmenter,/// Selected spot (Remove panel), by index into the active photo's spots.pub active_spot: Option<usize>,/// The persistent library this session writes to (`None` = in-memory only).@@ -237,6 +240,7 @@ impl Session {clock: Box::new(|| "2026-09-30T12:00:00".to_string()),depth: 0,active_mask: None,+ segmenter: segment::Segmenter::default(),active_spot: None,library: None,xmp: sidecar::XmpPrefs::default(),diff --git a/crates/engine/src/segment.rs b/crates/engine/src/segment.rsnew file mode 100644index 000000000..41aed09e9--- /dev/null+++ b/crates/engine/src/segment.rs@@ -0,0 +1,588 @@+//! AI masks: Object (clicks) and Describe (text) selections computed with SAM 3+//! (`lightcraft-segment`, cargo feature `sam`; the desktop app enables it).+//!+//! The model sees the photo as developed, uncropped and without masks, at 1008 px — the frame+//! mask coordinates live in — so a stored segmentation lines up with the photo at any size.+//! Encoding the image is the slow part (a few seconds on a laptop GPU); it runs once per photo+//! and look, on a worker thread when [`Session::segment_prepare`] is called (the Masking panel+//! does this when an AI mask is started), and each click then takes milliseconds.+//!+//! Without the feature (web, CLI) every request reports that this build has no model; stored+//! segmentations still render (they are plain data in the develop settings).++use std::path::PathBuf;++use lightcraft_catalog::PhotoId;+use lightcraft_develop::{MaskShape, SegMask};+use lightcraft_geom::Point;++use crate::Session;++/// Long edge of the image the model sees.+pub const INPUT_EDGE: usize = 1008;++/// One click of an Object selection (normalized image coordinates).+#[derive(Clone, Copy, Debug, PartialEq)]+pub struct Click {+ pub at: Point,+ pub include: bool,+}++/// What a request runs on: the photo, its look (settings without masks), the model input.+#[cfg_attr(not(feature = "sam"), allow(dead_code))]+struct Input {+ key: u64,+ rgb: Vec<u8>,+ w: usize,+ h: usize,+}++#[cfg(feature = "sam")]+mod model {+ use std::sync::atomic::{AtomicBool, Ordering};+ use std::sync::{Arc, Mutex, PoisonError};++ use lightcraft_segment::{Encoded, Sam3};++ #[derive(Default)]+ pub struct Inner {+ pub model: Option<Sam3>,+ /// The last encoded image and its key.+ pub cache: Option<(u64, Encoded)>,+ }++ #[derive(Clone, Default)]+ pub struct Shared {+ pub inner: Arc<Mutex<Inner>>,+ pub busy: Arc<AtomicBool>,+ /// A zoomed-in detail pass is running.+ pub detail_busy: Arc<AtomicBool>,+ /// Finished detail passes, waiting for [`crate::Session::segment_poll`].+ pub results: Arc<Mutex<Vec<super::DetailResult>>>,+ }++ impl Shared {+ pub fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {+ self.inner.lock().unwrap_or_else(PoisonError::into_inner)+ }++ pub fn busy(&self) -> bool {+ self.busy.load(Ordering::Relaxed)+ }++ pub fn set_busy(&self, v: bool) {+ self.busy.store(v, Ordering::Relaxed);+ }+ }+}++/// The model (loaded on first use) and the last encoded photo.+#[derive(Default)]+pub struct Segmenter {+ /// Where the `facebook/sam3` checkpoint lives (set by the app).+ pub dir: Option<PathBuf>,+ #[cfg(feature = "sam")]+ shared: model::Shared,+}++impl Segmenter {+ /// Whether this build can compute AI masks at all.+ pub const AVAILABLE: bool = cfg!(feature = "sam");++ /// Whether a worker is preparing a photo right now.+ pub fn busy(&self) -> bool {+ #[cfg(feature = "sam")]+ return self.shared.busy();+ #[cfg(not(feature = "sam"))]+ false+ }++ /// Whether a zoomed-in detail pass is running.+ pub fn detail_busy(&self) -> bool {+ #[cfg(feature = "sam")]+ return self.shared.detail_busy.load(std::sync::atomic::Ordering::Relaxed);+ #[cfg(not(feature = "sam"))]+ false+ }++ /// The model directory, if its files are there.+ pub fn model_dir(&self) -> Result<PathBuf, String> {+ if !Self::AVAILABLE {+ return Err("AI masks are not available in this build".into());+ }+ let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?;+ #[cfg(feature = "sam")]+ if !lightcraft_segment::is_model_dir(&dir) {+ return Err(format!(+ "the SAM 3 model is not installed: put the facebook/sam3 files (model.safetensors, vocab.json, merges.txt) in {}",+ dir.display()+ ));+ }+ Ok(dir)+ }+}++/// Mix `v` into a 64-bit hash.+fn mix(h: u64, v: u64) -> u64 {+ (h ^ v).wrapping_mul(0x0100_0000_01b3).rotate_left(29)+}++/// A finished zoomed-in pass: for which component (and the shape it was computed from, so a+/// component changed meanwhile is left alone), and the patch (`None`: nothing found there).+#[cfg_attr(not(feature = "sam"), allow(dead_code))]+pub struct DetailResult {+ photo: PhotoId,+ mask: u32,+ comp: usize,+ shape: MaskShape,+ patch: Option<SegMask>,+}++/// What a detail pass re-runs.+#[cfg_attr(not(feature = "sam"), allow(dead_code))]+enum Prompt {+ Clicks(Vec<Click>),+ Text(String),+}++/// Objects covering more than this much of the photo gain little from a zoomed pass.+const DETAIL_MAX_AREA: f64 = 0.4;++/// The part of the photo to zoom into for `seg`'s selection: its bounding box with a margin of+/// a quarter of its size, at least 12 % of the long edge, at most 2:1, inside the photo+/// (`aspect` = width / height). `None` when nothing is selected or it is most of the photo.+pub fn detail_region(seg: &SegMask, aspect: f64) -> Option<[f64; 4]> {+ let logits = seg.logits()?;+ let r = seg.bounds()?;+ let side = seg.side as usize;+ let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0, 0);+ for (i, l) in logits.iter().enumerate() {+ if *l > 0.0 {+ let (x, y) = (i % side, i / side);+ (x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x + 1), y1.max(y + 1));+ }+ }+ if x0 == usize::MAX {+ return None;+ }+ let (rw, rh) = (r[2] - r[0], r[3] - r[1]);+ let nx = |c: usize| r[0] + c as f64 / side as f64 * rw;+ let ny = |c: usize| r[1] + c as f64 / side as f64 * rh;+ let (bx0, by0, bx1, by1) = (nx(x0), ny(y0), nx(x1), ny(y1));+ if (bx1 - bx0) * (by1 - by0) > DETAIL_MAX_AREA {+ return None;+ }+ // in units of the photo's height: x × aspect+ let a = aspect.clamp(0.1, 10.0);+ let (cx, cy) = ((bx0 + bx1) / 2.0 * a, (by0 + by1) / 2.0);+ let size = ((bx1 - bx0) * a).max(by1 - by0);+ let m = 0.25 * size;+ let mut w = ((bx1 - bx0) * a + 2.0 * m).max(0.12 * a.max(1.0));+ let mut h = ((by1 - by0) + 2.0 * m).max(0.12 * a.max(1.0));+ w = w.max(h / 2.0).min(a);+ h = h.max(w / 2.0).min(1.0);+ let x = (cx - w / 2.0).clamp(0.0, (a - w).max(0.0));+ let y = (cy - h / 2.0).clamp(0.0, (1.0 - h).max(0.0));+ let out = [x / a, y, (x + w) / a, y + h];+ // a region nearly the whole photo isn't worth a second pass+ ((out[2] - out[0]) * (out[3] - out[1]) < 0.8).then_some(out)+}++impl Session {+ /// The key and settings of photo `id` as the model sees it: its look without masks, uncropped.+ fn segment_key(&self, id: PhotoId) -> Option<(u64, lightcraft_develop::DevelopSettings)> {+ let p = self.catalog.photo(id)?;+ let mut d = (*p.develop).clone();+ d.masks.clear();+ let content = crate::media::content_key(p);+ let mut h = mix(0xcbf2_9ce4_8422_2325, d.hash64());+ h = mix(h, id.0);+ for b in content.bytes() {+ h = mix(h, u64::from(b));+ }+ Some((h, d))+ }++ /// Render the model input for `id`.+ #[cfg_attr(not(feature = "sam"), allow(dead_code))]+ fn segment_input(&mut self, id: PhotoId) -> Result<Input, String> {+ let (key, d) = self.segment_key(id).ok_or("no such photo")?;+ let job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, false, &d).ok_or("no such photo")?;+ let r = job.run();+ self.accept(&r);+ let img = match r.rendered {+ Ok(r) => r.image,+ Err(e) => {+ // the usual reason: the file was moved or deleted (and there's no smart preview)+ if let Some(lightcraft_catalog::Source::File { path }) = self.catalog.photo(id).map(|p| &p.source)+ && !std::path::Path::new(path).exists()+ {+ return Err(format!(+ "This photo's file is missing (moved or deleted), so it can't be analyzed for AI masks: {path}. Library ▸ Find Missing Photos can relink it."+ ));+ }+ return Err(format!("couldn't render the photo for AI masks: {e}"));+ }+ };+ let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect();+ Ok(Input { key, rgb, w: img.width, h: img.height })+ }++ /// Start preparing photo `id` for AI masks on a worker thread (load the model, encode the+ /// image). Returns at once; does nothing when it is ready or being prepared.+ pub fn segment_prepare(&mut self, id: PhotoId) -> Result<(), String> {+ let dir = self.segmenter.model_dir()?;+ #[cfg(feature = "sam")]+ {+ let (key, _) = self.segment_key(id).ok_or("no such photo")?;+ let shared = self.segmenter.shared.clone();+ if shared.busy() {+ return Ok(());+ }+ if let Ok(inner) = shared.inner.try_lock()+ && inner.cache.as_ref().is_some_and(|c| c.0 == key)+ {+ return Ok(());+ }+ let input = self.segment_input(id)?;+ shared.set_busy(true);+ let spawned = std::thread::Builder::new().name("sam3-prepare".into()).spawn(move || {+ let mut inner = shared.lock();+ if let Err(e) = prepare(&mut inner, &dir, &input) {+ log::warn!("SAM 3: {e}");+ }+ shared.set_busy(false);+ });+ if let Err(e) = spawned {+ self.segmenter.shared.set_busy(false);+ return Err(format!("could not start the model: {e}"));+ }+ }+ #[cfg(not(feature = "sam"))]+ let _ = (dir, id);+ Ok(())+ }++ /// Segment the object `clicks` pick on photo `id` (waits for a running prepare).+ pub fn segment_clicks(&mut self, id: PhotoId, clicks: &[Click]) -> Result<SegMask, String> {+ let dir = self.segmenter.model_dir()?;+ if !clicks.iter().any(|c| c.include) {+ return Err("click the object to include it first".into());+ }+ #[cfg(feature = "sam")]+ {+ let input = self.segment_input_if_needed(id)?;+ let shared = self.segmenter.shared.clone();+ let mut inner = shared.lock();+ let enc = encoded(&mut inner, &dir, input)?;+ let clicks: Vec<lightcraft_segment::Click> =+ clicks.iter().map(|c| lightcraft_segment::Click { x: c.at.x as f32, y: c.at.y as f32, positive: c.include }).collect();+ let (model, enc) = enc;+ let p = model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?;+ Ok(SegMask::from_logits(lightcraft_segment::MASK_SIDE, &p.logits))+ }+ #[cfg(not(feature = "sam"))]+ {+ let _ = (dir, id);+ Err("AI masks are not available in this build".into())+ }+ }++ /// Segment everything `text` describes on photo `id`; `Ok(None)` when nothing matches.+ pub fn segment_text(&mut self, id: PhotoId, text: &str) -> Result<Option<SegMask>, String> {+ let dir = self.segmenter.model_dir()?;+ let text = text.trim();+ if text.is_empty() {+ return Err("describe what to select".into());+ }+ #[cfg(feature = "sam")]+ {+ let input = self.segment_input_if_needed(id)?;+ let shared = self.segmenter.shared.clone();+ let mut inner = shared.lock();+ let (model, enc) = encoded(&mut inner, &dir, input)?;+ // "car, road": each phrase on its own, merged (the per-pixel maximum)+ let mut merged: Option<Vec<f32>> = None;+ for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {+ if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {+ merged = Some(match merged {+ None => p.logits,+ Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(),+ });+ }+ }+ Ok(merged.map(|l| SegMask::from_logits(lightcraft_segment::MASK_SIDE, &l)))+ }+ #[cfg(not(feature = "sam"))]+ {+ let _ = (dir, id);+ Err("AI masks are not available in this build".into())+ }+ }++ /// Render the part `region` (normalized) of photo `id` for the model, from a source sharp+ /// enough for it.+ #[cfg_attr(not(feature = "sam"), allow(dead_code))]+ fn segment_region_input(&mut self, id: PhotoId, region: [f64; 4]) -> Result<Input, String> {+ let (_, mut d) = self.segment_key(id).ok_or("no such photo")?;+ d.crop.geometry = lightcraft_geom::CropGeometry { rect: lightcraft_geom::Rect::new(region[0], region[1], region[2], region[3]), angle: 0.0 };+ d.crop.flip_h = false;+ d.crop.flip_v = false;+ d.geometry.constrain_crop = false;+ let mut job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, true, &d).ok_or("no such photo")?;+ let frac = (region[2] - region[0]).max(region[3] - region[1]).max(0.01);+ let p = self.catalog.photo(id).ok_or("no such photo")?.clone();+ job.level = crate::media::SourceLevel::for_size((INPUT_EDGE as f64 / frac) as usize);+ job.source = self.media.source_ref(&p, job.level);+ let r = job.run();+ self.accept(&r);+ let img = r.rendered.map_err(|e| format!("couldn't render the photo for AI masks: {e}"))?.image;+ let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect();+ Ok(Input { key: 0, rgb, w: img.width, h: img.height })+ }++ /// Start a zoomed-in pass for component `comp` of mask `mask` on photo `id` (an Object or+ /// Describe selection): the photo around the selection is rendered at a higher resolution+ /// and segmented again, on a worker thread; [`Session::segment_poll`] applies the result.+ /// `Ok(false)` when there is nothing to refine (no selection yet, or it is most of the photo).+ pub fn segment_detail(&mut self, id: PhotoId, mask: u32, comp: usize) -> Result<bool, String> {+ let dir = self.segmenter.model_dir()?;+ let d = self.develop_of(id).ok_or("no such photo")?;+ let shape = d.masks.iter().find(|m| m.id == mask).and_then(|m| m.components.get(comp)).map(|c| c.shape.clone()).ok_or("no such mask")?;+ let (seg, prompt) = match &shape {+ MaskShape::Object { hint, exclude, seg: Some(seg), .. } if !hint.is_empty() => (+ seg.clone(),+ Prompt::Clicks(+ hint.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect(),+ ),+ ),+ MaskShape::Prompt { text, seg: Some(seg), .. } => (seg.clone(), Prompt::Text(text.clone())),+ _ => return Ok(false),+ };+ let p = self.catalog.photo(id).ok_or("no such photo")?;+ let (pw, ph) = (f64::from(p.width.max(1)), f64::from(p.height.max(1)));+ let aspect = if d.orientation.swaps_axes() { ph / pw } else { pw / ph };+ let Some(region) = detail_region(&seg, aspect) else { return Ok(false) };+ #[cfg(feature = "sam")]+ {+ let shared = self.segmenter.shared.clone();+ if shared.detail_busy.swap(true, std::sync::atomic::Ordering::Relaxed) {+ return Ok(false);+ }+ let input = match self.segment_region_input(id, region) {+ Ok(i) => i,+ Err(e) => {+ shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed);+ return Err(e);+ }+ };+ let shape = strip_detail(shape);+ let spawned = std::thread::Builder::new().name("sam3-detail".into()).spawn({+ let shared = shared.clone();+ move || {+ let patch = {+ let mut inner = shared.lock();+ detail_pass(&mut inner, &dir, &input, &prompt, region)+ };+ match patch {+ Ok(patch) => {+ let r = DetailResult { photo: id, mask, comp, shape, patch };+ shared.results.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(r);+ }+ Err(e) => log::warn!("SAM 3 detail pass: {e}"),+ }+ shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed);+ }+ });+ if let Err(e) = spawned {+ shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed);+ return Err(format!("could not start the detail pass: {e}"));+ }+ Ok(true)+ }+ #[cfg(not(feature = "sam"))]+ {+ let _ = (dir, prompt, region, mask);+ Ok(false)+ }+ }++ /// Apply finished detail passes (to components that haven't changed since). True when a+ /// mask changed.+ pub fn segment_poll(&mut self) -> bool {+ #[cfg(feature = "sam")]+ {+ let done: Vec<DetailResult> =+ std::mem::take(&mut *self.segmenter.shared.results.lock().unwrap_or_else(std::sync::PoisonError::into_inner));+ let mut changed = false;+ for r in done {+ let Some(patch) = r.patch else { continue };+ let Some(d) = self.develop_of(r.photo) else { continue };+ let mut d = (*d).clone();+ let Some(c) = d.masks.iter_mut().find(|m| m.id == r.mask).and_then(|m| m.components.get_mut(r.comp)) else { continue };+ if strip_detail(c.shape.clone()) != r.shape {+ continue; // clicked again meanwhile: a newer pass will follow+ }+ match &mut c.shape {+ MaskShape::Object { detail, .. } | MaskShape::Prompt { detail, .. } => *detail = vec![patch],+ _ => continue,+ }+ match self.set_develop(r.photo, d, "AI Mask Detail") {+ Ok(()) => changed = true,+ Err(e) => log::warn!("AI mask detail: {e}"),+ }+ }+ changed+ }+ #[cfg(not(feature = "sam"))]+ false+ }++ /// The model input for `id`, unless the cache already holds it (or a prepare is running,+ /// which will have it).+ #[cfg(feature = "sam")]+ fn segment_input_if_needed(&mut self, id: PhotoId) -> Result<Either, String> {+ let (key, _) = self.segment_key(id).ok_or("no such photo")?;+ if self.segmenter.shared.busy() {+ return Ok(Either::Key(key));+ }+ let cached = self.segmenter.shared.lock().cache.as_ref().is_some_and(|c| c.0 == key);+ if cached { Ok(Either::Key(key)) } else { Ok(Either::Input(self.segment_input(id)?)) }+ }+}++#[cfg(feature = "sam")]+enum Either {+ /// Already encoded (or being encoded) under this key.+ Key(u64),+ Input(Input),+}++#[cfg(feature = "sam")]+fn load(inner: &mut model::Inner, dir: &std::path::Path) -> Result<(), String> {+ if inner.model.is_none() {+ let t = web_time::Instant::now();+ inner.model = Some(lightcraft_segment::Sam3::load(dir).map_err(|e| e.to_string())?);+ log::info!("SAM 3 loaded in {:?}", t.elapsed());+ }+ Ok(())+}++/// Load the model and encode `input` into the cache (with the click features ready).+#[cfg(feature = "sam")]+fn prepare(inner: &mut model::Inner, dir: &std::path::Path, input: &Input) -> Result<(), String> {+ load(inner, dir)?;+ let model = inner.model.as_mut().ok_or("the model did not load")?;+ let t = web_time::Instant::now();+ let mut enc = model.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;+ model.prepare_clicks(&mut enc).map_err(|e| e.to_string())?;+ log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());+ inner.cache = Some((input.key, enc));+ Ok(())+}++/// The model and the encoded image for `want` (encoding it now when needed).+#[cfg(feature = "sam")]+fn encoded<'a>(+ inner: &'a mut model::Inner,+ dir: &std::path::Path,+ want: Either,+) -> Result<(&'a mut lightcraft_segment::Sam3, &'a mut lightcraft_segment::Encoded), String> {+ match want {+ Either::Input(input) => prepare(inner, dir, &input)?,+ Either::Key(key) => {+ if !inner.cache.as_ref().is_some_and(|c| c.0 == key) {+ return Err("the photo changed while it was being prepared; try again".into());+ }+ }+ }+ let model::Inner { model, cache } = inner;+ match (model.as_mut(), cache.as_mut()) {+ (Some(m), Some((_, e))) => Ok((m, e)),+ _ => Err("the model is not ready".into()),+ }+}++/// `shape` without its detail patches (what a detail pass was computed from)+#[cfg_attr(not(feature = "sam"), allow(dead_code))]+fn strip_detail(mut shape: MaskShape) -> MaskShape {+ // (nor its edge setting: moving the slider meanwhile keeps the pass)+ if let MaskShape::Object { detail, edge, .. } | MaskShape::Prompt { detail, edge, .. } = &mut shape {+ detail.clear();+ *edge = 0.0;+ }+ shape+}++/// Segment `input` (the photo's part `region`) with `prompt`, as a patch over `region`.+#[cfg(feature = "sam")]+fn detail_pass(inner: &mut model::Inner, dir: &std::path::Path, input: &Input, prompt: &Prompt, region: [f64; 4]) -> Result<Option<SegMask>, String> {+ load(inner, dir)?;+ let model = inner.model.as_mut().ok_or("the model did not load")?;+ let t = web_time::Instant::now();+ let mut enc = model.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;+ let (rw, rh) = (region[2] - region[0], region[3] - region[1]);+ let logits = match prompt {+ Prompt::Clicks(clicks) => {+ let inside: Vec<lightcraft_segment::Click> = clicks+ .iter()+ .map(|c| ((c.at.x - region[0]) / rw, (c.at.y - region[1]) / rh, c.include))+ .filter(|(x, y, _)| (0.0..=1.0).contains(x) && (0.0..=1.0).contains(y))+ .map(|(x, y, positive)| lightcraft_segment::Click { x: x as f32, y: y as f32, positive })+ .collect();+ if !inside.iter().any(|c| c.positive) {+ return Ok(None);+ }+ Some(model.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits)+ }+ Prompt::Text(text) => {+ let mut merged: Option<Vec<f32>> = None;+ for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {+ if let Some(p) = model.segment_text(&mut enc, phrase, 0.5).map_err(|e| e.to_string())? {+ merged = Some(match merged {+ None => p.logits,+ Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(),+ });+ }+ }+ merged+ }+ };+ log::info!("SAM 3 detail pass ({}×{}) in {:?}", input.w, input.h, t.elapsed());+ Ok(logits.map(|l| SegMask::from_logits_in(lightcraft_segment::MASK_SIDE, &l, region)))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask {+ let l: Vec<f32> =+ (0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect();+ SegMask::from_logits(side, &l)+ }++ #[test]+ fn detail_region_frames_the_selection_with_a_margin() {+ // a small object in the middle of a square photo+ let r = detail_region(&seg_box(100, 40, 45, 50, 55), 1.0).unwrap();+ assert!(r[0] < 0.4 && r[2] > 0.5 && r[1] < 0.45 && r[3] > 0.55, "contains it: {r:?}");+ assert!(r[2] - r[0] < 0.25 && r[3] - r[1] < 0.25, "zoomed in: {r:?}");+ assert!(r.iter().all(|v| (0.0..=1.0).contains(v)));+ // at the corner: shifted inside the photo, not cut+ let r = detail_region(&seg_box(100, 0, 0, 5, 5), 1.5).unwrap();+ assert!(r[0] >= 0.0 && r[1] >= 0.0 && r[0] < 1e-9 && r[1] < 1e-9);+ // a thin object stays at most 2:1 (in photo pixels, 1.5:1 photo)+ let r = detail_region(&seg_box(100, 10, 50, 90, 52), 1.5).unwrap();+ let (w, h) = ((r[2] - r[0]) * 1.5, r[3] - r[1]);+ assert!(w / h <= 2.0 + 1e-9, "{w} × {h}");+ }++ #[test]+ fn no_region_for_nothing_or_most_of_the_photo() {+ assert!(detail_region(&seg_box(50, 0, 0, 0, 0), 1.0).is_none());+ assert!(detail_region(&seg_box(50, 2, 2, 48, 48), 1.0).is_none());+ let damaged = SegMask { side: 50, data: "?".into(), rect: None };+ assert!(detail_region(&damaged, 1.0).is_none());+ }+}diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rsindex 58b7e643e..65104127f 100644--- a/crates/engine/src/tests.rs+++ b/crates/engine/src/tests.rs@@ -727,3 +727,37 @@ fn curve_reset_by_channel_and_whole() {assert_eq!(active_dev(&s).curve, lightcraft_develop::ToneCurve::default());assert!(s.execute("curve.reset", &json!({"channel": "alpha"})).is_err());}++#[test]+fn ai_masks_without_the_model_explain_themselves() {+ let mut s = demo();+ let id = s.active().unwrap();+ // an Object mask that could never be computed is refused, not left empty+ let e = s.execute("mask.add", &json!({"kind": "object"})).unwrap_err().to_string();+ assert!(e.contains("SAM 3") || e.contains("not available"), "{e}");+ assert!(!e.contains("invalid parameters"), "shown as is: {e}");+ assert!(s.develop_of(id).unwrap().masks.is_empty());+ // an empty Object component (as an older document may hold) still takes clicks+ let shape = lightcraft_develop::MaskShape::Object { hint: vec![], exclude: vec![], seg: None, detail: vec![], edge: 0.0 };+ let mut d = (*s.develop_of(id).unwrap()).clone();+ d.masks.push(lightcraft_develop::Mask {+ id: 1,+ components: vec![lightcraft_develop::MaskComponent { name: None, op: lightcraft_develop::MaskOp::Add, invert: false, shape }],+ ..Default::default()+ });+ s.set_develop(id, d, "test").unwrap();+ s.active_mask = Some(1);+ // clicks and descriptions need the model: an error that says what's missing, nothing changed+ let e = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap_err().to_string();+ assert!(e.contains("SAM 3") || e.contains("not available"), "{e}");+ let e = s.execute("mask.add", &json!({"kind": "prompt", "text": "sky"})).unwrap_err().to_string();+ assert!(e.contains("SAM 3") || e.contains("not available"), "{e}");+ let empty = std::env::temp_dir().join(format!("lc-no-sam3-{}", std::process::id()));+ s.segmenter.dir = Some(empty.clone());+ let e = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap_err().to_string();+ assert!(e.contains(&empty.display().to_string()) || e.contains("not available"), "names the folder: {e}");+ assert_eq!(s.develop_of(id).unwrap().masks.len(), 1);+ // bad input is an error, not a crash+ assert!(s.execute("mask.objectPoint", &json!({"x": 1.5, "y": 0.5})).is_err());+ assert!(s.execute("mask.add", &json!({"kind": "prompt", "text": " "})).is_err());+}diff --git a/crates/pipeline/src/masks.rs b/crates/pipeline/src/masks.rsindex 5c6e003cb..01985b5b2 100644--- a/crates/pipeline/src/masks.rs+++ b/crates/pipeline/src/masks.rs@@ -3,7 +3,7 @@//! Shapes are defined in normalized oriented-image coordinates; sizes are in "long-edge units".//! Components combine with Add (max), Subtract (a·(1−c)) and Intersect (a·c).-use lightcraft_develop::{BrushStroke, LocalAdjustments, Mask, MaskOp, MaskShape};+use lightcraft_develop::{BrushStroke, LocalAdjustments, Mask, MaskOp, MaskShape, SegMask};use lightcraft_geom::Point;use lightcraft_raster::{Plane, Rgb32f};@@ -153,6 +153,19 @@ pub fn shape_alpha(shape: &MaskShape, frame: &Frame, w: usize, h: usize, img: &R}smooth_plane(&mut out, 0.01 * frame_px(frame, w));}+ MaskShape::Object { seg, detail, edge, .. } | MaskShape::Prompt { seg, detail, edge, .. } if seg.is_some() || !detail.is_empty() => {+ // Edge: below 0 a steeper transition (up to 8× the logits), above 0 feathered (up to+ // 2 % of the long edge, so previews and exports match)+ let e = (*edge / 100.0).clamp(-1.0, 1.0) as f32;+ let gain = if e < 0.0 { 1.0 - 7.0 * e } else { 1.0 };+ sample_seg(seg.as_ref(), detail, gain, frame, w, h, &mut out);+ if e > 0.0 {+ smooth_plane(&mut out, e * 0.02 * frame_px(frame, w));+ }+ }+ // not computed yet (no clicks, or a build without the model): nothing selected+ MaskShape::Object { hint, .. } if hint.is_empty() => {}+ MaskShape::Prompt { .. } => {}MaskShape::Subject | MaskShape::Object { .. } | MaskShape::People { .. } => {// Saliency heuristic: centre-weighted local contrast (replaced by the segmenter in M12).let m = frame.out_to_norm(w, h);@@ -176,6 +189,60 @@ pub fn shape_alpha(shape: &MaskShape, frame: &Frame, w: usize, h: usize, img: &Rout}+/// A decoded [`SegMask`]: its logits and the normalized rectangle they cover.+struct SegGrid {+ logits: Vec<f32>,+ side: usize,+ r: [f64; 4],+}++impl SegGrid {+ fn new(seg: &SegMask) -> Option<SegGrid> {+ Some(SegGrid { logits: seg.logits()?, side: seg.side as usize, r: seg.bounds()? })+ }++ fn contains(&self, n: Point) -> bool {+ n.x >= self.r[0] && n.x <= self.r[2] && n.y >= self.r[1] && n.y <= self.r[3]+ }++ /// The bilinearly sampled logit at normalized image point `n` (cell centres at+ /// `(i + 0.5) / side` of the rectangle).+ fn logit(&self, n: Point) -> f32 {+ let side = self.side;+ let at = |x: usize, y: usize| self.logits.get(y.min(side - 1) * side + x.min(side - 1)).copied().unwrap_or(-16.0);+ let u = (n.x - self.r[0]) / (self.r[2] - self.r[0]);+ let v = (n.y - self.r[1]) / (self.r[3] - self.r[1]);+ let (fx, fy) = ((u * side as f64 - 0.5) as f32, (v * side as f64 - 0.5) as f32);+ let (fx, fy) = (fx.clamp(0.0, (side - 1) as f32), fy.clamp(0.0, (side - 1) as f32));+ let (x0, y0) = (fx.floor() as usize, fy.floor() as usize);+ let (tx, ty) = (fx - x0 as f32, fy - y0 as f32);+ let top = at(x0, y0) * (1.0 - tx) + at(x0 + 1, y0) * tx;+ let bot = at(x0, y0 + 1) * (1.0 - tx) + at(x0 + 1, y0 + 1) * tx;+ top * (1.0 - ty) + bot * ty+ }+}++/// Stored model segmentations at each output pixel: the zoomed-in `detail` passes inside+/// their rectangles (the highest of them where they overlap), the whole-photo `seg`+/// elsewhere; the logit (times `gain`: > 1 is a harder edge) through a sigmoid. Damaged+/// data selects nothing.+fn sample_seg(seg: Option<&SegMask>, detail: &[SegMask], gain: f32, frame: &Frame, w: usize, h: usize, out: &mut Plane) {+ let coarse = seg.and_then(SegGrid::new);+ let patches: Vec<SegGrid> = detail.iter().filter_map(SegGrid::new).collect();+ if coarse.is_none() && patches.is_empty() {+ return;+ }+ let m = frame.out_to_norm(w, h);+ for_rows(&mut out.data, w, |y, row| {+ for (x, v) in row.iter_mut().enumerate() {+ let n = m.apply(Point::new(x as f64 + 0.5, y as f64 + 0.5));+ let fine = patches.iter().filter(|p| p.contains(n)).map(|p| p.logit(n)).reduce(f32::max);+ let l = fine.or_else(|| coarse.as_ref().map(|c| c.logit(n))).unwrap_or(-16.0);+ *v = 1.0 / (1.0 + (-l * gain).exp());+ }+ });+}+fn frame_px(frame: &Frame, w: usize) -> f32 {frame.px_per_long(w) as f32}@@ -356,6 +423,78 @@ mod tests {assert!((a.get(50, 25) - 0.5).abs() < 0.05);}+ #[test]+ fn stored_segmentations_render_at_any_size_and_follow_the_crop() {+ // the left half selected, on a 16² logit grid+ let side = 16;+ let logits: Vec<f32> = (0..side * side).map(|i| if i % side < side / 2 { 12.0 } else { -12.0 }).collect();+ let seg = SegMask::from_logits(side, &logits);+ for shape in [+ MaskShape::Object { hint: vec![Point::new(0.25, 0.5)], exclude: vec![], seg: Some(seg.clone()), detail: vec![], edge: 0.0 },+ MaskShape::Prompt { text: "left".into(), seg: Some(seg.clone()), detail: vec![], edge: 0.0 },+ ] {+ for (w, h) in [(40, 20), (400, 200)] {+ let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0);+ assert!(a.get(1, h / 2) > 0.99 && a.get(w - 2, h / 2) < 0.01, "{w}×{h}");+ if w >= 400 {+ let soft = (0..w).filter(|x| (0.05..0.95).contains(&a.get(*x, h / 2))).count();+ assert!((2..30).contains(&soft), "an anti-aliased edge, not a ramp: {soft} px");+ }+ }+ }+ // cropped to the right half: nothing left selected+ let mut d = DevelopSettings::default();+ d.crop.geometry.rect = lightcraft_geom::Rect::from_center(Point::new(0.75, 0.5), 0.5, 1.0);+ let f = Frame::new(40, 20, &d, true);+ let shape = MaskShape::Prompt { text: "left".into(), seg: Some(seg), detail: vec![], edge: 0.0 };+ let a = shape_alpha(&shape, &f, 20, 20, &Rgb32f::new(20, 20), &Plane::new(20, 20), 0.0);+ assert!(a.data.iter().all(|v| *v < 0.05));+ }++ #[test]+ fn detail_patches_replace_the_coarse_mask_inside_their_rectangle() {+ // coarse: nothing; a patch over the right half selects its own left half (x 0.5..0.75)+ let coarse = SegMask::from_logits(4, &[-12.0; 16]);+ let side = 16;+ let l: Vec<f32> = (0..side * side).map(|i| if i % side < side / 2 { 12.0 } else { -12.0 }).collect();+ let patch = SegMask::from_logits_in(side, &l, [0.5, 0.0, 1.0, 1.0]);+ let shape = MaskShape::Object { hint: vec![Point::new(0.6, 0.5)], exclude: vec![], seg: Some(coarse), detail: vec![patch], edge: 0.0 };+ let (w, h) = (200, 100);+ let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0);+ assert!(a.get(20, 50) < 0.01, "outside the patch: the coarse mask");+ assert!(a.get(120, 50) > 0.99, "inside the patch, its selected half");+ assert!(a.get(190, 50) < 0.01, "inside the patch, its unselected half");+ }++ #[test]+ fn the_edge_setting_hardens_or_feathers_the_transition() {+ let side = 16;+ let l: Vec<f32> = (0..side * side).map(|i| ((i % side) as f32 - 7.5) * -0.6).collect(); // a gentle ramp+ let seg = SegMask::from_logits(side, &l);+ let (w, h) = (400, 40);+ let soft_px = |edge: f64| {+ let shape = MaskShape::Prompt { text: "x".into(), seg: Some(seg.clone()), detail: vec![], edge };+ let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0);+ assert!(a.get(2, h / 2) > 0.9 && a.get(w - 3, h / 2) < 0.1, "still the same selection at {edge}");+ (0..w).filter(|x| (0.1..0.9).contains(&a.get(*x, h / 2))).count()+ };+ let (hard, plain, soft) = (soft_px(-100.0), soft_px(0.0), soft_px(100.0));+ assert!(hard < plain && plain < soft, "transition widths {hard} < {plain} < {soft}");+ }++ #[test]+ fn ai_shapes_without_a_segmentation_select_nothing() {+ let f = frame(30, 30);+ let (img, l) = (Rgb32f::new(30, 30), Plane::new(30, 30));+ for shape in [+ MaskShape::Object { hint: vec![], exclude: vec![], seg: None, detail: vec![], edge: 0.0 },+ MaskShape::Prompt { text: "sky".into(), seg: None, detail: vec![], edge: 0.0 },+ MaskShape::Prompt { text: "sky".into(), seg: Some(SegMask { side: 4, data: "damaged!".into(), rect: None }), detail: vec![], edge: 0.0 },+ ] {+ assert!(shape_alpha(&shape, &f, 30, 30, &img, &l, 0.0).data.iter().all(|v| *v == 0.0));+ }+ }+#[test]fn radial_inside_and_invert() {let f = frame(100, 100);diff --git a/crates/segment/Cargo.toml b/crates/segment/Cargo.tomlnew file mode 100644index 000000000..336b052fe--- /dev/null+++ b/crates/segment/Cargo.toml@@ -0,0 +1,24 @@+[package]+name = "lightcraft-segment"+version.workspace = true+edition.workspace = true+license.workspace = true+description = "LightCraft segmentation: SAM 3 (Segment Anything with Concepts) inference in pure Rust (L3)"++[lints]+workspace = true++[dependencies]+serde = { workspace = true }+serde_json = { workspace = true }+thiserror = { workspace = true }+log = { workspace = true }++# the model runs natively only (the web build has no AI masks)+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]+candle-core = { workspace = true }+candle-nn = { workspace = true }++[target.'cfg(target_os = "macos")'.dependencies]+candle-core = { workspace = true, features = ["metal"] }+candle-nn = { workspace = true, features = ["metal"] }diff --git a/crates/segment/examples/sam3_bench.rs b/crates/segment/examples/sam3_bench.rsnew file mode 100644index 000000000..982f482e9--- /dev/null+++ b/crates/segment/examples/sam3_bench.rs@@ -0,0 +1,25 @@+//! Times SAM 3 on this machine (a synthetic gradient image; the first run includes compiling+//! the GPU kernels): `cargo run -p lightcraft-segment --release --example sam3_bench -- <model dir>`.++fn main() -> Result<(), Box<dyn std::error::Error>> {+ let mut args = std::env::args().skip(1);+ let dir = std::path::PathBuf::from(args.next().ok_or("usage: sam3_bench <model dir>")?);+ let t = std::time::Instant::now();+ let mut model = lightcraft_segment::Sam3::load(&dir)?;+ eprintln!("load: {:?} on {:?}", t.elapsed(), model.device());+ let (w, h) = (1200, 800);+ let rgb: Vec<u8> = (0..w * h * 3).map(|i| ((i / 3 % w) * 255 / w) as u8).collect();+ for run in 0..3 {+ let t = std::time::Instant::now();+ let mut enc = model.encode(&rgb, w, h)?;+ lightcraft_segment::sync(&model)?;+ let e = t.elapsed();+ let t = std::time::Instant::now();+ let p = model.segment_clicks(&mut enc, &[lightcraft_segment::Click { x: 0.5, y: 0.5, positive: true }])?;+ let c = t.elapsed();+ let t = std::time::Instant::now();+ let d = model.segment_text(&mut enc, "sky", 0.5)?;+ eprintln!("run {run}: encode {e:?}, click {c:?} (score {:.2}), text {:?} ({})", p.score, t.elapsed(), d.is_some());+ }+ Ok(())+}diff --git a/crates/segment/src/clip.rs b/crates/segment/src/clip.rsnew file mode 100644index 000000000..c392d5c5f--- /dev/null+++ b/crates/segment/src/clip.rs@@ -0,0 +1,77 @@+//! The CLIP text encoder (24 layers, 1024 wide, causal) that turns a prompt into the token+//! features SAM 3's detector attends to.++use candle_core::{Device, Module, Tensor};+use candle_nn::{Embedding, LayerNorm, VarBuilder};++use crate::Result;+use crate::nn::{Attention, Mlp, ln};+use crate::tokenizer::CONTEXT;++const HIDDEN: usize = 1024;+const LAYERS: usize = 24;+const HEADS: usize = 16;+const EPS: f64 = 1e-5;++struct Layer {+ ln1: LayerNorm,+ attn: Attention,+ ln2: LayerNorm,+ mlp: Mlp,+}++pub struct TextEncoder {+ tokens: Embedding,+ positions: Tensor,+ layers: Vec<Layer>,+ final_ln: LayerNorm,+ device: Device,+}++impl TextEncoder {+ /// `vb` at `detector_model.text_encoder.text_model`.+ pub fn new(vb: VarBuilder) -> Result<Self> {+ let e = vb.pp("embeddings");+ let mut layers = Vec::with_capacity(LAYERS);+ for i in 0..LAYERS {+ let l = vb.pp(format!("encoder.layers.{i}"));+ layers.push(Layer {+ ln1: ln(HIDDEN, EPS, l.pp("layer_norm1"))?,+ attn: Attention::named(HIDDEN, HIDDEN, HEADS, "out_proj", l.pp("self_attn"))?,+ ln2: ln(HIDDEN, EPS, l.pp("layer_norm2"))?,+ mlp: Mlp::new(HIDDEN, 4 * HIDDEN, true, l.pp("mlp"))?,+ });+ }+ Ok(Self {+ tokens: candle_nn::embedding(49408, HIDDEN, e.pp("token_embedding"))?,+ positions: e.get((CONTEXT, HIDDEN), "position_embedding.weight")?.unsqueeze(0)?,+ layers,+ final_ln: ln(HIDDEN, EPS, vb.pp("final_layer_norm"))?,+ device: vb.device().clone(),+ })+ }++ /// `[1, 32, 1024]` features for token `ids` (`valid` marks real tokens).+ pub fn forward(&self, ids: &[u32], valid: &[bool]) -> Result<Tensor> {+ let ids = Tensor::from_slice(ids, (1, ids.len()), &self.device)?;+ let mut x = self.tokens.forward(&ids)?.broadcast_add(&self.positions)?;+ // causal, and padding keys masked+ let n = valid.len();+ let mut bias = vec![0f32; n * n];+ for (q, row) in bias.chunks_mut(n).enumerate() {+ for (k, v) in row.iter_mut().enumerate() {+ if k > q || !valid.get(k).copied().unwrap_or(false) {+ *v = f32::MIN;+ }+ }+ }+ let bias = Tensor::from_vec(bias, (1, 1, n, n), &self.device)?;+ for l in &self.layers {+ let h = l.ln1.forward(&x)?;+ x = (&x + l.attn.forward(&h, &h, &h, Some(&bias))?)?;+ let h = l.ln2.forward(&x)?;+ x = (&x + l.mlp.forward(&h)?)?;+ }+ Ok(self.final_ln.forward(&x)?)+ }+}diff --git a/crates/segment/src/detector.rs b/crates/segment/src/detector.rsnew file mode 100644index 000000000..b21b32d84--- /dev/null+++ b/crates/segment/src/detector.rs@@ -0,0 +1,304 @@+//! Text prompts (SAM 3's concept detector): the prompt's CLIP features fused into the image+//! features by a DETR encoder, 200 object queries refined by a DETR decoder (with box+//! relative-position bias and a presence token), per-query masks from a pixel decoder.++use candle_core::{D, Device, IndexOp, Module, Tensor};+use candle_nn::{Conv2d, Conv2dConfig, GroupNorm, LayerNorm, Linear, VarBuilder};++use crate::clip::TextEncoder;+use crate::neck::{FPN, Neck};+use crate::nn::{Attention, DecoderMlp, Mlp, TORCH_LN_EPS, cxcywh_to_xyxy, inverse_sigmoid, linear, ln, padding_bias, sine_grid, to_vec};+use crate::{Error, Prediction, Result};++const HIDDEN: usize = FPN;+const HEADS: usize = 8;+const QUERIES: usize = 200;+const LAYERS: usize = 6;++/// Detector image features, computed once per image.+pub struct Features {+ /// Levels 0..=2 (288, 144, 72 per side).+ levels: Vec<Tensor>,+ /// Level 2 flattened `[1, 5184, 256]` and its sine position encoding.+ flat: Tensor,+ pos: Tensor,+ side: usize,+}++/// Everything the detector predicts for one prompt.+pub struct Output {+ /// Per-query mask logits `[200, 288, 288]` flattened.+ pub masks: Tensor,+ /// Per-query logits (before the presence score).+ pub logits: Vec<f32>,+ /// Whether the concept is in the image at all (logit).+ pub presence: f32,+ /// Boxes (x0, y0, x1, y1), normalized.+ pub boxes: Vec<[f32; 4]>,+ /// The text features `[1, 32, 256]` (tests).+ pub text: Tensor,+}++struct EncoderLayer {+ ln1: LayerNorm,+ self_attn: Attention,+ ln2: LayerNorm,+ cross_attn: Attention,+ ln3: LayerNorm,+ mlp: Mlp,+}++struct DecoderLayer {+ self_attn: Attention,+ self_ln: LayerNorm,+ text_attn: Attention,+ text_ln: LayerNorm,+ vision_attn: Attention,+ vision_ln: LayerNorm,+ mlp: Mlp,+ mlp_ln: LayerNorm,+}++pub struct Detector {+ device: Device,+ neck: Neck,+ text: TextEncoder,+ text_proj: Linear,+ encoder: Vec<EncoderLayer>,+ decoder: Vec<DecoderLayer>,+ out_ln: LayerNorm,+ box_head: DecoderMlp,+ query_embed: Tensor,+ reference_points: Tensor,+ presence_token: Tensor,+ presence_head: DecoderMlp,+ presence_ln: LayerNorm,+ ref_point_head: DecoderMlp,+ rpb_x: DecoderMlp,+ rpb_y: DecoderMlp,+ // dot-product scoring+ score_text_mlp: DecoderMlp,+ score_text_ln: LayerNorm,+ score_text_proj: Linear,+ score_query_proj: Linear,+ // mask decoder+ prompt_attn: Attention,+ prompt_ln: LayerNorm,+ pixel_convs: Vec<Conv2d>,+ pixel_norms: Vec<GroupNorm>,+ instance_proj: Conv2d,+ mask_embedder: Vec<Linear>,+}++impl Detector {+ /// `vb` at `detector_model`.+ pub fn new(vb: VarBuilder) -> Result<Self> {+ let e = vb.pp("detr_encoder");+ let mut encoder = Vec::with_capacity(LAYERS);+ for i in 0..LAYERS {+ let l = e.pp(format!("layers.{i}"));+ encoder.push(EncoderLayer {+ ln1: ln(HIDDEN, TORCH_LN_EPS, l.pp("layer_norm1"))?,+ self_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("self_attn"))?,+ ln2: ln(HIDDEN, TORCH_LN_EPS, l.pp("layer_norm2"))?,+ cross_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("cross_attn"))?,+ ln3: ln(HIDDEN, TORCH_LN_EPS, l.pp("layer_norm3"))?,+ mlp: Mlp::new(HIDDEN, 2048, false, l.pp("mlp"))?,+ });+ }+ let d = vb.pp("detr_decoder");+ let mut decoder = Vec::with_capacity(LAYERS);+ for i in 0..LAYERS {+ let l = d.pp(format!("layers.{i}"));+ decoder.push(DecoderLayer {+ self_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("self_attn"))?,+ self_ln: ln(HIDDEN, TORCH_LN_EPS, l.pp("self_attn_layer_norm"))?,+ text_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("text_cross_attn"))?,+ text_ln: ln(HIDDEN, TORCH_LN_EPS, l.pp("text_cross_attn_layer_norm"))?,+ vision_attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("vision_cross_attn"))?,+ vision_ln: ln(HIDDEN, TORCH_LN_EPS, l.pp("vision_cross_attn_layer_norm"))?,+ mlp: Mlp::new(HIDDEN, 2048, false, l.pp("mlp"))?,+ mlp_ln: ln(HIDDEN, TORCH_LN_EPS, l.pp("mlp_layer_norm"))?,+ });+ }+ let s = vb.pp("dot_product_scoring");+ let m = vb.pp("mask_decoder");+ let p = m.pp("pixel_decoder");+ let conv3 = Conv2dConfig { padding: 1, ..Default::default() };+ Ok(Self {+ device: vb.device().clone(),+ neck: Neck::new(vb.pp("vision_encoder.neck"))?,+ text: TextEncoder::new(vb.pp("text_encoder.text_model"))?,+ text_proj: linear(1024, HIDDEN, vb.pp("text_projection"))?,+ encoder,+ decoder,+ out_ln: ln(HIDDEN, TORCH_LN_EPS, d.pp("output_layer_norm"))?,+ box_head: DecoderMlp::new(HIDDEN, HIDDEN, 4, 3, d.pp("box_head"))?,+ query_embed: d.get((QUERIES, HIDDEN), "query_embed.weight")?.unsqueeze(0)?,+ reference_points: d.get((QUERIES, 4), "reference_points.weight")?.unsqueeze(0)?,+ presence_token: d.get((1, HIDDEN), "presence_token.weight")?.unsqueeze(0)?,+ presence_head: DecoderMlp::new(HIDDEN, HIDDEN, 1, 3, d.pp("presence_head"))?,+ presence_ln: ln(HIDDEN, TORCH_LN_EPS, d.pp("presence_layer_norm"))?,+ ref_point_head: DecoderMlp::new(2 * HIDDEN, HIDDEN, HIDDEN, 2, d.pp("ref_point_head"))?,+ rpb_x: DecoderMlp::new(2, HIDDEN, HEADS, 2, d.pp("box_rpb_embed_x"))?,+ rpb_y: DecoderMlp::new(2, HIDDEN, HEADS, 2, d.pp("box_rpb_embed_y"))?,+ score_text_mlp: DecoderMlp::new(HIDDEN, 2048, HIDDEN, 2, s.pp("text_mlp"))?,+ score_text_ln: ln(HIDDEN, TORCH_LN_EPS, s.pp("text_mlp_out_norm"))?,+ score_text_proj: linear(HIDDEN, HIDDEN, s.pp("text_proj"))?,+ score_query_proj: linear(HIDDEN, HIDDEN, s.pp("query_proj"))?,+ prompt_attn: Attention::new(HIDDEN, HIDDEN, HEADS, m.pp("prompt_cross_attn"))?,+ prompt_ln: ln(HIDDEN, TORCH_LN_EPS, m.pp("prompt_cross_attn_norm"))?,+ pixel_convs: (0..3).map(|i| Ok(candle_nn::conv2d(HIDDEN, HIDDEN, 3, conv3, p.pp(format!("conv_layers.{i}")))?)).collect::<Result<_>>()?,+ pixel_norms: (0..3).map(|i| Ok(candle_nn::group_norm(8, HIDDEN, 1e-5, p.pp(format!("norms.{i}")))?)).collect::<Result<_>>()?,+ instance_proj: candle_nn::conv2d(HIDDEN, HIDDEN, 1, Conv2dConfig::default(), m.pp("instance_projection"))?,+ mask_embedder: (0..3).map(|i| linear(HIDDEN, HIDDEN, m.pp(format!("mask_embedder.layers.{i}")))).collect::<Result<_>>()?,+ })+ }++ /// The detector's feature pyramid for the backbone output `[1, 1024, 72, 72]`.+ pub fn features(&self, backbone: &Tensor) -> Result<Features> {+ let levels = self.neck.forward(backbone, 3)?;+ let last = levels.last().ok_or_else(|| Error::Model("detector neck".into()))?;+ let (_, _, side, _) = last.dims4()?;+ let flat = last.flatten_from(2)?.transpose(1, 2)?.contiguous()?;+ let pos = sine_grid(side, side, HIDDEN / 2, &self.device)?.flatten_from(2)?.transpose(1, 2)?.contiguous()?;+ Ok(Features { levels, flat, pos, side })+ }++ /// Box relative-position bias `[1, heads, 1 + 200, side²]` for reference boxes (cx, cy, w, h).+ fn rpb(&self, boxes: &Tensor, side: usize) -> Result<Tensor> {+ let xyxy = cxcywh_to_xyxy(boxes)?.squeeze(0)?; // [200, 4]+ let coords = Tensor::arange(0u32, side as u32, &self.device)?.to_dtype(candle_core::DType::F32)?.affine(1.0 / side as f64, 0.0)?;+ let coords = coords.reshape((1, side, 1))?;+ let log_scale = |d: Tensor| -> Result<Tensor> {+ let d = (d * 8.0)?;+ Ok(d.sign()?.mul(&((d.abs()? + 1.0)?.log()? / (8f64.ln()))?)?)+ };+ let pick = |a: usize, b: usize| -> Result<Tensor> { Ok(Tensor::cat(&[xyxy.narrow(1, a, 1)?, xyxy.narrow(1, b, 1)?], 1)?.unsqueeze(1)?) };+ let dy = log_scale(coords.broadcast_sub(&pick(1, 3)?)?)?; // [200, side, 2]+ let dx = log_scale(coords.broadcast_sub(&pick(0, 2)?)?)?;+ let ey = self.rpb_y.forward(&dy)?; // [200, side, heads]+ let ex = self.rpb_x.forward(&dx)?;+ let m = ey.unsqueeze(2)?.broadcast_add(&ex.unsqueeze(1)?)?; // [200, side, side, heads]+ let m = m.reshape((QUERIES, side * side, HEADS))?.permute((2, 0, 1))?; // [heads, 200, side²]+ let m = m.pad_with_zeros(1, 1, 0)?; // the presence token attends everywhere+ Ok(m.unsqueeze(0)?.contiguous()?)+ }++ /// Sine embedding of boxes (cx, cy, w, h) `[1, 200, 512]` (y, x, w, h features).+ fn box_sine(&self, boxes: &Tensor) -> Result<Tensor> {+ let feats = HIDDEN / 2;+ let dim_t: Vec<f32> = (0..feats).map(|i| 10000f64.powf(2.0 * (i / 2) as f64 / feats as f64) as f32).collect();+ let dim_t = Tensor::from_vec(dim_t, (1, 1, feats), &self.device)?;+ let even: Vec<f32> = (0..feats).map(|i| if i % 2 == 0 { 1.0 } else { 0.0 }).collect();+ let even = Tensor::from_vec(even, (1, 1, feats), &self.device)?;+ let one = |i: usize| -> Result<Tensor> {+ let p = (boxes.narrow(D::Minus1, i, 1)? * (2.0 * std::f64::consts::PI))?.broadcast_div(&dim_t)?;+ Ok((p.sin()?.broadcast_mul(&even)? + p.cos()?.broadcast_mul(&(1.0 - &even)?)?)?)+ };+ Ok(Tensor::cat(&[one(1)?, one(0)?, one(2)?, one(3)?], D::Minus1)?)+ }++ /// Run the detector for one tokenized prompt.+ pub fn predict(&self, f: &Features, ids: &[u32], valid: &[bool]) -> Result<Output> {+ let text = self.text_proj.forward(&self.text.forward(ids, valid)?)?; // [1, 32, 256]+ let text_bias = padding_bias(valid, &self.device)?;++ // DETR encoder: image tokens attend to themselves and to the prompt+ let mut x = f.flat.clone();+ for l in &self.encoder {+ let h = l.ln1.forward(&x)?;+ let hp = (&h + &f.pos)?;+ x = (&x + l.self_attn.forward(&hp, &hp, &h, None)?)?;+ let h = l.ln2.forward(&x)?;+ x = (&x + l.cross_attn.forward(&h, &text, &text, Some(&text_bias))?)?;+ let h = l.ln3.forward(&x)?;+ x = (&x + l.mlp.forward(&h)?)?;+ }+ let memory = x;++ // DETR decoder: presence token + 200 queries with iterative box refinement+ let mut boxes = candle_nn::ops::sigmoid(&self.reference_points)?;+ let mut h = Tensor::cat(&[self.presence_token.clone(), self.query_embed.clone()], 1)?;+ let key_pos = (&memory + &f.pos)?;+ let mut queries = None;+ let mut presence = None;+ for l in &self.decoder {+ let qpos = self.ref_point_head.forward(&self.box_sine(&boxes)?)?.pad_with_zeros(1, 1, 0)?;+ let bias = self.rpb(&boxes, f.side)?;+ let q = (&h + &qpos)?;+ h = l.self_ln.forward(&(&h + l.self_attn.forward(&q, &q, &h, None)?)?)?;+ let q = (&h + &qpos)?;+ h = l.text_ln.forward(&(&h + l.text_attn.forward(&q, &text, &text, Some(&text_bias))?)?)?;+ let q = (&h + &qpos)?;+ h = l.vision_ln.forward(&(&h + l.vision_attn.forward(&q, &key_pos, &memory, Some(&bias))?)?)?;+ h = l.mlp_ln.forward(&(&h + l.mlp.forward(&h)?)?)?;+ let qh = self.out_ln.forward(&h.narrow(1, 1, QUERIES)?)?;+ boxes = candle_nn::ops::sigmoid(&(self.box_head.forward(&qh)? + inverse_sigmoid(&boxes)?)?)?;+ let p = self.presence_head.forward(&self.presence_ln.forward(&h.narrow(1, 0, 1)?)?)?;+ presence = Some(p.clamp(-10f32, 10f32)?);+ queries = Some(qh);+ }+ let (Some(queries), Some(presence)) = (queries, presence) else { return Err(Error::Model("decoder has no layers".into())) };++ // dot-product scoring against the mean of the (refined) prompt tokens+ let t = self.score_text_ln.forward(&(self.score_text_mlp.forward(&text)? + &text)?)?;+ let w: Vec<f32> = valid.iter().map(|v| if *v { 1.0 } else { 0.0 }).collect();+ let count = w.iter().sum::<f32>().max(1.0);+ let w = Tensor::from_vec(w, (1, valid.len(), 1), &self.device)?;+ let pooled = (t.broadcast_mul(&w)?.sum(1)? / f64::from(count))?; // [1, 256]+ let pt = self.score_text_proj.forward(&pooled)?; // [1, 256]+ let pq = self.score_query_proj.forward(&queries)?; // [1, 200, 256]+ let logits = (pq.matmul(&pt.unsqueeze(2)?)? / (HIDDEN as f64).sqrt())?.clamp(-12f32, 12f32)?;++ // masks: the encoder memory attends to the prompt, then a pixel decoder up to 288²+ let mem = (&memory + self.prompt_attn.forward(&self.prompt_ln.forward(&memory)?, &text, &text, Some(&text_bias))?)?;+ let [l0, l1, _] = &f.levels[..] else { return Err(Error::Model("detector levels".into())) };+ let mut prev = mem.transpose(1, 2)?.reshape((1, HIDDEN, f.side, f.side))?;+ for (i, skip) in [l1, l0].into_iter().enumerate() {+ let (_, _, sh, sw) = skip.dims4()?;+ prev = (prev.upsample_nearest2d(sh, sw)? + skip)?;+ let (Some(conv), Some(norm)) = (self.pixel_convs.get(i), self.pixel_norms.get(i)) else {+ return Err(Error::Model("pixel decoder".into()));+ };+ prev = norm.forward(&conv.forward(&prev)?)?.relu()?;+ }+ let inst = self.instance_proj.forward(&prev)?; // [1, 256, 288, 288]+ let (_, c, mh, mw) = inst.dims4()?;+ let mut e = queries.clone();+ for (i, l) in self.mask_embedder.iter().enumerate() {+ e = l.forward(&e)?;+ if i + 1 < self.mask_embedder.len() {+ e = e.relu()?;+ }+ }+ let masks = e.matmul(&inst.reshape((1, c, mh * mw))?)?.reshape((QUERIES, mh, mw))?;+ let boxes = cxcywh_to_xyxy(&boxes)?;+ let bv = to_vec(&boxes)?;+ Ok(Output {+ masks,+ logits: to_vec(&logits)?,+ presence: to_vec(&presence)?.first().copied().unwrap_or(-10.0),+ boxes: bv.as_chunks::<4>().0.to_vec(),+ text,+ })+ }+}++/// The instances of `out` scoring above `threshold` (sigmoid(logit) × sigmoid(presence)),+/// merged into one mask (the per-pixel maximum of their probabilities, as logits).+pub fn merge(out: &Output, threshold: f32) -> Option<Prediction> {+ let s = |x: f32| 1.0 / (1.0 + (-x).exp());+ let presence = s(out.presence);+ let keep: Vec<(usize, f32)> = out.logits.iter().enumerate().map(|(i, l)| (i, s(*l) * presence)).filter(|(_, p)| *p > threshold).collect();+ let best = keep.iter().map(|k| k.1).fold(0f32, f32::max);+ let mut merged: Option<Vec<f32>> = None;+ for (i, _) in &keep {+ let m = out.masks.i(*i).ok().and_then(|t| to_vec(&t).ok())?;+ merged = Some(match merged {+ None => m,+ Some(acc) => acc.iter().zip(&m).map(|(a, b)| a.max(*b)).collect(),+ });+ }+ merged.map(|logits| Prediction { logits, score: best })+}diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rsnew file mode 100644index 000000000..83bcbbafe--- /dev/null+++ b/crates/segment/src/lib.rs@@ -0,0 +1,252 @@+//! SAM 3 (Segment Anything with Concepts, Meta, 2025) inference in pure Rust on candle.+//!+//! - **Point prompts** ([`Sam3::segment_clicks`]): positive and negative clicks select one+//! object, via SAM 3's interactive (SAM 2 style) prompt encoder and mask decoder.+//! - **Text prompts** ([`Sam3::segment_text`]): a noun phrase ("the red car", "sky") selects+//! every instance of that concept, via the CLIP text encoder and the DETR detector.+//!+//! The expensive part, the 32-layer ViT over the 1008 × 1008 input, runs once per image+//! ([`Sam3::encode`]); each click or prompt then reuses the [`Encoded`] features.+//!+//! On macOS the model runs on the GPU through Metal; elsewhere (for now) on the CPU. The+//! weights are not part of LightCraft: they are the `facebook/sam3` checkpoint (SAM License),+//! read from a directory the user downloads them to ([`Sam3::load`]).+//!+//! Ported from the Hugging Face `transformers` implementation (Apache-2.0); see `NOTICE`.+//! No UI dependencies (L3).+#![forbid(unsafe_code)]+#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]++#[cfg(not(target_arch = "wasm32"))]+mod clip;+#[cfg(not(target_arch = "wasm32"))]+mod detector;+pub mod mask;+#[cfg(not(target_arch = "wasm32"))]+mod neck;+#[cfg(not(target_arch = "wasm32"))]+mod nn;+pub mod tokenizer;+#[cfg(not(target_arch = "wasm32"))]+mod tracker;+#[cfg(not(target_arch = "wasm32"))]+mod vit;+#[cfg(not(target_arch = "wasm32"))]+mod weights;++use std::path::{Path, PathBuf};+#[cfg(not(target_arch = "wasm32"))]+use std::sync::Arc;++#[cfg(not(target_arch = "wasm32"))]+use candle_core::{DType, Device, Tensor};+#[cfg(not(target_arch = "wasm32"))]+use candle_nn::VarBuilder;++#[cfg(not(target_arch = "wasm32"))]+pub use tracker::Click;++#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("SAM 3 model: {0}")]+ Model(String),+ #[error("SAM 3 model files not found in {0} (expected model.safetensors, vocab.json, merges.txt)")]+ Missing(PathBuf),+ #[cfg(not(target_arch = "wasm32"))]+ #[error(transparent)]+ Candle(#[from] candle_core::Error),+}++pub type Result<T> = std::result::Result<T, Error>;++/// Side of the square model input.+pub const INPUT: usize = 1008;+/// Side of the low-resolution masks the model predicts.+pub const MASK_SIDE: usize = 288;+#[cfg(not(target_arch = "wasm32"))]+const _: () = assert!(MASK_SIDE == tracker::LOW_RES && INPUT == vit::IMAGE);++/// The files of a model directory.+pub const WEIGHTS_FILE: &str = "model.safetensors";++/// Whether `dir` holds a usable checkpoint.+pub fn is_model_dir(dir: &Path) -> bool {+ dir.join(WEIGHTS_FILE).is_file() && dir.join("vocab.json").is_file() && dir.join("merges.txt").is_file()+}++#[cfg(not(target_arch = "wasm32"))]+/// The best device here: Metal on macOS (when a GPU is available), else the CPU.+pub fn best_device() -> Device {+ #[cfg(target_os = "macos")]+ if let Ok(d) = Device::new_metal(0) {+ return d;+ }+ Device::Cpu+}++#[cfg(not(target_arch = "wasm32"))]+/// A loaded model. The vision backbone loads with it; the point and text heads load on first use.+pub struct Sam3 {+ dir: PathBuf,+ device: Device,+ weights: Arc<weights::Weights>,+ backbone: vit::Vit,+ tracker_neck: Option<neck::Neck>,+ tracker: Option<tracker::Tracker>,+ detector: Option<detector::Detector>,+ tokenizer: Option<tokenizer::Tokenizer>,+}++#[cfg(not(target_arch = "wasm32"))]+/// An image run through the backbone, with the head-specific features computed on demand.+pub struct Encoded {+ backbone: Tensor,+ tracker: Option<tracker::Features>,+ detector: Option<detector::Features>,+}++/// A predicted mask: `MASK_SIDE²` logits over the model's square input (which the image was+/// stretched to), and a confidence 0..1.+#[derive(Clone, Debug)]+pub struct Prediction {+ pub logits: Vec<f32>,+ pub score: f32,+}++#[cfg(not(target_arch = "wasm32"))]+impl Sam3 {+ /// Load from a directory holding `model.safetensors` (the `facebook/sam3` checkpoint in+ /// Hugging Face format), `vocab.json` and `merges.txt`.+ pub fn load(dir: &Path) -> Result<Self> {+ Self::load_on(dir, best_device())+ }++ pub fn load_on(dir: &Path, device: Device) -> Result<Self> {+ if !is_model_dir(dir) {+ return Err(Error::Missing(dir.to_path_buf()));+ }+ let weights = weights::Weights::open(&dir.join(WEIGHTS_FILE))?;+ let backbone = vit::Vit::new(Self::vb(&weights, &device).pp("detector_model.vision_encoder.backbone"))?;+ Ok(Self { dir: dir.to_path_buf(), device, weights, backbone, tracker_neck: None, tracker: None, detector: None, tokenizer: None })+ }++ fn vb<'a>(w: &Arc<weights::Weights>, device: &Device) -> VarBuilder<'a> {+ VarBuilder::from_backend(Box::new(weights::Backend(w.clone())), DType::F32, device.clone())+ }++ pub fn device(&self) -> &Device {+ &self.device+ }++ /// Run the backbone on an RGB image (8-bit, row-major, `w × h`), stretched to the square+ /// input like the reference preprocessing.+ pub fn encode(&self, rgb: &[u8], w: usize, h: usize) -> Result<Encoded> {+ let pixels = mask::preprocess(rgb, w, h, INPUT)?;+ let pixels = Tensor::from_vec(pixels, (1, 3, INPUT, INPUT), &self.device)?;+ self.encode_pixels(&pixels)+ }++ /// Run the backbone on preprocessed pixels `[1, 3, 1008, 1008]` (−1..1).+ pub fn encode_pixels(&self, pixels: &Tensor) -> Result<Encoded> {+ Ok(Encoded { backbone: self.backbone.forward(&pixels.to_device(&self.device)?)?, tracker: None, detector: None })+ }++ fn tracker_parts(&mut self) -> Result<(&neck::Neck, &tracker::Tracker)> {+ if self.tracker_neck.is_none() {+ self.tracker_neck = Some(neck::Neck::new(Self::vb(&self.weights, &self.device).pp("tracker_neck"))?);+ }+ if self.tracker.is_none() {+ self.tracker = Some(tracker::Tracker::new(Self::vb(&self.weights, &self.device).pp("tracker_model"))?);+ }+ match (&self.tracker_neck, &self.tracker) {+ (Some(n), Some(t)) => Ok((n, t)),+ _ => Err(Error::Model("tracker failed to load".into())),+ }+ }++ /// Compute the click decoder's features for `enc` now (so the first click is fast).+ pub fn prepare_clicks(&mut self, enc: &mut Encoded) -> Result<()> {+ let (neck, tracker) = self.tracker_parts()?;+ if enc.tracker.is_none() {+ let levels = neck.forward(&enc.backbone, 3)?;+ enc.tracker = Some(tracker.features(&levels)?);+ }+ Ok(())+ }++ /// Segment one object from clicks (positions normalized to the image, 0..1).+ pub fn segment_clicks(&mut self, enc: &mut Encoded, clicks: &[Click]) -> Result<Prediction> {+ if clicks.is_empty() {+ return Err(Error::Model("no clicks".into()));+ }+ let (neck, tracker) = self.tracker_parts()?;+ if enc.tracker.is_none() {+ let levels = neck.forward(&enc.backbone, 3)?;+ enc.tracker = Some(tracker.features(&levels)?);+ }+ let f = enc.tracker.as_ref().ok_or_else(|| Error::Model("tracker features".into()))?;+ // one click: SAM's three alternatives, best by predicted IoU; more: the single mask+ let p = tracker.predict(f, clicks, clicks.len() == 1)?;+ Ok(Prediction { logits: p.logits, score: p.iou * sigmoid(p.object_score) })+ }++ fn detector_parts(&mut self) -> Result<(&detector::Detector, &tokenizer::Tokenizer)> {+ if self.detector.is_none() {+ self.detector = Some(detector::Detector::new(Self::vb(&self.weights, &self.device).pp("detector_model"))?);+ }+ if self.tokenizer.is_none() {+ self.tokenizer = Some(tokenizer::Tokenizer::load(&self.dir)?);+ }+ match (&self.detector, &self.tokenizer) {+ (Some(d), Some(t)) => Ok((d, t)),+ _ => Err(Error::Model("detector failed to load".into())),+ }+ }++ /// Segment every instance of the concept `text` names. Instances scoring above+ /// `threshold` (0..1, 0.5 is the reference default) are merged into one mask; `None` when+ /// nothing matches.+ pub fn segment_text(&mut self, enc: &mut Encoded, text: &str, threshold: f32) -> Result<Option<Prediction>> {+ let (det, tok) = self.detector_parts()?;+ if enc.detector.is_none() {+ enc.detector = Some(det.features(&enc.backbone)?);+ }+ let f = enc.detector.as_ref().ok_or_else(|| Error::Model("detector features".into()))?;+ let (ids, valid) = tok.encode(text);+ let out = det.predict(f, &ids, &valid)?;+ Ok(detector::merge(&out, threshold))+ }++ /// Raw detector outputs for `text` (tests and tools).+ pub fn detect(&mut self, enc: &mut Encoded, text: &str) -> Result<detector::Output> {+ let (det, tok) = self.detector_parts()?;+ if enc.detector.is_none() {+ enc.detector = Some(det.features(&enc.backbone)?);+ }+ let f = enc.detector.as_ref().ok_or_else(|| Error::Model("detector features".into()))?;+ let (ids, valid) = tok.encode(text);+ det.predict(f, &ids, &valid)+ }+}++#[cfg(not(target_arch = "wasm32"))]+impl Encoded {+ /// The backbone output `[1, 1024, 72, 72]` (tests).+ pub fn backbone(&self) -> &Tensor {+ &self.backbone+ }+}++#[cfg(not(target_arch = "wasm32"))]+fn sigmoid(x: f32) -> f32 {+ 1.0 / (1.0 + (-x).exp())+}++#[cfg(not(target_arch = "wasm32"))]+pub use detector::Output as DetectorOutput;++#[cfg(not(target_arch = "wasm32"))]+/// Wait for queued GPU work to finish (benchmarks).+pub fn sync(model: &Sam3) -> Result<()> {+ Ok(model.device.synchronize()?)+}diff --git a/crates/segment/src/mask.rs b/crates/segment/src/mask.rsnew file mode 100644index 000000000..1e61c1f7c--- /dev/null+++ b/crates/segment/src/mask.rs@@ -0,0 +1,126 @@+//! Image preprocessing and mask post-processing (CPU, no model needed).++use crate::{Error, Result};++/// Separable resampling weights for `src → dst` samples with an antialiased triangle filter+/// (Pillow's BILINEAR: the support widens with the downscale factor).+fn weights(src: usize, dst: usize) -> Vec<(usize, Vec<f32>)> {+ let scale = src as f64 / dst as f64;+ let ss = scale.max(1.0);+ let support = ss;+ (0..dst)+ .map(|o| {+ let center = (o as f64 + 0.5) * scale;+ let lo = ((center - support + 0.5).floor().max(0.0)) as usize;+ let hi = ((center + support + 0.5).floor() as usize).min(src);+ let mut w: Vec<f32> = (lo..hi).map(|i| (1.0 - ((i as f64 - center + 0.5) / ss).abs()).max(0.0) as f32).collect();+ let sum: f32 = w.iter().sum();+ if sum > 0.0 {+ w.iter_mut().for_each(|v| *v /= sum);+ }+ (lo, w)+ })+ .collect()+}++/// An 8-bit RGB image (`w × h`, row-major) stretched to `side × side`, scaled to −1..1 and laid+/// out channel-first (`[3, side, side]`), as SAM 3's processor does.+pub fn preprocess(rgb: &[u8], w: usize, h: usize, side: usize) -> Result<Vec<f32>> {+ let n = w.checked_mul(h).and_then(|n| n.checked_mul(3)).ok_or_else(|| Error::Model("image too large".into()))?;+ if w == 0 || h == 0 || rgb.len() < n {+ return Err(Error::Model(format!("image buffer of {} bytes for {w} × {h}", rgb.len())));+ }+ let (wx, wy) = (weights(w, side), weights(h, side));+ // horizontal pass: h rows × side columns × 3+ let mut tmp = vec![0f32; h * side * 3];+ for y in 0..h {+ for (ox, (lo, ws)) in wx.iter().enumerate() {+ let mut acc = [0f32; 3];+ for (k, wt) in ws.iter().enumerate() {+ let i = (y * w + lo + k) * 3;+ for (c, a) in acc.iter_mut().enumerate() {+ *a += wt * f32::from(rgb.get(i + c).copied().unwrap_or(0));+ }+ }+ if let Some(dst) = tmp.get_mut((y * side + ox) * 3..(y * side + ox) * 3 + 3) {+ dst.copy_from_slice(&acc);+ }+ }+ }+ let mut out = vec![0f32; 3 * side * side];+ for (oy, (lo, ws)) in wy.iter().enumerate() {+ for ox in 0..side {+ let mut acc = [0f32; 3];+ for (k, wt) in ws.iter().enumerate() {+ let i = ((lo + k) * side + ox) * 3;+ for (c, a) in acc.iter_mut().enumerate() {+ *a += wt * tmp.get(i + c).copied().unwrap_or(0.0);+ }+ }+ for (c, a) in acc.iter().enumerate() {+ // Pillow rounds to 8 bits; then /255 and (x − 0.5) / 0.5+ let v = a.round().clamp(0.0, 255.0) / 255.0;+ if let Some(d) = out.get_mut((c * side + oy) * side + ox) {+ *d = (v - 0.5) / 0.5;+ }+ }+ }+ }+ Ok(out)+}++/// Mask `logits` (`side × side`) resampled bilinearly (half-pixel centres) to `w × h` and+/// mapped to 0..1 coverage (sigmoid).+pub fn alpha(logits: &[f32], side: usize, w: usize, h: usize) -> Vec<f32> {+ let mut out = vec![0f32; w * h];+ if side == 0 || logits.len() < side * side {+ return out;+ }+ let sample = |x: usize, y: usize| logits.get(y.min(side - 1) * side + x.min(side - 1)).copied().unwrap_or(-20.0);+ let (sx, sy) = (side as f32 / w.max(1) as f32, side as f32 / h.max(1) as f32);+ for y in 0..h {+ let fy = ((y as f32 + 0.5) * sy - 0.5).max(0.0);+ let (y0, ty) = (fy.floor() as usize, fy.fract());+ for x in 0..w {+ let fx = ((x as f32 + 0.5) * sx - 0.5).max(0.0);+ let (x0, tx) = (fx.floor() as usize, fx.fract());+ let top = sample(x0, y0) * (1.0 - tx) + sample(x0 + 1, y0) * tx;+ let bot = sample(x0, y0 + 1) * (1.0 - tx) + sample(x0 + 1, y0 + 1) * tx;+ let l = top * (1.0 - ty) + bot * ty;+ if let Some(o) = out.get_mut(y * w + x) {+ *o = 1.0 / (1.0 + (-l).exp());+ }+ }+ }+ out+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn preprocess_is_bounded_and_channel_first() {+ let rgb: Vec<u8> = (0..20 * 10 * 3).map(|i| (i % 256) as u8).collect();+ let p = preprocess(&rgb, 20, 10, 8).unwrap();+ assert_eq!(p.len(), 3 * 64);+ assert!(p.iter().all(|v| (-1.0..=1.0).contains(v)));+ // a flat image stays flat+ let flat = vec![255u8; 30 * 30 * 3];+ assert!(preprocess(&flat, 30, 30, 7).unwrap().iter().all(|v| (*v - 1.0).abs() < 1e-6));+ }++ #[test]+ fn short_buffers_are_errors() {+ assert!(preprocess(&[0; 5], 2, 2, 4).is_err());+ assert!(preprocess(&[], 0, 0, 4).is_err());+ }++ #[test]+ fn alpha_follows_the_logits() {+ let logits: Vec<f32> = (0..16).map(|i| if i % 4 < 2 { 10.0 } else { -10.0 }).collect();+ let a = alpha(&logits, 4, 8, 2);+ assert!(a[0] > 0.99 && a[7] < 0.01);+ assert!(alpha(&[], 4, 3, 3).iter().all(|v| *v == 0.0));+ }+}diff --git a/crates/segment/src/neck.rs b/crates/segment/src/neck.rsnew file mode 100644index 000000000..df4db88d1--- /dev/null+++ b/crates/segment/src/neck.rs@@ -0,0 +1,73 @@+//! The feature pyramid (`Sam3VisionNeck`): the backbone's 72 × 72 map at four scales+//! (288, 144, 72 and 36 per side), each projected to 256 channels.++use candle_core::{Module, Tensor};+use candle_nn::{Conv2d, Conv2dConfig, ConvTranspose2d, ConvTranspose2dConfig, VarBuilder};++use crate::Result;+use crate::vit::HIDDEN;++pub const FPN: usize = 256;++enum Scale {+ /// Two 2× transposed convolutions with GELU between (4×).+ Up4(ConvTranspose2d, ConvTranspose2d),+ Up2(ConvTranspose2d),+ Same,+ /// 2 × 2 max pooling (½×).+ Down2,+}++struct Level {+ scale: Scale,+ proj1: Conv2d,+ proj2: Conv2d,+}++pub struct Neck {+ levels: Vec<Level>,+}++impl Neck {+ /// `vb` at `…neck` (detector) or `tracker_neck`.+ pub fn new(vb: VarBuilder) -> Result<Self> {+ let up = ConvTranspose2dConfig { stride: 2, ..Default::default() };+ let mut levels = Vec::new();+ for (i, factor) in [4.0f32, 2.0, 1.0, 0.5].into_iter().enumerate() {+ let l = vb.pp(format!("fpn_layers.{i}"));+ let s = l.pp("scale_layers");+ let (scale, inter) = if factor == 4.0 {+ let a = candle_nn::conv_transpose2d(HIDDEN, HIDDEN / 2, 2, up, s.pp("0"))?;+ let b = candle_nn::conv_transpose2d(HIDDEN / 2, HIDDEN / 4, 2, up, s.pp("2"))?;+ (Scale::Up4(a, b), HIDDEN / 4)+ } else if factor == 2.0 {+ (Scale::Up2(candle_nn::conv_transpose2d(HIDDEN, HIDDEN / 2, 2, up, s.pp("0"))?), HIDDEN / 2)+ } else if factor == 1.0 {+ (Scale::Same, HIDDEN)+ } else {+ (Scale::Down2, HIDDEN)+ };+ levels.push(Level {+ scale,+ proj1: candle_nn::conv2d(inter, FPN, 1, Conv2dConfig::default(), l.pp("proj1"))?,+ proj2: candle_nn::conv2d(FPN, FPN, 3, Conv2dConfig { padding: 1, ..Default::default() }, l.pp("proj2"))?,+ });+ }+ Ok(Self { levels })+ }++ /// `x`: the backbone's `[1, 1024, 72, 72]`; `n`: how many levels (from the finest) to compute.+ pub fn forward(&self, x: &Tensor, n: usize) -> Result<Vec<Tensor>> {+ let mut out = Vec::with_capacity(n);+ for l in self.levels.iter().take(n) {+ let y = match &l.scale {+ Scale::Up4(a, b) => b.forward(&a.forward(x)?.gelu_erf()?)?,+ Scale::Up2(a) => a.forward(x)?,+ Scale::Same => x.clone(),+ Scale::Down2 => x.max_pool2d(2)?,+ };+ out.push(l.proj2.forward(&l.proj1.forward(&y)?)?);+ }+ Ok(out)+ }+}diff --git a/crates/segment/src/nn.rs b/crates/segment/src/nn.rsnew file mode 100644index 000000000..cec6a58b1--- /dev/null+++ b/crates/segment/src/nn.rs@@ -0,0 +1,238 @@+//! Shared building blocks: attention, MLPs, layer norms, sine position encodings.++use candle_core::{D, DType, Device, Module, Tensor};+use candle_nn::{LayerNorm, Linear, VarBuilder};++use crate::Result;++/// `nn.LayerNorm(d)` (PyTorch's default eps).+pub const TORCH_LN_EPS: f64 = 1e-5;++pub fn ln(d: usize, eps: f64, vb: VarBuilder) -> Result<LayerNorm> {+ Ok(candle_nn::layer_norm(d, eps, vb)?)+}++pub fn linear(i: usize, o: usize, vb: VarBuilder) -> Result<Linear> {+ Ok(candle_nn::linear(i, o, vb)?)+}++/// Largest attention-score block computed at once (elements): the 5184-token global layers+/// would otherwise need ~1.7 GB for all 16 heads.+const SCORE_BLOCK: usize = 64 << 20;++/// Scaled dot-product attention on `[B, H, L, d]` tensors, with an optional additive bias+/// broadcastable to `[B, H, Lq, Lk]`. Heads are processed in groups to bound memory.+pub fn sdpa(q: &Tensor, k: &Tensor, v: &Tensor, bias: Option<&Tensor>, scale: f64) -> Result<Tensor> {+ let (b, h, lq, _) = q.dims4()?;+ let lk = k.dim(2)?;+ let per_head = b.saturating_mul(lq).saturating_mul(lk).max(1);+ let group = (SCORE_BLOCK / per_head).clamp(1, h);+ let bias_heads = match bias {+ Some(t) => t.dim(1)?,+ None => 1,+ };+ let mut outs = Vec::with_capacity(h.div_ceil(group));+ let mut start = 0;+ while start < h {+ let n = group.min(h - start);+ let qs = q.narrow(1, start, n)?;+ let ks = k.narrow(1, start, n)?;+ let vs = v.narrow(1, start, n)?;+ let mut att = (qs.matmul(&ks.t()?)? * scale)?;+ if let Some(bias) = bias {+ let bs = if bias_heads > 1 { bias.narrow(1, start, n)? } else { bias.clone() };+ att = att.broadcast_add(&bs)?;+ }+ let att = candle_nn::ops::softmax_last_dim(&att)?;+ outs.push(att.matmul(&vs)?);+ start += n;+ }+ Ok(Tensor::cat(&outs, 1)?)+}++/// Multi-head attention with q/k/v/o projections (`Sam3Attention`, `Sam3TrackerAttention`+/// with `internal` = hidden / downsample rate).+pub struct Attention {+ q: Linear,+ k: Linear,+ v: Linear,+ o: Linear,+ heads: usize,+ head_dim: usize,+}++impl Attention {+ pub fn new(hidden: usize, internal: usize, heads: usize, vb: VarBuilder) -> Result<Self> {+ Self::named(hidden, internal, heads, "o_proj", vb)+ }++ /// With the output projection called `out` (CLIP's `out_proj`).+ pub fn named(hidden: usize, internal: usize, heads: usize, out: &str, vb: VarBuilder) -> Result<Self> {+ Ok(Self {+ q: linear(hidden, internal, vb.pp("q_proj"))?,+ k: linear(hidden, internal, vb.pp("k_proj"))?,+ v: linear(hidden, internal, vb.pp("v_proj"))?,+ o: linear(internal, hidden, vb.pp(out))?,+ heads,+ head_dim: internal / heads,+ })+ }++ fn split(&self, x: &Tensor) -> Result<Tensor> {+ let (b, l, _) = x.dims3()?;+ Ok(x.reshape((b, l, self.heads, self.head_dim))?.transpose(1, 2)?.contiguous()?)+ }++ /// The q, k and v projections of `x` (for attention variants that transform them).+ pub fn qkv(&self, x: &Tensor) -> Result<(Tensor, Tensor, Tensor)> {+ Ok((self.q.forward(x)?, self.k.forward(x)?, self.v.forward(x)?))+ }++ /// The output projection.+ pub fn out(&self, x: &Tensor) -> Result<Tensor> {+ Ok(self.o.forward(x)?)+ }++ /// `q`, `k`, `v`: `[B, L, hidden]`; `bias` broadcastable to `[B, heads, Lq, Lk]`.+ pub fn forward(&self, q: &Tensor, k: &Tensor, v: &Tensor, bias: Option<&Tensor>) -> Result<Tensor> {+ let (b, lq, _) = q.dims3()?;+ let q = self.split(&self.q.forward(q)?)?;+ let k = self.split(&self.k.forward(k)?)?;+ let v = self.split(&self.v.forward(v)?)?;+ let o = sdpa(&q, &k, &v, bias, (self.head_dim as f64).powf(-0.5))?;+ let o = o.transpose(1, 2)?.reshape((b, lq, self.heads * self.head_dim))?;+ Ok(self.o.forward(&o)?)+ }+}++/// An additive key-padding bias `[1, 1, 1, L]` (0 for valid tokens, the f32 minimum for padding).+pub fn padding_bias(valid: &[bool], device: &Device) -> Result<Tensor> {+ let v: Vec<f32> = valid.iter().map(|&ok| if ok { 0.0 } else { f32::MIN }).collect();+ Ok(Tensor::from_vec(v, (1, 1, 1, valid.len()), device)?)+}++/// `Sam3MLP`: fc1 → activation → fc2.+pub struct Mlp {+ fc1: Linear,+ fc2: Linear,+ gelu: bool,+}++impl Mlp {+ pub fn new(hidden: usize, inter: usize, gelu: bool, vb: VarBuilder) -> Result<Self> {+ Ok(Self { fc1: linear(hidden, inter, vb.pp("fc1"))?, fc2: linear(inter, hidden, vb.pp("fc2"))?, gelu })+ }++ pub fn forward(&self, x: &Tensor) -> Result<Tensor> {+ let x = self.fc1.forward(x)?;+ let x = if self.gelu { x.gelu_erf()? } else { x.relu()? };+ Ok(self.fc2.forward(&x)?)+ }+}++/// `Sam3DecoderMLP`: `layer1 … layerN` with ReLU between (2 or 3 layers).+pub struct DecoderMlp {+ layers: Vec<Linear>,+}++impl DecoderMlp {+ pub fn new(input: usize, hidden: usize, output: usize, n: usize, vb: VarBuilder) -> Result<Self> {+ let mut layers = Vec::with_capacity(n);+ for i in 0..n {+ let (a, b) = (if i == 0 { input } else { hidden }, if i + 1 == n { output } else { hidden });+ layers.push(linear(a, b, vb.pp(format!("layer{}", i + 1)))?);+ }+ Ok(Self { layers })+ }++ pub fn forward(&self, x: &Tensor) -> Result<Tensor> {+ let mut x = x.clone();+ for (i, l) in self.layers.iter().enumerate() {+ x = l.forward(&x)?;+ if i + 1 < self.layers.len() {+ x = x.relu()?;+ }+ }+ Ok(x)+ }+}++/// `Sam3TrackerFeedForward`: proj_in → act → (layers.i → act)* → proj_out (→ sigmoid).+pub struct FeedForward {+ proj_in: Linear,+ layers: Vec<Linear>,+ proj_out: Linear,+ sigmoid: bool,+}++impl FeedForward {+ pub fn new(input: usize, hidden: usize, output: usize, n: usize, sigmoid: bool, vb: VarBuilder) -> Result<Self> {+ let mut layers = Vec::new();+ for i in 0..n.saturating_sub(2) {+ layers.push(linear(hidden, hidden, vb.pp(format!("layers.{i}")))?);+ }+ Ok(Self { proj_in: linear(input, hidden, vb.pp("proj_in"))?, layers, proj_out: linear(hidden, output, vb.pp("proj_out"))?, sigmoid })+ }++ pub fn forward(&self, x: &Tensor) -> Result<Tensor> {+ let mut x = self.proj_in.forward(x)?.relu()?;+ for l in &self.layers {+ x = l.forward(&x)?.relu()?;+ }+ let x = self.proj_out.forward(&x)?;+ Ok(if self.sigmoid { candle_nn::ops::sigmoid(&x)? } else { x })+ }+}++/// Layer norm over the channel axis of an NCHW tensor (`Sam3TrackerLayerNorm`, channels_first).+pub fn ln_channels(x: &Tensor, norm: &LayerNorm) -> Result<Tensor> {+ let x = x.permute((0, 2, 3, 1))?;+ Ok(norm.forward(&x)?.permute((0, 3, 1, 2))?.contiguous()?)+}++/// The DETR sine embedding of an `h × w` grid, normalized (`Sam3SinePositionEmbedding` with+/// `normalize=True`, scale 2π): `[1, 2·feats, h, w]`, y features first.+pub fn sine_grid(h: usize, w: usize, feats: usize, device: &Device) -> Result<Tensor> {+ let scale = 2.0 * std::f64::consts::PI;+ let dim_t: Vec<f64> = (0..feats).map(|i| 10000f64.powf(2.0 * (i / 2) as f64 / feats as f64)).collect();+ let mut out = vec![0f32; 2 * feats * h * w];+ for y in 0..h {+ let ye = (y + 1) as f64 / (h as f64 + 1e-6) * scale;+ for x in 0..w {+ let xe = (x + 1) as f64 / (w as f64 + 1e-6) * scale;+ for (i, d) in dim_t.iter().enumerate() {+ let (py, px) = (ye / d, xe / d);+ let (vy, vx) = if i % 2 == 0 { (py.sin(), px.sin()) } else { (py.cos(), px.cos()) };+ if let Some(v) = out.get_mut((i * h + y) * w + x) {+ *v = vy as f32;+ }+ if let Some(v) = out.get_mut(((feats + i) * h + y) * w + x) {+ *v = vx as f32;+ }+ }+ }+ }+ Ok(Tensor::from_vec(out, (1, 2 * feats, h, w), device)?)+}++/// `inverse_sigmoid` with the reference's clamping (eps 1e-3).+pub fn inverse_sigmoid(x: &Tensor) -> Result<Tensor> {+ let x = x.clamp(0f32, 1f32)?;+ let x1 = x.clamp(1e-3f32, f32::MAX)?;+ let x2 = (1.0 - &x)?.clamp(1e-3f32, f32::MAX)?;+ Ok((x1 / x2)?.log()?)+}++/// `[..., 4]` boxes (cx, cy, w, h) → (x0, y0, x1, y1).+pub fn cxcywh_to_xyxy(b: &Tensor) -> Result<Tensor> {+ let cx = b.narrow(D::Minus1, 0, 1)?;+ let cy = b.narrow(D::Minus1, 1, 1)?;+ let w = (b.narrow(D::Minus1, 2, 1)? * 0.5)?;+ let h = (b.narrow(D::Minus1, 3, 1)? * 0.5)?;+ Ok(Tensor::cat(&[(&cx - &w)?, (&cy - &h)?, (&cx + &w)?, (&cy + &h)?], D::Minus1)?)+}++/// A tensor's values as `f32` on the CPU.+pub fn to_vec(t: &Tensor) -> Result<Vec<f32>> {+ Ok(t.to_dtype(DType::F32)?.flatten_all()?.to_vec1::<f32>()?)+}diff --git a/crates/segment/src/tokenizer.rs b/crates/segment/src/tokenizer.rsnew file mode 100644index 000000000..c1a9f3c10--- /dev/null+++ b/crates/segment/src/tokenizer.rs@@ -0,0 +1,173 @@+//! The CLIP byte-pair-encoding tokenizer (lower-cased, `</w>` word ends), padded to the+//! 32-token context SAM 3's text encoder takes.++use std::collections::HashMap;+use std::path::Path;++use crate::{Error, Result};++pub const CONTEXT: usize = 32;+const BOS: u32 = 49406;+const EOS: u32 = 49407;++pub struct Tokenizer {+ vocab: HashMap<String, u32>,+ ranks: HashMap<(String, String), usize>,+ byte_char: [char; 256],+}++/// GPT-2's reversible byte → printable character map.+fn bytes_to_unicode() -> [char; 256] {+ let mut map = ['\0'; 256];+ let mut n = 0u32;+ for b in 0..=255u8 {+ let printable = (b'!'..=b'~').contains(&b) || (0xA1..=0xAC).contains(&b) || (0xAE..=0xFF).contains(&b);+ let c = if printable {+ u32::from(b)+ } else {+ n += 1;+ 255 + n+ };+ if let Some(slot) = map.get_mut(usize::from(b)) {+ *slot = char::from_u32(c).unwrap_or('?');+ }+ }+ map+}++impl Tokenizer {+ /// From `vocab.json` and `merges.txt` in `dir`.+ pub fn load(dir: &Path) -> Result<Self> {+ let read = |name: &str| std::fs::read_to_string(dir.join(name)).map_err(|e| Error::Model(format!("{name}: {e}")));+ let vocab: HashMap<String, u32> = serde_json::from_str(&read("vocab.json")?).map_err(|e| Error::Model(format!("vocab.json: {e}")))?;+ let merges = read("merges.txt")?;+ Ok(Self::new(vocab, &merges))+ }++ pub fn new(vocab: HashMap<String, u32>, merges: &str) -> Self {+ let ranks = merges+ .lines()+ .filter(|l| !l.starts_with("#version") && !l.trim().is_empty())+ .filter_map(|l| l.split_once(' ').map(|(a, b)| (a.to_string(), b.to_string())))+ .enumerate()+ .map(|(i, p)| (p, i))+ .collect();+ Self { vocab, ranks, byte_char: bytes_to_unicode() }+ }++ /// Token ids (start, words, end, padding) and which positions are real tokens.+ pub fn encode(&self, text: &str) -> (Vec<u32>, Vec<bool>) {+ let mut ids = vec![BOS];+ for word in pre_tokenize(&text.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase()) {+ let mapped: String = word.bytes().map(|b| self.byte_char.get(usize::from(b)).copied().unwrap_or('?')).collect();+ for piece in self.bpe(&mapped) {+ ids.push(self.vocab.get(&piece).copied().unwrap_or(EOS));+ }+ }+ ids.truncate(CONTEXT - 1);+ ids.push(EOS);+ let n = ids.len();+ ids.resize(CONTEXT, EOS);+ let valid = (0..CONTEXT).map(|i| i < n).collect();+ (ids, valid)+ }++ fn bpe(&self, word: &str) -> Vec<String> {+ let mut syms: Vec<String> = word.chars().map(String::from).collect();+ if let Some(last) = syms.last_mut() {+ last.push_str("</w>");+ }+ loop {+ let best = syms+ .windows(2)+ .enumerate()+ .filter_map(|(i, w)| match w {+ [a, b] => self.ranks.get(&(a.clone(), b.clone())).map(|r| (*r, i)),+ _ => None,+ })+ .min();+ let Some((_, i)) = best else { break };+ let (Some(a), Some(b)) = (syms.get(i).cloned(), syms.get(i + 1).cloned()) else { break };+ let merged = a.clone() + &b;+ // merge every occurrence of the pair, left to right+ let mut out = Vec::with_capacity(syms.len());+ let mut j = 0;+ while j < syms.len() {+ if j + 1 < syms.len() && syms.get(j) == Some(&a) && syms.get(j + 1) == Some(&b) {+ out.push(merged.clone());+ j += 2;+ } else {+ out.extend(syms.get(j).cloned());+ j += 1;+ }+ }+ syms = out;+ if syms.len() == 1 {+ break;+ }+ }+ syms+ }+}++/// CLIP's pre-tokenizer: contractions, letter runs, single digits, and runs of other+/// non-space characters.+fn pre_tokenize(text: &str) -> Vec<String> {+ let chars: Vec<char> = text.chars().collect();+ let mut out = Vec::new();+ let mut i = 0;+ while let Some(&c) = chars.get(i) {+ if c.is_whitespace() {+ i += 1;+ continue;+ }+ if c == '\'' {+ let rest: String = chars.get(i + 1..(i + 3).min(chars.len())).map(|s| s.iter().collect()).unwrap_or_default();+ if let Some(k) = ["re", "ve", "ll"]+ .iter()+ .find(|k| rest.starts_with(**k))+ .map(|k| k.len())+ .or_else(|| rest.chars().next().filter(|c| matches!(c, 's' | 't' | 'm' | 'd')).map(|_| 1))+ {+ out.push(chars.get(i..i + 1 + k).map(|s| s.iter().collect()).unwrap_or_default());+ i += 1 + k;+ continue;+ }+ }+ let start = i;+ if c.is_alphabetic() {+ while chars.get(i).is_some_and(|c| c.is_alphabetic()) {+ i += 1;+ }+ } else if c.is_numeric() {+ i += 1;+ } else {+ while chars.get(i).is_some_and(|c| !c.is_whitespace() && !c.is_alphabetic() && !c.is_numeric()) {+ i += 1;+ }+ }+ out.push(chars.get(start..i).map(|s| s.iter().collect()).unwrap_or_default());+ }+ out+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn splits_like_clip() {+ assert_eq!(pre_tokenize("a dog's toy, 25 cats!"), ["a", "dog", "'s", "toy", ",", "2", "5", "cats", "!"]);+ }++ #[test]+ fn merges_by_rank() {+ let vocab: HashMap<String, u32> =+ [("c", 1), ("a", 2), ("t</w>", 3), ("ca", 4), ("cat</w>", 5)].into_iter().map(|(k, v)| (k.to_string(), v)).collect();+ let t = Tokenizer::new(vocab, "#version: 0.2\nc a\nca t</w>\n");+ let (ids, valid) = t.encode(" CAT ");+ assert_eq!(&ids[..3], &[BOS, 5, EOS]);+ assert_eq!(ids.len(), CONTEXT);+ assert_eq!(valid.iter().filter(|v| **v).count(), 3);+ }+}diff --git a/crates/segment/src/tracker.rs b/crates/segment/src/tracker.rsnew file mode 100644index 000000000..828d56b66--- /dev/null+++ b/crates/segment/src/tracker.rs@@ -0,0 +1,249 @@+//! Point prompts (SAM 3's interactive "tracker" head, SAM 2 style): a prompt encoder for+//! positive/negative clicks and a two-way-transformer mask decoder over the image features.++use candle_core::{DType, Device, IndexOp, Module, Tensor};+use candle_nn::{Conv2d, Conv2dConfig, ConvTranspose2d, ConvTranspose2dConfig, LayerNorm, VarBuilder};++use crate::nn::{Attention, FeedForward, TORCH_LN_EPS, ln, ln_channels};+use crate::vit::{GRID, IMAGE};+use crate::{Result, nn::to_vec};++const HIDDEN: usize = 256;+const MASK_TOKENS: usize = 4;+/// Low-resolution mask side (4 × the 72 grid).+pub const LOW_RES: usize = 288;++/// The image features the decoder needs, computed once per image.+pub struct Features {+ /// Level 2 (72 × 72, 256 channels).+ pub image: Tensor,+ /// `conv_s0`(level 0): `[1, 32, 288, 288]`.+ pub s0: Tensor,+ /// `conv_s1`(level 1): `[1, 64, 144, 144]`.+ pub s1: Tensor,+}++/// One click: position normalized to the image (0..1), and whether it includes or excludes.+#[derive(Clone, Copy, Debug, PartialEq)]+pub struct Click {+ pub x: f32,+ pub y: f32,+ pub positive: bool,+}++/// A predicted mask: 288 × 288 logits over the (squared) model input, and its quality.+pub struct Prediction {+ pub logits: Vec<f32>,+ pub iou: f32,+ pub object_score: f32,+}++/// Random-Fourier positional encoding of points normalized to 0..1 (`Sam3TrackerPositionalEmbedding`).+struct Fourier {+ pe: Vec<f32>, // [2, 128]+}++impl Fourier {+ fn encode(&self, x: f32, y: f32) -> Vec<f32> {+ let half = self.pe.len() / 2;+ let (cx, cy) = (2.0 * x - 1.0, 2.0 * y - 1.0);+ let proj: Vec<f32> = (0..half)+ .map(|i| {+ let a = cx * self.pe.get(i).copied().unwrap_or(0.0) + cy * self.pe.get(half + i).copied().unwrap_or(0.0);+ 2.0 * std::f32::consts::PI * a+ })+ .collect();+ proj.iter().map(|a| a.sin()).chain(proj.iter().map(|a| a.cos())).collect()+ }+}++struct TwoWayBlock {+ self_attn: Attention,+ ln1: LayerNorm,+ t2i: Attention,+ ln2: LayerNorm,+ mlp: FeedForward,+ ln3: LayerNorm,+ ln4: LayerNorm,+ i2t: Attention,+ skip_first_pe: bool,+}++/// `Sam3TrackerAttention` on `[1, L, C]` (the point batch of one is folded away).+fn attn(a: &Attention, q: &Tensor, k: &Tensor, v: &Tensor) -> Result<Tensor> {+ a.forward(q, k, v, None)+}++impl TwoWayBlock {+ fn forward(&self, queries: &Tensor, keys: &Tensor, qpe: &Tensor, kpe: &Tensor) -> Result<(Tensor, Tensor)> {+ let queries = if self.skip_first_pe {+ attn(&self.self_attn, queries, queries, queries)?+ } else {+ let q = (queries + qpe)?;+ (queries + attn(&self.self_attn, &q, &q, queries)?)?+ };+ let queries = self.ln1.forward(&queries)?;+ let (q, k) = ((&queries + qpe)?, (keys + kpe)?);+ let queries = self.ln2.forward(&(&queries + attn(&self.t2i, &q, &k, keys)?)?)?;+ let queries = self.ln3.forward(&(&queries + self.mlp.forward(&queries)?)?)?;+ let (q, k) = ((&queries + qpe)?, (keys + kpe)?);+ let keys = self.ln4.forward(&(keys + attn(&self.i2t, &k, &q, &queries)?)?)?;+ Ok((queries, keys))+ }+}++pub struct Tracker {+ device: Device,+ point_pe: Fourier,+ image_pe: Tensor,+ point_embed: Tensor,+ not_a_point: Tensor,+ no_mask: Tensor,+ conv_s0: Conv2d,+ conv_s1: Conv2d,+ output_tokens: Tensor,+ blocks: Vec<TwoWayBlock>,+ final_t2i: Attention,+ ln_final: LayerNorm,+ up1: ConvTranspose2d,+ up_ln: LayerNorm,+ up2: ConvTranspose2d,+ hyper: Vec<FeedForward>,+ iou_head: FeedForward,+ obj_head: FeedForward,+}++fn read_vec(vb: &VarBuilder, shape: (usize, usize), name: &str) -> Result<Vec<f32>> {+ to_vec(&vb.get(shape, name)?)+}++impl Tracker {+ /// `vb` at `tracker_model`.+ pub fn new(vb: VarBuilder) -> Result<Self> {+ let device = vb.device().clone();+ let pe = vb.pp("prompt_encoder");+ let md = vb.pp("mask_decoder");+ let image_fourier = Fourier { pe: read_vec(&vb, (2, HIDDEN / 2), "shared_image_embedding.positional_embedding")? };+ // image-wide positional embedding: pixel centres of the 72 × 72 grid+ let mut grid = Vec::with_capacity(GRID * GRID * HIDDEN);+ for r in 0..GRID {+ for c in 0..GRID {+ grid.extend(image_fourier.encode((c as f32 + 0.5) / GRID as f32, (r as f32 + 0.5) / GRID as f32));+ }+ }+ let image_pe = Tensor::from_vec(grid, (1, GRID * GRID, HIDDEN), &device)?;+ let t = md.pp("transformer");+ let mut blocks = Vec::new();+ for i in 0..2 {+ let b = t.pp(format!("layers.{i}"));+ blocks.push(TwoWayBlock {+ self_attn: Attention::new(HIDDEN, HIDDEN, 8, b.pp("self_attn"))?,+ ln1: ln(HIDDEN, TORCH_LN_EPS, b.pp("layer_norm1"))?,+ t2i: Attention::new(HIDDEN, HIDDEN / 2, 8, b.pp("cross_attn_token_to_image"))?,+ ln2: ln(HIDDEN, TORCH_LN_EPS, b.pp("layer_norm2"))?,+ mlp: FeedForward::new(HIDDEN, 2048, HIDDEN, 2, false, b.pp("mlp"))?,+ ln3: ln(HIDDEN, TORCH_LN_EPS, b.pp("layer_norm3"))?,+ ln4: ln(HIDDEN, TORCH_LN_EPS, b.pp("layer_norm4"))?,+ i2t: Attention::new(HIDDEN, HIDDEN / 2, 8, b.pp("cross_attn_image_to_token"))?,+ skip_first_pe: i == 0,+ });+ }+ let output_tokens = Tensor::cat(+ &[+ md.get((1, HIDDEN), "obj_score_token.weight")?,+ md.get((1, HIDDEN), "iou_token.weight")?,+ md.get((MASK_TOKENS, HIDDEN), "mask_tokens.weight")?,+ ],+ 0,+ )?;+ let up = ConvTranspose2dConfig { stride: 2, ..Default::default() };+ let one = Conv2dConfig::default();+ Ok(Self {+ point_pe: Fourier { pe: read_vec(&pe, (2, HIDDEN / 2), "shared_embedding.positional_embedding")? },+ image_pe,+ point_embed: pe.get((4, HIDDEN), "point_embed.weight")?.to_device(&Device::Cpu)?,+ not_a_point: pe.get((1, HIDDEN), "not_a_point_embed.weight")?.to_device(&Device::Cpu)?,+ no_mask: pe.get((1, HIDDEN), "no_mask_embed.weight")?.reshape((1, HIDDEN, 1, 1))?,+ conv_s0: candle_nn::conv2d(HIDDEN, HIDDEN / 8, 1, one, md.pp("conv_s0"))?,+ conv_s1: candle_nn::conv2d(HIDDEN, HIDDEN / 4, 1, one, md.pp("conv_s1"))?,+ output_tokens,+ blocks,+ final_t2i: Attention::new(HIDDEN, HIDDEN / 2, 8, t.pp("final_attn_token_to_image"))?,+ ln_final: ln(HIDDEN, TORCH_LN_EPS, t.pp("layer_norm_final_attn"))?,+ up1: candle_nn::conv_transpose2d(HIDDEN, HIDDEN / 4, 2, up, md.pp("upscale_conv1"))?,+ up_ln: ln(HIDDEN / 4, 1e-6, md.pp("upscale_layer_norm"))?,+ up2: candle_nn::conv_transpose2d(HIDDEN / 4, HIDDEN / 8, 2, up, md.pp("upscale_conv2"))?,+ hyper: (0..MASK_TOKENS)+ .map(|i| FeedForward::new(HIDDEN, HIDDEN, HIDDEN / 8, 3, false, md.pp(format!("output_hypernetworks_mlps.{i}"))))+ .collect::<Result<_>>()?,+ iou_head: FeedForward::new(HIDDEN, HIDDEN, MASK_TOKENS, 3, true, md.pp("iou_prediction_head"))?,+ obj_head: FeedForward::new(HIDDEN, HIDDEN, 1, 3, false, md.pp("pred_obj_score_head"))?,+ device,+ })+ }++ /// The decoder's image features from the tracker neck's levels (288, 144, 72 per side).+ pub fn features(&self, levels: &[Tensor]) -> Result<Features> {+ let [l0, l1, l2, ..] = levels else { return Err(crate::Error::Model("tracker needs three feature levels".into())) };+ Ok(Features { image: l2.clone(), s0: self.conv_s0.forward(l0)?, s1: self.conv_s1.forward(l1)? })+ }++ /// Sparse prompt embeddings `[1, n + 1, 256]` (a padding point closes the list).+ fn embed_clicks(&self, clicks: &[Click]) -> Result<Tensor> {+ let point_embed = to_vec(&self.point_embed)?;+ let not_a_point = to_vec(&self.not_a_point)?;+ let mut rows = Vec::with_capacity((clicks.len() + 1) * HIDDEN);+ for c in clicks {+ // the reference shifts pixel coordinates (in the 1008² input) by half a pixel+ let (x, y) = ((c.x * IMAGE as f32 + 0.5) / IMAGE as f32, (c.y * IMAGE as f32 + 0.5) / IMAGE as f32);+ let e = self.point_pe.encode(x, y);+ let label = usize::from(c.positive);+ rows.extend(e.iter().enumerate().map(|(i, v)| v + point_embed.get(label * HIDDEN + i).copied().unwrap_or(0.0)));+ }+ rows.extend_from_slice(¬_a_point);+ Ok(Tensor::from_vec(rows, (1, clicks.len() + 1, HIDDEN), &self.device)?)+ }++ /// Predict masks for `clicks`. `multimask`: the three alternatives (best by predicted IoU is+ /// returned), else the single-mask output with SAM 2's stability fallback.+ pub fn predict(&self, f: &Features, clicks: &[Click], multimask: bool) -> Result<Prediction> {+ let sparse = self.embed_clicks(clicks)?;+ let tokens = Tensor::cat(&[self.output_tokens.unsqueeze(0)?, sparse], 1)?; // [1, T, 256]+ let image = f.image.broadcast_add(&self.no_mask)?; // dense prompt: "no mask"+ let (b, c, h, w) = image.dims4()?;+ let keys = image.flatten_from(2)?.transpose(1, 2)?.contiguous()?; // [1, HW, 256]+ let (mut q, mut k) = (tokens.clone(), keys);+ for blk in &self.blocks {+ (q, k) = blk.forward(&q, &k, &tokens, &self.image_pe)?;+ }+ let qf = (&q + &tokens)?;+ let kf = (&k + &self.image_pe)?;+ let q = self.ln_final.forward(&(&q + attn(&self.final_t2i, &qf, &kf, &k)?)?)?;+ let iou_token = q.i((.., 1, ..))?;+ let image = k.transpose(1, 2)?.reshape((b, c, h, w))?;+ let up = (self.up1.forward(&image)? + &f.s1)?;+ let up = ln_channels(&up, &self.up_ln)?.gelu_erf()?;+ let up = (self.up2.forward(&up)? + &f.s0)?.gelu_erf()?; // [1, 32, 288, 288]+ let (_, uc, uh, uw) = up.dims4()?;+ let hyper = (0..MASK_TOKENS)+ .map(|i| self.hyper.get(i).map_or(Err(crate::Error::Model("hypernetwork".into())), |m| m.forward(&q.i((.., 2 + i, ..))?)))+ .collect::<Result<Vec<_>>>()?;+ let hyper = Tensor::stack(&hyper, 1)?; // [1, 4, 32]+ let masks = hyper.matmul(&up.reshape((b, uc, uh * uw))?)?.reshape((b, MASK_TOKENS, uh, uw))?;+ let iou = to_vec(&self.iou_head.forward(&iou_token)?)?;+ let object_score = to_vec(&self.obj_head.forward(&q.i((.., 0, ..))?)?)?.first().copied().unwrap_or(0.0);+ let pick = |i: usize| -> Result<Prediction> {+ Ok(Prediction { logits: to_vec(&masks.i((0, i))?)?, iou: iou.get(i).copied().unwrap_or(0.0), object_score })+ };+ let best_multi = || (1..MASK_TOKENS).max_by(|a, b| iou.get(*a).partial_cmp(&iou.get(*b)).unwrap_or(std::cmp::Ordering::Equal)).unwrap_or(1);+ if multimask {+ return pick(best_multi());+ }+ // single-mask output unless it is unstable (area changes > 2 % when the threshold moves ±0.05)+ let single = masks.i((0, 0))?;+ let inter = single.gt(0.05)?.to_dtype(DType::F32)?.sum_all()?.to_scalar::<f32>()?;+ let union = single.gt(-0.05)?.to_dtype(DType::F32)?.sum_all()?.to_scalar::<f32>()?;+ let stability = if union > 0.0 { inter / union } else { 1.0 };+ if stability >= 0.98 { pick(0) } else { pick(best_multi()) }+ }+}diff --git a/crates/segment/src/vit.rs b/crates/segment/src/vit.rsnew file mode 100644index 000000000..ad9018269--- /dev/null+++ b/crates/segment/src/vit.rs@@ -0,0 +1,166 @@+//! The SAM 3 vision backbone: a 32-layer ViT (Perception Encoder) with 2-D axial rotary+//! position encoding, windowed attention (24 × 24) in most layers and global attention in four.++use candle_core::{D, Device, Module, Tensor};+use candle_nn::{Conv2d, Conv2dConfig, LayerNorm, VarBuilder};++use crate::nn::{Attention, Mlp, ln, sdpa};+use crate::{Error, Result};++pub const IMAGE: usize = 1008;+pub const PATCH: usize = 14;+/// Patches per side (72).+pub const GRID: usize = IMAGE / PATCH;+pub const HIDDEN: usize = 1024;+const HEADS: usize = 16;+const HEAD_DIM: usize = HIDDEN / HEADS;+const LAYERS: usize = 32;+const WINDOW: usize = 24;+const GLOBAL: [usize; 4] = [7, 15, 23, 31];+const PRETRAIN_GRID: usize = 336 / PATCH;+const EPS: f64 = 1e-6;++/// Cosine and sine tables `[N, head_dim]` for a `side × side` grid whose coordinates are scaled+/// by `scale` (`Sam3ViTRotaryEmbedding`: x frequencies then y, each duplicated pairwise).+fn rope_tables(side: usize, scale: f64, device: &Device) -> Result<(Tensor, Tensor)> {+ let spatial = HEAD_DIM / 2;+ let inv: Vec<f64> = (0..spatial).step_by(2).map(|i| 1.0 / 10000f64.powf(i as f64 / spatial as f64)).collect();+ let n = side * side;+ let (mut cos, mut sin) = (Vec::with_capacity(n * HEAD_DIM), Vec::with_capacity(n * HEAD_DIM));+ for r in 0..side {+ for c in 0..side {+ let (x, y) = (c as f64 * scale, r as f64 * scale);+ for pos in [x, y] {+ for f in &inv {+ let a = (pos * f) as f32 as f64;+ let (s, co) = (a.sin() as f32, a.cos() as f32);+ cos.extend([co, co]);+ sin.extend([s, s]);+ }+ }+ }+ }+ Ok((Tensor::from_vec(cos, (n, HEAD_DIM), device)?, Tensor::from_vec(sin, (n, HEAD_DIM), device)?))+}++/// `x·cos + rotate_pairwise(x)·sin`, where rotate_pairwise maps (a, b) pairs to (−b, a).+fn apply_rope(x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor> {+ let dims = x.dims().to_vec();+ let last = *dims.last().ok_or_else(|| Error::Model("rope on a scalar".into()))?;+ let mut paired = dims.clone();+ paired.pop();+ paired.extend([last / 2, 2]);+ let p = x.reshape(paired.as_slice())?;+ let a = p.narrow(D::Minus1, 0, 1)?;+ let b = p.narrow(D::Minus1, 1, 1)?;+ let rot = Tensor::cat(&[b.neg()?, a], D::Minus1)?.reshape(dims.as_slice())?;+ Ok((x.broadcast_mul(cos)? + rot.broadcast_mul(sin)?)?)+}++struct Layer {+ ln1: LayerNorm,+ attn: Attention,+ ln2: LayerNorm,+ mlp: Mlp,+ window: usize,+}++impl Layer {+ /// `x`: `[B, H, W, C]`.+ fn forward(&self, x: &Tensor, rope: &(Tensor, Tensor)) -> Result<Tensor> {+ let (b, h, w, c) = x.dims4()?;+ let y = self.ln1.forward(x)?;+ let y = match (h.checked_div(self.window), w.checked_div(self.window)) {+ (Some(nh), Some(nw)) => {+ // non-overlapping windows (72 is a multiple of 24: no padding)+ let ws = self.window;+ let y = y.reshape((b, nh, ws, nw, ws, c))?.permute((0, 1, 3, 2, 4, 5))?.reshape((b * nh * nw, ws, ws, c))?;+ let y = self.attend(&y, rope)?;+ y.reshape((b, nh, nw, ws, ws, c))?.permute((0, 1, 3, 2, 4, 5))?.reshape((b, h, w, c))?+ }+ // global attention+ _ => self.attend(&y, rope)?,+ };+ let x = (x + y)?;+ let y = self.mlp.forward(&self.ln2.forward(&x)?)?;+ Ok((x + y)?)+ }++ /// RoPE self-attention over `[B, H, W, C]`.+ fn attend(&self, x: &Tensor, rope: &(Tensor, Tensor)) -> Result<Tensor> {+ let (b, h, w, c) = x.dims4()?;+ let seq = x.reshape((b, h * w, c))?;+ Ok(self.attn_rope(&seq, rope)?.reshape((b, h, w, c))?)+ }++ fn attn_rope(&self, x: &Tensor, rope: &(Tensor, Tensor)) -> Result<Tensor> {+ let (b, n, _) = x.dims3()?;+ let split = |t: Tensor| -> Result<Tensor> { Ok(t.reshape((b, n, HEADS, HEAD_DIM))?.transpose(1, 2)?.contiguous()?) };+ let (q, k, v) = self.attn.qkv(x)?;+ let q = apply_rope(&split(q)?, &rope.0, &rope.1)?;+ let k = apply_rope(&split(k)?, &rope.0, &rope.1)?;+ let v = split(v)?;+ let o = sdpa(&q, &k, &v, None, (HEAD_DIM as f64).powf(-0.5))?;+ let o = o.transpose(1, 2)?.reshape((b, n, HIDDEN))?;+ self.attn.out(&o)+ }+}++pub struct Vit {+ patch: Conv2d,+ pos: Tensor,+ ln_pre: LayerNorm,+ layers: Vec<Layer>,+ rope_window: (Tensor, Tensor),+ rope_global: (Tensor, Tensor),+}++impl Vit {+ /// `vb` at `detector_model.vision_encoder.backbone`.+ pub fn new(vb: VarBuilder) -> Result<Self> {+ let device = vb.device().clone();+ let cfg = Conv2dConfig { stride: PATCH, ..Default::default() };+ let patch = candle_nn::conv2d_no_bias(3, HIDDEN, PATCH, cfg, vb.pp("embeddings.patch_embeddings.projection"))?;+ let pos = vb.get((1, PRETRAIN_GRID * PRETRAIN_GRID, HIDDEN), "embeddings.position_embeddings")?;+ // tile (not interpolate) the 24 × 24 pretraining table over the 72 × 72 grid+ let reps = GRID.div_ceil(PRETRAIN_GRID);+ let pos = pos+ .reshape((1, PRETRAIN_GRID, PRETRAIN_GRID, HIDDEN))?+ .repeat((1, reps, reps, 1))?+ .narrow(1, 0, GRID)?+ .narrow(2, 0, GRID)?+ .reshape((1, GRID * GRID, HIDDEN))?;+ let mut layers = Vec::with_capacity(LAYERS);+ for i in 0..LAYERS {+ let l = vb.pp(format!("layers.{i}"));+ layers.push(Layer {+ ln1: ln(HIDDEN, EPS, l.pp("layer_norm1"))?,+ attn: Attention::new(HIDDEN, HIDDEN, HEADS, l.pp("attention"))?,+ ln2: ln(HIDDEN, EPS, l.pp("layer_norm2"))?,+ mlp: Mlp::new(HIDDEN, 4736, true, l.pp("mlp"))?,+ window: if GLOBAL.contains(&i) { 0 } else { WINDOW },+ });+ }+ Ok(Self {+ patch,+ pos,+ ln_pre: ln(HIDDEN, EPS, vb.pp("layer_norm"))?,+ layers,+ rope_window: rope_tables(WINDOW, 1.0, &device)?,+ rope_global: rope_tables(GRID, WINDOW as f64 / GRID as f64, &device)?,+ })+ }++ /// `pixels`: `[1, 3, 1008, 1008]` normalized to −1..1. Returns `[1, 1024, 72, 72]`.+ pub fn forward(&self, pixels: &Tensor) -> Result<Tensor> {+ let x = self.patch.forward(pixels)?; // [1, C, 72, 72]+ let b = x.dim(0)?;+ let x = x.flatten_from(2)?.transpose(1, 2)?.broadcast_add(&self.pos)?;+ let mut x = self.ln_pre.forward(&x.reshape((b, GRID, GRID, HIDDEN))?)?;+ for l in &self.layers {+ let rope = if l.window > 0 { &self.rope_window } else { &self.rope_global };+ x = l.forward(&x, rope)?;+ }+ Ok(x.permute((0, 3, 1, 2))?.contiguous()?)+ }+}diff --git a/crates/segment/src/weights.rs b/crates/segment/src/weights.rsnew file mode 100644index 000000000..15de564ae--- /dev/null+++ b/crates/segment/src/weights.rs@@ -0,0 +1,108 @@+//! Safetensors weights read on demand.+//!+//! Tensors are read from the file when a module asks for them (positional reads, no memory+//! map: `unsafe` stays in `lightcraft-sysmem`), so building only the point-prompt model never+//! touches the text encoder's bytes.++use std::collections::HashMap;+use std::fs::File;+use std::io::{Read, Seek, SeekFrom};+use std::path::Path;+use std::sync::{Arc, Mutex};++use candle_core::{DType, Device, Shape, Tensor};+use candle_nn::var_builder::SimpleBackend;++use crate::{Error, Result};++struct Entry {+ shape: Vec<usize>,+ start: u64,+ end: u64,+}++/// An open safetensors file: its header, and the file to read tensor bytes from.+pub struct Weights {+ file: Mutex<File>,+ entries: HashMap<String, Entry>,+ data_start: u64,+}++impl Weights {+ pub fn open(path: &Path) -> Result<Arc<Self>> {+ let mut file = File::open(path).map_err(|e| Error::Model(format!("{}: {e}", path.display())))?;+ let mut len = [0u8; 8];+ file.read_exact(&mut len).map_err(|e| Error::Model(format!("{}: {e}", path.display())))?;+ let n = u64::from_le_bytes(len);+ if n > 100 << 20 {+ return Err(Error::Model(format!("{}: not a safetensors file (header of {n} bytes)", path.display())));+ }+ let mut header = vec![0u8; n as usize];+ file.read_exact(&mut header).map_err(|e| Error::Model(format!("{}: {e}", path.display())))?;+ let header: serde_json::Map<String, serde_json::Value> =+ serde_json::from_slice(&header).map_err(|e| Error::Model(format!("{}: bad header: {e}", path.display())))?;+ let mut entries = HashMap::new();+ for (name, v) in header {+ if name == "__metadata__" {+ continue;+ }+ let dtype = match v.get("dtype").and_then(|d| d.as_str()) {+ Some("F32") => DType::F32,+ other => return Err(Error::Model(format!("{name}: unsupported dtype {other:?} (the SAM 3 checkpoint is F32)"))),+ };+ let shape: Vec<usize> = v+ .get("shape")+ .and_then(|s| s.as_array())+ .map(|a| a.iter().filter_map(|x| x.as_u64().map(|x| x as usize)).collect())+ .unwrap_or_default();+ let offsets: Vec<u64> =+ v.get("data_offsets").and_then(|s| s.as_array()).map(|a| a.iter().filter_map(|x| x.as_u64()).collect()).unwrap_or_default();+ let [start, end] = offsets[..] else { return Err(Error::Model(format!("{name}: bad data_offsets"))) };+ let want = shape.iter().try_fold(dtype.size_in_bytes(), |a, d| a.checked_mul(*d));+ if end.checked_sub(start) != want.map(|w| w as u64) {+ return Err(Error::Model(format!("{name}: size does not match its shape")));+ }+ entries.insert(name, Entry { shape, start, end });+ }+ Ok(Arc::new(Self { file: Mutex::new(file), entries, data_start: 8 + n }))+ }++ pub fn contains(&self, name: &str) -> bool {+ self.entries.contains_key(name)+ }++ /// Tensor `name` as stored, on the CPU.+ pub fn read(&self, name: &str) -> Result<Tensor> {+ let e = self.entries.get(name).ok_or_else(|| Error::Model(format!("missing tensor `{name}`")))?;+ let mut bytes = vec![0u8; e.end.saturating_sub(e.start) as usize];+ {+ let mut f = self.file.lock().unwrap_or_else(std::sync::PoisonError::into_inner);+ f.seek(SeekFrom::Start(self.data_start + e.start)).map_err(|err| Error::Model(format!("{name}: {err}")))?;+ f.read_exact(&mut bytes).map_err(|err| Error::Model(format!("{name}: {err}")))?;+ }+ let v: Vec<f32> = bytes.as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect();+ Ok(Tensor::from_vec(v, e.shape.as_slice(), &Device::Cpu)?)+ }+}++/// [`Weights`] as a candle `VarBuilder` backend.+pub struct Backend(pub Arc<Weights>);++impl SimpleBackend for Backend {+ fn get(&self, s: Shape, name: &str, _h: candle_nn::Init, dtype: DType, dev: &Device) -> candle_core::Result<Tensor> {+ let t = self.get_unchecked(name, dtype, dev)?;+ if t.shape() != &s {+ candle_core::bail!("shape mismatch for {name}: expected {s:?}, got {:?}", t.shape())+ }+ Ok(t)+ }++ fn get_unchecked(&self, name: &str, dtype: DType, dev: &Device) -> candle_core::Result<Tensor> {+ let t = self.0.read(name).map_err(|e| candle_core::Error::Msg(e.to_string()))?;+ t.to_dtype(dtype)?.to_device(dev)+ }++ fn contains_tensor(&self, name: &str) -> bool {+ self.0.contains(name)+ }+}diff --git a/crates/segment/tests/reference.rs b/crates/segment/tests/reference.rsnew file mode 100644index 000000000..a73fcaa7c--- /dev/null+++ b/crates/segment/tests/reference.rs@@ -0,0 +1,97 @@+//! The port against the reference implementation (Hugging Face `transformers`, fp32).+//!+//! Needs the checkpoint and a reference file made by `tools/sam3_reference.py`:+//! `LIGHTCRAFT_SAM3_DIR=<model dir> LIGHTCRAFT_SAM3_REF=<ref.safetensors> cargo test -p lightcraft-segment --release -- --nocapture`.+//! Skipped (passes) when either is missing.++use std::collections::HashMap;+use std::path::PathBuf;++use candle_core::{Device, IndexOp, Tensor};+use lightcraft_segment::{Click, Sam3, tokenizer::Tokenizer};++fn setup() -> Option<(PathBuf, HashMap<String, Tensor>)> {+ let dir = PathBuf::from(std::env::var_os("LIGHTCRAFT_SAM3_DIR")?);+ let reference = PathBuf::from(std::env::var_os("LIGHTCRAFT_SAM3_REF")?);+ let r = candle_core::safetensors::load(&reference, &Device::Cpu).ok()?;+ Some((dir, r))+}++fn vec(t: &Tensor) -> Vec<f32> {+ t.to_device(&Device::Cpu).unwrap().flatten_all().unwrap().to_vec1::<f32>().unwrap()+}++/// Max |a − b| relative to the reference's max |b|.+fn rel_err(name: &str, a: &[f32], b: &[f32]) -> f32 {+ assert_eq!(a.len(), b.len(), "{name}: length");+ let scale = b.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-6);+ let err = a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs()));+ eprintln!("{name:>18}: max abs err {err:.2e}, rel {:.2e} (scale {scale:.2e})", err / scale);+ err / scale+}++/// Fraction of pixels where the two logit maps disagree on the mask (> 0).+fn mask_disagreement(a: &[f32], b: &[f32]) -> f32 {+ a.iter().zip(b).filter(|(x, y)| (**x > 0.0) != (**y > 0.0)).count() as f32 / a.len() as f32+}++#[test]+fn matches_reference() {+ let Some((dir, r)) = setup() else {+ eprintln!("skipped: set LIGHTCRAFT_SAM3_DIR and LIGHTCRAFT_SAM3_REF");+ return;+ };+ let mut model = Sam3::load(&dir).unwrap();+ eprintln!("device: {:?}", model.device());+ let t = std::time::Instant::now();+ let mut enc = model.encode_pixels(&r["pixel_values"]).unwrap();+ let backbone = enc.backbone().permute((0, 2, 3, 1)).unwrap();+ assert!(rel_err("backbone", &vec(&backbone), &vec(&r["backbone"])) < 1e-3);+ eprintln!("backbone: {:?}", t.elapsed());++ // points: two clicks (single-mask output), one click (best of three)+ let clicks = [Click { x: 0.5, y: 0.55, positive: true }, Click { x: 0.2, y: 0.2, positive: false }];+ let t = std::time::Instant::now();+ let p = model.segment_clicks(&mut enc, &clicks).unwrap();+ eprintln!("clicks (incl. tracker load + features): {:?}", t.elapsed());+ let want = vec(&r["trk_masks"]);+ assert!(rel_err("2-click logits", &p.logits, &want) < 1e-2);+ assert!(mask_disagreement(&p.logits, &want) < 1e-3);+ let t = std::time::Instant::now();+ let p = model.segment_clicks(&mut enc, &clicks[..1]).unwrap();+ eprintln!("one more click: {:?}", t.elapsed());+ let iou = vec(&r["trk_multi_iou"]);+ let best = (0..3).max_by(|a, b| iou[*a].partial_cmp(&iou[*b]).unwrap()).unwrap();+ let want = vec(&r["trk_multi_masks"].i((0, 0, best)).unwrap());+ assert!(rel_err("1-click logits", &p.logits, &want) < 1e-2);+ assert!(mask_disagreement(&p.logits, &want) < 1e-3);++ // tokenizer+ let tok = Tokenizer::load(&dir).unwrap();+ let cases = ["person", "Red Car", "a dog's toy", "sky, clouds!", "café 2 people", "tree-line"];+ for (i, c) in cases.iter().enumerate() {+ let (ids, _) = tok.encode(c);+ let want: Vec<i64> = r[&format!("tok{i}")].flatten_all().unwrap().to_vec1().unwrap();+ assert_eq!(ids.iter().map(|v| *v as i64).collect::<Vec<_>>(), want, "tokens of {c:?}");+ }++ // text prompts+ for (i, text) in ["moon", "sky", "bird"].iter().enumerate() {+ let t = std::time::Instant::now();+ let out = model.detect(&mut enc, text).unwrap();+ eprintln!("detect {text:?}: {:?}", t.elapsed());+ assert!(rel_err("text features", &vec(&out.text), &vec(&r[&format!("det{i}_text")])) < 1e-3);+ assert!(rel_err("query logits", &out.logits, &vec(&r[&format!("det{i}_logits")])) < 1e-2);+ let presence = vec(&r[&format!("det{i}_presence")])[0];+ assert!((out.presence - presence).abs() < 1e-2, "presence {} vs {presence}", out.presence);+ let boxes: Vec<f32> = out.boxes.iter().flatten().copied().collect();+ assert!(rel_err("boxes", &boxes, &vec(&r[&format!("det{i}_boxes")])) < 1e-2);+ // the best instance's mask+ let logits = vec(&r[&format!("det{i}_logits")]);+ let q = (0..logits.len()).max_by(|a, b| logits[*a].partial_cmp(&logits[*b]).unwrap()).unwrap();+ let ours = vec(&out.masks.i(q).unwrap());+ let want = vec(&r[&format!("det{i}_masks")].i((0, q)).unwrap());+ assert!(rel_err("best mask", &ours, &want) < 2e-2);+ assert!(mask_disagreement(&ours, &want) < 2e-3);+ }+}diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rsindex 8addc61b5..0afdd915d 100644--- a/crates/ui-egui/src/lib.rs+++ b/crates/ui-egui/src/lib.rs@@ -260,6 +260,7 @@ impl LightcraftApp {}let r = self.session.execute(id, ¶ms).map_err(|e| e.to_string());if let Err(e) = &r {+ log::warn!("{id}: {e}");self.ui.status = e.clone();}r@@ -348,6 +349,34 @@ impl LightcraftApp {self.ui.toast = Some((text.into(), t + 1.4));}+ /// AI masks: start a zoomed-in detail pass once the clicking stops (`ui.detail_due`), and+ /// apply finished passes.+ fn ai_mask_detail(&mut self, ctx: &egui::Context) {+ if self.session.segment_poll() {+ ctx.request_repaint();+ }+ let now = ctx.input(|i| i.time);+ if let Some((due, mask)) = self.ui.detail_due {+ if now >= due && !self.session.segmenter.busy() && !self.session.segmenter.detail_busy() {+ self.ui.detail_due = None;+ if let Err(e) = self.run("mask.refineDetail", serde_json::json!({"id": mask})) {+ log::warn!("detail pass: {e}");+ }+ } else {+ ctx.request_repaint_after(std::time::Duration::from_millis(150));+ }+ }+ if self.session.segmenter.detail_busy() {+ ctx.request_repaint_after(std::time::Duration::from_millis(200));+ }+ }++ /// A toast for an error the user has to read and act on (stays 6 s).+ pub fn toast_error(&mut self, ctx: &egui::Context, text: impl Into<String>) {+ let t = ctx.input(|i| i.time);+ self.ui.toast = Some((text.into(), t + 6.0));+ }+fn drain_control(&mut self, ctx: &egui::Context) {let Some(rx) = self.control_rx.take() else { return };while let Ok(req) = rx.try_recv() {@@ -673,6 +702,7 @@ impl LightcraftApp {}// panels set it again this frame while the pointer rests on a preset or profileself.hover_preview = None;+ self.ai_mask_detail(&ctx);if self.ui.fullscreen {// full-screen preview: the photo alone on blackegui::CentralPanel::default().frame(egui::Frame::NONE.fill(egui::Color32::BLACK)).show(ui, |ui| panels::detail::show(self, ui));diff --git a/crates/ui-egui/src/panels/detail.rs b/crates/ui-egui/src/panels/detail.rsindex 0697b75cc..e8341a2af 100644--- a/crates/ui-egui/src/panels/detail.rs+++ b/crates/ui-egui/src/panels/detail.rs@@ -526,6 +526,9 @@ fn targeted_drag(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respon}}+/// The colour of a mask hovered in the Masks list.+const HOVER_RED: [u8; 3] = [230, 30, 40];+/// The diagnostic overlay the loupe shows (the selected mask, Point Color's visualized range,/// Visualize Spots).pub(crate) fn view_overlay(app: &LightcraftApp, d: &DevelopSettings) -> lightcraft_pipeline::Overlay {@@ -533,6 +536,13 @@ pub(crate) fn view_overlay(app: &LightcraftApp, d: &DevelopSettings) -> lightcraif app.ui.fullscreen {return Overlay::None;}+ // a mask hovered in the Masks list shows in red; the selected one only with the overlay on (O)+ if app.ui.right == RightPanel::Masking+ && let Some(m) = app.ui.hover_mask.and_then(|id| d.masks.iter().find(|m| m.id == id))+ && !m.components.is_empty()+ {+ return Overlay::Mask { id: m.id.min(u16::MAX as u32) as u16, view: MaskView::default(), color: HOVER_RED, opacity: 55 };+ }if app.ui.right == RightPanel::Masking&& app.ui.mask_overlay&& let Some(m) = app.session.active_mask.and_then(|id| d.masks.iter().find(|m| m.id == id))@@ -941,6 +951,41 @@ fn mask_overlay(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respons}}}+ // Object selections (SAM 3) of the selected mask: each click, green to include, red to exclude+ for m in d.masks.iter().filter(|m| Some(m.id) == active) {+ for c in &m.components {+ if let MaskShape::Object { hint, exclude, .. } = &c.shape {+ for (pts, col) in [(hint, Color32::from_rgb(40, 200, 90)), (exclude, Color32::from_rgb(230, 60, 60))] {+ for q in pts {+ let q = map.screen(*q);+ p.circle_filled(q, 5.0, col);+ p.circle_stroke(q, 5.0, Stroke::new(1.5, Color32::WHITE));+ }+ }+ }+ }+ }+ // Object tool: a click includes what's under it, ⌥-click leaves it out+ if app.ui.tool == "object" {+ if resp.hovered() {+ ui.ctx().set_cursor_icon(egui::CursorIcon::Crosshair);+ }+ if resp.clicked()+ && let Some(q) = resp.interact_pointer_pos()+ {+ let n = map.norm(q);+ let exclude = ui.input(|i| i.modifiers.alt);+ match app.run("mask.objectPoint", json!({"x": n.x, "y": n.y, "exclude": exclude})) {+ Ok(_) => {+ if let Some(mid) = app.session.active_mask {+ app.ui.detail_due = Some((ui.input(|i| i.time) + 1.0, mid));+ }+ }+ Err(e) => app.toast_error(ui.ctx(), e),+ }+ }+ return;+ }// brush toolif app.ui.tool == "brush" {let r = (app.ui.brush_size as f64 * long) as f32;diff --git a/crates/ui-egui/src/panels/masking.rs b/crates/ui-egui/src/panels/masking.rsindex f938dc8d7..86fa7aad7 100644--- a/crates/ui-egui/src/panels/masking.rs+++ b/crates/ui-egui/src/panels/masking.rs@@ -53,6 +53,7 @@ fn kind_label(s: &MaskShape) -> (&'static str, Icon) {MaskShape::Sky => ("Sky", Icon::Sky),MaskShape::Background => ("Background", Icon::Subject),MaskShape::Object { .. } => ("Object", Icon::Subject),+ MaskShape::Prompt { .. } => ("Describe", Icon::Subject),MaskShape::People { .. } => ("People", Icon::Subject),MaskShape::Landscape { .. } => ("Landscape", Icon::Sky),}@@ -75,7 +76,9 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {egui::Frame::NONE.inner_margin(egui::Margin { left: 24, right: 22, top: 0, bottom: 10 }).show(ui, |ui| {ui.label(egui::RichText::new(crate::i18n::tr("Create New Mask")).color(t.text_dim));ui.add_space(6.0);- let tiles: [(&str, &str, Icon); 8] = [+ let tiles: [(&str, &str, Icon); 10] = [+ ("object", "Object", Icon::Subject),+ ("prompt", "Describe", Icon::Subject),("subject", "Subject", Icon::Subject),("sky", "Sky", Icon::Sky),("background", "Background", Icon::Subject),@@ -102,6 +105,8 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {app.ui.tool = "colorRange".into();app.toast(ui.ctx(), "Click the photo to pick a colour · ⇧-click adds more");}+ "object" => start_object(app, ui.ctx(), "new"),+ "prompt" => app.ui.describe = Some(("new".into(), String::new())),"brush" | "linear" | "radial" => {app.ui.tool = kind.to_string();if *kind != "brush" {@@ -118,6 +123,19 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {}}});+ describe_field(app, ui, true);+ let status = if app.session.segmenter.busy() {+ Some("Analyzing the photo for AI masks…")+ } else if app.session.segmenter.detail_busy() || app.ui.detail_due.is_some() {+ Some("Refining the mask's detail…")+ } else {+ None+ };+ if let Some(status) = status {+ ui.add_space(4.0);+ ui.label(egui::RichText::new(crate::i18n::tr(status)).color(t.text_dim));+ ui.ctx().request_repaint_after(std::time::Duration::from_millis(250));+ }});divider(ui);// mask list@@ -126,7 +144,17 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {ui.label(egui::RichText::new(crate::i18n::tr("Masks")).font(t.semibold(13.0)).color(t.text_label));ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {let on = app.ui.mask_overlay;- if icon_button(ui, "maskOverlay", Icon::Eye, vec2(24.0, 24.0), on, true, "Show overlay (O)").clicked() {+ if icon_button(+ ui,+ "maskOverlay",+ Icon::Eye,+ vec2(24.0, 24.0),+ on,+ true,+ "Always show the selected mask (O); hovering a mask shows it",+ )+ .clicked()+ {app.ui.mask_overlay = !on;}});@@ -135,6 +163,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {ui.label(egui::RichText::new(crate::i18n::tr("No masks yet. Choose a mask type above.")).color(t.text_dim));}let count = d.masks.len();+ app.ui.hover_mask = None;for (index, m) in d.masks.iter().enumerate() {let sel = app.session.active_mask == Some(m.id);if let Some((rid, name)) = app.ui.renaming_mask.as_mut().filter(|(rid, _)| *rid == m.id) {@@ -158,6 +187,10 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {}let (r, resp) = ui.allocate_exact_size(vec2(ui.available_width(), 30.0), Sense::click());register(ui.ctx(), format!("mask:{}", m.id), r);+ // the mask under the pointer shows on the photo (red overlay)+ if ui.rect_contains_pointer(r) {+ app.ui.hover_mask = Some(m.id);+ }ui.painter().rect_filled(r,4.0,@@ -201,6 +234,18 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {app.ui.renaming_mask = Some((m.id, m.name.clone()));}resp.context_menu(|ui| mask_menu(app, ui, m.id, &m.name, m.visible, index, count));+ if sel {+ // right under the selected mask: + adds to it, − takes away from it (any mask+ // type; Describe… opens its field here)+ ui.horizontal(|ui| {+ ui.add_space(30.0);+ let plus = text_button(ui, &format!("maskPlus:{}", m.id), "+", false).on_hover_text(crate::i18n::tr("Add to this mask"));+ egui::Popup::menu(&plus).show(|ui| component_menu(app, ui, "add"));+ let minus = text_button(ui, &format!("maskMinus:{}", m.id), "−", false).on_hover_text(crate::i18n::tr("Subtract from this mask"));+ egui::Popup::menu(&minus).show(|ui| component_menu(app, ui, "subtract"));+ });+ describe_field(app, ui, false);+ }}if !d.masks.is_empty() {ui.add_space(6.0);@@ -213,7 +258,10 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {egui::Frame::NONE.inner_margin(egui::Margin { left: 24, right: 22, top: 8, bottom: 8 }).show(ui, |ui| {for (i, c) in m.components.iter().enumerate() {let (kind, icon) = kind_label(&c.shape);- let label = c.name.clone().unwrap_or_else(|| kind.to_string());+ let label = c.name.clone().unwrap_or_else(|| match &c.shape {+ MaskShape::Prompt { text, .. } => format!("“{text}”"),+ _ => kind.to_string(),+ });ui.horizontal(|ui| {let (r, _) = ui.allocate_exact_size(vec2(16.0, 16.0), Sense::hover());paint(ui.painter(), r, icon, t.text_label);@@ -249,6 +297,9 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {});let resp = resp.inner;register(ui.ctx(), format!("component:{i}"), resp.rect);+ if ui.rect_contains_pointer(resp.rect) {+ app.ui.hover_mask = Some(m.id);+ }if resp.double_clicked() {app.ui.renaming_component = Some((m.id, i, label.clone()));}@@ -486,6 +537,30 @@ fn range_controls(app: &mut LightcraftApp, ui: &mut egui::Ui, comp: usize, shape}}}+ MaskShape::Object { edge, .. } | MaskShape::Prompt { edge, .. } => {+ // Edge: how crisp the selection's border is (−100 hard … 0 as computed … 100 soft)+ let spec = ControlSpec {+ id: "edge",+ label: "Edge",+ section: Section::Light,+ min: -100.0,+ max: 100.0,+ default: 0.0,+ step: 1.0,+ decimals: 0,+ track: Track::Centered,+ };+ let out = slider(ui, &spec, *edge, true, None);+ let base = shape.clone();+ apply_slider_out(app, &spec, out, |app, v| {+ let mut s = base.clone();+ if let MaskShape::Object { edge, .. } | MaskShape::Prompt { edge, .. } = &mut s {+ *edge = v.clamp(-100.0, 100.0);+ }+ app.run("mask.update", json!({"component": comp, "shape": s}))+ });+ ui.label(egui::RichText::new(crate::i18n::tr("− harder border · + softer border")).color(t.text_dim).size(11.0));+ }MaskShape::ColorRange { samples, refine } => {ui.horizontal(|ui| {ui.label(@@ -560,7 +635,84 @@ fn component_row_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, mask: u32, k:}}+/// Start an Object selection (SAM 3 clicks): a new mask, or a component of the selected one+/// combined by `op`; the photo is analyzed meanwhile.+fn start_object(app: &mut LightcraftApp, ctx: &egui::Context, op: &str) {+ let r =+ if op == "new" { app.run("mask.add", json!({"kind": "object"})) } else { app.run("mask.addComponent", json!({"op": op, "kind": "object"})) };+ match r {+ Ok(_) => {+ app.ui.tool = "object".into();+ // a missing model is reported now rather than at the first click+ app.toast(ctx, "Click the object to select it · ⌥-click leaves a part out");+ }+ Err(e) => app.toast_error(ctx, e),+ }+}++/// The Describe field: type what to select ("sky", "the red car") and press Return.+fn describe_field(app: &mut LightcraftApp, ui: &mut egui::Ui, new_mask: bool) {+ let Some((op, mut text)) = app.ui.describe.clone() else { return };+ // a new mask's field sits under the tiles; one that combines, under the selected mask+ if (op == "new") != new_mask {+ return;+ }+ ui.add_space(8.0);+ let prompt = match op.as_str() {+ "subtract" => "Describe what to leave out",+ "intersect" => "Describe what to keep",+ _ => "Describe what to select",+ };+ ui.label(crate::i18n::tr(prompt));+ let mut submit = false;+ ui.horizontal(|ui| {+ let r =+ ui.add(egui::TextEdit::singleline(&mut text).hint_text("e.g. sky · the red car · car, road").desired_width(ui.available_width() - 64.0));+ register(ui.ctx(), "maskDescribe", r.rect);+ if !r.has_focus() && !r.lost_focus() && text.is_empty() {+ r.request_focus();+ }+ if r.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)) {+ submit = true;+ }+ if text_button(ui, "maskDescribeGo", crate::i18n::tr("Select"), false).clicked() {+ submit = true;+ }+ });+ if ui.input(|i| i.key_pressed(egui::Key::Escape)) {+ app.ui.describe = None;+ return;+ }+ app.ui.describe = Some((op.clone(), text.clone()));+ if submit && !text.trim().is_empty() {+ let r = if op == "new" {+ app.run("mask.add", json!({"kind": "prompt", "text": text}))+ } else {+ app.run("mask.addComponent", json!({"op": op, "kind": "prompt", "text": text}))+ };+ match r {+ Ok(_) => {+ app.ui.describe = None;+ if let Some(mid) = app.session.active_mask {+ app.ui.detail_due = Some((ui.input(|i| i.time) + 0.3, mid));+ }+ }+ Err(e) => app.toast_error(ui.ctx(), e),+ }+ }+}+fn component_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, op: &str) {+ let b = ui.button(crate::i18n::tr("Object"));+ register(ui.ctx(), format!("maskComp:{op}:object"), b.rect);+ if b.clicked() {+ start_object(app, ui.ctx(), op);+ }+ let b = ui.button(crate::i18n::tr("Describe…"));+ register(ui.ctx(), format!("maskComp:{op}:prompt"), b.rect);+ if b.clicked() {+ app.ui.describe = Some((op.to_string(), String::new()));+ }for (kind, label) in [("brush", "Brush"),("linear", "Linear Gradient"),diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rsindex 1a4446e3a..c7e6b2258 100644--- a/crates/ui-egui/src/state.rs+++ b/crates/ui-egui/src/state.rs@@ -283,6 +283,17 @@ pub struct UiState {/// A mask being renamed in the Masks list: its id and the edited name.#[serde(skip)]pub renaming_mask: Option<(u32, String)>,+ /// The Describe field (AI mask from a text prompt) while open: how the selection combines+ /// (`new` mask, or `add`/`subtract`/`intersect` on the selected one) and the text typed.+ #[serde(skip)]+ pub describe: Option<(String, String)>,+ /// The mask under the pointer in the Masks list (shown on the photo in red).+ #[serde(skip)]+ pub hover_mask: Option<u32>,+ /// When to start the zoomed-in detail pass of an AI mask (app time) and which mask: set by+ /// each click or description, so the pass runs once the clicking stops.+ #[serde(skip)]+ pub detail_due: Option<(f64, u32)>,/// A mask component being renamed inline: (mask id, component index, name).pub renaming_component: Option<(u32, usize, String)>,/// Close the window on the next frame (File → Quit).@@ -538,7 +549,7 @@ impl Default for UiState {histogram: true,soft_proof: false,proof: lightcraft_engine::pipeline::Proof { dest_warning: false, ..Default::default() },- mask_overlay: true,+ mask_overlay: false,mask_overlay_mode: "color".into(),mask_overlay_color: [230, 30, 40],mask_overlay_opacity: 50.0,@@ -556,6 +567,9 @@ impl Default for UiState {search: String::new(),focus_search: false,renaming_mask: None,+ describe: None,+ hover_mask: None,+ detail_due: None,renaming_component: None,quit: false,dragging_photos: None,diff --git a/crates/ui-egui/src/tests_masking.rs b/crates/ui-egui/src/tests_masking.rsindex 1e4c0d143..5be82af44 100644--- a/crates/ui-egui/src/tests_masking.rs+++ b/crates/ui-egui/src/tests_masking.rs@@ -47,14 +47,20 @@ fn mask_overlay_keys_and_pins() {exec(&mut h, "mask.add", json!({"kind": "radial", "center": [0.3, 0.4], "rx": 0.1, "ry": 0.1}));exec(&mut h, "mask.add", json!({"kind": "linear", "start": [0.7, 0.2], "end": [0.7, 0.6]}));assert_eq!(h.app.session.active_mask, Some(2));- // the loupe asks the renderer for the selected mask's overlay+ // no overlay until a mask is hovered in the Masks list; then that mask, in redlet d = develop(&h);+ assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::None);+ h.app.ui.hover_mask = Some(1);+ assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::Mask { id: 1, view: MaskView::Color, color: [230, 30, 40], opacity: 55 });+ h.app.ui.hover_mask = None;+ // O shows the selected mask all the time (and toggles it off again)+ h.request("ui.key", json!({"key": "o"}), T);+ assert!(h.app.ui.mask_overlay);let o = crate::panels::detail::view_overlay(&h.app, &d);assert_eq!(o, Overlay::Mask { id: 2, view: MaskView::Color, color: [230, 30, 40], opacity: 50 });- // O toggles it, Shift+O cycles the colour (and leaves the crop overlay alone)h.request("ui.key", json!({"key": "o"}), T);- assert!(!h.app.ui.mask_overlay);assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::None);+ // Shift+O cycles the colour (and leaves the crop overlay alone)h.request("ui.key", json!({"key": "o"}), T);let crop = h.app.ui.crop_overlay;let colour = h.app.ui.mask_overlay_color;diff --git a/docs/ai-masks.md b/docs/ai-masks.mdnew file mode 100644index 000000000..dc3e6726e--- /dev/null+++ b/docs/ai-masks.md@@ -0,0 +1,103 @@+# AI masks (Object and Describe)++LightCraft selects objects with **SAM 3** (Segment Anything with Concepts, Meta 2025), run in+pure Rust by `crates/segment` on [candle](https://github.com/huggingface/candle): on the GPU+through Metal on macOS, on the CPU elsewhere (for now).++## Using them++In the Masking panel (M):++- **Object**: click the thing you want; every click refines the selection. ⌥-click (Alt) a part+ to leave it out. Clicks show as green (include) and red (leave out) dots.+- **Describe**: type what to select — `sky`, `trees`, `the red car` — and press Return. Every+ instance the model finds (score above 0.5) joins the mask; if nothing matches, nothing is+ created and you are told so.+- Several things at once: `car, road` (comma-separated) selects each and merges them.+- **+ / −** under the selected mask (and the Add / Subtract / Intersect menus) combine+ selections: Describe `car`, click the mask, + ▸ Describe… `road`; − ▸ Describe… `people` takes+ the people out of a sky mask; − ▸ Object then clicks remove one object.+- **Edge** (per selection, −100…100): below 0 a crisper border, above 0 a feathered one (up to 2 %+ of the long edge, so previews and exports match).+- Hover a mask in the Masks list to see it in red on the photo; O shows the selected one all the+ time.+- **Detail:** about a second after the last click (or right after a description), the photo+ around the selection is analyzed again, zoomed in, from a sharper source (~5 s on an M4 Pro,+ in the background); the result is kept as a high-resolution patch, so small objects get 5–10×+ finer edges than one pass over the whole photo gives.++The first Object or Describe on a photo analyzes it (the image encoder runs once per photo and+look: ~3 s on an Apple M4 Pro, plus a one-time ~6 s for compiling GPU kernels and loading the+model after launch). That starts in the background as soon as you pick Object; after it, each+click takes ~35 ms and each description ~0.6 s.++The result is stored with the mask: the model's 288 × 288 mask logits over the uncropped photo+(quantized, compressed, ~10–30 KB). Renders and exports sample them at any resolution and never+need the model, so masks survive moving the library to a machine without it, and the web build+renders them. Editing the photo's look later does not recompute a mask (click again to update).++## Installing the model++The weights are not part of LightCraft: they are Meta's `facebook/sam3` checkpoint on Hugging+Face, under the SAM License, and access is gated. Once:++1. Open <https://huggingface.co/facebook/sam3>, accept the license and wait for approval.+2. Create a read token (Hugging Face ▸ Settings ▸ Access Tokens), or log in with `hf auth login`.+3. Run the installer from the repository:++```sh+HF_TOKEN=hf_... tools/install-sam3.sh # macOS, Linux+tools/install-sam3.sh --check # verify an installation+```++```powershell+$env:HF_TOKEN = "hf_..."; .\tools\install-sam3.ps1 # Windows+```++It downloads `model.safetensors` (3.4 GB, fp32; resumes when interrupted, checked against the+official SHA-256), `vocab.json`, `merges.txt` and the small config files into the folder+LightCraft looks in: `~/Library/Application Support/LightCraft/models/sam3/` (macOS),+`%APPDATA%\LightCraft\models\sam3` (Windows), `~/.config/lightcraft/models/sam3` (Linux).+`--dir` (or `LIGHTCRAFT_SAM3_DIR`, which LightCraft also reads) picks another folder; `--repo`+another Hugging Face repository with the same files. Restart LightCraft afterwards.++Nothing else is needed at run time: the model runs in LightCraft itself (pure Rust, no Python).+Without the files, Object and Describe say where they are expected.++**Developers:** the accuracy test against the Python reference needs `torch` and+`transformers` (`tools/requirements-sam3-reference.txt`):++```sh+python3 -m venv .venv-sam3 && .venv-sam3/bin/pip install -r tools/requirements-sam3-reference.txt+.venv-sam3/bin/python tools/sam3_reference.py photo.jpg ref.safetensors+LIGHTCRAFT_SAM3_DIR=<model dir> LIGHTCRAFT_SAM3_REF=ref.safetensors cargo test -p lightcraft-segment --release -- --nocapture+```++## Commands (control channel, MCP)++| Command | Params | |+|---|---|---|+| `mask.add` | `{kind: "object", points?: [[x,y],…], exclude?: [[x,y],…]}` | an Object mask (empty until clicked) |+| `mask.add` | `{kind: "prompt", text}` | a Describe mask |+| `mask.addComponent` | `{op: add\|subtract\|intersect, kind: "object"\|"prompt", …}` | the same as a component |+| `mask.objectPoint` | `{x, y, exclude?: bool, id?}` | one click on the selected mask's Object selection |+| `mask.refineDetail` | `{id?, component?}` | the zoomed-in detail pass, in the background → `{started}` |+| `segment.prepare` | `{}` | analyze the active photo in the background → `{busy}` |++Coordinates are normalized to the uncropped, oriented photo, like every mask shape.++## Implementation notes++- `crates/segment` ports the Hugging Face `transformers` implementation (Apache-2.0; see+ `NOTICE`): the 32-layer ViT backbone with 2-D RoPE and windowed attention, the feature+ pyramids, the SAM 2-style prompt encoder and two-way mask decoder for clicks, and the CLIP text+ encoder, DETR encoder/decoder (box relative-position bias, presence token) and pixel decoder+ for text. The CLIP tokenizer is a small BPE in `tokenizer.rs`.+- candle is pinned at 0.9.2: later releases make `candle-core` depend on `tokenizers` with the+ Oniguruma C library, and the product is pure Rust. Weights are read with positional reads (no+ memory map: `unsafe` stays in `lightcraft-sysmem`), and only the tensors a path needs.+- Accuracy: `crates/segment/tests/reference.rs` compares every stage against the Python+ reference (`tools/sam3_reference.py`); on Metal the backbone matches to 7e-5 relative error+ and masks to ~5e-5. f16 was tried and rejected (overflow, ~7 % faster).+- The engine feature `sam` (on in the desktop app, off for web and CLI) gates the model; without+ it the commands report that the build has no AI masks.diff --git a/docs/parity.md b/docs/parity.mdindex fc94d08a7..2967d3115 100644--- a/docs/parity.md+++ b/docs/parity.md@@ -69,8 +69,9 @@ Take the first one nobody is working on.then tune tone, highlights, texture/clarity/dehaze, NR and sharpening against it.5. **LR-EDIT-OPTICS-PROFILE** (P1): a lens-profile database of our own (embedded DNG/maker corrections work today).6. **AI masks and Enhance** (LR-MASK-SUBJECT / SKY / PEOPLE / OBJECTS, LR-EDIT-DETAIL-DENOISE, SUPERRES, LENSBLUR):- blocked on a model strategy (permissively licensed weights or our own training, pure-Rust inference). A maintainer- decision, not just engineering.+ Object and Describe masks run SAM 3 in pure Rust (`crates/segment`) with user-installed weights; Subject / Sky /+ People could use the same model with fixed prompts. Bundling or recommending weights (SAM License) and denoise /+ super-resolution models remain a maintainer decision.7. **HDR** (Q. HDR, LR-EXP-HDR), **video** (R. Video), **Classic output modules** (Map view, Book, Slideshow module,Print, publish): large, well understood, lower priority than 1–5.@@ -328,7 +329,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.| LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model || LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`) | heuristic || LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject heuristic |-| LR-MASK-OBJECTS | Object selection | P2 | ⬜ | | shape exists (falls back to the subject heuristic); no UI |+| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:segment.prepare`, `crates/segment`, `crates/engine/src/segment.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components. The segmentation is stored with the mask (288² logits), so renders and exports never need the model. Weights are not bundled: the user installs the `facebook/sam3` checkpoint (SAM License) — see docs/ai-masks.md. + / − under the selected mask combine selections; comma lists (`car, road`); a zoomed-in detail pass (`cmd:mask.refineDetail`, background) for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft); hovering a mask shows it in red. No brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro) || LR-MASK-PEOPLE | People parts | P2 | ⬜ | | || LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty || LR-MASK-BRUSH | Brush mask | P0 | ✅ | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs` | size/feather/flow/density/erase; Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); no A/B brushes, no pressure |diff --git a/tools/install-sam3.ps1 b/tools/install-sam3.ps1new file mode 100644index 000000000..bd43fd98b--- /dev/null+++ b/tools/install-sam3.ps1@@ -0,0 +1,40 @@+# Install the SAM 3 model for LightCraft's Object and Describe masks on Windows+# (see docs/ai-masks.md; tools/install-sam3.sh is the macOS/Linux version).+#+# facebook/sam3 is gated: accept the SAM License at https://huggingface.co/facebook/sam3, wait+# for approval, then: $env:HF_TOKEN = "hf_..." ; .\tools\install-sam3.ps1+param(+ [string]$Dir = $(if ($env:LIGHTCRAFT_SAM3_DIR) { $env:LIGHTCRAFT_SAM3_DIR } else { Join-Path $env:APPDATA "LightCraft\models\sam3" }),+ [string]$Repo = "facebook/sam3",+ [switch]$Check+)+$ErrorActionPreference = "Stop"+$Sha256 = "6d06f0a5f84e435071fe6603e61d0b4cc7b40e0d39d487cfd4d67d8cc11cc14a"+$Files = "config.json", "vocab.json", "merges.txt", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "processor_config.json"++function Test-Install {+ foreach ($f in "model.safetensors", "vocab.json", "merges.txt") {+ if (-not (Test-Path (Join-Path $Dir $f))) { throw "missing: $(Join-Path $Dir $f)" }+ }+ Write-Host "checking model.safetensors (SHA-256 of 3.4 GB)..."+ $got = (Get-FileHash (Join-Path $Dir "model.safetensors") -Algorithm SHA256).Hash.ToLower()+ if ($got -ne $Sha256) { throw "model.safetensors does not match the official checkpoint (got $got)" }+ Write-Host "SAM 3 is installed in $Dir"+}++if ($Check) { Test-Install; exit 0 }++$Token = if ($env:HF_TOKEN) { $env:HF_TOKEN } else { $tf = Join-Path $HOME ".cache\huggingface\token"; if (Test-Path $tf) { (Get-Content $tf -Raw).Trim() } }+if (-not $Token -and $Repo -eq "facebook/sam3") {+ throw "No Hugging Face token. Accept the license at https://huggingface.co/facebook/sam3, then set HF_TOKEN or run 'hf auth login'."+}+New-Item -ItemType Directory -Force -Path $Dir | Out-Null+$Headers = @{}+if ($Token) { $Headers["Authorization"] = "Bearer $Token" }+$Base = "https://huggingface.co/$Repo/resolve/main"+foreach ($f in $Files + "model.safetensors") {+ Write-Host "fetching $f"+ Invoke-WebRequest -Uri "$Base/$f" -Headers $Headers -OutFile (Join-Path $Dir $f)+}+Test-Install+Write-Host "Restart LightCraft; Object and Describe in the Masking panel now use it."diff --git a/tools/install-sam3.sh b/tools/install-sam3.shnew file mode 100755index 000000000..86554c39d--- /dev/null+++ b/tools/install-sam3.sh@@ -0,0 +1,110 @@+#!/bin/sh+# Install the SAM 3 model for LightCraft's Object and Describe masks (see docs/ai-masks.md).+#+# The weights are not part of LightCraft: they are Meta's facebook/sam3 checkpoint on Hugging+# Face, under the SAM License. Access is gated: open https://huggingface.co/facebook/sam3, accept+# the license, wait for approval, then run this with a token that can read it:+#+# HF_TOKEN=hf_... tools/install-sam3.sh+#+# (or log in once with `hf auth login`; the token it saves is used). Downloads resume when+# interrupted, and the 3.4 GB weights are checked against the official SHA-256.+#+# Options:+# --dir DIR where to install (default: the folder LightCraft looks in; or LIGHTCRAFT_SAM3_DIR)+# --repo NAME another Hugging Face repo with the same files (default: facebook/sam3)+# --check only verify an existing installation+set -eu++REPO="facebook/sam3"+CHECK_ONLY=0+SHA256="6d06f0a5f84e435071fe6603e61d0b4cc7b40e0d39d487cfd4d67d8cc11cc14a"+SIZE=3439938512+FILES="config.json vocab.json merges.txt tokenizer.json tokenizer_config.json special_tokens_map.json processor_config.json"++case "$(uname -s)" in+ Darwin) DEFAULT_DIR="$HOME/Library/Application Support/LightCraft/models/sam3" ;;+ *) DEFAULT_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/lightcraft/models/sam3" ;;+esac+DIR="${LIGHTCRAFT_SAM3_DIR:-$DEFAULT_DIR}"++while [ $# -gt 0 ]; do+ case "$1" in+ --dir) DIR="$2"; shift 2 ;;+ --repo) REPO="$2"; shift 2 ;;+ --check) CHECK_ONLY=1; shift ;;+ -h|--help) sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;+ *) echo "unknown option: $1 (see --help)" >&2; exit 2 ;;+ esac+done++sha256() {+ if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | cut -d' ' -f1+ else sha256sum "$1" | cut -d' ' -f1; fi+}++verify() {+ for f in model.safetensors vocab.json merges.txt; do+ [ -s "$DIR/$f" ] || { echo "missing: $DIR/$f" >&2; return 1; }+ done+ echo "checking model.safetensors (SHA-256 of 3.4 GB, takes a few seconds)…"+ got=$(sha256 "$DIR/model.safetensors")+ if [ "$got" != "$SHA256" ]; then+ echo "model.safetensors does not match the official checkpoint (got $got)" >&2+ return 1+ fi+ echo "SAM 3 is installed in $DIR"+}++if [ "$CHECK_ONLY" = 1 ]; then verify; exit $?; fi++command -v curl >/dev/null 2>&1 || { echo "curl is required" >&2; exit 1; }++TOKEN="${HF_TOKEN:-${HUGGING_FACE_HUB_TOKEN:-}}"+if [ -z "$TOKEN" ]; then+ for t in "${HF_HOME:-$HOME/.cache/huggingface}/token" "$HOME/.huggingface/token"; do+ [ -s "$t" ] && TOKEN=$(cat "$t") && break+ done+fi+if [ -z "$TOKEN" ] && [ "$REPO" = "facebook/sam3" ]; then+ echo "No Hugging Face token found. facebook/sam3 is gated: accept the license at" >&2+ echo "https://huggingface.co/facebook/sam3, then set HF_TOKEN or run \`hf auth login\`." >&2+ exit 1+fi++mkdir -p "$DIR"+BASE="https://huggingface.co/$REPO/resolve/main"++fetch() { # fetch FILE DEST [resume]+ set -- "$1" "$2" "${3:-}"+ if [ -n "$TOKEN" ]; then auth="Authorization: Bearer $TOKEN"; else auth="X-No-Auth: 1"; fi+ code=$(curl -sSL ${3:+-C -} -H "$auth" -w '%{http_code}' -o "$2" "$BASE/$1") || code=$?+ case "$code" in+ 200|206|416) return 0 ;;+ 401|403) echo "access denied to $REPO/$1 (has your request for facebook/sam3 been approved?)" >&2; return 1 ;;+ *) echo "download of $REPO/$1 failed ($code)" >&2; return 1 ;;+ esac+}++for f in $FILES; do+ echo "fetching $f"+ fetch "$f" "$DIR/$f"+done++if [ -s "$DIR/model.safetensors" ] && [ "$(sha256 "$DIR/model.safetensors")" = "$SHA256" ]; then+ echo "model.safetensors already installed"+else+ echo "fetching model.safetensors (3.4 GB; resumes if interrupted)"+ part="$DIR/model.safetensors.part"+ [ -s "$DIR/model.safetensors" ] && mv -f "$DIR/model.safetensors" "$part"+ fetch model.safetensors "$part" resume+ have=$(wc -c < "$part" | tr -d ' ')+ if [ "$have" != "$SIZE" ]; then+ echo "incomplete download ($have of $SIZE bytes); run the script again to resume" >&2+ exit 1+ fi+ mv -f "$part" "$DIR/model.safetensors"+fi++verify+echo "Restart LightCraft; Object and Describe in the Masking panel now use it."diff --git a/tools/requirements-sam3-reference.txt b/tools/requirements-sam3-reference.txtnew file mode 100644index 000000000..cf8f263aa--- /dev/null+++ b/tools/requirements-sam3-reference.txt@@ -0,0 +1,8 @@+# Python packages for tools/sam3_reference.py (reference tensors for crates/segment/tests/reference.rs).+# python3 -m venv .venv-sam3 && .venv-sam3/bin/pip install -r tools/requirements-sam3-reference.txt+torch>=2.6+torchvision>=0.21+transformers>=4.57+pillow+safetensors+numpydiff --git a/tools/sam3_reference.py b/tools/sam3_reference.pynew file mode 100644index 000000000..66d9def90--- /dev/null+++ b/tools/sam3_reference.py@@ -0,0 +1,71 @@+"""Reference tensors for the Rust SAM 3 port (crates/segment/tests/reference.rs).++Runs the Hugging Face transformers implementation (fp32, CPU) on one photo and writes the+shared pixel input plus intermediate and final outputs of the click (tracker) and text+(detector) paths. Usage (needs torch, torchvision, transformers, pillow, safetensors):+ LIGHTCRAFT_SAM3_DIR=<model dir> python tools/sam3_reference.py photo.jpg ref.safetensors+The text prompts below suit a photo of the moon in a blue sky; change them for other photos.+"""+import os, sys, time+import numpy as np+import torch+from PIL import Image+from safetensors.torch import save_file+from transformers import Sam3TrackerModel, Sam3Model, CLIPTokenizer++MODEL = os.environ.get("LIGHTCRAFT_SAM3_DIR", os.path.expanduser("~/Library/Application Support/LightCraft/models/sam3"))+img_path, out_path = sys.argv[1], sys.argv[2]+torch.set_grad_enabled(False)++img = Image.open(img_path).convert("RGB")+W, H = img.size+# Same preprocessing as the Rust side: bilinear resize to 1008², /255, (x-0.5)/0.5.+r = img.resize((1008, 1008), Image.BILINEAR)+px = torch.from_numpy(np.asarray(r, dtype=np.float32) / 255.0).permute(2, 0, 1)[None]+px = (px - 0.5) / 0.5+out = {"pixel_values": px.contiguous()}++t = time.time()+trk = Sam3TrackerModel.from_pretrained(MODEL).eval()+vis = trk.vision_encoder(px)+out["backbone"] = vis.last_hidden_state.contiguous()+for i, f in enumerate(vis.fpn_hidden_states):+ out[f"trk_fpn{i}"] = f.contiguous()+# points in 1008 space: one positive, one negative+pts = torch.tensor([[[[0.5 * 1008, 0.55 * 1008], [0.2 * 1008, 0.2 * 1008]]]], dtype=torch.float32)+lbl = torch.tensor([[[1, 0]]], dtype=torch.int32)+o = trk(pixel_values=px, input_points=pts, input_labels=lbl, multimask_output=False)+out["trk_points"] = pts.contiguous()+out["trk_masks"] = o.pred_masks.contiguous()+out["trk_iou"] = o.iou_scores.contiguous()+o3 = trk(pixel_values=px, input_points=pts[:, :, :1], input_labels=lbl[:, :, :1], multimask_output=True)+out["trk_multi_masks"] = o3.pred_masks.contiguous()+out["trk_multi_iou"] = o3.iou_scores.contiguous()+print("tracker", time.time() - t, o.pred_masks.shape, o.iou_scores, file=sys.stderr)+del trk++t = time.time()+det = Sam3Model.from_pretrained(MODEL).eval()+tok = CLIPTokenizer.from_pretrained(MODEL)+for i, text in enumerate(["moon", "sky", "bird"]):+ enc = tok(text, padding="max_length", max_length=32, return_tensors="pt")+ out[f"det{i}_ids"] = enc.input_ids.to(torch.int64).contiguous()+ out[f"det{i}_mask"] = enc.attention_mask.to(torch.int64).contiguous()+ tf = det.get_text_features(input_ids=enc.input_ids, attention_mask=enc.attention_mask)+ out[f"det{i}_text"] = tf.pooler_output.contiguous()+ o = det(pixel_values=px, input_ids=enc.input_ids, attention_mask=enc.attention_mask)+ out[f"det{i}_logits"] = o.pred_logits.contiguous()+ out[f"det{i}_presence"] = o.presence_logits.contiguous()+ out[f"det{i}_boxes"] = o.pred_boxes.contiguous()+ out[f"det{i}_masks"] = o.pred_masks.contiguous()+ sc = (o.pred_logits.sigmoid() * o.presence_logits.sigmoid())[0]+ print(text, "presence", o.presence_logits.sigmoid().item(), "top", sc.topk(3).values.tolist(), file=sys.stderr)+for i, f in enumerate(det.vision_encoder(px).fpn_hidden_states):+ out[f"det_fpn{i}"] = f.contiguous()+print("detector", time.time() - t, file=sys.stderr)+# tokenizer cases for the Rust BPE+cases = ["person", "Red Car", "a dog's toy", "sky, clouds!", "café 2 people", "tree-line"]+for i, c in enumerate(cases):+ out[f"tok{i}"] = tok(c, padding="max_length", max_length=32, return_tensors="pt").input_ids.to(torch.int64).contiguous()+save_file(out, out_path, metadata={"image": img_path, "W": str(W), "H": str(H), "cases": "|".join(cases)})+print("saved", out_path, file=sys.stderr)diff --git a/xtask/src/layers.rs b/xtask/src/layers.rsindex b5c631422..28c994622 100644--- a/xtask/src/layers.rs+++ b/xtask/src/layers.rs@@ -49,6 +49,7 @@ pub const TABLE: &[(&str, Class)] = &[("preview", Class::Layer(3)),("export", Class::Layer(3)),("merge", Class::Layer(3)),+ ("segment", Class::Layer(3)),("engine", Class::Layer(4)),("ui-egui", Class::Layer(5)),("mcp", Class::Layer(5)),