Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

LR-EDIT-DETAIL-DENOISE: non-destructive AI RAW denoise with pure-Rust CPU/GPU

Daniel Voshart7935999d9449
diff --git a/Cargo.lock b/Cargo.lock
index 86d4bca64..6de52a46f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -209,7 +209,7 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b"
dependencies = [
- "equator",
+ "equator 0.4.2",
]
[[package]]
@@ -967,6 +967,12 @@ dependencies = [
"unicode-width",
]
+[[package]]
+name = "coe-rs"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698"
+
[[package]]
name = "color"
version = "0.3.3"
@@ -1256,6 +1262,16 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
+[[package]]
+name = "dyn-stack"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"
+dependencies = [
+ "bytemuck",
+ "reborrow",
+]
+
[[package]]
name = "dyn-stack"
version = "0.13.2"
@@ -1537,13 +1553,33 @@ version = "0.36.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13"
+[[package]]
+name = "equator"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3b0a88aa91d0ad2b9684e4479aed31a17d3f9051bdbbc634bd2c01bc5a5eee8"
+dependencies = [
+ "equator-macro 0.1.9",
+]
+
[[package]]
name = "equator"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc"
dependencies = [
- "equator-macro",
+ "equator-macro 0.4.2",
+]
+
+[[package]]
+name = "equator-macro"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60d08acb9849f7fb4401564f251be5a526829183a3645a90197dea8e786cf3ae"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
]
[[package]]
@@ -1608,6 +1644,42 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "faer-core"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32da0f6262445261106407bdfb548df4f8f1c89bcc3cff1bb6d08e7f525b575a"
+dependencies = [
+ "bytemuck",
+ "coe-rs",
+ "dyn-stack 0.10.0",
+ "equator 0.1.10",
+ "faer-entity",
+ "gemm 0.17.1",
+ "matrixcompare-core",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "pulp 0.18.22",
+ "reborrow",
+ "seq-macro",
+]
+
+[[package]]
+name = "faer-entity"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fb12f36da3162b3fefc0c7343d905d5bd2c2bf444a7f63ceeaa2bc003311d3"
+dependencies = [
+ "bytemuck",
+ "coe-rs",
+ "libm",
+ "num-complex",
+ "num-traits",
+ "pulp 0.18.22",
+ "reborrow",
+]
+
[[package]]
name = "fastrand"
version = "2.5.0"
@@ -1783,13 +1855,33 @@ dependencies = [
"slab",
]
+[[package]]
+name = "gemm"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32"
+dependencies = [
+ "dyn-stack 0.10.0",
+ "gemm-c32 0.17.1",
+ "gemm-c64 0.17.1",
+ "gemm-common 0.17.1",
+ "gemm-f16 0.17.1",
+ "gemm-f32 0.17.1",
+ "gemm-f64 0.17.1",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid 10.7.0",
+ "seq-macro",
+]
+
[[package]]
name = "gemm"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-c32 0.18.2",
"gemm-c64 0.18.2",
"gemm-common 0.18.2",
@@ -1799,7 +1891,7 @@ dependencies = [
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"seq-macro",
]
@@ -1809,7 +1901,7 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-c32 0.19.0",
"gemm-c64 0.19.0",
"gemm-common 0.19.0",
@@ -1819,7 +1911,22 @@ dependencies = [
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-c32"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0"
+dependencies = [
+ "dyn-stack 0.10.0",
+ "gemm-common 0.17.1",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid 10.7.0",
"seq-macro",
]
@@ -1829,12 +1936,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"seq-macro",
]
@@ -1844,12 +1951,27 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-c64"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a"
+dependencies = [
+ "dyn-stack 0.10.0",
+ "gemm-common 0.17.1",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid 10.7.0",
"seq-macro",
]
@@ -1859,12 +1981,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"seq-macro",
]
@@ -1874,13 +1996,32 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-common"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8"
+dependencies = [
+ "bytemuck",
+ "dyn-stack 0.10.0",
+ "half",
+ "num-complex",
+ "num-traits",
+ "once_cell",
+ "paste",
+ "pulp 0.18.22",
+ "raw-cpuid 10.7.0",
"seq-macro",
+ "sysctl 0.5.5",
]
[[package]]
@@ -1890,7 +2031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3"
dependencies = [
"bytemuck",
- "dyn-stack",
+ "dyn-stack 0.13.2",
"half",
"libm",
"num-complex",
@@ -1898,10 +2039,10 @@ dependencies = [
"once_cell",
"paste",
"pulp 0.21.5",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"rayon",
"seq-macro",
- "sysctl",
+ "sysctl 0.6.0",
]
[[package]]
@@ -1911,7 +2052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e"
dependencies = [
"bytemuck",
- "dyn-stack",
+ "dyn-stack 0.13.2",
"half",
"libm",
"num-complex",
@@ -1919,10 +2060,27 @@ dependencies = [
"once_cell",
"paste",
"pulp 0.22.3",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"rayon",
"seq-macro",
- "sysctl",
+ "sysctl 0.6.0",
+]
+
+[[package]]
+name = "gemm-f16"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4"
+dependencies = [
+ "dyn-stack 0.10.0",
+ "gemm-common 0.17.1",
+ "gemm-f32 0.17.1",
+ "half",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid 10.7.0",
+ "seq-macro",
]
[[package]]
@@ -1931,14 +2089,14 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.18.2",
"gemm-f32 0.18.2",
"half",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"rayon",
"seq-macro",
]
@@ -1949,30 +2107,45 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.19.0",
"gemm-f32 0.19.0",
"half",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"rayon",
"seq-macro",
]
+[[package]]
+name = "gemm-f32"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113"
+dependencies = [
+ "dyn-stack 0.10.0",
+ "gemm-common 0.17.1",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid 10.7.0",
+ "seq-macro",
+]
+
[[package]]
name = "gemm-f32"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"seq-macro",
]
@@ -1982,12 +2155,27 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-f64"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0"
+dependencies = [
+ "dyn-stack 0.10.0",
+ "gemm-common 0.17.1",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid 10.7.0",
"seq-macro",
]
@@ -1997,12 +2185,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"seq-macro",
]
@@ -2012,12 +2200,12 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a"
dependencies = [
- "dyn-stack",
+ "dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"seq-macro",
]
@@ -2859,6 +3047,22 @@ dependencies = [
"serde",
]
+[[package]]
+name = "lightcraft-denoise"
+version = "0.4.0"
+dependencies = [
+ "faer-core",
+ "lightcraft-raster",
+ "miniz_oxide 0.8.9",
+ "proptest",
+ "rayon",
+ "serde",
+ "serde_json",
+ "sha2",
+ "thiserror 2.0.21",
+ "web-time",
+]
+
[[package]]
name = "lightcraft-develop"
version = "0.4.0"
@@ -2881,7 +3085,9 @@ dependencies = [
"lightcraft-catalog",
"lightcraft-codecs",
"lightcraft-color",
+ "lightcraft-denoise",
"lightcraft-develop",
+ "lightcraft-fetch",
"lightcraft-geom",
"lightcraft-gpu",
"lightcraft-merge",
@@ -2928,6 +3134,7 @@ version = "0.4.0"
dependencies = [
"bytemuck",
"lightcraft-color",
+ "lightcraft-denoise",
"lightcraft-develop",
"lightcraft-geom",
"lightcraft-pipeline",
@@ -3082,6 +3289,7 @@ dependencies = [
"lightcraft-catalog",
"lightcraft-codecs",
"lightcraft-color",
+ "lightcraft-denoise",
"lightcraft-develop",
"lightcraft-engine",
"lightcraft-geom",
@@ -3174,6 +3382,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "matrixcompare-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73"
+
[[package]]
name = "maybe-rayon"
version = "0.1.1"
@@ -4291,6 +4505,18 @@ dependencies = [
"unarray",
]
+[[package]]
+name = "pulp"
+version = "0.18.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6"
+dependencies = [
+ "bytemuck",
+ "libm",
+ "num-complex",
+ "reborrow",
+]
+
[[package]]
name = "pulp"
version = "0.21.5"
@@ -4317,7 +4543,7 @@ dependencies = [
"num-complex",
"paste",
"pulp-wasm-simd-flag",
- "raw-cpuid",
+ "raw-cpuid 11.6.0",
"reborrow",
"version_check",
]
@@ -4518,6 +4744,15 @@ dependencies = [
"rgb",
]
+[[package]]
+name = "raw-cpuid"
+version = "10.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
[[package]]
name = "raw-cpuid"
version = "11.6.0"
@@ -5245,6 +5480,20 @@ dependencies = [
"syn 3.0.6",
]
+[[package]]
+name = "sysctl"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
+dependencies = [
+ "bitflags 2.13.2",
+ "byteorder",
+ "enum-as-inner",
+ "libc",
+ "thiserror 1.0.69",
+ "walkdir",
+]
+
[[package]]
name = "sysctl"
version = "0.6.0"
diff --git a/Cargo.toml b/Cargo.toml
index 77b206669..b23fae21c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,7 @@ repository = "https://github.com/storytold/lightcraft"
[workspace.dependencies]
# internal
+lightcraft-denoise = { path = "crates/denoise" }
lightcraft-geom = { path = "crates/geom" }
lightcraft-sysmem = { path = "crates/sysmem" }
lightcraft-color = { path = "crates/color" }
@@ -34,6 +35,7 @@ lightcraft-segment = { path = "crates/segment" }
# foundation
thiserror = "2"
+sha2 = "0.10"
serde = { version = "1", features = ["derive", "rc"] }
serde_json = { version = "1", features = ["float_roundtrip"] }
log = "0.4"
diff --git a/ROADMAP.md b/ROADMAP.md
index 5e793e8be..18d4dae10 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -25,7 +25,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
| **Feature checklist** | 79.7% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P0: full-resolution zoom detail; P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim |
| **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) |
| **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW, NEF, RW2, RAF, CR3, CR2 and PEF have guarded camera-JPEG colour estimates, with bundled ILCE-7M4, X-H2S and X-T4 profiles (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |
-| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics | Real segmentation masks (subject, sky, people, objects, landscape, depth), AI denoise, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
+| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics; non-destructive Bayer RAW denoise with opt-in weights on pure-Rust CPU/GPU | Real segmentation masks (subject, sky, people, objects, landscape, depth), denoise model policy / linear RGB / quality verification, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
| **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering |
| **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic) | **Map view, Book, Print, Slideshow module, Web, publish services**: ~40 tracker rows ⬜ |
| **HDR & video** | 0% | | HDR edit/display/export; video play/trim/edit/export |
@@ -40,7 +40,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
| Fujifilm raw shooter | ~65% | Uncompressed and lossless/lossy compressed RAF decoded; camera colour calibration remains missing. Verified on 13 bodies, including 14/16-bit GFX; see `docs/raf-compression.md` |
| Canon CR3 / Olympus compressed-raw shooter | ~45% | CR3 lossless Bayer and version 0x100/0x200 C-RAW develop from sensor data (M50, R100, R8 verified pixel for pixel); other CRX variants and compressed ORF open as embedded JPEG previews. Camera colour and broader model verification remain |
| Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering |
-| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise |
+| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); Bayer AI denoise needs separately installed weights; X-Trans/linear RGB, model policy and quality parity remain open |
## Where we're going
@@ -59,9 +59,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par
(references stay in the local `plan/`), then tune against the numbers.
5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1).
6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for
- segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Object / Describe masks now
+ segmentation masks and denoise. Bayer denoise inference is implemented in pure Rust with an opt-in GPL model; release policy and independently licensed weights remain open. Unblocks M12 and Enhance. Object / Describe masks now
run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still
- to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open.
+ to be set up, docs/ai-masks.md); Subject / Sky / People, Raw Details and Super Resolution are still open.
7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation
and accessibility.
@@ -86,7 +86,7 @@ yet guarantee original-pixel detail; visible-region rendering remains a gap (LR-
| M4 | Crop, geometry, optics | crop tool + overlays, straighten, Upright (auto/level/vertical/full/guided), manual transforms, CA, defringe, manual lens corrections | 6–10 | ✅ |
| M5 | Performance | source pyramids, wgpu compute pipeline (CPU oracle), draft/full renders, prefetch, budgets (16 ms slider updates on 24 MP) | 10–15 | 🚧 (stage cache, source pyramid, wgpu pipeline, prefetch, memory budget ✅; colour NR at half resolution, GPU histogram ⬜) |
| M6 | Masking | brush, linear/radial gradients, colour/luminance/depth range, add/subtract/intersect/invert, all local adjustments, masks panel | 8–12 | 🚧 (brush/linear/radial/colour/luminance range, add/subtract/intersect, masks panel ✅; depth range, AI masks ⬜) |
-| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise, Raw Details, Super Resolution ⬜) |
+| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise 🟡 (Bayer, optional weights; policy/fidelity open); Raw Details, Super Resolution ⬜) |
| M8 | Heal / Remove | content-aware remove (PatchMatch), heal, clone, brush spots, visualize spots, red/pet eye | 6–10 | 🚧 (heal, clone, auto source, visualize spots, red/pet eye ✅; PatchMatch remove ⬜) |
| M9 | Presets, profiles, versions, sync | preset browser + amount, create/import presets, profile browser, versions, history, copy/paste/sync settings | 5–8 | ✅ |
| M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs ✅; JXL encode, HDR export ⬜) |
diff --git a/apps/lightcraft-cli/Cargo.toml b/apps/lightcraft-cli/Cargo.toml
index 3f60fe32a..72d468586 100644
--- a/apps/lightcraft-cli/Cargo.toml
+++ b/apps/lightcraft-cli/Cargo.toml
@@ -25,6 +25,8 @@ serde_json = { workspace = true }
dhat = { workspace = true, optional = true }
[features]
+default = ["denoise"]
+denoise = ["lightcraft-engine/denoise"]
# Heap profiling: `cargo run --release -p lightcraft-cli --features dhat-heap -- snapshot …` counts
# every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json
# (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work.
diff --git a/apps/lightcraft-cli/src/main.rs b/apps/lightcraft-cli/src/main.rs
index 6c5e6e442..c6dfdc2ac 100644
--- a/apps/lightcraft-cli/src/main.rs
+++ b/apps/lightcraft-cli/src/main.rs
@@ -391,7 +391,7 @@ fn merge(args: &[String]) -> Result<(), String> {
}
i += 1;
}
- let mut s = Session::new().with_fs();
+ let mut s = Session::new().with_fs().with_default_denoise_models();
let paths = expand_paths(&files);
let r = s.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;
let mut ids: Vec<u64> = r["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default();
@@ -652,7 +652,7 @@ fn render(args: &[String]) -> Result<(), String> {
}
let input = input.ok_or("render: missing input file")?;
let output = output.ok_or("render: missing -o OUTPUT")?;
- let mut s = Session::new().with_fs();
+ let mut s = Session::new().with_fs().with_default_denoise_models();
let abs = expand_paths(std::slice::from_ref(&input));
let r = s.execute("library.import", &json!({"paths": abs})).map_err(|e| e.to_string())?;
let id = r["imported"][0].as_u64().ok_or_else(|| format!("{input}: not a readable photo"))?;
@@ -734,12 +734,12 @@ fn snapshot(args: &[String]) -> Result<(), String> {
let t0 = Instant::now();
let mut session = match &library {
Some(dir) => {
- let mut s = Session::new().with_fs();
+ let mut s = Session::new().with_fs().with_default_denoise_models();
s.open_library(dir, false).map_err(|e| library_error(dir, e))?;
s
}
- None if files.is_empty() => Session::with_demo().with_fs(),
- None => Session::new().with_fs(),
+ None if files.is_empty() => Session::with_demo().with_fs().with_default_denoise_models(),
+ None => Session::new().with_fs().with_default_denoise_models(),
};
if !files.is_empty() {
let ti = Instant::now();
diff --git a/apps/lightcraft/Cargo.toml b/apps/lightcraft/Cargo.toml
index a29299da0..3f3f40ba0 100644
--- a/apps/lightcraft/Cargo.toml
+++ b/apps/lightcraft/Cargo.toml
@@ -9,6 +9,10 @@ default-run = "lightcraft"
[lints]
workspace = true
+[features]
+default = ["denoise"]
+denoise = ["lightcraft-engine/denoise"]
+
[dependencies]
lightcraft-engine = { workspace = true, features = ["sam"] }
lightcraft-sysmem = { workspace = true }
diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs
index 4c7bb4242..44f0548dd 100644
--- a/apps/lightcraft/src/main.rs
+++ b/apps/lightcraft/src/main.rs
@@ -453,15 +453,15 @@ fn services(ctx: egui::Context, log_file: Option<&std::path::Path>) -> Services
/// and offers Try Again / Choose Another Library… / Continue Without Saving / Quit (issue #100).
fn open_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: bool) -> (Session, Option<LibraryProblem>) {
if in_memory {
- return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock(), None);
+ return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_default_denoise_models().with_system_clock(), None);
}
- let unopened = || Session::new().with_fs().with_system_clock();
+ let unopened = || Session::new().with_fs().with_default_denoise_models().with_system_clock();
let Some(dir) = dir else {
log::warn!("no library location (set --library or LIGHTCRAFT_LIBRARY)");
return (unopened(), Some(LibraryProblem::new("", "There is no home folder to keep the library in. Choose a folder for it.")));
};
let t0 = std::time::Instant::now();
- let mut s = Session::new().with_fs().with_system_clock();
+ let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock();
match s.open_library(&dir, seed_demo) {
Ok(r) => {
let (replayed, torn) = (r.replayed, r.torn_bytes);
diff --git a/crates/denoise/Cargo.toml b/crates/denoise/Cargo.toml
new file mode 100644
index 000000000..55c44a144
--- /dev/null
+++ b/crates/denoise/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "lightcraft-denoise"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "LightCraft AI denoise (L1): denoise-model manifests, Bayer packing, tiling and blending, the cache file, and pure-Rust CPU inference"
+
+[lints]
+workspace = true
+
+[features]
+default = []
+
+[dependencies]
+# Safe views over Rust-intrinsic gemm kernels. Pin the core-only release: newer backends
+# generate assembly. No BLAS, cc/build scripts, nightly/AMX, or inner thread pool.
+faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"] }
+lightcraft-raster = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+sha2 = { workspace = true }
+thiserror = { workspace = true }
+miniz_oxide = { workspace = true }
+rayon = { workspace = true }
+web-time = { workspace = true }
+
+[dev-dependencies]
+proptest = { workspace = true }
diff --git a/crates/denoise/README.md b/crates/denoise/README.md
new file mode 100644
index 000000000..cb69e3822
--- /dev/null
+++ b/crates/denoise/README.md
@@ -0,0 +1,14 @@
+# lightcraft-denoise
+
+L1 pure-Rust AI RAW denoise primitives and CPU inference. No weights are bundled.
+See [user workflow and limitations](../../docs/denoise.md).
+
+`manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset
+into `net`; `cpu` executes it and `reference` supplies scalar checks. `bayer`, `tiles`
+and `run` pack sensor data and blend overlapping tiles while preserving clipped
+highlights. `product` reads/writes the disposable `.lcdn` cache. `runtime` supplies
+the installation self-test. Synthetic networks are generated in Rust for tests.
+
+Run `cargo test -p lightcraft-denoise`. Real-model tests are ignored by default and
+require an explicitly supplied local model. The GPU twin is `lightcraft-gpu::nn`;
+model management, background jobs and export integration live in `lightcraft-engine`.
diff --git a/crates/denoise/src/archive.rs b/crates/denoise/src/archive.rs
new file mode 100644
index 000000000..ddaf7084d
--- /dev/null
+++ b/crates/denoise/src/archive.rs
@@ -0,0 +1,271 @@
+//! Taking one file out of a `.zip` (a `.dtmodel` is one), the little of the format a model download needs: stored and
+//! deflated entries, found by name, checked against their CRC-32 and a size limit, written next to where they are
+//! going and moved into place. It is a stranger's file, so every length and offset in it is checked, a name is only
+//! ever compared (never used as a path), and zip64, encrypted and multi-disk archives are refused.
+
+use std::io::{Read, Seek, SeekFrom, Write};
+use std::path::Path;
+
+/// How far from the end the end-of-central-directory record can start (a 22-byte record and a 64 KiB comment).
+const EOCD_SEARCH: u64 = 22 + 65535;
+/// Largest central directory we read.
+const MAX_DIRECTORY: u64 = 1 << 20;
+
+#[derive(Debug, PartialEq, thiserror::Error)]
+pub enum ArchiveError {
+ #[error("the archive could not be read: {0}")]
+ Io(String),
+ #[error("not a usable archive: {0}")]
+ Format(String),
+ #[error("the archive has no file called `{0}`")]
+ Missing(String),
+}
+
+impl From<std::io::Error> for ArchiveError {
+ fn from(e: std::io::Error) -> Self {
+ ArchiveError::Io(e.to_string())
+ }
+}
+
+fn format<T>(why: &str) -> Result<T, ArchiveError> {
+ Err(ArchiveError::Format(why.into()))
+}
+
+fn u16_at(b: &[u8], at: usize) -> Option<u16> {
+ Some(u16::from_le_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?))
+}
+
+fn u32_at(b: &[u8], at: usize) -> Option<u32> {
+ Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
+}
+
+/// CRC-32 (IEEE) of `bytes`.
+fn crc32(bytes: &[u8]) -> u32 {
+ let mut table = [0u32; 256];
+ for (i, t) in table.iter_mut().enumerate() {
+ let mut c = i as u32;
+ for _ in 0..8 {
+ c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
+ }
+ *t = c;
+ }
+ !bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))
+}
+
+struct Entry {
+ method: u16,
+ crc: u32,
+ compressed: u64,
+ size: u64,
+ local_header: u64,
+}
+
+/// Find the entry called `name` (exactly, or as the last part of a longer name: `folder/name`) in the central directory.
+fn find(f: &mut (impl Read + Seek), name: &str) -> Result<Entry, ArchiveError> {
+ let len = f.seek(SeekFrom::End(0))?;
+ let tail_len = len.min(EOCD_SEARCH);
+ f.seek(SeekFrom::Start(len - tail_len))?;
+ let mut tail = vec![0u8; tail_len as usize];
+ f.read_exact(&mut tail)?;
+ let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| u32_at(&tail, i) == Some(0x0605_4b50));
+ let Some(e) = eocd else { return format("this is not a zip file") };
+ let (entries, size, offset) = (u16_at(&tail, e + 10), u32_at(&tail, e + 12), u32_at(&tail, e + 16));
+ let (Some(entries), Some(size), Some(offset)) = (entries, size, offset) else { return format("the end of the archive is cut short") };
+ if entries == 0xffff || size == u32::MAX || offset == u32::MAX {
+ return format("zip64 archives are not supported");
+ }
+ if u64::from(size) > MAX_DIRECTORY || u64::from(offset) + u64::from(size) > len {
+ return format("the list of files is not where it says");
+ }
+ f.seek(SeekFrom::Start(u64::from(offset)))?;
+ let mut dir = vec![0u8; size as usize];
+ f.read_exact(&mut dir)?;
+ let mut at = 0usize;
+ for _ in 0..entries {
+ if u32_at(&dir, at) != Some(0x0201_4b50) {
+ return format("the list of files is damaged");
+ }
+ let get = |o: usize| at.checked_add(o);
+ let (flags, method, crc) = (u16_at(&dir, at + 8), u16_at(&dir, at + 10), u32_at(&dir, at + 16));
+ let (compressed, uncompressed) = (u32_at(&dir, at + 20), u32_at(&dir, at + 24));
+ let (n, x, c) = (u16_at(&dir, at + 28), u16_at(&dir, at + 30), u16_at(&dir, at + 32));
+ let local = u32_at(&dir, at + 42);
+ let (Some(flags), Some(method), Some(crc), Some(compressed), Some(uncompressed), Some(n), Some(x), Some(c), Some(local), Some(start)) =
+ (flags, method, crc, compressed, uncompressed, n, x, c, local, get(46))
+ else {
+ return format("the list of files is cut short");
+ };
+ let Some(entry_name) = dir.get(start..start + usize::from(n)) else { return format("a file name is cut short") };
+ let matches = entry_name == name.as_bytes() || entry_name.strip_suffix(name.as_bytes()).is_some_and(|p| p.ends_with(b"/"));
+ if matches {
+ if flags & 1 != 0 {
+ return format("encrypted files are not supported");
+ }
+ if compressed == u32::MAX || uncompressed == u32::MAX {
+ return format("zip64 files are not supported");
+ }
+ return Ok(Entry { method, crc, compressed: u64::from(compressed), size: u64::from(uncompressed), local_header: u64::from(local) });
+ }
+ at = start + usize::from(n) + usize::from(x) + usize::from(c);
+ }
+ Err(ArchiveError::Missing(name.into()))
+}
+
+/// Take the file called `name` out of the archive at `archive` and write it to `dest` (through a temporary file next
+/// to it); `max_bytes` bounds its size. Returns the number of bytes written.
+pub fn extract(archive: &Path, name: &str, dest: &Path, max_bytes: u64) -> Result<u64, ArchiveError> {
+ let mut f = std::io::BufReader::new(std::fs::File::open(archive)?);
+ let e = find(&mut f, name)?;
+ if e.size > max_bytes || e.compressed > max_bytes {
+ return format("the file in the archive is larger than allowed");
+ }
+ f.seek(SeekFrom::Start(e.local_header))?;
+ let mut head = [0u8; 30];
+ f.read_exact(&mut head)?;
+ if u32_at(&head, 0) != Some(0x0403_4b50) {
+ return format("the file's header is not where it says");
+ }
+ let skip = u64::from(u16_at(&head, 26).unwrap_or(0)) + u64::from(u16_at(&head, 28).unwrap_or(0));
+ f.seek_relative(i64::try_from(skip).map_err(|_| ArchiveError::Format("a header is too long".into()))?)?;
+ let mut packed = vec![0u8; e.compressed as usize];
+ f.read_exact(&mut packed)?;
+ let data = match e.method {
+ 0 => packed,
+ 8 => {
+ miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes as usize).map_err(|err| ArchiveError::Format(format!("{err:?}")))?
+ }
+ m => return format(&format!("compression method {m} is not supported")),
+ };
+ if data.len() as u64 != e.size || crc32(&data) != e.crc {
+ return format("the file in the archive is damaged");
+ }
+ let tmp = dest.with_extension("part");
+ let written = (|| -> std::io::Result<()> {
+ let mut out = std::fs::File::create(&tmp)?;
+ out.write_all(&data)?;
+ out.flush()?;
+ drop(out);
+ std::fs::rename(&tmp, dest)
+ })();
+ if let Err(err) = written {
+ let _ = std::fs::remove_file(&tmp);
+ return Err(err.into());
+ }
+ Ok(data.len() as u64)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// A zip with the given (name, bytes, deflate) entries, written by hand.
+ pub(crate) fn make_zip(files: &[(&str, &[u8], bool)]) -> Vec<u8> {
+ let mut out = Vec::new();
+ let mut dir = Vec::new();
+ for (name, data, deflate) in files {
+ let packed = if *deflate { miniz_oxide::deflate::compress_to_vec(data, 6) } else { data.to_vec() };
+ let offset = out.len() as u32;
+ let method: u16 = if *deflate { 8 } else { 0 };
+ out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
+ out.extend_from_slice(&[20, 0, 0, 0]);
+ out.extend_from_slice(&method.to_le_bytes());
+ out.extend_from_slice(&[0, 0, 0, 0]);
+ out.extend_from_slice(&crc32(data).to_le_bytes());
+ out.extend_from_slice(&(packed.len() as u32).to_le_bytes());
+ out.extend_from_slice(&(data.len() as u32).to_le_bytes());
+ out.extend_from_slice(&(name.len() as u16).to_le_bytes());
+ out.extend_from_slice(&[0, 0]);
+ out.extend_from_slice(name.as_bytes());
+ out.extend_from_slice(&packed);
+ dir.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
+ dir.extend_from_slice(&[20, 0, 20, 0, 0, 0]);
+ dir.extend_from_slice(&method.to_le_bytes());
+ dir.extend_from_slice(&[0, 0, 0, 0]);
+ dir.extend_from_slice(&crc32(data).to_le_bytes());
+ dir.extend_from_slice(&(packed.len() as u32).to_le_bytes());
+ dir.extend_from_slice(&(data.len() as u32).to_le_bytes());
+ dir.extend_from_slice(&(name.len() as u16).to_le_bytes());
+ dir.extend_from_slice(&[0; 12]);
+ dir.extend_from_slice(&offset.to_le_bytes());
+ dir.extend_from_slice(name.as_bytes());
+ }
+ let cd_offset = out.len() as u32;
+ out.extend_from_slice(&dir);
+ out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
+ out.extend_from_slice(&[0, 0, 0, 0]);
+ out.extend_from_slice(&(files.len() as u16).to_le_bytes());
+ out.extend_from_slice(&(files.len() as u16).to_le_bytes());
+ out.extend_from_slice(&(dir.len() as u32).to_le_bytes());
+ out.extend_from_slice(&cd_offset.to_le_bytes());
+ out.extend_from_slice(&[0, 0]);
+ out
+ }
+
+ fn temp(name: &str) -> std::path::PathBuf {
+ std::env::temp_dir().join(format!("lc-denoise-zip-{name}-{}", std::process::id()))
+ }
+
+ #[test]
+ fn a_stored_and_a_deflated_file_come_out_whole() {
+ let big: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();
+ let zip = make_zip(&[("m/config.json", b"{}", false), ("m/model.onnx", &big, true), ("m/other.onnx", b"zz", true)]);
+ let (a, out) = (temp("a.zip"), temp("a.onnx"));
+ std::fs::write(&a, &zip).unwrap();
+ assert_eq!(extract(&a, "model.onnx", &out, 1 << 20).unwrap(), big.len() as u64);
+ assert_eq!(std::fs::read(&out).unwrap(), big);
+ assert_eq!(extract(&a, "m/config.json", &out, 1 << 20).unwrap(), 2);
+ assert_eq!(std::fs::read(&out).unwrap(), b"{}");
+ // a name must match whole path parts: `del.onnx` is not `model.onnx`
+ assert!(matches!(extract(&a, "del.onnx", &out, 1 << 20), Err(ArchiveError::Missing(_))));
+ assert!(matches!(extract(&a, "nothing", &out, 1 << 20), Err(ArchiveError::Missing(_))));
+ // the size limit applies
+ assert!(matches!(extract(&a, "model.onnx", &out, 1000), Err(ArchiveError::Format(_))));
+ for p in [a, out] {
+ let _ = std::fs::remove_file(p);
+ }
+ }
+
+ /// Opt-in: `LC_DENOISE_DTMODEL=<rawdenoise-nind.dtmodel> cargo test -p lightcraft-denoise -- --ignored archive`
+ #[test]
+ #[ignore = "needs the real download: set LC_DENOISE_DTMODEL"]
+ fn the_real_download_gives_the_model_it_should() {
+ let Some(path) = std::env::var_os("LC_DENOISE_DTMODEL") else { return };
+ let known = crate::known::find("rawnind-bayer").unwrap();
+ let (d, entry) = (known.download.unwrap(), known.entry.unwrap());
+ assert_eq!(crate::hash::sha256_file(Path::new(&path)).unwrap(), d.sha256);
+ let out = temp("real.onnx");
+ let n = extract(Path::new(&path), entry, &out, crate::manifest::MAX_MODEL_BYTES).unwrap();
+ assert_eq!(Some(n), known.manifest.size_bytes);
+ assert_eq!(Some(crate::hash::sha256_file(&out).unwrap()), known.manifest.sha256);
+ let _ = std::fs::remove_file(out);
+ }
+
+ #[test]
+ fn damaged_and_hostile_archives_are_errors_not_panics() {
+ let zip = make_zip(&[("m/model.onnx", &vec![9u8; 5000], true)]);
+ let (a, out) = (temp("b.zip"), temp("b.onnx"));
+ // cut anywhere
+ for n in (0..zip.len()).step_by(3) {
+ std::fs::write(&a, &zip[..n]).unwrap();
+ assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err(), "cut at {n}");
+ }
+ // one byte changed anywhere: an error, or (when it falls in an unused field) the right file, never a panic
+ for at in 0..zip.len() {
+ let mut bad = zip.clone();
+ bad[at] ^= 0x5a;
+ std::fs::write(&a, &bad).unwrap();
+ if extract(&a, "model.onnx", &out, 1 << 20).is_ok() {
+ assert_eq!(std::fs::read(&out).unwrap(), vec![9u8; 5000], "a changed byte at {at} gave another file");
+ }
+ }
+ // not a zip at all, and an empty file
+ for junk in [Vec::new(), b"PK".to_vec(), vec![0u8; 100_000]] {
+ std::fs::write(&a, junk).unwrap();
+ assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err());
+ }
+ assert!(extract(&temp("missing.zip"), "model.onnx", &out, 1 << 20).is_err());
+ for p in [a, out] {
+ let _ = std::fs::remove_file(p);
+ }
+ }
+}
diff --git a/crates/denoise/src/bayer.rs b/crates/denoise/src/bayer.rs
new file mode 100644
index 000000000..a4a00d947
--- /dev/null
+++ b/crates/denoise/src/bayer.rs
@@ -0,0 +1,191 @@
+//! Bayer geometry: which way a mosaic is laid out, and how a patch of it is packed into the four planes
+//! (red, green on the red row, green on the blue row, blue) a denoise model takes.
+//!
+//! Every Bayer layout is the same 2 × 2 cell shifted by one pixel at most, so a layout is just where its
+//! first red sample is. The cell grid starts one pixel before the picture when that red sample is in
+//! column or row 1, and the picture is read with its edge mirrored (without repeating the edge sample, so
+//! the colour of a mirrored sample is the colour that belongs at that place).
+
+/// Where the first red sample of a Bayer mosaic sits, `(dx, dy)` in `{0, 1}²`: RGGB is `(0, 0)`, GRBG `(1, 0)`,
+/// GBRG `(0, 1)`, BGGR `(1, 1)`.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Layout {
+ pub dx: usize,
+ pub dy: usize,
+}
+
+impl Layout {
+ /// The layout of a 2 × 2 colour pattern, row-major, colours 0 = red, 1 = green, 2 = blue; `None` when
+ /// it is not a Bayer cell (one red, one blue, the greens on the other diagonal).
+ pub fn from_cell(cell: [u8; 4]) -> Option<Layout> {
+ let at = |c: u8| cell.iter().position(|&v| v == c);
+ let red = at(0)?;
+ let blue = at(2)?;
+ let greens = cell.iter().filter(|&&v| v == 1).count();
+ // red and blue sit on one diagonal: indices 0+3 or 1+2
+ if greens != 2 || red + blue != 3 {
+ return None;
+ }
+ Some(Layout { dx: red % 2, dy: red / 2 })
+ }
+
+ /// The cell grid's origin: `-1` along an axis whose first red sample is at 1, so that cell 0 starts on a red column or row.
+ fn origin(self) -> (isize, isize) {
+ (isize::try_from(self.dx).unwrap_or(isize::MAX).saturating_neg(), isize::try_from(self.dy).unwrap_or(isize::MAX).saturating_neg())
+ }
+
+ /// Cells across and down covering a `width × height` mosaic.
+ pub fn cells(self, width: usize, height: usize) -> (usize, usize) {
+ let cells = |n: usize, phase: usize| n / 2 + (n % 2 + phase.min(1)).div_ceil(2);
+ (cells(width, self.dx), cells(height, self.dy))
+ }
+
+ /// The mosaic position of cell `(cx, cy)`'s top-left sample (may lie outside the picture).
+ pub fn position(self, cx: isize, cy: isize) -> (isize, isize) {
+ let (ox, oy) = self.origin();
+ (ox.saturating_add(cx.saturating_mul(2)), oy.saturating_add(cy.saturating_mul(2)))
+ }
+}
+
+/// `i` mirrored into `0..n` (ping-pong, edge not repeated), so `-1` is `1` and `n` is `n - 2`. `n == 0` gives 0.
+pub fn reflect(i: isize, n: usize) -> usize {
+ if n <= 1 {
+ return 0;
+ }
+ if let Some(period) = isize::try_from(n).ok().and_then(|n| (n - 1).checked_mul(2)) {
+ let m = i.rem_euclid(period);
+ return (if m < n as isize { m } else { period - m }) as usize;
+ }
+ let period = 2 * (n as i128 - 1);
+ let m = (i as i128).rem_euclid(period);
+ // The reflected value is in 0..n, so this conversion always fits usize.
+ (if m < n as i128 { m } else { period - m }) as usize
+}
+
+/// Pack the `tile × tile` cells whose top-left cell is `(cx0, cy0)` into `out` as four planes `[R, G1, G2, B]`
+/// (`4 · tile²` values, NCHW). G1 is the green beside the red sample, G2 the green under it. Samples outside
+/// the picture are mirrored. Returns `false` (and writes nothing) when the sizes do not fit.
+pub fn pack_tile(mosaic: &[f32], width: usize, height: usize, layout: Layout, cx0: isize, cy0: isize, tile: usize, out: &mut [f32]) -> bool {
+ let Some(plane) = tile.checked_mul(tile) else { return false };
+ let Some(values) = plane.checked_mul(4) else { return false };
+ if width == 0
+ || height == 0
+ || tile == 0
+ || layout.dx > 1
+ || layout.dy > 1
+ || width.checked_mul(height) != Some(mosaic.len())
+ || out.len() != values
+ {
+ return false;
+ }
+ let Ok(last) = isize::try_from(tile - 1) else { return false };
+ let (ox, oy) = layout.origin();
+ for (start, origin) in [(cx0, ox), (cy0, oy)] {
+ if start.checked_mul(2).and_then(|v| v.checked_add(origin)).is_none()
+ || start.checked_add(last).and_then(|v| v.checked_mul(2)).and_then(|v| v.checked_add(origin)).and_then(|v| v.checked_add(1)).is_none()
+ {
+ return false;
+ }
+ }
+ // the four samples of a cell, as (dx, dy) from its top-left sample
+ const SUB: [(isize, isize); 4] = [(0, 0), (1, 0), (0, 1), (1, 1)];
+ for (p, &(sx, sy)) in SUB.iter().enumerate() {
+ let Some(dst) = out.get_mut(p * plane..(p + 1) * plane) else { return false };
+ for (j, row) in dst.chunks_exact_mut(tile).enumerate() {
+ let (_, y) = layout.position(cx0, cy0 + j as isize);
+ let ry = reflect(y + sy, height);
+ let Some(src) = mosaic.get(ry * width..(ry + 1) * width) else { return false };
+ for (i, v) in row.iter_mut().enumerate() {
+ let (x, _) = layout.position(cx0 + i as isize, cy0);
+ *v = src.get(reflect(x + sx, width)).copied().unwrap_or(0.0);
+ }
+ }
+ }
+ true
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn hostile_sizes_and_positions_never_panic_or_write_output() {
+ let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();
+ let mut out = [7.0; 4];
+ for (w, h, tile, x) in [(usize::MAX, 2, 1, 0), (1, 1, usize::MAX, 0), (1, 1, 0, 0), (1, 1, 1, isize::MAX), (1, 1, 1, isize::MIN)] {
+ assert!(!pack_tile(&[0.5], w, h, layout, x, 0, tile, &mut out));
+ assert_eq!(out, [7.0; 4]);
+ }
+ for n in [1, 2, isize::MAX as usize, usize::MAX] {
+ for i in [isize::MIN, -1, 0, isize::MAX] {
+ assert!(reflect(i, n) < n);
+ }
+ }
+ assert_eq!(layout.cells(usize::MAX, usize::MAX), (usize::MAX / 2 + 1, usize::MAX / 2 + 1));
+ }
+
+ #[test]
+ fn layouts_from_cells() {
+ assert_eq!(Layout::from_cell([0, 1, 1, 2]), Some(Layout { dx: 0, dy: 0 }));
+ assert_eq!(Layout::from_cell([1, 0, 2, 1]), Some(Layout { dx: 1, dy: 0 }));
+ assert_eq!(Layout::from_cell([1, 2, 0, 1]), Some(Layout { dx: 0, dy: 1 }));
+ assert_eq!(Layout::from_cell([2, 1, 1, 0]), Some(Layout { dx: 1, dy: 1 }));
+ // not Bayer: two reds, a missing blue, greens on the wrong diagonal
+ assert_eq!(Layout::from_cell([0, 0, 1, 2]), None);
+ assert_eq!(Layout::from_cell([0, 1, 2, 1]), None);
+ assert_eq!(Layout::from_cell([1, 1, 1, 1]), None);
+ }
+
+ #[test]
+ fn reflection_keeps_the_colour_of_a_site() {
+ assert_eq!(reflect(-1, 10), 1);
+ assert_eq!(reflect(-2, 10), 2);
+ assert_eq!(reflect(10, 10), 8);
+ assert_eq!(reflect(11, 10), 7);
+ assert_eq!(reflect(25, 10), 7);
+ assert_eq!(reflect(5, 0), 0);
+ assert_eq!(reflect(5, 1), 0);
+ for i in -40..60 {
+ assert_eq!(reflect(i, 11) % 2, (i.rem_euclid(2)) as usize, "parity at {i}");
+ }
+ }
+
+ #[test]
+ fn packing_puts_each_colour_in_its_plane_for_every_layout() {
+ // a mosaic whose sample value says which colour it is: R = 10, G = 20, B = 30
+ for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
+ let layout = Layout::from_cell(cell).unwrap();
+ let (w, h) = (10usize, 8usize);
+ let mosaic: Vec<f32> = (0..w * h).map(|i| [10.0, 20.0, 30.0][cell[(i / w % 2) * 2 + i % w % 2] as usize]).collect();
+ let (nx, ny) = layout.cells(w, h);
+ let tile = nx.max(ny);
+ let mut out = vec![0.0; 4 * tile * tile];
+ assert!(pack_tile(&mosaic, w, h, layout, 0, 0, tile, &mut out));
+ let plane = tile * tile;
+ for (p, want) in [10.0, 20.0, 20.0, 30.0].into_iter().enumerate() {
+ assert!(out[p * plane..(p + 1) * plane].iter().all(|&v| v == want), "layout {cell:?} plane {p}");
+ }
+ }
+ }
+
+ #[test]
+ fn g1_is_beside_the_red_sample_and_g2_under_it() {
+ // RGGB with a distinct value at every site
+ let (w, h) = (4, 4);
+ let mosaic: Vec<f32> = (0..16).map(|i| i as f32).collect();
+ let mut out = vec![0.0; 4 * 4];
+ assert!(pack_tile(&mosaic, w, h, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
+ // cell (0,0): R = site (0,0)=0, G1 = site (1,0)=1, G2 = site (0,1)=4, B = site (1,1)=5
+ assert_eq!([out[0], out[4], out[8], out[12]], [0.0, 1.0, 4.0, 5.0]);
+ // cell (1,0): sites (2,0)=2, (3,0)=3, (2,1)=6, (3,1)=7
+ assert_eq!([out[1], out[5], out[9], out[13]], [2.0, 3.0, 6.0, 7.0]);
+ }
+
+ #[test]
+ fn packing_rejects_wrong_sizes() {
+ let mut out = vec![0.0; 16];
+ assert!(!pack_tile(&[0.0; 15], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
+ assert!(!pack_tile(&[0.0; 16], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 3, &mut out));
+ assert!(!pack_tile(&[], 0, 0, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
+ }
+}
diff --git a/crates/denoise/src/cpu.rs b/crates/denoise/src/cpu.rs
new file mode 100644
index 000000000..a856fc256
--- /dev/null
+++ b/crates/denoise/src/cpu.rs
@@ -0,0 +1,386 @@
+//! Single-thread CPU execution of the checked denoise network.
+//!
+//! Convolutions lower at most 256 spatial positions at a time, then use faer's safe views over pure-Rust GEMM.
+//! No inner thread pool: `run::denoise_bayer` already runs tiles on the application's Rayon pool. Tensor lifetimes
+//! are planned once; buffers and im2col storage are recycled between calls, independently for concurrent callers.
+
+use crate::net::{Net, NetError, Op};
+use crate::run::{Error, TileRunner};
+use faer::{Parallelism, mat};
+use std::sync::{Arc, Mutex};
+
+const BLOCK: usize = 256;
+/// Cap all tensor buffers and convolution scratch for one tile together (512 MiB).
+const MAX_VALUES: usize = 128 * 1024 * 1024;
+const MAX_TILE: usize = 2048;
+const MAX_IDLE: usize = 16;
+
+type Result<T> = std::result::Result<T, NetError>;
+fn bad(why: &str) -> NetError {
+ NetError::Invalid(why.into())
+}
+fn count(c: usize, side: usize) -> Result<usize> {
+ c.checked_mul(side)
+ .and_then(|n| n.checked_mul(side))
+ .filter(|&n| n > 0 && n <= MAX_VALUES)
+ .ok_or_else(|| bad("a tensor has no size or is too large"))
+}
+fn zeros(n: usize) -> Result<Vec<f32>> {
+ let mut v = Vec::new();
+ v.try_reserve_exact(n).map_err(|_| bad("not enough memory for a denoise tile"))?;
+ v.resize(n, 0.0);
+ Ok(v)
+}
+fn sources(op: &Op) -> (usize, Option<usize>) {
+ match op {
+ Op::Conv(c) => (c.src, c.src2),
+ Op::ConvT2(c) => (c.src, None),
+ Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => (*src, None),
+ }
+}
+#[derive(Clone, Copy)]
+struct Tensor {
+ slot: usize,
+ side: usize,
+ channels: usize,
+}
+struct Plan {
+ net: Net,
+ tensors: Vec<Tensor>,
+ sizes: Vec<usize>,
+ lower: usize,
+ product: usize,
+ up_weights: Vec<Option<Vec<f32>>>,
+}
+struct Workspace {
+ buffers: Vec<Vec<f32>>,
+ lower: Vec<f32>,
+ product: Vec<f32>,
+}
+
+/// A checked Net on square CHW tiles. Clones share immutable weights and the idle workspace cache.
+#[derive(Clone)]
+pub struct NetRunner {
+ plan: Arc<Plan>,
+ idle: Arc<Mutex<Vec<Workspace>>>,
+}
+
+impl NetRunner {
+ pub fn new(net: &Net, tile: usize) -> Result<Self> {
+ if tile == 0
+ || tile > MAX_TILE
+ || !net.fits_tile(tile)
+ || net.in_channels != 4
+ || net.out_channels() != 3
+ || net.shape(0).is_none_or(|s| s.channels != net.in_channels)
+ {
+ return Err(bad("the tile or Bayer-to-RGB contract does not fit the network"));
+ }
+ let mut last = vec![0; net.ops().len() + 1];
+ for (i, op) in net.ops().iter().enumerate() {
+ let (a, b) = sources(op);
+ for id in std::iter::once(a).chain(b) {
+ *last.get_mut(id).ok_or_else(|| bad("missing tensor"))? = i + 1;
+ }
+ }
+ let mut tensors = vec![Tensor { slot: 0, side: tile, channels: net.in_channels }];
+ let mut sizes = vec![count(net.in_channels, tile)?];
+ let mut busy = vec![true];
+ let (mut lower, mut product) = (0, 0);
+ let mut up_weights = Vec::new();
+ for (i, op) in net.ops().iter().enumerate() {
+ let (src, src2) = sources(op);
+ let a = *tensors.get(src).ok_or_else(|| bad("missing tensor"))?;
+ let (channels, side) = match op {
+ Op::Conv(c) => {
+ if let Some(s) = src2
+ && tensors.get(s).is_none_or(|b| b.side != a.side)
+ {
+ return Err(bad("a join has different sizes"));
+ }
+ (c.cout, a.side)
+ }
+ Op::ConvT2(c) => (c.cout, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),
+ Op::MaxPool2 { .. } => {
+ if a.side < 2 || !a.side.is_multiple_of(2) {
+ return Err(bad("pooling does not divide the tile"));
+ }
+ (a.channels, a.side / 2)
+ }
+ Op::DepthToSpace2 { .. } => (a.channels / 4, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),
+ };
+ let n = count(channels, side)?;
+ let slot = busy
+ .iter()
+ .zip(&sizes)
+ .enumerate()
+ .filter(|(_, (used, _))| !**used)
+ .min_by_key(|(_, (_, size))| (n.saturating_sub(**size), **size))
+ .map_or(busy.len(), |(i, _)| i);
+ if slot == sizes.len() {
+ sizes.push(n);
+ busy.push(true);
+ } else {
+ let sz = sizes.get_mut(slot).ok_or_else(|| bad("missing buffer"))?;
+ *sz = (*sz).max(n);
+ *busy.get_mut(slot).ok_or_else(|| bad("missing buffer"))? = true;
+ }
+ tensors.push(Tensor { slot, side, channels });
+ // Release sources only after the output has its own buffer (also handles reading the same tensor twice).
+ for id in std::iter::once(src).chain(src2) {
+ if last.get(id) == Some(&(i + 1)) {
+ let t = tensors.get(id).ok_or_else(|| bad("missing tensor"))?;
+ *busy.get_mut(t.slot).ok_or_else(|| bad("missing buffer"))? = false;
+ }
+ }
+ let (k, m, positions) = match op {
+ Op::Conv(c) => (c.cin * c.k * c.k, c.cout, a.side * a.side),
+ Op::ConvT2(c) => (c.cin, c.cout * 4, a.side * a.side),
+ _ => (0, 0, 0),
+ };
+ lower = lower.max(k.saturating_mul(BLOCK.min(positions)));
+ product = product.max(m.saturating_mul(BLOCK.min(positions)));
+ let packed = if let Op::ConvT2(c) = op {
+ let mut w = zeros(c.weight.len())?;
+ // [cin][cout][dy][dx] -> [cout*4][cin], so an up-convolution is one GEMM and a shuffle.
+ for (ci, row) in c.weight.chunks_exact(c.cout * 4).enumerate() {
+ for (co, &v) in row.iter().enumerate() {
+ *w.get_mut(co * c.cin + ci).ok_or_else(|| bad("invalid transposed weights"))? = v;
+ }
+ }
+ Some(w)
+ } else {
+ None
+ };
+ up_weights.push(packed);
+ }
+ let total = sizes
+ .iter()
+ .try_fold(lower.saturating_add(product), |n, &s| n.checked_add(s))
+ .filter(|&n| n <= MAX_VALUES)
+ .ok_or_else(|| bad("the tile needs too much working memory"))?;
+ let _ = total;
+ Ok(Self { plan: Arc::new(Plan { net: net.clone(), tensors, sizes, lower, product, up_weights }), idle: Arc::new(Mutex::new(Vec::new())) })
+ }
+ /// Immutable data for the GPU runner, without re-reading the model.
+ pub fn net(&self) -> &Net {
+ &self.plan.net
+ }
+ pub fn bytes_per_tile(&self) -> usize {
+ (self.plan.sizes.iter().sum::<usize>() + self.plan.lower + self.plan.product) * 4
+ }
+ fn workspace(&self) -> Result<Workspace> {
+ if let Some(w) = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop() {
+ return Ok(w);
+ }
+ Ok(Workspace {
+ buffers: self.plan.sizes.iter().map(|&n| zeros(n)).collect::<Result<_>>()?,
+ lower: zeros(self.plan.lower)?,
+ product: zeros(self.plan.product)?,
+ })
+ }
+ fn execute(&self, input: &[f32], w: &mut Workspace) -> Result<Vec<f32>> {
+ let first = self.plan.tensors.first().ok_or_else(|| bad("no input"))?;
+ let want = count(first.channels, first.side)?;
+ w.buffers.first_mut().and_then(|b| b.get_mut(..want)).ok_or_else(|| bad("missing input buffer"))?.copy_from_slice(input);
+ let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some();
+ for (i, op) in self.plan.net.ops().iter().enumerate() {
+ let started = web_time::Instant::now();
+ let out = *self.plan.tensors.get(i + 1).ok_or_else(|| bad("missing output tensor"))?;
+ let (before, rest) = w.buffers.split_at_mut_checked(out.slot).ok_or_else(|| bad("missing output buffer"))?;
+ let (dest, after) = rest.split_first_mut().ok_or_else(|| bad("missing output buffer"))?;
+ let get = |id: usize| -> Result<(&[f32], Tensor)> {
+ let t = *self.plan.tensors.get(id).ok_or_else(|| bad("missing source tensor"))?;
+ let data = if t.slot < out.slot {
+ before.get(t.slot)
+ } else {
+ after.get(t.slot.checked_sub(out.slot + 1).ok_or_else(|| bad("aliased buffers"))?)
+ }
+ .and_then(|b| b.get(..count(t.channels, t.side).ok()?))
+ .ok_or_else(|| bad("missing source buffer"))?;
+ Ok((data, t))
+ };
+ let (src, src2) = sources(op);
+ let (a, shape) = get(src)?;
+ let b = src2.map(get).transpose()?.map(|v| v.0);
+ let dest = dest.get_mut(..count(out.channels, out.side)?).ok_or_else(|| bad("short output buffer"))?;
+ match op {
+ Op::Conv(c) => conv(c, a, b, shape.side, dest, &mut w.lower, &mut w.product)?,
+ Op::ConvT2(c) => up(
+ c,
+ a,
+ shape.side,
+ dest,
+ self.plan.up_weights.get(i).and_then(Option::as_deref).ok_or_else(|| bad("missing up weights"))?,
+ &mut w.lower,
+ &mut w.product,
+ )?,
+ Op::MaxPool2 { .. } => pool(a, shape.side, dest)?,
+ Op::DepthToSpace2 { .. } => shuffle(a, shape.side, dest)?,
+ }
+ if dest.iter().any(|v| !v.is_finite()) {
+ return Err(bad("the model produced non-finite values"));
+ }
+ if profile {
+ eprintln!("[denoise-cpu] layer {i}: {}x{}x{} {:.2} ms", out.channels, out.side, out.side, started.elapsed().as_secs_f64() * 1e3);
+ }
+ }
+ let t = self.plan.tensors.last().ok_or_else(|| bad("no output"))?;
+ let data = w.buffers.get(t.slot).and_then(|b| b.get(..count(t.channels, t.side).ok()?)).ok_or_else(|| bad("missing result"))?;
+ let mut out = zeros(data.len())?;
+ out.copy_from_slice(data);
+ Ok(out)
+ }
+}
+impl TileRunner for NetRunner {
+ fn run(&self, input: &[f32]) -> std::result::Result<Vec<f32>, Error> {
+ let first = self.plan.tensors.first().ok_or_else(|| Error::Runtime("missing input shape".into()))?;
+ let want = count(first.channels, first.side).map_err(|e| Error::Runtime(e.to_string()))?;
+ if input.len() != want || input.iter().any(|v| !v.is_finite()) {
+ return Err(Error::Input("the tile has the wrong size or non-finite values".into()));
+ }
+ let mut w = self.workspace().map_err(|e| Error::Runtime(e.to_string()))?;
+ let result = self.execute(input, &mut w).map_err(|e| Error::Runtime(e.to_string()));
+ let mut idle = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+ if idle.len() < MAX_IDLE {
+ idle.push(w);
+ }
+ result
+ }
+}
+fn activate(x: f32, bias: f32, alpha: Option<f32>) -> f32 {
+ let x = x + bias;
+ if x < 0.0 { x * alpha.unwrap_or(1.0) } else { x }
+}
+
+/// All views are over exact, checked slices. The matrix library's assertions therefore never see file-derived sizes.
+fn multiply(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) -> Result<()> {
+ if m.checked_mul(k) != Some(w.len()) || k.checked_mul(n) != Some(x.len()) || m.checked_mul(n) != Some(y.len()) || m == 0 || k == 0 || n == 0 {
+ return Err(bad("invalid matrix dimensions"));
+ }
+ faer::mul::matmul(
+ mat::from_row_major_slice_mut::<f32>(y, m, n),
+ mat::from_row_major_slice::<f32>(w, m, k),
+ mat::from_row_major_slice::<f32>(x, k, n),
+ None,
+ 1.0,
+ Parallelism::None,
+ );
+ Ok(())
+}
+fn lower(a: &[f32], b: Option<&[f32]>, side: usize, k: usize, start: usize, n: usize, dest: &mut [f32]) -> Result<()> {
+ let pad = k / 2;
+ let pixels = side * side;
+ // Whole row segments are copied, so bounds checks and edge branches stay out of the multiply loop.
+ for (ci, plane) in a.chunks_exact(pixels).chain(b.into_iter().flat_map(|b| b.chunks_exact(pixels))).enumerate() {
+ for ky in 0..k {
+ for kx in 0..k {
+ let row = (ci * k * k + ky * k + kx) * n;
+ let out = dest.get_mut(row..row + n).ok_or_else(|| bad("short im2col buffer"))?;
+ let mut at = 0;
+ while at < n {
+ let p = start + at;
+ let (y, x) = (p / side, p % side);
+ let len = (side - x).min(n - at);
+ if let Some(yy) = y.checked_add(ky).and_then(|v| v.checked_sub(pad)).filter(|&v| v < side) {
+ let lo = x.max(pad.saturating_sub(kx));
+ let hi = (x + len).min((side + pad).saturating_sub(kx));
+ if lo < hi {
+ let from = yy * side + lo + kx - pad;
+ let to = at + lo - x;
+ out.get_mut(at..to).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);
+ out.get_mut(to..to + hi - lo)
+ .ok_or_else(|| bad("short im2col row"))?
+ .copy_from_slice(plane.get(from..from + hi - lo).ok_or_else(|| bad("short source row"))?);
+ out.get_mut(to + hi - lo..at + len).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);
+ } else {
+ out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);
+ }
+ } else {
+ out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);
+ }
+ at += len;
+ }
+ }
+ }
+ }
+ Ok(())
+}
+fn conv(c: &crate::net::Conv, a: &[f32], b: Option<&[f32]>, side: usize, dest: &mut [f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {
+ let pixels = side * side;
+ let k = c.cin * c.k * c.k;
+ for start in (0..pixels).step_by(BLOCK) {
+ let n = BLOCK.min(pixels - start);
+ let x = scratch.get_mut(..k * n).ok_or_else(|| bad("short im2col storage"))?;
+ lower(a, b, side, c.k, start, n, x)?;
+ let y = product.get_mut(..c.cout * n).ok_or_else(|| bad("short product storage"))?;
+ multiply(&c.weight, x, y, c.cout, k, n)?;
+ for ((row, &bias), out) in y.chunks_exact(n).zip(&c.bias).zip(dest.chunks_exact_mut(pixels)) {
+ let to = out.get_mut(start..start + n).ok_or_else(|| bad("short convolution output"))?;
+ for (d, &v) in to.iter_mut().zip(row) {
+ *d = activate(v, bias, c.leaky);
+ }
+ }
+ }
+ Ok(())
+}
+fn up(c: &crate::net::ConvT2, a: &[f32], side: usize, dest: &mut [f32], weights: &[f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {
+ let pixels = side * side;
+ let out_side = 2 * side;
+ for start in (0..pixels).step_by(BLOCK) {
+ let n = BLOCK.min(pixels - start);
+ let x = scratch.get_mut(..c.cin * n).ok_or_else(|| bad("short up input storage"))?;
+ for (to, from) in x.chunks_exact_mut(n).zip(a.chunks_exact(pixels)) {
+ to.copy_from_slice(from.get(start..start + n).ok_or_else(|| bad("short up input"))?);
+ }
+ let y = product.get_mut(..c.cout * 4 * n).ok_or_else(|| bad("short up product storage"))?;
+ multiply(weights, x, y, c.cout * 4, c.cin, n)?;
+ for (co, (&bias, out)) in c.bias.iter().zip(dest.chunks_exact_mut(4 * pixels)).enumerate() {
+ for d in 0..4 {
+ let row = y.get((4 * co + d) * n..(4 * co + d + 1) * n).ok_or_else(|| bad("short up product"))?;
+ for (j, &v) in row.iter().enumerate() {
+ let p = start + j;
+ let at = (2 * (p / side) + d / 2) * out_side + 2 * (p % side) + d % 2;
+ *out.get_mut(at).ok_or_else(|| bad("short up output"))? = activate(v, bias, c.leaky);
+ }
+ }
+ }
+ }
+ Ok(())
+}
+fn pool(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {
+ let s = side / 2;
+ for (input, out) in a.chunks_exact(side * side).zip(dest.chunks_exact_mut(s * s)) {
+ for (y, row) in out.chunks_exact_mut(s).enumerate() {
+ let lo = input.get(2 * y * side..(2 * y + 1) * side).ok_or_else(|| bad("short pool input"))?;
+ let hi = input.get((2 * y + 1) * side..(2 * y + 2) * side).ok_or_else(|| bad("short pool input"))?;
+ for ((d, lo), hi) in row.iter_mut().zip(lo.as_chunks::<2>().0).zip(hi.as_chunks::<2>().0) {
+ let [a, b] = *lo;
+ let [c, e] = *hi;
+ *d = a.max(b).max(c).max(e);
+ }
+ }
+ }
+ Ok(())
+}
+fn shuffle(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {
+ let pixels = side * side;
+ for (input, out) in a.chunks_exact(4 * pixels).zip(dest.chunks_exact_mut(4 * pixels)) {
+ for d in 0..4 {
+ let plane = input.get(d * pixels..(d + 1) * pixels).ok_or_else(|| bad("short shuffle input"))?;
+ for (y, row) in plane.chunks_exact(side).enumerate() {
+ let at = (2 * y + d / 2) * 2 * side;
+ let to = out.get_mut(at..at + 2 * side).ok_or_else(|| bad("short shuffle output"))?;
+ for (to, &v) in to.as_chunks_mut::<2>().0.iter_mut().zip(row) {
+ if let Some(t) = to.get_mut(d % 2) {
+ *t = v;
+ }
+ }
+ }
+ }
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/crates/denoise/src/cpu/tests.rs b/crates/denoise/src/cpu/tests.rs
new file mode 100644
index 000000000..16a496b53
--- /dev/null
+++ b/crates/denoise/src/cpu/tests.rs
@@ -0,0 +1,151 @@
+use super::*;
+use crate::{
+ net::{Conv, ConvT2},
+ reference, synthetic,
+};
+
+fn random(n: usize, seed: u32) -> Vec<f32> {
+ let mut s = seed | 1;
+ (0..n)
+ .map(|_| {
+ s ^= s << 13;
+ s ^= s >> 17;
+ s ^= s << 5;
+ (s >> 8) as f32 / (1u32 << 23) as f32 - 1.0
+ })
+ .collect()
+}
+fn net(tile: usize, ch: u64, depth: usize, seed: u64) -> Net {
+ let p = std::env::temp_dir().join(format!("lc-purecpu-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id()));
+ std::fs::write(&p, synthetic::unet_onnx(tile as u64, ch, depth, seed)).unwrap();
+ let n = crate::onnx::read(&p).unwrap();
+ std::fs::remove_file(p).unwrap();
+ n
+}
+fn agree(a: &[f32], b: &[f32]) {
+ assert_eq!(a.len(), b.len());
+ assert!(a.iter().chain(b).all(|v| v.is_finite()));
+ let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs()));
+ let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
+ assert!(error <= 1e-3, "relative maximum error {error:e}");
+}
+#[test]
+fn all_operators_match_scalar_on_random_nets_and_odd_and_tiny_shapes() {
+ for (t, ch, d) in [(1, 3, 0), (3, 5, 0), (5, 7, 0), (6, 3, 1), (20, 5, 2), (36, 9, 2), (64, 8, 3)] {
+ for seed in [1, 73] {
+ let n = net(t, ch, d, seed);
+ let r = NetRunner::new(&n, t).unwrap();
+ let x = random(4 * t * t, seed as u32);
+ let want = reference::run(&n, t, &x).unwrap();
+ agree(&r.run(&x).unwrap(), &want);
+ assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());
+ }
+ }
+}
+
+#[test]
+fn malformed_values_fail_before_execution_and_workspace_sizes_are_bounded() {
+ let mut n = net(4, 3, 1, 11);
+ for t in [0, 1, 3, usize::MAX, 4096] {
+ assert!(NetRunner::new(&n, t).is_err());
+ }
+ for ch in [0, 3, usize::MAX] {
+ n.in_channels = ch;
+ assert!(NetRunner::new(&n, 4).is_err());
+ }
+ let n = Net::new(
+ 4,
+ vec![
+ Op::Conv(Conv { src: 0, src2: None, cin: 4, cout: 4096, k: 1, weight: vec![0.1; 4 * 4096], bias: vec![0.0; 4096], leaky: None }),
+ Op::Conv(Conv { src: 1, src2: None, cin: 4096, cout: 12, k: 1, weight: vec![0.1; 4096 * 12], bias: vec![0.0; 12], leaky: None }),
+ Op::DepthToSpace2 { src: 2 },
+ ],
+ )
+ .unwrap();
+ assert!(NetRunner::new(&n, 2048).is_err());
+ let ops = (0..100).map(|i| Op::MaxPool2 { src: i }).collect();
+ let n = Net::new(4, ops).unwrap();
+ assert!(NetRunner::new(&n, 16).is_err());
+ let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![0.25; 48], bias: vec![0.0; 12], leaky: None };
+ for mode in 0..6 {
+ let mut bad = c.clone();
+ match mode {
+ 0 => bad.src = usize::MAX,
+ 1 => bad.weight.pop().map(|_| ()).unwrap(),
+ 2 => bad.bias[0] = f32::NAN,
+ 3 => bad.leaky = Some(f32::INFINITY),
+ 4 => bad.cout = usize::MAX,
+ _ => bad.k = usize::MAX,
+ }
+ assert!(Net::new(4, vec![Op::Conv(bad), Op::DepthToSpace2 { src: 1 }]).is_err());
+ }
+}
+#[test]
+fn bad_tiles_and_arithmetic_overflow_are_errors_and_do_not_poison_reuse() {
+ let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![f32::MAX; 48], bias: vec![0.0; 12], leaky: None };
+ let n = Net::new(4, vec![Op::Conv(c), Op::DepthToSpace2 { src: 1 }]).unwrap();
+ let r = NetRunner::new(&n, 1).unwrap();
+ assert!(r.run(&[]).is_err());
+ assert!(r.run(&[f32::NAN; 4]).is_err());
+ assert!(r.run(&[f32::INFINITY; 4]).is_err());
+ assert_eq!(r.idle.lock().unwrap().len(), 0, "invalid input never allocates a workspace");
+ assert!(r.run(&[2.0; 4]).is_err());
+ assert_eq!(r.run(&[0.0; 4]).unwrap(), vec![0.0; 12]);
+ // Matrix shape checks are the boundary in front of the dependency's asserting constructors.
+ assert!(multiply(&[], &[], &mut [], usize::MAX, 2, 2).is_err());
+}
+#[test]
+fn shared_runner_serves_concurrent_tiles_without_cross_contamination() {
+ let n = net(12, 5, 1, 97);
+ let r = NetRunner::new(&n, 12).unwrap();
+ let inputs: Vec<_> = (0..8).map(|s| random(4 * 12 * 12, s)).collect();
+ let answers: Vec<_> = inputs.iter().map(|x| reference::run(&n, 12, x).unwrap()).collect();
+ std::thread::scope(|scope| {
+ let mut handles = Vec::new();
+ for (x, want) in inputs.iter().zip(&answers) {
+ let r = &r;
+ handles.push(scope.spawn(move || {
+ for _ in 0..3 {
+ agree(&r.run(x).unwrap(), want);
+ }
+ }));
+ }
+ for h in handles {
+ h.join().unwrap();
+ }
+ });
+}
+#[test]
+fn join_may_read_the_same_tensor_twice_and_leaky_up_convolution_is_supported() {
+ let c = ConvT2 { src: 0, cin: 4, cout: 3, weight: random(48, 2), bias: vec![0.1, -0.1, 0.0], leaky: Some(0.13) };
+ let n = Net::new(
+ 4,
+ vec![
+ Op::ConvT2(c),
+ Op::Conv(Conv { src: 1, src2: Some(1), cin: 6, cout: 3, k: 3, weight: random(162, 3), bias: vec![0.0; 3], leaky: Some(0.0) }),
+ ],
+ )
+ .unwrap();
+ let x = random(4 * 3 * 3, 7);
+ agree(&NetRunner::new(&n, 3).unwrap().run(&x).unwrap(), &reference::run(&n, 3, &x).unwrap());
+}
+
+#[test]
+fn reused_im2col_clears_all_edge_padding_between_images() {
+ for t in [1, 3, 5, 36] {
+ let n = net(t, 5, 0, 121);
+ let r = NetRunner::new(&n, t).unwrap();
+ for x in [vec![1.0; 4 * t * t], vec![0.0; 4 * t * t], random(4 * t * t, 9)] {
+ agree(&r.run(&x).unwrap(), &reference::run(&n, t, &x).unwrap());
+ }
+ }
+}
+#[test]
+#[ignore = "needs LC_DENOISE_MODEL; one tile with LIGHTCRAFT_PROFILE=1"]
+fn profile_one_real_tile() {
+ let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL");
+ let m = crate::known::find("rawnind-bayer").unwrap().manifest;
+ let r = crate::runtime::CpuRunner::load(std::path::Path::new(&model), &m).unwrap();
+ println!("workspace {:.1} MB", r.bytes_per_tile() as f64 / 1e6);
+ r.run(&crate::runtime::check_tile(512)).unwrap();
+}
diff --git a/crates/denoise/src/hash.rs b/crates/denoise/src/hash.rs
new file mode 100644
index 000000000..50f83be39
--- /dev/null
+++ b/crates/denoise/src/hash.rs
@@ -0,0 +1,46 @@
+//! SHA-256 of a model file, streamed (a model can be hundreds of megabytes).
+
+use std::io::{self, Read};
+use std::path::Path;
+
+use sha2::{Digest, Sha256};
+
+/// Lowercase hex SHA-256 of everything `r` yields.
+pub fn sha256_reader<R: Read>(mut r: R) -> io::Result<String> {
+ let mut h = Sha256::new();
+ let mut buf = vec![0u8; 1 << 20];
+ loop {
+ let n = r.read(&mut buf)?;
+ if n == 0 {
+ break;
+ }
+ h.update(buf.get(..n).unwrap_or(&[]));
+ }
+ Ok(hex(&h.finalize()))
+}
+
+/// Lowercase hex SHA-256 of a file.
+pub fn sha256_file(path: &Path) -> io::Result<String> {
+ sha256_reader(io::BufReader::new(std::fs::File::open(path)?))
+}
+
+/// Lowercase hex SHA-256 of `bytes`.
+pub fn sha256_hex(bytes: &[u8]) -> String {
+ hex(&Sha256::digest(bytes))
+}
+
+fn hex(bytes: &[u8]) -> String {
+ bytes.iter().map(|b| format!("{b:02x}")).collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn known_digests() {
+ assert_eq!(sha256_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+ assert_eq!(sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
+ assert_eq!(sha256_reader(&b"abc"[..]).unwrap(), sha256_hex(b"abc"));
+ }
+}
diff --git a/crates/denoise/src/known.rs b/crates/denoise/src/known.rs
new file mode 100644
index 000000000..9e0e7ea1e
--- /dev/null
+++ b/crates/denoise/src/known.rs
@@ -0,0 +1,112 @@
+//! The denoise models LightCraft knows about: what each one is, who may use it, and where to fetch it.
+//!
+//! None is part of LightCraft. A model with an address here can be downloaded when the user asks and has accepted
+//! its terms; anything else is brought by the user as an `.onnx` file with a `denoise-model.json` beside it.
+
+use crate::licence::{Commercial, Licence};
+
+use crate::manifest::{DenoiserManifest, Domain, Gain};
+
+/// A known model, and how to get it.
+pub struct Known {
+ pub manifest: DenoiserManifest,
+ /// What to fetch (an archive or the `.onnx` itself); `None` when the user must bring the file.
+ pub download: Option<Download>,
+ /// The `.onnx` inside the downloaded archive (`None`: the download is the `.onnx`).
+ pub entry: Option<&'static str>,
+}
+
+/// RawNIND's UtNet2 for Bayer sensors, as published for darktable (`rawdenoise-nind.dtmodel`, a zip with a Bayer and a
+/// linear model). Its weights are GPL-3.0; the training pictures are CC BY 4.0 and CC0.
+fn rawnind_bayer() -> Known {
+ let manifest = DenoiserManifest {
+ id: "rawnind-bayer".into(),
+ name: "RawNIND UtNet2 (Bayer)".into(),
+ version: "1.0".into(),
+ licence: Licence {
+ name: "GPL-3.0".into(),
+ commercial: Commercial::Yes,
+ url: Some("https://www.gnu.org/licenses/gpl-3.0.html".into()),
+ notice: "The weights are published under the GNU General Public License v3. LightCraft does not include them: they \
+ are downloaded to your computer at your request and used by it only. If you pass the model file on, the \
+ GPL applies to that copy."
+ .into(),
+ },
+ source: Some("https://github.com/darktable-org/darktable-ai/tree/master/models/rawdenoise-nind".into()),
+ sha256: Some("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901".into()),
+ size_bytes: Some(31_056_425),
+ provenance: "Trained on RawNIND, real photographs of the same scenes taken noisy and clean (images CC BY 4.0 and CC0, \
+ published on Wikimedia Commons and by the University of Louvain). Paper: arXiv 2501.08924."
+ .into(),
+ domain: Domain::BayerToRgb,
+ tile: 512,
+ overlap: 64,
+ gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
+ };
+ let download = Download {
+ id: manifest.id.clone(),
+ url: "https://github.com/darktable-org/darktable-ai/releases/download/release-5.6.0/rawdenoise-nind.dtmodel".into(),
+ file_name: "rawdenoise-nind.dtmodel".into(),
+ size_bytes: 57_700_134,
+ sha256: "d71b5f1e727c85a359e6f74dca9e2016c9d8fc3e2f7ac3e9b347d80ceca969af".into(),
+ };
+ Known { manifest, download: Some(download), entry: Some("model_bayer.onnx") }
+}
+
+/// Every model LightCraft knows.
+pub fn all() -> Vec<Known> {
+ vec![rawnind_bayer()]
+}
+
+/// The known model with this id.
+pub fn find(id: &str) -> Option<Known> {
+ all().into_iter().find(|k| k.manifest.id == id)
+}
+
+/// The known model whose `.onnx` has this SHA-256.
+pub fn by_sha256(sha256: &str) -> Option<DenoiserManifest> {
+ all().into_iter().map(|k| k.manifest).find(|m| m.sha256.as_deref() == Some(sha256))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::manifest::validate;
+
+ #[test]
+ fn known_models_are_valid_unique_and_downloaded_over_https() {
+ let all = all();
+ assert!(!all.is_empty());
+ for (i, k) in all.iter().enumerate() {
+ validate(&k.manifest).unwrap();
+ assert!(all.iter().skip(i + 1).all(|o| o.manifest.id != k.manifest.id), "unique ids");
+ assert!(!k.manifest.licence.name.is_empty() && !k.manifest.licence.notice.is_empty(), "terms are shown before use");
+ if let Some(d) = &k.download {
+ assert_eq!(d.id, k.manifest.id);
+ assert!(d.url.starts_with("https://") && d.sha256.len() == 64 && d.size_bytes > 0);
+ // a name that cannot leave the staging folder
+ assert!(!d.file_name.contains(['/', '\\']) && !d.file_name.starts_with('.'));
+ }
+ // a model that may not be used commercially is never offered for download
+ if k.manifest.licence.commercial == Commercial::No {
+ assert!(k.download.is_none(), "{}", k.manifest.id);
+ }
+ }
+ assert!(find("rawnind-bayer").is_some() && find("nothing").is_none());
+ assert_eq!(by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id), Some("rawnind-bayer".into()));
+ }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct Download {
+ pub id: String,
+ pub url: String,
+ /// The name the file is kept under while it waits for the user to accept its terms.
+ pub file_name: String,
+ pub size_bytes: u64,
+ pub sha256: String,
+}
+
+pub fn host(url: &str) -> &str {
+ url.strip_prefix("https://").and_then(|r| r.split('/').next()).unwrap_or("")
+}
diff --git a/crates/denoise/src/lib.rs b/crates/denoise/src/lib.rs
new file mode 100644
index 000000000..c7c76b585
--- /dev/null
+++ b/crates/denoise/src/lib.rs
@@ -0,0 +1,42 @@
+//! AI denoise for LightCraft, as a step between the sensor and the demosaic that the rest of the develop
+//! pipeline never sees: the raw file stays untouched, the denoised picture is cached data that can always be
+//! made again, and the amount is an ordinary develop setting.
+//!
+//! - [`manifest`]: what a denoise model takes and gives, and who may use it (untrusted JSON, validated).
+//! - [`known`] and [`archive`]: the models LightCraft can fetch, and taking the model file out of the `.zip` it comes in.
+//! - [`bayer`]: Bayer layouts and the packing of a mosaic into the four planes a model takes.
+//! - [`tiles`] and [`run`]: cutting a picture into overlapping tiles for a fixed-size model, bringing each tile's
+//! output scale back to the input's, keeping clipped highlights as they were, and blending the tiles together.
+//! The model itself is a [`run::TileRunner`], so all of this is tested without one.
+//! - [`product`]: the cache file the denoised picture is kept in (half floats, compressed in strips) and the
+//! blend with the plain demosaic that the Amount slider controls.
+//! - [`cpu`] and [`runtime`]: a single-thread pure-Rust CPU runner and its installation self-test.
+//! - [`net`] and [`onnx`]: checked plain network data and bounded ONNX decoding, shared by CPU and GPU.
+//!
+//! Everything here treats model files, manifests and cache files as hostile input: sizes are capped, numbers must
+//! be finite, and a malformed file is an error, never a panic.
+#![forbid(unsafe_code)]
+#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
+
+pub mod archive;
+pub mod bayer;
+pub mod cpu;
+pub mod known;
+pub mod manifest;
+pub mod net;
+pub mod onnx;
+mod onnx_proto;
+pub mod product;
+pub mod reference;
+pub mod run;
+pub mod runtime;
+#[doc(hidden)]
+pub mod synthetic;
+pub mod tiles;
+
+pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError};
+pub use run::{Control, Error, Params, TileRunner, denoise_bayer};
+
+pub mod hash;
+pub mod licence;
+pub mod synthetic_proto;
diff --git a/crates/denoise/src/licence.rs b/crates/denoise/src/licence.rs
new file mode 100644
index 000000000..1d89146b1
--- /dev/null
+++ b/crates/denoise/src/licence.rs
@@ -0,0 +1,34 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum Commercial {
+ Yes,
+ /// Research or personal use only. Never bundled or redistributed by LightCraft.
+ No,
+ /// The licence or the training data is unclear: the user is told so before installing.
+ Unknown,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+pub struct Licence {
+ /// SPDX id or a short name ("MIT", "Apache-2.0", "InsightFace non-commercial research").
+ pub name: String,
+ pub commercial: Commercial,
+ pub url: Option<String>,
+ /// Plain-language terms shown before the model is enabled.
+ pub notice: String,
+}
+
+impl Default for Licence {
+ fn default() -> Self {
+ Self { name: String::new(), commercial: Commercial::Unknown, url: None, notice: String::new() }
+ }
+}
+
+pub fn valid_id(id: &str) -> bool {
+ let mut bytes = id.bytes();
+ let first_ok = bytes.next().is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit());
+ first_ok && id.len() <= 64 && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-'))
+}
diff --git a/crates/denoise/src/manifest.rs b/crates/denoise/src/manifest.rs
new file mode 100644
index 000000000..9fbc9f49f
--- /dev/null
+++ b/crates/denoise/src/manifest.rs
@@ -0,0 +1,217 @@
+//! A denoise model's manifest: what the model takes and gives, how big its tiles are and who may use it, as
+//! plain data.
+//!
+//! It comes from a `denoise-model.json` next to the `.onnx` file or from the user's `catalog.json`. Both are
+//! untrusted: [`validate`] must pass before a manifest is used, and [`parse`] applies it after reading JSON of
+//! bounded size. The licence, id and address rules are the face models' (`crate::licence`).
+
+use crate::licence::{Licence, valid_id};
+use serde::{Deserialize, Serialize};
+
+/// Largest `denoise-model.json` we read.
+pub const MAX_MANIFEST_BYTES: usize = 64 * 1024;
+/// Largest model file we accept (bytes).
+pub const MAX_MODEL_BYTES: u64 = 2 * 1024 * 1024 * 1024;
+
+/// What the model takes and what it gives.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum Domain {
+ /// The sensor's Bayer mosaic (black subtracted, white at 1, not white balanced) packed as four planes
+ /// `[R, G1, G2, B]` of its 2 × 2 cells, forced to RGGB; gives camera RGB at the mosaic's resolution, so
+ /// the picture is denoised and demosaiced in one step and must not be demosaiced again.
+ BayerToRgb,
+}
+
+/// How the model's output scale is brought back to the input's.
+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "camelCase")]
+pub enum Gain {
+ /// The output already has the input's scale.
+ None,
+ /// The output has a scale of its own, `nominal` times the input's (a million, for one model) and a little
+ /// different from tile to tile: each tile is scaled so that its channel means equal the input's. A tile too
+ /// dark to measure, or whose measured scale is further than `maxDeviation` (a fraction, 0.05 = 5 %) from
+ /// `nominal`, is scaled as if it were within that limit.
+ MatchMean { nominal: f32, max_deviation: f32 },
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DenoiserManifest {
+ /// Lowercase letters, digits, `.` `_` `-`.
+ pub id: String,
+ pub name: String,
+ pub version: String,
+ #[serde(default)]
+ pub licence: Licence,
+ /// Where the weights come from (an `https://` page or file), for the user's information.
+ #[serde(default)]
+ pub source: Option<String>,
+ /// Lowercase hex SHA-256 of the `.onnx` file, when known.
+ #[serde(default)]
+ pub sha256: Option<String>,
+ #[serde(default)]
+ pub size_bytes: Option<u64>,
+ /// What the model was trained on, in plain words; says "undisclosed" when it is.
+ #[serde(default)]
+ pub provenance: String,
+ pub domain: Domain,
+ /// Side of the square the model runs on, in packed cells (the file's fixed input size).
+ pub tile: u32,
+ /// Cells neighbouring tiles share; the picture is blended across them.
+ pub overlap: u32,
+ pub gain: Gain,
+}
+
+#[derive(Debug, PartialEq, thiserror::Error)]
+pub enum ManifestError {
+ #[error("`{0}` is not valid: {1}")]
+ Field(&'static str, String),
+ #[error("not a denoise-model manifest: {0}")]
+ Parse(String),
+ #[error("a manifest may be at most {MAX_MANIFEST_BYTES} bytes")]
+ TooLarge,
+}
+
+fn bad<T>(field: &'static str, why: impl Into<String>) -> Result<T, ManifestError> {
+ Err(ManifestError::Field(field, why.into()))
+}
+
+fn text(field: &'static str, s: &str, min: usize, max: usize) -> Result<(), ManifestError> {
+ let n = s.chars().count();
+ if n < min || n > max {
+ return bad(field, format!("must be {min} to {max} characters"));
+ }
+ if s.chars().any(|c| c.is_control() && c != '\n') {
+ return bad(field, "must not contain control characters");
+ }
+ Ok(())
+}
+
+fn url(field: &'static str, u: &Option<String>) -> Result<(), ManifestError> {
+ match u {
+ None => Ok(()),
+ Some(u) if u.len() <= 500 && u.starts_with("https://") && !u.chars().any(|c| c.is_control() || c.is_whitespace()) => Ok(()),
+ Some(_) => bad(field, "must be an https:// address of at most 500 characters"),
+ }
+}
+
+/// Check a manifest from outside before trusting it.
+pub fn validate(m: &DenoiserManifest) -> Result<(), ManifestError> {
+ if !valid_id(&m.id) {
+ return bad("id", "1 to 64 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit");
+ }
+ text("name", &m.name, 1, 80)?;
+ text("version", &m.version, 1, 40)?;
+ text("provenance", &m.provenance, 0, 1000)?;
+ text("licence.name", &m.licence.name, 0, 80)?;
+ text("licence.notice", &m.licence.notice, 0, 2000)?;
+ url("licence.url", &m.licence.url)?;
+ url("source", &m.source)?;
+ if let Some(h) = &m.sha256
+ && !(h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
+ {
+ return bad("sha256", "must be 64 lowercase hex digits");
+ }
+ if let Some(n) = m.size_bytes
+ && (n == 0 || n > MAX_MODEL_BYTES)
+ {
+ return bad("sizeBytes", format!("must be between 1 and {MAX_MODEL_BYTES}"));
+ }
+ // a U-Net with four poolings wants a multiple of 16; nothing smaller than 64 or larger than 2048 is a tile
+ if !(64..=2048).contains(&m.tile) || !m.tile.is_multiple_of(16) {
+ return bad("tile", "must be a multiple of 16 from 64 to 2048");
+ }
+ // tiles must overlap by less than half, or one tile's blend ramps would run into its other neighbour's
+ if !m.overlap.is_multiple_of(2) || m.overlap < 16 || m.overlap.saturating_mul(2) >= m.tile {
+ return bad("overlap", "must be even, at least 16 and less than half the tile");
+ }
+ if let Gain::MatchMean { nominal, max_deviation } = m.gain {
+ if !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal)) {
+ return bad("gain.nominal", "must be between 0.000001 and 1000000000000");
+ }
+ if !(max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) {
+ return bad("gain.maxDeviation", "must be between 0 and 0.5");
+ }
+ }
+ Ok(())
+}
+
+/// Read and validate a `denoise-model.json`.
+pub fn parse(bytes: &[u8]) -> Result<DenoiserManifest, ManifestError> {
+ if bytes.len() > MAX_MANIFEST_BYTES {
+ return Err(ManifestError::TooLarge);
+ }
+ let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes);
+ let m: DenoiserManifest = serde_json::from_slice(bytes).map_err(|e| ManifestError::Parse(e.to_string()))?;
+ validate(&m)?;
+ Ok(m)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ pub(crate) fn sample() -> DenoiserManifest {
+ DenoiserManifest {
+ id: "nind-bayer".into(),
+ name: "Bayer denoiser".into(),
+ version: "1".into(),
+ licence: Licence::default(),
+ source: Some("https://example.org/model.onnx".into()),
+ sha256: None,
+ size_bytes: Some(31_000_000),
+ provenance: "trained on real raw noise pairs".into(),
+ domain: Domain::BayerToRgb,
+ tile: 512,
+ overlap: 64,
+ gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
+ }
+ }
+
+ #[test]
+ fn a_good_manifest_round_trips() {
+ let m = sample();
+ let json = serde_json::to_vec(&m).unwrap();
+ assert_eq!(parse(&json).unwrap(), m);
+ }
+
+ #[test]
+ fn bad_fields_are_named() {
+ let cases: Vec<(&str, Box<dyn Fn(&mut DenoiserManifest)>)> = vec![
+ ("id", Box::new(|m| m.id = "../evil".into())),
+ ("id", Box::new(|m| m.id = String::new())),
+ ("name", Box::new(|m| m.name = "x\u{7}".into())),
+ ("source", Box::new(|m| m.source = Some("http://insecure".into()))),
+ ("sha256", Box::new(|m| m.sha256 = Some("XYZ".into()))),
+ ("sizeBytes", Box::new(|m| m.size_bytes = Some(0))),
+ ("tile", Box::new(|m| m.tile = 500)),
+ ("tile", Box::new(|m| m.tile = 4096)),
+ ("overlap", Box::new(|m| m.overlap = 300)),
+ ("overlap", Box::new(|m| m.overlap = 63)),
+ ("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN })),
+ ("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: 2.0 })),
+ ("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: 0.0, max_deviation: 0.1 })),
+ ("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 })),
+ ];
+ for (field, edit) in cases {
+ let mut m = sample();
+ edit(&mut m);
+ match validate(&m) {
+ Err(ManifestError::Field(f, _)) => assert_eq!(f, field),
+ other => panic!("{field}: {other:?}"),
+ }
+ }
+ }
+
+ #[test]
+ fn parse_is_bounded_and_tolerant() {
+ assert_eq!(parse(&vec![b' '; MAX_MANIFEST_BYTES + 1]), Err(ManifestError::TooLarge));
+ assert!(matches!(parse(b"{ nope"), Err(ManifestError::Parse(_))));
+ assert!(matches!(parse(b""), Err(ManifestError::Parse(_))));
+ let mut with_bom = vec![0xEF, 0xBB, 0xBF];
+ with_bom.extend(serde_json::to_vec(&sample()).unwrap());
+ assert!(parse(&with_bom).is_ok());
+ }
+}
diff --git a/crates/denoise/src/net.rs b/crates/denoise/src/net.rs
new file mode 100644
index 000000000..422418336
--- /dev/null
+++ b/crates/denoise/src/net.rs
@@ -0,0 +1,345 @@
+//! A denoise network as plain data: the handful of layer kinds a U-Net denoiser is made of, with their weights.
+//!
+//! [`onnx`](crate::onnx) reads the file into this description, shared by the pure-Rust CPU and GPU runners.
+//! Unsupported operators, groups, dilation or activations are reported as errors when loading the model.
+//!
+//! The description is checked ([`Net::new`]) because it comes from a stranger's file: channel counts must chain,
+//! weights must have the sizes the shapes need and be finite, and the depth must stay small.
+
+/// Most layers a network may have.
+pub const MAX_OPS: usize = 512;
+/// Most channels a layer may have.
+pub const MAX_CHANNELS: usize = 4096;
+/// Most weights (numbers) a network may have in all.
+pub const MAX_WEIGHTS: usize = 128 * 1024 * 1024;
+
+#[derive(Debug, PartialEq, thiserror::Error)]
+pub enum NetError {
+ /// The model uses something this denoise network description cannot express.
+ #[error("not supported by the denoise runners: {0}")]
+ Unsupported(String),
+ /// The model is not a well-formed network.
+ #[error("not a valid network: {0}")]
+ Invalid(String),
+}
+
+fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {
+ Err(NetError::Invalid(why.into()))
+}
+
+/// A tensor is named by its number: 0 is the network's input, and layer `i` makes tensor `i + 1`.
+pub type TensorId = usize;
+
+/// A convolution with stride 1 and "same" padding (`k` is 1 or 3), over one tensor or two joined along the channels
+/// (`src` first), then a bias and optionally a leaky ReLU.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Conv {
+ pub src: TensorId,
+ /// A second source, whose channels follow `src`'s (a skip connection's concatenation).
+ pub src2: Option<TensorId>,
+ pub cin: usize,
+ pub cout: usize,
+ pub k: usize,
+ /// `[cout][cin][k][k]`, as ONNX has it.
+ pub weight: Vec<f32>,
+ pub bias: Vec<f32>,
+ /// `Some(alpha)`: a leaky ReLU with this slope follows.
+ pub leaky: Option<f32>,
+}
+
+/// A transposed convolution with a 2 × 2 kernel and stride 2: every input pixel becomes a 2 × 2 block.
+#[derive(Clone, Debug, PartialEq)]
+pub struct ConvT2 {
+ pub src: TensorId,
+ pub cin: usize,
+ pub cout: usize,
+ /// `[cin][cout][2][2]`, as ONNX has it.
+ pub weight: Vec<f32>,
+ pub bias: Vec<f32>,
+ pub leaky: Option<f32>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum Op {
+ Conv(Conv),
+ ConvT2(ConvT2),
+ /// 2 × 2 maximum, stride 2.
+ MaxPool2 {
+ src: TensorId,
+ },
+ /// Depth-to-space with block size 2 in "CRD" order: output channel `c` at `(2y + dy, 2x + dx)` is input channel
+ /// `4c + 2dy + dx` at `(y, x)`.
+ DepthToSpace2 {
+ src: TensorId,
+ },
+}
+
+/// What a tensor looks like: its channels, and its resolution as a power of two below the input's (a negative
+/// level is above it).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Shape {
+ pub channels: usize,
+ pub level: i32,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct Net {
+ pub in_channels: usize,
+ ops: Vec<Op>,
+ shapes: Vec<Shape>,
+}
+
+impl Net {
+ /// Check `ops` (layer `i` makes tensor `i + 1`) over an input of `in_channels` and describe every tensor.
+ pub fn new(in_channels: usize, ops: Vec<Op>) -> Result<Net, NetError> {
+ if in_channels == 0 || in_channels > MAX_CHANNELS {
+ return invalid("the input has no channels or too many");
+ }
+ if ops.is_empty() || ops.len() > MAX_OPS {
+ return invalid("the network has no layers or too many");
+ }
+ let mut shapes = vec![Shape { channels: in_channels, level: 0 }];
+ let mut weights = 0usize;
+ for (i, op) in ops.iter().enumerate() {
+ let shape_of = |t: TensorId| {
+ shapes
+ .get(t)
+ .copied()
+ .filter(|_| t <= i)
+ .ok_or_else(|| NetError::Invalid(format!("layer {i} reads tensor {t}, which does not exist yet")))
+ };
+ let out = match op {
+ Op::Conv(c) => {
+ let a = shape_of(c.src)?;
+ let (cin, level) = match c.src2 {
+ Some(s2) => {
+ let b = shape_of(s2)?;
+ if b.level != a.level {
+ return invalid(format!("layer {i} joins tensors of different resolutions"));
+ }
+ (a.channels + b.channels, a.level)
+ }
+ None => (a.channels, a.level),
+ };
+ if c.cin != cin || c.cout == 0 || c.cout > MAX_CHANNELS || !(c.k == 1 || c.k == 3) {
+ return invalid(format!(
+ "layer {i}: a convolution of {} to {} channels with a {}-wide kernel does not fit",
+ c.cin, c.cout, c.k
+ ));
+ }
+ check_weights(
+ i,
+ &c.weight,
+ c.cout.checked_mul(cin).and_then(|n| n.checked_mul(c.k * c.k)),
+ &c.bias,
+ c.cout,
+ c.leaky,
+ &mut weights,
+ )?;
+ Shape { channels: c.cout, level }
+ }
+ Op::ConvT2(c) => {
+ let a = shape_of(c.src)?;
+ if c.cin != a.channels || c.cout == 0 || c.cout > MAX_CHANNELS {
+ return invalid(format!("layer {i}: a transposed convolution of {} to {} channels does not fit", c.cin, c.cout));
+ }
+ check_weights(i, &c.weight, c.cin.checked_mul(c.cout).and_then(|n| n.checked_mul(4)), &c.bias, c.cout, c.leaky, &mut weights)?;
+ Shape { channels: c.cout, level: a.level - 1 }
+ }
+ Op::MaxPool2 { src } => {
+ let a = shape_of(*src)?;
+ Shape { channels: a.channels, level: a.level + 1 }
+ }
+ Op::DepthToSpace2 { src } => {
+ let a = shape_of(*src)?;
+ if !a.channels.is_multiple_of(4) {
+ return invalid(format!("layer {i}: depth-to-space needs a multiple of 4 channels"));
+ }
+ Shape { channels: a.channels / 4, level: a.level - 1 }
+ }
+ };
+ shapes.push(out);
+ }
+ Ok(Net { in_channels, ops, shapes })
+ }
+
+ /// The layers: layer `i` makes tensor `i + 1`.
+ pub fn ops(&self) -> &[Op] {
+ &self.ops
+ }
+
+ /// The shape of tensor `t` (0 is the input).
+ pub fn shape(&self, t: TensorId) -> Option<Shape> {
+ self.shapes.get(t).copied()
+ }
+
+ /// The tensor the network gives (the last layer's).
+ pub fn output(&self) -> TensorId {
+ self.ops.len()
+ }
+
+ pub fn out_channels(&self) -> usize {
+ self.shapes.last().map_or(0, |s| s.channels)
+ }
+
+ /// The deepest resolution level: a tile must be a multiple of `2^this` cells wide.
+ pub fn depth(&self) -> u32 {
+ self.shapes.iter().map(|s| s.level.max(0) as u32).max().unwrap_or(0)
+ }
+
+ /// Can the network run on a `tile × tile` input and give `[out_channels, 2·tile, 2·tile]`? (Pooling must divide
+ /// the tile, and the output must be one level above the input.)
+ pub fn fits_tile(&self, tile: usize) -> bool {
+ let Some(multiple) = 1usize.checked_shl(self.depth()) else { return false };
+ tile > 0 && tile.is_multiple_of(multiple) && self.shapes.last().is_some_and(|s| s.level == -1)
+ }
+
+ /// Multiply-adds for one tile, for estimating time.
+ pub fn macs(&self, tile: usize) -> u64 {
+ let pixels = |level: i32| -> u64 {
+ let side = if level >= 0 {
+ tile.checked_shr(level as u32).unwrap_or(0)
+ } else {
+ tile.saturating_mul(1usize.checked_shl(level.unsigned_abs()).unwrap_or(usize::MAX))
+ };
+ (side as u64).saturating_mul(side as u64)
+ };
+ self.ops
+ .iter()
+ .enumerate()
+ .map(|(i, op)| {
+ let out = self.shapes.get(i + 1).map_or(0, |s| pixels(s.level));
+ match op {
+ Op::Conv(c) => out.saturating_mul((c.cin * c.cout * c.k * c.k) as u64),
+ // every input pixel makes a 2 × 2 block: as many multiply-adds as the 2 × 2-wide kernel has weights
+ Op::ConvT2(c) => pixels(self.shapes.get(c.src).map_or(0, |s| s.level)).saturating_mul((c.cin * c.cout * 4) as u64),
+ _ => 0,
+ }
+ })
+ .fold(0u64, u64::saturating_add)
+ }
+}
+
+fn check_weights(
+ layer: usize,
+ weight: &[f32],
+ want: Option<usize>,
+ bias: &[f32],
+ cout: usize,
+ leaky: Option<f32>,
+ total: &mut usize,
+) -> Result<(), NetError> {
+ if want != Some(weight.len()) || bias.len() != cout {
+ return invalid(format!("layer {layer}: the weights are not the size its shape needs"));
+ }
+ *total = total.saturating_add(weight.len()).saturating_add(bias.len());
+ if *total > MAX_WEIGHTS {
+ return invalid("the network has too many weights");
+ }
+ if weight.iter().chain(bias).any(|v| !v.is_finite()) || leaky.is_some_and(|a| !a.is_finite()) {
+ return invalid(format!("layer {layer}: a weight is not a finite number"));
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn conv(src: TensorId, src2: Option<TensorId>, cin: usize, cout: usize, k: usize) -> Op {
+ Op::Conv(Conv { src, src2, cin, cout, k, weight: vec![0.1; cout * cin * k * k], bias: vec![0.0; cout], leaky: Some(0.2) })
+ }
+
+ fn convt(src: TensorId, cin: usize, cout: usize) -> Op {
+ Op::ConvT2(ConvT2 { src, cin, cout, weight: vec![0.1; cin * cout * 4], bias: vec![0.0; cout], leaky: None })
+ }
+
+ /// A two-level network of the shape the denoisers have: 4 → 8 channels, pool, 16 channels, up, join, 12, and the
+ /// 1 × 1 convolution to the colours.
+ fn unet() -> Vec<Op> {
+ vec![
+ conv(0, None, 4, 8, 3), // 1: 8 ch, level 0
+ Op::MaxPool2 { src: 1 }, // 2: level 1
+ conv(2, None, 8, 16, 3), // 3: 16 ch, level 1
+ convt(3, 16, 8), // 4: 8 ch, level 0
+ conv(4, Some(1), 16, 8, 3), // 5: join [up, skip]
+ conv(5, None, 8, 12, 1), // 6: 12 ch
+ Op::DepthToSpace2 { src: 6 }, // 7: 3 ch, level -1
+ ]
+ }
+
+ #[test]
+ fn deep_networks_and_huge_cost_queries_never_overflow_or_truncate_depth() {
+ let n = Net::new(4, unet()).unwrap();
+ assert_eq!(n.macs(usize::MAX), u64::MAX);
+ assert_eq!(n.macs(0), 0);
+ let mut ops = Vec::new();
+ for i in 0..21 {
+ ops.push(Op::MaxPool2 { src: i });
+ }
+ for i in 21..42 {
+ ops.push(convt(i, 4, 4));
+ }
+ ops.push(conv(42, None, 4, 12, 1));
+ ops.push(Op::DepthToSpace2 { src: 43 });
+ let n = Net::new(4, ops).unwrap();
+ assert!(!n.fits_tile(1 << 20));
+ assert!(n.fits_tile(1 << 21));
+ let _ = n.macs(16);
+ }
+
+ #[test]
+ fn a_u_net_is_described_with_its_shapes_and_costs() {
+ let net = Net::new(4, unet()).unwrap();
+ assert_eq!(net.out_channels(), 3);
+ assert_eq!(net.shape(3), Some(Shape { channels: 16, level: 1 }));
+ assert_eq!(net.shape(7), Some(Shape { channels: 3, level: -1 }));
+ assert_eq!(net.depth(), 1);
+ assert!(net.fits_tile(64) && !net.fits_tile(63) && !net.fits_tile(0));
+ // 64 × 64 cells: layer 1 = 4096·4·8·9, and so on; just check it is the sum of what the layers cost
+ let t = 64u64;
+ let want = t * t * 4 * 8 * 9 + (t / 2) * (t / 2) * 8 * 16 * 9 + (t / 2) * (t / 2) * 16 * 8 * 4 + t * t * 16 * 8 * 9 + t * t * 8 * 12;
+ assert_eq!(net.macs(64), want);
+ }
+
+ #[test]
+ fn a_network_that_does_not_chain_is_refused() {
+ // wrong input channel count for the first convolution
+ assert!(Net::new(3, unet()).is_err());
+ // a layer reading a tensor that does not exist yet
+ let mut ops = unet();
+ ops[2] = conv(9, None, 8, 16, 3);
+ assert!(Net::new(4, ops).is_err());
+ // joining tensors of different resolutions
+ let mut ops = unet();
+ ops[4] = conv(3, Some(1), 24, 8, 3);
+ assert!(Net::new(4, ops).is_err());
+ // weights of the wrong size, and not finite
+ let mut ops = unet();
+ if let Op::Conv(c) = &mut ops[0] {
+ c.weight.pop();
+ }
+ assert!(Net::new(4, ops).is_err());
+ let mut ops = unet();
+ if let Op::Conv(c) = &mut ops[0] {
+ c.bias[0] = f32::NAN;
+ }
+ assert!(Net::new(4, ops).is_err());
+ // a kernel that is neither 1 nor 3 wide, no layers, depth-to-space on a channel count it cannot split
+ let mut ops = unet();
+ if let Op::Conv(c) = &mut ops[0] {
+ c.k = 5;
+ }
+ assert!(Net::new(4, ops).is_err());
+ assert!(Net::new(4, vec![]).is_err());
+ assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }]).is_ok());
+ assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }, Op::DepthToSpace2 { src: 1 }]).is_err());
+ assert!(Net::new(2, vec![Op::DepthToSpace2 { src: 0 }]).is_err());
+ }
+
+ #[test]
+ fn a_network_that_does_not_end_one_level_up_does_not_fit_a_tile() {
+ let net = Net::new(4, unet()[..6].to_vec()).unwrap();
+ assert!(!net.fits_tile(64), "no depth-to-space: the output is not at twice the resolution");
+ }
+}
diff --git a/crates/denoise/src/onnx.rs b/crates/denoise/src/onnx.rs
new file mode 100644
index 000000000..fc84a36cf
--- /dev/null
+++ b/crates/denoise/src/onnx.rs
@@ -0,0 +1,369 @@
+//! Reading an `.onnx` file into a [`Net`] with bounded, pure-Rust protobuf decoding.
+//!
+//! Only what the denoise U-Nets are made of is understood: `Conv` (3 × 3 "same" or 1 × 1, stride 1, no groups, no
+//! dilation) optionally followed by `LeakyRelu`, `ConvTranspose` (2 × 2, stride 2) likewise, `MaxPool` (2 × 2, stride 2),
+//! a two-input `Concat` on the channels that feeds a convolution, and a final `DepthToSpace` (2, `CRD`). Any other
+//! operator or attribute is [`NetError::Unsupported`], which is reported to the user by both runners.
+
+use std::collections::HashMap;
+use std::path::Path;
+
+use crate::onnx_proto::{self, AttributeProto, NodeProto, TensorProto, ValueInfoProto};
+use std::io::Read;
+
+use crate::net::{Conv, ConvT2, MAX_WEIGHTS, Net, NetError, Op, TensorId};
+
+fn unsupported<T>(why: impl Into<String>) -> Result<T, NetError> {
+ Err(NetError::Unsupported(why.into()))
+}
+
+fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {
+ Err(NetError::Invalid(why.into()))
+}
+
+/// Read the model at `path`.
+pub fn read(path: &Path) -> Result<Net, NetError> {
+ let file = std::fs::File::open(path).map_err(|e| NetError::Invalid(e.to_string()))?;
+ let size = usize::try_from(file.metadata().map_err(|e| NetError::Invalid(e.to_string()))?.len())
+ .ok()
+ .filter(|&n| n <= onnx_proto::MAX_BYTES)
+ .ok_or_else(|| NetError::Invalid("model file too large".into()))?;
+ let mut bytes = Vec::new();
+ bytes.try_reserve_exact(size).map_err(|_| NetError::Invalid("not enough memory to load the model".into()))?;
+ file.take(onnx_proto::MAX_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|e| NetError::Invalid(e.to_string()))?;
+ read_bytes(&bytes)
+}
+
+fn read_bytes(bytes: &[u8]) -> Result<Net, NetError> {
+ let graph = onnx_proto::decode(bytes)?;
+
+ let initialisers: HashMap<&str, &TensorProto> = graph.initializer.iter().map(|t| (t.name.as_str(), t)).collect();
+ if initialisers.len() != graph.initializer.len() {
+ return invalid("duplicate initializer names");
+ }
+ let inputs: Vec<_> = graph.input.iter().filter(|i| !initialisers.contains_key(i.name.as_str())).collect();
+ let ([input], [output]) = (inputs.as_slice(), graph.output.as_slice()) else { return unsupported("not one input and one output") };
+ let in_channels = match channel_dim(input) {
+ Some(c) => c,
+ None => return unsupported("the input's channel count is not written in the file"),
+ };
+
+ // how many nodes read each name (a graph output counts as a reader)
+ let mut readers: HashMap<&str, usize> = HashMap::new();
+ for n in &graph.node {
+ for i in &n.input {
+ *readers.entry(i.as_str()).or_default() += 1;
+ }
+ }
+ *readers.entry(output.name.as_str()).or_default() += 1;
+
+ let mut ids: HashMap<String, TensorId> = HashMap::new();
+ ids.insert(input.name.clone(), 0);
+ // a Concat is not a layer of its own: the convolution that reads it takes both sources
+ let mut joins: HashMap<String, (TensorId, TensorId)> = HashMap::new();
+ let mut ops: Vec<Op> = Vec::new();
+ let mut weights_total = 0usize;
+
+ let nodes = &graph.node;
+ let mut i = 0;
+ while i < nodes.len() {
+ let Some(node) = nodes.get(i) else { break };
+ i += 1;
+ if node.output.len() != 1 || node.output.first().is_none_or(|n| n.is_empty() || ids.contains_key(n) || joins.contains_key(n)) {
+ return invalid("a layer has missing, repeated or multiple outputs");
+ }
+ let allowed: &[&str] = match node.op_type.as_str() {
+ "Conv" => &["kernel_shape", "pads", "strides", "dilations", "group", "auto_pad"],
+ "ConvTranspose" => &["kernel_shape", "pads", "strides", "dilations", "group", "output_padding", "auto_pad"],
+ "MaxPool" => &["kernel_shape", "pads", "strides", "dilations", "ceil_mode", "storage_order", "auto_pad"],
+ "Concat" => &["axis"],
+ "DepthToSpace" => &["blocksize", "mode"],
+ _ => &[],
+ };
+ if node.attribute.iter().any(|a| !allowed.contains(&a.name.as_str())) {
+ return unsupported("an unknown operator attribute");
+ }
+ let this = ops.len() + 1;
+ let lookup = |name: &str| -> Result<TensorId, NetError> {
+ ids.get(name).copied().ok_or_else(|| NetError::Invalid(format!("`{}` reads `{name}`, which nothing makes", node.op_type)))
+ };
+ // a LeakyRelu straight after this layer, which nothing else reads, is part of it
+ let mut fuse_leaky = |out: &str| -> Result<(Option<f32>, String), NetError> {
+ if let Some(next) = nodes.get(i)
+ && next.op_type == "LeakyRelu"
+ && next.input.first().map(String::as_str) == Some(out)
+ && readers.get(out).copied() == Some(1)
+ {
+ if next.input.len() != 1 || next.output.len() != 1 || next.attribute.iter().any(|a| a.name != "alpha") {
+ return invalid("malformed LeakyRelu");
+ }
+ let alpha = attr(next, "alpha").map_or(0.01, |a| a.f);
+ i += 1;
+ let name = next.output.first().cloned().ok_or_else(|| NetError::Invalid("LeakyRelu without an output".into()))?;
+ return Ok((Some(alpha), name));
+ }
+ Ok((None, out.to_string()))
+ };
+ match node.op_type.as_str() {
+ "Concat" => {
+ if attr(node, "axis").map(|a| a.i) != Some(1) {
+ return unsupported("Concat on another axis than the channels");
+ }
+ let [a, b] = node.input.as_slice() else { return unsupported("Concat of other than two tensors") };
+ let out = node.output.first().ok_or_else(|| NetError::Invalid("Concat without an output".into()))?;
+ if readers.get(out.as_str()).copied() != Some(1) {
+ return unsupported("a Concat that is read by more than one layer");
+ }
+ joins.insert(out.clone(), (lookup(a)?, lookup(b)?));
+ }
+ "Conv" => {
+ let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;
+ let (cout, cin, kh, kw) = four(&w.0)?;
+ let b = if b.is_empty() { vec![0.0; cout] } else { b };
+ if kh != kw || !(kh == 1 || kh == 3) {
+ return unsupported(format!("a {kh} × {kw} convolution"));
+ }
+ if ints_or(node, "strides", &[1, 1]) != [1, 1]
+ || ints_or(node, "dilations", &[1, 1]) != [1, 1]
+ || attr(node, "group").is_some_and(|a| a.i != 1)
+ {
+ return unsupported("a convolution with strides, dilation or groups");
+ }
+ if attr(node, "kernel_shape").is_some_and(|a| a.ints != [kh as i64, kw as i64]) {
+ return invalid("kernel shape does not match the weights");
+ }
+ let pad = (kh as i64 - 1) / 2;
+ if ints_or(node, "pads", &[0; 4]) != [pad; 4] || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {
+ return unsupported("a convolution that is not padded to keep its size");
+ }
+ let src_name = node.input.first().ok_or_else(|| NetError::Invalid("Conv without an input".into()))?;
+ let (src, src2) = match joins.remove(src_name) {
+ Some((a, b)) => (a, Some(b)),
+ None => (lookup(src_name)?, None),
+ };
+ let out = node.output.first().ok_or_else(|| NetError::Invalid("Conv without an output".into()))?;
+ let (leaky, out_name) = fuse_leaky(out)?;
+ ids.insert(out_name, this);
+ ops.push(Op::Conv(Conv { src, src2, cin, cout, k: kh, weight: w.1, bias: b, leaky }));
+ }
+ "ConvTranspose" => {
+ let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;
+ let (cin, cout, kh, kw) = four(&w.0)?;
+ let b = if b.is_empty() { vec![0.0; cout] } else { b };
+ if (kh, kw) != (2, 2)
+ || ints_or(node, "strides", &[1, 1]) != [2, 2]
+ || ints_or(node, "pads", &[0; 4]) != [0; 4]
+ || ints_or(node, "dilations", &[1, 1]) != [1, 1]
+ || attr(node, "group").is_some_and(|a| a.i != 1)
+ || attr(node, "output_padding").is_some_and(|a| a.ints.iter().any(|&v| v != 0))
+ {
+ return unsupported("a transposed convolution other than 2 × 2 with stride 2");
+ }
+ if attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {
+ return unsupported("transposed convolution auto padding");
+ }
+ if attr(node, "kernel_shape").is_some_and(|a| a.ints != [2, 2]) {
+ return invalid("transposed kernel shape does not match the weights");
+ }
+ let src = lookup(node.input.first().map_or("", String::as_str))?;
+ let out = node.output.first().ok_or_else(|| NetError::Invalid("ConvTranspose without an output".into()))?;
+ let (leaky, out_name) = fuse_leaky(out)?;
+ ids.insert(out_name, this);
+ ops.push(Op::ConvT2(ConvT2 { src, cin, cout, weight: w.1, bias: b, leaky }));
+ }
+ "MaxPool" => {
+ if ints_or(node, "kernel_shape", &[]) != [2, 2]
+ || ints_or(node, "strides", &[1, 1]) != [2, 2]
+ || ints_or(node, "pads", &[0; 4]) != [0; 4]
+ || ints_or(node, "dilations", &[1, 1]) != [1, 1]
+ || attr(node, "ceil_mode").is_some_and(|a| a.i != 0)
+ || attr(node, "storage_order").is_some_and(|a| a.i != 0)
+ || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET")
+ {
+ return unsupported("a max-pool other than 2 × 2 with stride 2");
+ }
+ let src = lookup(node.input.first().map_or("", String::as_str))?;
+ let out = node.output.first().ok_or_else(|| NetError::Invalid("MaxPool without an output".into()))?;
+ ids.insert(out.clone(), this);
+ ops.push(Op::MaxPool2 { src });
+ }
+ "DepthToSpace" => {
+ if attr(node, "blocksize").map(|a| a.i) != Some(2) || attr(node, "mode").map(|a| a.s.as_slice()) != Some(&b"CRD"[..]) {
+ return unsupported("depth-to-space other than block size 2 in CRD order");
+ }
+ let src = lookup(node.input.first().map_or("", String::as_str))?;
+ let out = node.output.first().ok_or_else(|| NetError::Invalid("DepthToSpace without an output".into()))?;
+ ids.insert(out.clone(), this);
+ ops.push(Op::DepthToSpace2 { src });
+ }
+ other => return unsupported(format!("the `{other}` operator")),
+ }
+ }
+ if !joins.is_empty() {
+ return unsupported("a Concat whose result no convolution reads");
+ }
+ if ids.get(output.name.as_str()) != Some(&ops.len()) {
+ return unsupported("the output is not the last layer");
+ }
+ Net::new(in_channels, ops)
+}
+
+fn attr<'a>(n: &'a NodeProto, name: &str) -> Option<&'a AttributeProto> {
+ n.attribute.iter().find(|a| a.name == name)
+}
+
+/// The integer list attribute `name`, or `default` when the node does not have it.
+fn ints_or(n: &NodeProto, name: &str, default: &[i64]) -> Vec<i64> {
+ attr(n, name).map_or_else(|| default.to_vec(), |a| a.ints.clone())
+}
+
+fn four(dims: &[i64]) -> Result<(usize, usize, usize, usize), NetError> {
+ match dims {
+ [a, b, c, d] if dims.iter().all(|&v| v > 0 && v <= 1 << 20) => Ok((*a as usize, *b as usize, *c as usize, *d as usize)),
+ _ => invalid("a weight tensor that is not four-dimensional"),
+ }
+}
+
+type Weights = ((Vec<i64>, Vec<f32>), Vec<f32>);
+
+/// The weight and bias initialisers of a convolution node.
+fn weights_of(node: &NodeProto, initialisers: &HashMap<&str, &TensorProto>, total: &mut usize) -> Result<Weights, NetError> {
+ let mut get = |idx: usize| -> Result<(Vec<i64>, Vec<f32>), NetError> {
+ let name = node.input.get(idx).ok_or_else(|| NetError::Invalid(format!("`{}` without its weights", node.op_type)))?;
+ let t = initialisers.get(name.as_str()).ok_or_else(|| NetError::Unsupported(format!("weights that are computed (`{name}`)")))?;
+ let values = floats(t)?;
+ *total = total.saturating_add(values.len());
+ if *total > MAX_WEIGHTS {
+ return invalid("the network has too many weights");
+ }
+ Ok((t.dims.clone(), values))
+ };
+ let w = get(1)?;
+ let b = if node.input.len() > 2 {
+ let (dims, values) = get(2)?;
+ if dims.len() != 1 || values.is_empty() {
+ return invalid("a bias tensor must be a nonempty vector");
+ }
+ values
+ } else {
+ Vec::new()
+ };
+ Ok((w, b))
+}
+
+fn floats(t: &TensorProto) -> Result<Vec<f32>, NetError> {
+ if t.data_type != 1 {
+ return unsupported("weights that are not 32-bit floats");
+ }
+ let want: usize = t
+ .dims
+ .iter()
+ .try_fold(1usize, |n, &d| usize::try_from(d).ok().and_then(|d| n.checked_mul(d)))
+ .ok_or_else(|| NetError::Invalid("a tensor with a negative or huge size".into()))?;
+ if want > MAX_WEIGHTS {
+ return invalid("a tensor with too many numbers");
+ }
+ if !t.raw_data.is_empty() && (!t.float_data.is_empty() || !t.raw_data.len().is_multiple_of(4)) {
+ return invalid("ambiguous or truncated tensor data");
+ }
+ let actual = if t.raw_data.is_empty() { t.float_data.len() } else { t.raw_data.len() / 4 };
+ if actual != want {
+ return invalid("tensor data does not match its dimensions");
+ }
+ let mut values = Vec::new();
+ values.try_reserve_exact(want).map_err(|_| NetError::Invalid("not enough memory for weights".into()))?;
+ if !t.raw_data.is_empty() {
+ values.extend(t.raw_data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)));
+ } else {
+ values.extend_from_slice(&t.float_data);
+ }
+ if values.len() != want {
+ return invalid(format!("tensor `{}` has {} numbers, not the {want} its shape says", t.name, values.len()));
+ }
+ Ok(values)
+}
+
+/// The channel count in an input's declared shape `[batch, channels, …]`.
+fn channel_dim(v: &ValueInfoProto) -> Option<usize> {
+ v.channels
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::synthetic::unet_onnx;
+
+ fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {
+ let p = std::env::temp_dir().join(format!("lc-denoise-onnx-{name}-{}.onnx", std::process::id()));
+ std::fs::write(&p, bytes).unwrap();
+ p
+ }
+
+ #[test]
+ fn a_u_net_is_read_with_its_layers_fused() {
+ let (tile, depth) = (32usize, 2usize);
+ let p = temp_model("unet", &unet_onnx(tile as u64, 8, depth, 7));
+ let net = read(&p).unwrap();
+ // per level two convolutions; a pool per level but the last; going up a transposed convolution and two convolutions
+ // per level (the join is part of the first); then the 1 × 1 convolution and depth-to-space
+ assert_eq!(net.ops().len(), 2 * (depth + 1) + depth + depth * 3 + 2);
+ assert!(matches!(net.ops().last(), Some(crate::net::Op::DepthToSpace2 { .. })));
+ assert!(net.ops().iter().any(|o| matches!(o, crate::net::Op::Conv(c) if c.src2.is_some())), "the joins became two-source convolutions");
+ assert!(
+ net.ops().iter().all(|o| !matches!(o, crate::net::Op::Conv(c) if c.leaky.is_none() && c.k == 3)),
+ "every 3 × 3 convolution has its activation"
+ );
+ assert_eq!((net.in_channels, net.out_channels(), net.depth()), (4, 3, 2));
+ assert!(net.fits_tile(tile) && !net.fits_tile(30));
+
+ let _ = std::fs::remove_file(p);
+ }
+
+ #[test]
+ fn an_explicit_empty_or_nonvector_bias_is_not_treated_as_missing() {
+ let w = TensorProto { name: "w".into(), dims: vec![12, 4, 1, 1], data_type: 1, float_data: vec![0.1; 48], ..Default::default() };
+ let n = NodeProto { input: vec!["data".into(), "w".into(), "b".into()], ..Default::default() };
+ for dims in [vec![0], vec![3, 4]] {
+ let b = TensorProto {
+ name: "b".into(),
+ float_data: vec![0.0; dims.iter().product::<i64>() as usize],
+ dims,
+ data_type: 1,
+ ..Default::default()
+ };
+ let inits = HashMap::from([("w", &w), ("b", &b)]);
+ assert!(weights_of(&n, &inits, &mut 0).is_err());
+ }
+ }
+
+ #[test]
+ fn truncated_and_mutated_protobuf_is_bounded_and_never_panics() {
+ let good = unet_onnx(4, 3, 1, 7);
+ for n in (0..good.len()).step_by(31) {
+ let _ = read_bytes(&good[..n]);
+ }
+ for at in (0..good.len()).step_by(37) {
+ let mut bad = good.clone();
+ bad[at] ^= 0xff;
+ let _ = read_bytes(&bad);
+ }
+ assert!(read_bytes(&[0x3a, 0xff, 0xff, 0xff, 0xff, 0x7f]).is_err());
+ assert!(read_bytes(&[0x3a, 0, 0x3a, 0]).is_err());
+ }
+ proptest::proptest! {
+ #[test]
+ fn arbitrary_model_bytes_never_panic(bytes in proptest::collection::vec(proptest::prelude::any::<u8>(), 0..4096)) {
+ let _ = read_bytes(&bytes);
+ }
+ }
+
+ #[test]
+ fn what_the_description_cannot_say_is_unsupported_and_garbage_is_an_error() {
+ // not a model
+ let p = temp_model("junk", b"this is not an onnx file");
+ assert!(matches!(read(&p), Err(NetError::Invalid(_)) | Err(NetError::Unsupported(_))));
+ let _ = std::fs::remove_file(p);
+ // a missing file
+ assert!(read(Path::new("/does/not/exist.onnx")).is_err());
+ }
+}
diff --git a/crates/denoise/src/onnx_proto.rs b/crates/denoise/src/onnx_proto.rs
new file mode 100644
index 000000000..d8bbba411
--- /dev/null
+++ b/crates/denoise/src/onnx_proto.rs
@@ -0,0 +1,295 @@
+//! Bounded, borrowed decoding of the ONNX fields used by denoise, from the public Apache-2.0 schema:
+//! https://github.com/onnx/onnx/blob/main/onnx/onnx.proto3
+//! No generated code, protoc, recursive graph decoding, or external tensor files.
+
+use crate::net::{MAX_OPS, MAX_WEIGHTS, NetError};
+
+const MAX_ITEMS: usize = MAX_OPS * 4;
+const MAX_STRING: usize = 4096;
+pub(super) const MAX_BYTES: usize = MAX_WEIGHTS * 4 + 16 * 1024 * 1024;
+
+type Result<T> = std::result::Result<T, NetError>;
+fn bad() -> NetError {
+ NetError::Invalid("malformed or oversized ONNX protobuf".into())
+}
+
+struct Fields<'a> {
+ rest: &'a [u8],
+ count: usize,
+}
+enum Wire<'a> {
+ Int(u64),
+ Bytes(&'a [u8]),
+ F32(f32),
+ Other,
+}
+
+fn varint(rest: &mut &[u8]) -> Result<u64> {
+ let mut value = 0u64;
+ for shift in (0..70).step_by(7) {
+ let (b, tail) = rest.split_first().ok_or_else(bad)?;
+ *rest = tail;
+ if shift == 63 && *b > 1 {
+ return Err(bad());
+ }
+ value |= u64::from(*b & 127) << shift;
+ if b & 128 == 0 {
+ return Ok(value);
+ }
+ }
+ Err(bad())
+}
+
+impl<'a> Fields<'a> {
+ fn new(rest: &'a [u8]) -> Self {
+ Self { rest, count: 0 }
+ }
+ fn take(&mut self, n: usize) -> Result<&'a [u8]> {
+ let out = self.rest.get(..n).ok_or_else(bad)?;
+ self.rest = self.rest.get(n..).ok_or_else(bad)?;
+ Ok(out)
+ }
+ fn next(&mut self) -> Result<Option<(u32, Wire<'a>)>> {
+ if self.rest.is_empty() {
+ return Ok(None);
+ }
+ self.count += 1;
+ if self.count > MAX_WEIGHTS {
+ return Err(bad());
+ }
+ let tag = varint(&mut self.rest)?;
+ let number = u32::try_from(tag >> 3).map_err(|_| bad())?;
+ if number == 0 || number >= 1 << 29 {
+ return Err(bad());
+ }
+ let wire = match tag & 7 {
+ 0 => Wire::Int(varint(&mut self.rest)?),
+ 1 => {
+ self.take(8)?;
+ Wire::Other
+ }
+ 2 => {
+ let n = usize::try_from(varint(&mut self.rest)?).map_err(|_| bad())?;
+ Wire::Bytes(self.take(n)?)
+ }
+ 5 => {
+ let b: [u8; 4] = self.take(4)?.try_into().map_err(|_| bad())?;
+ Wire::F32(f32::from_le_bytes(b))
+ }
+ _ => return Err(bad()),
+ };
+ Ok(Some((number, wire)))
+ }
+}
+
+fn string(b: &[u8]) -> Result<String> {
+ if b.len() > MAX_STRING {
+ return Err(bad());
+ }
+ std::str::from_utf8(b).map(str::to_owned).map_err(|_| bad())
+}
+fn push<T>(v: &mut Vec<T>, value: T, cap: usize) -> Result<()> {
+ if v.len() >= cap {
+ return Err(bad());
+ }
+ v.try_reserve(1).map_err(|_| bad())?;
+ v.push(value);
+ Ok(())
+}
+fn ints(w: Wire<'_>, out: &mut Vec<i64>, cap: usize) -> Result<()> {
+ match w {
+ Wire::Int(v) => push(out, v as i64, cap),
+ Wire::Bytes(mut b) => {
+ while !b.is_empty() {
+ let v = varint(&mut b)?;
+ push(out, v as i64, cap)?;
+ }
+ Ok(())
+ }
+ _ => Err(bad()),
+ }
+}
+
+#[derive(Default)]
+pub(super) struct AttributeProto {
+ pub name: String,
+ pub f: f32,
+ pub i: i64,
+ pub s: Vec<u8>,
+ pub ints: Vec<i64>,
+}
+#[derive(Default)]
+pub(super) struct NodeProto {
+ pub input: Vec<String>,
+ pub output: Vec<String>,
+ pub op_type: String,
+ pub attribute: Vec<AttributeProto>,
+}
+#[derive(Default)]
+pub(super) struct TensorProto<'a> {
+ pub name: String,
+ pub dims: Vec<i64>,
+ pub data_type: i32,
+ pub raw_data: &'a [u8],
+ pub float_data: Vec<f32>,
+}
+pub(super) struct ValueInfoProto {
+ pub name: String,
+ pub channels: Option<usize>,
+}
+#[derive(Default)]
+pub(super) struct Graph<'a> {
+ pub node: Vec<NodeProto>,
+ pub initializer: Vec<TensorProto<'a>>,
+ pub input: Vec<ValueInfoProto>,
+ pub output: Vec<ValueInfoProto>,
+}
+
+fn attribute(b: &[u8]) -> Result<AttributeProto> {
+ let mut a = AttributeProto::default();
+ let mut fields = Fields::new(b);
+ while let Some((n, w)) = fields.next()? {
+ match (n, w) {
+ (1, Wire::Bytes(b)) => a.name = string(b)?,
+ (2, Wire::F32(v)) => a.f = v,
+ (3, Wire::Int(v)) => a.i = v as i64,
+ (4, Wire::Bytes(b)) => {
+ if b.len() > MAX_STRING {
+ return Err(bad());
+ }
+ a.s = b.to_vec();
+ }
+ (8, w) => ints(w, &mut a.ints, 16)?,
+ (20, Wire::Int(1 | 2 | 3 | 7)) => {}
+ _ => return Err(NetError::Unsupported("unknown attribute value".into())),
+ }
+ }
+ Ok(a)
+}
+fn node(b: &[u8]) -> Result<NodeProto> {
+ let mut a = NodeProto::default();
+ let mut fields = Fields::new(b);
+ while let Some((n, w)) = fields.next()? {
+ match (n, w) {
+ (1, Wire::Bytes(b)) => push(&mut a.input, string(b)?, 4)?,
+ (2, Wire::Bytes(b)) => push(&mut a.output, string(b)?, 2)?,
+ (4, Wire::Bytes(b)) => a.op_type = string(b)?,
+ (5, Wire::Bytes(b)) => push(&mut a.attribute, attribute(b)?, 16)?,
+ (7, Wire::Bytes(b)) if !b.is_empty() => return Err(NetError::Unsupported("a custom operator domain".into())),
+ _ => {}
+ }
+ }
+ Ok(a)
+}
+fn tensor(b: &[u8]) -> Result<TensorProto<'_>> {
+ let mut a = TensorProto::default();
+ let mut fields = Fields::new(b);
+ while let Some((n, w)) = fields.next()? {
+ match (n, w) {
+ (1, w) => ints(w, &mut a.dims, 8)?,
+ (2, Wire::Int(v)) => a.data_type = i32::try_from(v).map_err(|_| bad())?,
+ (4, Wire::F32(v)) => push(&mut a.float_data, v, MAX_WEIGHTS)?,
+ (4, Wire::Bytes(b)) => {
+ if !b.len().is_multiple_of(4) {
+ return Err(bad());
+ }
+ for chunk in b.as_chunks::<4>().0 {
+ push(&mut a.float_data, f32::from_le_bytes(*chunk), MAX_WEIGHTS)?;
+ }
+ }
+ (8, Wire::Bytes(b)) => a.name = string(b)?,
+ (9, Wire::Bytes(b)) => {
+ if b.len() > MAX_WEIGHTS * 4 {
+ return Err(bad());
+ }
+ a.raw_data = b;
+ }
+ (13, _) | (14, Wire::Int(1..)) => return Err(NetError::Unsupported("external tensor data".into())),
+ _ => {}
+ }
+ }
+ Ok(a)
+}
+fn channel_shape(b: &[u8]) -> Result<Option<usize>> {
+ let mut fields = Fields::new(b);
+ let mut dim = 0;
+ let mut channels = None;
+ while let Some((n, w)) = fields.next()? {
+ if let (1, Wire::Bytes(b)) = (n, w) {
+ if dim >= 8 {
+ return Err(bad());
+ }
+ let mut df = Fields::new(b);
+ while let Some((n, w)) = df.next()? {
+ if let (1, Wire::Int(v)) = (n, w)
+ && dim == 1
+ {
+ channels = usize::try_from(v).ok().filter(|&v| v > 0);
+ }
+ }
+ dim += 1;
+ }
+ }
+ Ok(channels)
+}
+fn channel_type(b: &[u8]) -> Result<Option<usize>> {
+ let mut fields = Fields::new(b);
+ while let Some((n, w)) = fields.next()? {
+ if let (1, Wire::Bytes(b)) = (n, w) {
+ let mut tf = Fields::new(b);
+ let (mut elem, mut channels) = (0, None);
+ while let Some((n, w)) = tf.next()? {
+ match (n, w) {
+ (1, Wire::Int(v)) => elem = v,
+ (2, Wire::Bytes(b)) => channels = channel_shape(b)?,
+ _ => {}
+ }
+ }
+ return Ok(if elem == 1 { channels } else { None });
+ }
+ }
+ Ok(None)
+}
+fn value_info(b: &[u8]) -> Result<ValueInfoProto> {
+ let mut out = ValueInfoProto { name: String::new(), channels: None };
+ let mut fields = Fields::new(b);
+ while let Some((n, w)) = fields.next()? {
+ match (n, w) {
+ (1, Wire::Bytes(b)) => out.name = string(b)?,
+ (2, Wire::Bytes(b)) => out.channels = channel_type(b)?,
+ _ => {}
+ }
+ }
+ Ok(out)
+}
+fn graph(b: &[u8]) -> Result<Graph<'_>> {
+ let mut out = Graph::default();
+ let mut fields = Fields::new(b);
+ while let Some((n, w)) = fields.next()? {
+ match (n, w) {
+ (1, Wire::Bytes(b)) => push(&mut out.node, node(b)?, MAX_ITEMS)?,
+ (5, Wire::Bytes(b)) => push(&mut out.initializer, tensor(b)?, MAX_ITEMS)?,
+ (11, Wire::Bytes(b)) => push(&mut out.input, value_info(b)?, MAX_ITEMS)?,
+ (12, Wire::Bytes(b)) => push(&mut out.output, value_info(b)?, 1)?,
+ (15, _) => return Err(NetError::Unsupported("sparse initializers".into())),
+ _ => {}
+ }
+ }
+ Ok(out)
+}
+pub(super) fn decode(b: &[u8]) -> Result<Graph<'_>> {
+ if b.len() > MAX_BYTES {
+ return Err(bad());
+ }
+ let mut fields = Fields::new(b);
+ let mut out = None;
+ while let Some((n, w)) = fields.next()? {
+ if let (7, Wire::Bytes(b)) = (n, w) {
+ if out.is_some() {
+ return Err(bad());
+ }
+ out = Some(graph(b)?);
+ }
+ }
+ out.ok_or_else(bad)
+}
diff --git a/crates/denoise/src/product.rs b/crates/denoise/src/product.rs
new file mode 100644
index 000000000..1e3938855
--- /dev/null
+++ b/crates/denoise/src/product.rs
@@ -0,0 +1,595 @@
+//! The denoised picture as cached data, and the blend the Amount slider controls.
+//!
+//! A product is camera RGB at the sensor's resolution (what the model gave, before the raw's own opcodes and
+//! crop), kept as half floats in strips of rows, each strip byte-shuffled and deflated with its own checksum.
+//! It is derived data: any problem reading it (missing, truncated, corrupt, made for another photo or model) is
+//! just a cache miss, never a panic, and the picture can be made again.
+//!
+//! File: `LCDN`, version, header length, header JSON (`width`, `height`, `strip`, `key`), then for each strip its
+//! compressed length and CRC-32, then the strips' bytes. `key` says what the product was made from (the photo's
+//! content, the model, this code's version, the settings it depends on); a file with another key is stale.
+
+use std::io::{Read, Write};
+use std::path::Path;
+
+use lightcraft_raster::Rgb32f;
+use rayon::prelude::*;
+use serde::{Deserialize, Serialize};
+
+const MAGIC: &[u8; 4] = b"LCDN";
+const VERSION: u8 = 1;
+const MAX_HEADER: usize = 4096;
+const MAX_SIDE: usize = 1 << 17;
+/// Most pixels a product may have (the denoiser's own limit, [`crate::run::MAX_PIXELS`]).
+const MAX_PIXELS: usize = crate::run::MAX_PIXELS;
+/// Rows per strip: about a megapixel of three planes at 24 MP widths.
+const STRIP_ROWS: usize = 128;
+/// Bumped when the meaning of a product changes (included in keys by the caller).
+pub const ALGORITHM: u32 = 1;
+
+#[derive(Debug, thiserror::Error)]
+pub enum ProductError {
+ #[error("the denoised picture could not be read or written: {0}")]
+ Io(String),
+ #[error("not a usable denoised picture: {0}")]
+ Format(String),
+ /// A product made from something else (another photo state, model or version).
+ #[error("the denoised picture is out of date")]
+ Stale,
+}
+
+impl From<std::io::Error> for ProductError {
+ fn from(e: std::io::Error) -> Self {
+ ProductError::Io(e.to_string())
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Header {
+ pub width: usize,
+ pub height: usize,
+ pub strip: usize,
+ pub key: String,
+}
+
+/// `f32` as IEEE half-float bits, rounded to nearest even; beyond the range it saturates, NaN becomes 0.
+pub fn to_f16(v: f32) -> u16 {
+ if !v.is_finite() {
+ return if v.is_nan() {
+ 0
+ } else if v > 0.0 {
+ 0x7bff
+ } else {
+ 0xfbff
+ };
+ }
+ let b = v.to_bits();
+ let sign = ((b >> 16) & 0x8000) as u16;
+ let exp = ((b >> 23) & 0xff) as i32 - 127 + 15;
+ let man = b & 0x7f_ffff;
+ if exp >= 31 {
+ return sign | 0x7bff;
+ }
+ if exp <= 0 {
+ if exp < -10 {
+ return sign;
+ }
+ let man = man | 0x80_0000;
+ let shift = (14 - exp) as u32;
+ let (half, rem, halfway) = (man >> shift, man & ((1u32 << shift) - 1), 1u32 << (shift - 1));
+ let mut h = half as u16;
+ if rem > halfway || (rem == halfway && h & 1 == 1) {
+ h += 1;
+ }
+ return sign | h;
+ }
+ let mut h = (((exp as u32) << 10) | (man >> 13)) as u16;
+ let rem = man & 0x1fff;
+ if rem > 0x1000 || (rem == 0x1000 && h & 1 == 1) {
+ h += 1;
+ }
+ if h >= 0x7c00 {
+ h = 0x7bff;
+ }
+ sign | h
+}
+
+/// Half-float bits as `f32`.
+pub fn from_f16(h: u16) -> f32 {
+ let sign = if h & 0x8000 != 0 { -1.0f32 } else { 1.0 };
+ let exp = i32::from((h >> 10) & 0x1f);
+ let man = f32::from(h & 0x3ff);
+ match exp {
+ 0 => sign * man * 2f32.powi(-24),
+ 31 => sign * 65504.0,
+ _ => sign * (1.0 + man / 1024.0) * 2f32.powi(exp - 15),
+ }
+}
+
+fn crc(bytes: &[u8]) -> u32 {
+ // CRC-32 (IEEE), bitwise: strips are small and this runs once per strip
+ let mut table = [0u32; 256];
+ for (i, t) in table.iter_mut().enumerate() {
+ let mut c = i as u32;
+ for _ in 0..8 {
+ c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
+ }
+ *t = c;
+ }
+ !bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))
+}
+
+/// One strip as bytes: for each plane (R, G, B) the low bytes of its half floats, then the high bytes.
+fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Vec<u8> {
+ let n = (y1 - y0) * img.width;
+ let mut raw = vec![0u8; 6 * n];
+ let rows = img.data.get(y0 * img.width..y1 * img.width).unwrap_or(&[]);
+ for (i, px) in rows.iter().enumerate() {
+ for (c, &v) in px.iter().enumerate() {
+ let h = to_f16(v);
+ if let Some(lo) = raw.get_mut(2 * c * n + i) {
+ *lo = h as u8;
+ }
+ if let Some(hi) = raw.get_mut((2 * c + 1) * n + i) {
+ *hi = (h >> 8) as u8;
+ }
+ }
+ }
+ miniz_oxide::deflate::compress_to_vec(&raw, 3)
+}
+
+/// Write `img` to `path` (through a temporary file next to it, so a reader never sees half a product).
+pub fn write(path: &Path, img: &Rgb32f, key: &str) -> Result<(), ProductError> {
+ if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || img.data.len() != img.width * img.height {
+ return Err(ProductError::Format("the picture has no usable size".into()));
+ }
+ let strips: Vec<(usize, usize)> = (0..img.height).step_by(STRIP_ROWS).map(|y| (y, (y + STRIP_ROWS).min(img.height))).collect();
+ let packed: Vec<Vec<u8>> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect();
+ let header = serde_json::to_vec(&Header { width: img.width, height: img.height, strip: STRIP_ROWS, key: key.to_string() })
+ .map_err(|e| ProductError::Format(e.to_string()))?;
+ let tmp = path.with_extension("tmp");
+ let result = (|| -> Result<(), ProductError> {
+ if let Some(dir) = path.parent() {
+ std::fs::create_dir_all(dir)?;
+ }
+ let mut f = std::io::BufWriter::new(std::fs::File::create(&tmp)?);
+ f.write_all(MAGIC)?;
+ f.write_all(&[VERSION, 0, 0, 0])?;
+ f.write_all(&(header.len() as u32).to_le_bytes())?;
+ f.write_all(&header)?;
+ for p in &packed {
+ f.write_all(&(p.len() as u32).to_le_bytes())?;
+ f.write_all(&crc(p).to_le_bytes())?;
+ }
+ for p in &packed {
+ f.write_all(p)?;
+ }
+ f.flush()?;
+ drop(f);
+ std::fs::rename(&tmp, path)?;
+ Ok(())
+ })();
+ if result.is_err() {
+ let _ = std::fs::remove_file(&tmp);
+ }
+ result
+}
+
+/// The parsed start of a product file: header, each strip's (compressed length, CRC-32), and the file length the
+/// start says the whole file has.
+struct Start {
+ header: Header,
+ table: Vec<(usize, u32)>,
+ expected_len: u64,
+}
+
+fn read_start(r: &mut impl Read) -> Result<Start, ProductError> {
+ let mut fixed = [0u8; 12];
+ r.read_exact(&mut fixed)?;
+ if fixed.get(0..4) != Some(&MAGIC[..]) || fixed.get(4) != Some(&VERSION) {
+ return Err(ProductError::Format("not a denoised picture of this version".into()));
+ }
+ let len = fixed.get(8..12).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_le_bytes).unwrap_or(u32::MAX) as usize;
+ if len > MAX_HEADER {
+ return Err(ProductError::Format("the header is too large".into()));
+ }
+ let mut hb = vec![0u8; len];
+ r.read_exact(&mut hb)?;
+ let h: Header = serde_json::from_slice(&hb).map_err(|e| ProductError::Format(e.to_string()))?;
+ let pixels = h.width.saturating_mul(h.height);
+ if h.width == 0 || h.height == 0 || h.width > MAX_SIDE || h.height > MAX_SIDE || pixels > MAX_PIXELS || h.strip != STRIP_ROWS {
+ return Err(ProductError::Format("the size is not usable".into()));
+ }
+ let n = h.height.div_ceil(h.strip);
+ let mut table = Vec::with_capacity(n);
+ let mut total = 12 + len as u64 + 8 * n as u64;
+ for _ in 0..n {
+ let mut e = [0u8; 8];
+ r.read_exact(&mut e)?;
+ let l = u32::from_le_bytes([e[0], e[1], e[2], e[3]]) as usize;
+ let c = u32::from_le_bytes([e[4], e[5], e[6], e[7]]);
+ // a strip never compresses to much more than its raw size
+ if l > 12 * h.strip * h.width + 4096 {
+ return Err(ProductError::Format("a strip is larger than it can be".into()));
+ }
+ total += l as u64;
+ table.push((l, c));
+ }
+ Ok(Start { header: h, table, expected_len: total })
+}
+
+/// The header of the product at `path`, when it is one.
+pub fn read_header_at(path: &Path) -> Result<Header, ProductError> {
+ let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
+ Ok(read_start(&mut f)?.header)
+}
+
+/// Whether `path` holds a complete-looking product made from `key` (the header and the file's length are checked;
+/// strips are checked when read).
+pub fn is_current(path: &Path, key: &str) -> bool {
+ let check = || -> Result<bool, ProductError> {
+ let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
+ let s = read_start(&mut f)?;
+ Ok(s.header.key == key && std::fs::metadata(path)?.len() == s.expected_len)
+ };
+ check().unwrap_or(false)
+}
+
+/// A rectangle of a product, in the product's pixels.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Window {
+ pub x: usize,
+ pub y: usize,
+ pub width: usize,
+ pub height: usize,
+}
+
+/// Half floats as `f32`, for all 65536 bit patterns (a table is much faster than computing each).
+fn f16_table() -> &'static [f32] {
+ static TABLE: std::sync::OnceLock<Vec<f32>> = std::sync::OnceLock::new();
+ TABLE.get_or_init(|| (0..=u16::MAX).map(from_f16).collect())
+}
+
+/// Strips decoded together (they are inflated in parallel, then folded in order).
+const GROUP: usize = 8;
+
+/// One strip, checked and inflated: for each plane the low bytes of its half floats, then the high bytes.
+fn decode_strip(index: usize, packed: &[u8], expect_crc: u32, n: usize) -> Result<Vec<u8>, ProductError> {
+ if crc(packed) != expect_crc {
+ return Err(ProductError::Format(format!("strip {index} is damaged")));
+ }
+ let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(packed, 6 * n).map_err(|e| ProductError::Format(format!("strip {index}: {e:?}")))?;
+ if raw.len() != 6 * n {
+ return Err(ProductError::Format(format!("strip {index} has the wrong size")));
+ }
+ Ok(raw)
+}
+
+/// Read the product at `path`, made from `key`, with every `factor × factor` block of pixels averaged into one
+/// (`factor` 1 = full size). The result is camera RGB.
+///
+/// With a `window`, only that rectangle is read and the blocks start at its corner; only whole blocks are made, so
+/// the result is `window.width / factor × window.height / factor` (the same blocks a binned development of the same
+/// crop makes). Without one the whole product is read and blocks at the right and bottom edges may be smaller.
+///
+/// With `clip` (white is 1), a block in which any sample of a colour is at or above it takes that colour's
+/// maximum instead of the mean, so a clipped highlight stays clipped for highlight reconstruction.
+pub fn read_window(path: &Path, key: &str, window: Option<Window>, factor: usize, clip: Option<f32>) -> Result<Rgb32f, ProductError> {
+ let factor = factor.clamp(1, 1 << 12);
+ let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
+ let Start { header: h, table, expected_len } = read_start(&mut f)?;
+ if h.key != key {
+ return Err(ProductError::Stale);
+ }
+ if std::fs::metadata(path)?.len() != expected_len {
+ return Err(ProductError::Format("the file is not the length its header says".into()));
+ }
+ let w = window.unwrap_or(Window { x: 0, y: 0, width: h.width, height: h.height });
+ let fits = |a: usize, b: usize, limit: usize| a.checked_add(b).is_some_and(|e| e <= limit);
+ if w.width == 0 || w.height == 0 || !fits(w.x, w.width, h.width) || !fits(w.y, w.height, h.height) {
+ return Err(ProductError::Format("the window does not lie inside the picture".into()));
+ }
+ // whole blocks for a window; partial ones at the edges when the whole picture is read
+ let whole = window.is_none();
+ let (ow, oh) = if whole { (w.width.div_ceil(factor), w.height.div_ceil(factor)) } else { (w.width / factor, w.height / factor) };
+ if ow == 0 || oh == 0 {
+ return Err(ProductError::Format("the window is smaller than one block".into()));
+ }
+ // the picture rows and columns the blocks cover
+ let (x_end, y_end) = if whole { (w.x + w.width, w.y + w.height) } else { (w.x + ow * factor, w.y + oh * factor) };
+ let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] };
+ let lut = f16_table();
+ let value = |raw: &[u8], c: usize, n: usize, i: usize| -> f32 {
+ let (lo, hi) = (raw.get(2 * c * n + i).copied().unwrap_or(0), raw.get((2 * c + 1) * n + i).copied().unwrap_or(0));
+ lut.get(usize::from(lo) | usize::from(hi) << 8).copied().unwrap_or(0.0)
+ };
+ // which strips are needed, and where each starts in the file (after the header and the strip table)
+ let wanted = |s: usize| s * h.strip < y_end && (s + 1) * h.strip > w.y;
+ let (mut sum, mut peak) = (vec![[0f32; 3]; ow], vec![[f32::MIN; 3]; ow]);
+ let (mut rows_in, mut oy) = (0usize, 0usize);
+ let strips: Vec<usize> = (0..table.len()).filter(|&s| wanted(s)).collect();
+ // skip the strips before the first one wanted
+ let skipped: usize = table.iter().take(strips.first().copied().unwrap_or(0)).map(|e| e.0).sum();
+ f.seek_relative(i64::try_from(skipped).map_err(|_| ProductError::Format("the file is too large".into()))?)?;
+ for group in strips.chunks(GROUP) {
+ let mut packed = Vec::with_capacity(group.len());
+ for &s in group {
+ let Some(&(len, _)) = table.get(s) else { return Err(ProductError::Format("a strip is missing".into())) };
+ let mut p = vec![0u8; len];
+ f.read_exact(&mut p)?;
+ packed.push(p);
+ }
+ let decoded: Vec<Result<Vec<u8>, ProductError>> = group
+ .par_iter()
+ .zip(packed.par_iter())
+ .map(|(&s, p)| {
+ let rows = h.height.saturating_sub(s * h.strip).min(h.strip);
+ decode_strip(s, p, table.get(s).map_or(0, |e| e.1), rows * h.width)
+ })
+ .collect();
+ for (&s, raw) in group.iter().zip(decoded) {
+ let raw = raw?;
+ let y0 = s * h.strip;
+ let rows = h.height.saturating_sub(y0).min(h.strip);
+ let n = rows * h.width;
+ for r in 0..rows {
+ let y = y0 + r;
+ if y < w.y || y >= y_end {
+ continue;
+ }
+ let base = r * h.width;
+ for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {
+ let x0 = w.x + ox * factor;
+ let x1 = (x0 + factor).min(x_end);
+ for x in x0..x1 {
+ for c in 0..3 {
+ let v = value(&raw, c, n, base + x);
+ if let (Some(s_), Some(p_)) = (sm.get_mut(c), pk.get_mut(c)) {
+ *s_ += v;
+ *p_ = p_.max(v);
+ }
+ }
+ }
+ }
+ rows_in += 1;
+ if rows_in == factor || y + 1 == y_end {
+ for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {
+ // the block's size at the right and bottom edges may be smaller
+ let bw = (x_end - (w.x + ox * factor)).min(factor);
+ let k = 1.0 / (rows_in * bw).max(1) as f32;
+ if let Some(o) = out.data.get_mut(oy * ow + ox) {
+ for c in 0..3 {
+ let (s_, p_) = (sm.get(c).copied().unwrap_or(0.0), pk.get(c).copied().unwrap_or(0.0));
+ if let Some(slot) = o.get_mut(c) {
+ *slot = if factor > 1 && clip.is_some_and(|t| p_ >= t) { p_ } else { s_ * k };
+ }
+ }
+ }
+ *sm = [0.0; 3];
+ *pk = [f32::MIN; 3];
+ }
+ rows_in = 0;
+ oy += 1;
+ }
+ }
+ }
+ }
+ Ok(out)
+}
+
+/// [`read_window`] of the whole product: every `factor × factor` block averaged into one.
+pub fn read(path: &Path, key: &str, factor: usize) -> Result<Rgb32f, ProductError> {
+ read_window(path, key, None, factor, None)
+}
+
+/// `base + (denoised − base) · amount` for every pixel (`amount` 0 to 1), in place in `base`. The two pictures must
+/// be the same size; if they are not, `base` is left as it was and `false` is returned.
+pub fn blend(base: &mut Rgb32f, denoised: &Rgb32f, amount: f32) -> bool {
+ if base.width != denoised.width || base.height != denoised.height {
+ return false;
+ }
+ let a = if amount.is_finite() { amount.clamp(0.0, 1.0) } else { 0.0 };
+ base.data.par_iter_mut().zip(denoised.data.par_iter()).for_each(|(b, d)| {
+ for c in 0..3 {
+ b[c] += (d[c] - b[c]) * a;
+ }
+ });
+ true
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn temp(name: &str) -> std::path::PathBuf {
+ std::env::temp_dir().join(format!("lc-denoise-{name}-{}.lcdn", std::process::id()))
+ }
+
+ fn picture(w: usize, h: usize) -> Rgb32f {
+ Rgb32f {
+ width: w,
+ height: h,
+ data: (0..w * h).map(|i| [(i % w) as f32 / w as f32, (i / w) as f32 / h as f32, 0.001 + 0.5 * ((i % 7) as f32 / 7.0)]).collect(),
+ }
+ }
+
+ #[test]
+ fn half_floats_round_trip_closely() {
+ for v in [0.0f32, 1.0, -1.0, 0.5, 0.333_333, 1e-4, 6.1e-5, 3e-6, 123.456, 65000.0, -0.002] {
+ let back = from_f16(to_f16(v));
+ assert!((back - v).abs() <= v.abs() * 0.001 + 1e-7, "{v} -> {back}");
+ }
+ assert_eq!(from_f16(to_f16(1e10)), 65504.0);
+ assert_eq!(to_f16(f32::NAN), 0);
+ assert_eq!(from_f16(to_f16(f32::NEG_INFINITY)), -65504.0);
+ }
+
+ #[test]
+ fn a_product_survives_a_round_trip_at_full_and_reduced_size() {
+ let (w, h) = (301, 517);
+ let img = picture(w, h);
+ let p = temp("round");
+ write(&p, &img, "k1").unwrap();
+ assert!(is_current(&p, "k1") && !is_current(&p, "k2"));
+ assert_eq!(read_header_at(&p).unwrap().width, w);
+ let full = read(&p, "k1", 1).unwrap();
+ assert_eq!((full.width, full.height), (w, h));
+ for (a, b) in full.data.iter().zip(&img.data) {
+ for c in 0..3 {
+ assert!((a[c] - b[c]).abs() < 0.002, "{a:?} vs {b:?}");
+ }
+ }
+ // averaged into blocks of 4: the sizes round up, and the mean of the picture is kept
+ let small = read(&p, "k1", 4).unwrap();
+ assert_eq!((small.width, small.height), (w.div_ceil(4), h.div_ceil(4)));
+ let mean = |i: &Rgb32f| i.data.iter().map(|p| f64::from(p[1])).sum::<f64>() / i.data.len() as f64;
+ assert!((mean(&small) - mean(&img)).abs() < 0.01);
+ // a factor that does not divide the strip height is the same average
+ let odd = read(&p, "k1", 5).unwrap();
+ assert_eq!((odd.width, odd.height), (w.div_ceil(5), h.div_ceil(5)));
+ assert!((mean(&odd) - mean(&img)).abs() < 0.01);
+ assert!(matches!(read(&p, "other", 1), Err(ProductError::Stale)));
+ std::fs::remove_file(&p).unwrap();
+ }
+
+ /// What binning a window of `img` by `k` means, written the slow obvious way.
+ fn blocks(img: &Rgb32f, win: Window, k: usize, clip: Option<f32>) -> Rgb32f {
+ let (ow, oh) = (win.width / k, win.height / k);
+ let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] };
+ for by in 0..oh {
+ for bx in 0..ow {
+ for c in 0..3 {
+ let vals: Vec<f32> = (0..k * k).map(|i| img.data[(win.y + by * k + i / k) * img.width + win.x + bx * k + i % k][c]).collect();
+ let max = vals.iter().copied().fold(f32::MIN, f32::max);
+ out.data[by * ow + bx][c] =
+ if k > 1 && clip.is_some_and(|t| max >= t) { max } else { vals.iter().sum::<f32>() / vals.len() as f32 };
+ }
+ }
+ }
+ out
+ }
+
+ #[test]
+ fn a_window_is_read_as_the_blocks_of_that_crop() {
+ let (w, h) = (301, 517);
+ let mut img = picture(w, h);
+ // a clipped sample in the middle of a block
+ img.data[(130 + 2) * w + 61 + 1] = [1.0, 0.4, 0.4];
+ let p = temp("window");
+ write(&p, &img, "k1").unwrap();
+ let near = |a: &Rgb32f, b: &Rgb32f| {
+ assert_eq!((a.width, a.height), (b.width, b.height));
+ for (x, y) in a.data.iter().zip(&b.data) {
+ for c in 0..3 {
+ assert!((x[c] - y[c]).abs() < 0.003, "{x:?} vs {y:?}");
+ }
+ }
+ };
+ // windows at odd places, across strip boundaries (128 rows), with factors that do and do not divide them
+ for (win, k) in [
+ (Window { x: 3, y: 5, width: 200, height: 300 }, 1),
+ (Window { x: 3, y: 5, width: 200, height: 300 }, 4),
+ (Window { x: 61, y: 130, width: 121, height: 250 }, 5),
+ (Window { x: 0, y: 0, width: w, height: h }, 2),
+ (Window { x: 1, y: 380, width: 90, height: 137 }, 3),
+ ] {
+ let got = read_window(&p, "k1", Some(win), k, None).unwrap();
+ near(&got, &blocks(&img, win, k, None));
+ }
+ // with a clip level the block with the clipped sample keeps its highest value, the others stay means
+ let win = Window { x: 61, y: 130, width: 40, height: 20 };
+ let clipped = read_window(&p, "k1", Some(win), 4, Some(0.99)).unwrap();
+ near(&clipped, &blocks(&img, win, 4, Some(0.99)));
+ assert!(clipped.data[0][0] > 0.99, "the clipped block stays clipped: {:?}", clipped.data[0]);
+ let plain = read_window(&p, "k1", Some(win), 4, None).unwrap();
+ assert!(plain.data[0][0] < 0.99, "an average falls below the clip level");
+ // windows that do not fit, or are smaller than a block, are errors
+ for bad in [
+ Window { x: 0, y: 0, width: 0, height: 5 },
+ Window { x: 290, y: 0, width: 20, height: 5 },
+ Window { x: 0, y: 510, width: 5, height: 10 },
+ Window { x: usize::MAX, y: 0, width: 2, height: 2 },
+ Window { x: 0, y: 0, width: 3, height: 3 },
+ ] {
+ assert!(read_window(&p, "k1", Some(bad), 4, None).is_err(), "{bad:?}");
+ }
+ std::fs::remove_file(&p).unwrap();
+ }
+
+ #[test]
+ fn a_smooth_picture_compresses_well() {
+ let img = Rgb32f {
+ width: 512,
+ height: 512,
+ data: (0..512 * 512)
+ .map(|i| {
+ let v = 0.2 + 0.0001 * (i % 512) as f32;
+ [v, v * 0.8, v * 0.6]
+ })
+ .collect(),
+ };
+ let p = temp("size");
+ write(&p, &img, "k").unwrap();
+ let bytes = std::fs::metadata(&p).unwrap().len() as usize;
+ assert!(bytes < 512 * 512 * 6 / 2, "{bytes} bytes for a smooth picture");
+ std::fs::remove_file(&p).unwrap();
+ }
+
+ #[test]
+ fn damaged_files_are_errors_not_panics() {
+ let img = picture(100, 300);
+ let p = temp("damage");
+ write(&p, &img, "k").unwrap();
+ let good = std::fs::read(&p).unwrap();
+ // truncated at every length
+ for cut in (0..good.len()).step_by(37) {
+ std::fs::write(&p, &good[..cut]).unwrap();
+ assert!(read(&p, "k", 1).is_err(), "cut at {cut}");
+ assert!(!is_current(&p, "k"));
+ }
+ // a flipped byte anywhere is caught by a checksum, the header parse, or the key
+ for at in (0..good.len()).step_by(53) {
+ let mut bad = good.clone();
+ bad[at] ^= 0xff;
+ std::fs::write(&p, &bad).unwrap();
+ let _ = read(&p, "k", 1); // must not panic; most are errors
+ }
+ // a header that claims an absurd size
+ let mut huge = b"LCDN\x01\0\0\0".to_vec();
+ let hdr = br#"{"width":4000000000,"height":4000000000,"strip":128,"key":"k"}"#;
+ huge.extend((hdr.len() as u32).to_le_bytes());
+ huge.extend(hdr);
+ std::fs::write(&p, &huge).unwrap();
+ assert!(matches!(read(&p, "k", 1), Err(ProductError::Format(_))));
+ std::fs::write(&p, b"").unwrap();
+ assert!(read(&p, "k", 1).is_err());
+ assert!(read(&temp("missing"), "k", 1).is_err());
+ std::fs::remove_file(&p).unwrap();
+ }
+
+ #[test]
+ fn writing_rejects_unusable_pictures() {
+ let p = temp("bad");
+ assert!(write(&p, &Rgb32f { width: 0, height: 0, data: vec![] }, "k").is_err());
+ assert!(write(&p, &Rgb32f { width: 3, height: 3, data: vec![[0.0; 3]; 8] }, "k").is_err());
+ assert!(!p.exists());
+ }
+
+ #[test]
+ fn blend_mixes_by_amount_and_rejects_other_sizes() {
+ let mut base = Rgb32f { width: 2, height: 1, data: vec![[0.0; 3], [1.0; 3]] };
+ let dn = Rgb32f { width: 2, height: 1, data: vec![[1.0; 3], [0.0; 3]] };
+ assert!(blend(&mut base, &dn, 0.25));
+ assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);
+ assert!(blend(&mut base, &dn, f32::NAN));
+ assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);
+ assert!(!blend(&mut base, &Rgb32f { width: 1, height: 1, data: vec![[0.0; 3]] }, 1.0));
+ assert!(blend(&mut base, &dn, 1.0));
+ assert_eq!(base.data, dn.data);
+ }
+
+ #[test]
+ fn crc_matches_the_standard_check_value() {
+ assert_eq!(crc(b"123456789"), 0xcbf4_3926);
+ }
+}
diff --git a/crates/denoise/src/reference.rs b/crates/denoise/src/reference.rs
new file mode 100644
index 000000000..ef58b5b9c
--- /dev/null
+++ b/crates/denoise/src/reference.rs
@@ -0,0 +1,133 @@
+//! A plain-loop interpreter of a [`Net`], slow and obviously right: the reference the parser and the GPU runner are
+//! tested against (tract's answer on the real model is the other). It is not used on photos.
+
+use crate::net::{Net, NetError, Op, TensorId};
+use crate::run::{Error, TileRunner};
+
+/// A tensor in `[channels][side][side]` order.
+struct T {
+ channels: usize,
+ side: usize,
+ data: Vec<f32>,
+}
+
+/// Run `net` on `input` (`in_channels` planes of `tile × tile`).
+pub fn run(net: &Net, tile: usize, input: &[f32]) -> Result<Vec<f32>, NetError> {
+ if !net.fits_tile(tile) || input.len() != net.in_channels * tile * tile {
+ return Err(NetError::Invalid("the tile does not fit the network".into()));
+ }
+ let mut tensors: Vec<T> = vec![T { channels: net.in_channels, side: tile, data: input.to_vec() }];
+ let get = |tensors: &[T], id: TensorId| -> Result<usize, NetError> {
+ if id < tensors.len() { Ok(id) } else { Err(NetError::Invalid("a layer reads a tensor that does not exist yet".into())) }
+ };
+ for op in net.ops() {
+ let out = match op {
+ Op::Conv(c) => {
+ let a = get(&tensors, c.src)?;
+ let b = c.src2.map(|s| get(&tensors, s)).transpose()?;
+ let side = tensors[a].side;
+ let pad = (c.k - 1) / 2;
+ let mut data = vec![0f32; c.cout * side * side];
+ // the input channels in order: the first source's, then the second's
+ let planes: Vec<&[f32]> =
+ std::iter::once(&tensors[a]).chain(b.map(|b| &tensors[b])).flat_map(|t| t.data.chunks(side * side)).collect();
+ for co in 0..c.cout {
+ for y in 0..side {
+ for x in 0..side {
+ let mut sum = c.bias[co];
+ for (ci, plane) in planes.iter().enumerate() {
+ for ky in 0..c.k {
+ for kx in 0..c.k {
+ let (yy, xx) = ((y + ky) as isize - pad as isize, (x + kx) as isize - pad as isize);
+ if yy < 0 || xx < 0 || yy >= side as isize || xx >= side as isize {
+ continue;
+ }
+ sum += plane[yy as usize * side + xx as usize] * c.weight[((co * c.cin + ci) * c.k + ky) * c.k + kx];
+ }
+ }
+ }
+ data[(co * side + y) * side + x] = leaky(sum, c.leaky);
+ }
+ }
+ }
+ T { channels: c.cout, side, data }
+ }
+ Op::ConvT2(c) => {
+ let a = get(&tensors, c.src)?;
+ let side = tensors[a].side;
+ let out_side = 2 * side;
+ let mut data = vec![0f32; c.cout * out_side * out_side];
+ for co in 0..c.cout {
+ for y in 0..side {
+ for x in 0..side {
+ for dy in 0..2 {
+ for dx in 0..2 {
+ let mut sum = c.bias[co];
+ for ci in 0..c.cin {
+ sum += tensors[a].data[(ci * side + y) * side + x] * c.weight[((ci * c.cout + co) * 2 + dy) * 2 + dx];
+ }
+ data[(co * out_side + 2 * y + dy) * out_side + 2 * x + dx] = leaky(sum, c.leaky);
+ }
+ }
+ }
+ }
+ }
+ T { channels: c.cout, side: out_side, data }
+ }
+ Op::MaxPool2 { src } => {
+ let a = get(&tensors, *src)?;
+ let (channels, side) = (tensors[a].channels, tensors[a].side / 2);
+ let mut data = vec![0f32; channels * side * side];
+ for ch in 0..channels {
+ for y in 0..side {
+ for x in 0..side {
+ let at = |dy: usize, dx: usize| tensors[a].data[(ch * 2 * side + 2 * y + dy) * 2 * side + 2 * x + dx];
+ data[(ch * side + y) * side + x] = at(0, 0).max(at(0, 1)).max(at(1, 0)).max(at(1, 1));
+ }
+ }
+ }
+ T { channels, side, data }
+ }
+ Op::DepthToSpace2 { src } => {
+ let a = get(&tensors, *src)?;
+ let (channels, side) = (tensors[a].channels / 4, tensors[a].side);
+ let out_side = 2 * side;
+ let mut data = vec![0f32; channels * out_side * out_side];
+ for ch in 0..channels {
+ for y in 0..side {
+ for x in 0..side {
+ for dy in 0..2 {
+ for dx in 0..2 {
+ data[(ch * out_side + 2 * y + dy) * out_side + 2 * x + dx] =
+ tensors[a].data[((4 * ch + 2 * dy + dx) * side + y) * side + x];
+ }
+ }
+ }
+ }
+ }
+ T { channels, side: out_side, data }
+ }
+ };
+ tensors.push(out);
+ }
+ tensors.pop().map(|t| t.data).ok_or_else(|| NetError::Invalid("no layers".into()))
+}
+
+fn leaky(v: f32, alpha: Option<f32>) -> f32 {
+ match alpha {
+ Some(a) if v < 0.0 => v * a,
+ _ => v,
+ }
+}
+
+/// The reference as a [`TileRunner`] (for tests of the code around the model).
+pub struct ReferenceRunner {
+ pub net: Net,
+ pub tile: usize,
+}
+
+impl TileRunner for ReferenceRunner {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
+ run(&self.net, self.tile, input).map_err(|e| Error::Runtime(e.to_string()))
+ }
+}
diff --git a/crates/denoise/src/run.rs b/crates/denoise/src/run.rs
new file mode 100644
index 000000000..5bc763ce5
--- /dev/null
+++ b/crates/denoise/src/run.rs
@@ -0,0 +1,582 @@
+//! Denoising a whole Bayer mosaic with a tile model: cut it into overlapping tiles, run the model on each, bring
+//! each tile's scale back to the input's, keep clipped highlights as they were, and blend the tiles together.
+//!
+//! The model itself is a [`TileRunner`], so everything around it (packing, tiling, scale matching, highlight
+//! protection, blending, cancelling, progress) is tested without one.
+
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use lightcraft_raster::Rgb32f;
+use rayon::prelude::*;
+
+use crate::bayer::{Layout, pack_tile};
+use crate::manifest::Gain;
+use crate::tiles::{starts, weight};
+
+/// Most pixels one picture may have (guards the allocations that follow from its size).
+pub const MAX_PIXELS: usize = 400_000_000;
+/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from.
+const MIN_SIGNAL: f32 = 0.003;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("{0}")]
+ Input(String),
+ #[error("the model gave {0}")]
+ Model(String),
+ #[error("the model could not run: {0}")]
+ Runtime(String),
+ #[error("the model could not be loaded: {0}")]
+ Load(String),
+ #[error("cancelled")]
+ Cancelled,
+}
+
+/// Something that runs the model on one tile.
+pub trait TileRunner: Sync {
+ /// `input` is the tile as four planes `[R, G1, G2, B]` of `tile × tile` cells (NCHW, batch 1); the answer is
+ /// camera RGB, three planes of `2·tile × 2·tile` pixels (CHW), in the model's own scale.
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error>;
+}
+
+pub struct Params {
+ /// Side of the model's square, in cells.
+ pub tile: usize,
+ /// Cells neighbouring tiles share.
+ pub overlap: usize,
+ pub gain: Gain,
+ /// White is 1; samples at or above this (minus a little) are clipped and keep their value.
+ pub clip: Option<f32>,
+ /// Tiles run at once (each model call may itself be single-threaded).
+ pub parallel: usize,
+}
+
+#[derive(Default)]
+pub struct Control<'a> {
+ pub cancel: Option<&'a AtomicBool>,
+ /// `(tiles done, tiles in all)`.
+ pub progress: Option<&'a (dyn Fn(usize, usize) + Sync)>,
+}
+
+/// The tile grid of a picture: `(starts across, starts down)`.
+pub fn grid(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> (Vec<usize>, Vec<usize>) {
+ let (nx, ny) = layout.cells(width, height);
+ (starts(nx, tile, overlap), starts(ny, tile, overlap))
+}
+
+/// How many model calls [`denoise_bayer`] makes for a picture.
+pub fn tile_count(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> usize {
+ let (sx, sy) = grid(width, height, layout, tile, overlap);
+ sx.len().saturating_mul(sy.len())
+}
+
+/// Denoise (and demosaic) the normalised mosaic `mosaic` of `width × height` samples, laid out as `layout`.
+/// Gives camera RGB at the mosaic's resolution.
+pub fn denoise_bayer(
+ mosaic: &[f32],
+ width: usize,
+ height: usize,
+ layout: Layout,
+ runner: &dyn TileRunner,
+ p: &Params,
+ ctl: &Control<'_>,
+) -> Result<Rgb32f, Error> {
+ let pixels = width
+ .checked_mul(height)
+ .filter(|&n| n > 0 && n <= MAX_PIXELS)
+ .ok_or_else(|| Error::Input("the picture has no size, or is too large".into()))?;
+ if mosaic.len() != pixels {
+ return Err(Error::Input("the mosaic does not have width × height samples".into()));
+ }
+ if !(16..=2048).contains(&p.tile) || p.overlap >= p.tile.div_ceil(2) || layout.dx > 1 || layout.dy > 1 {
+ return Err(Error::Input("the tile size and overlap do not fit".into()));
+ }
+ if let Gain::MatchMean { nominal, max_deviation } = p.gain
+ && !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal) && max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation))
+ {
+ return Err(Error::Input("the model gain is not finite or within its supported range".into()));
+ }
+ if p.clip.is_some_and(|v| !v.is_finite() || v <= 0.0) {
+ return Err(Error::Input("the highlight clip must be finite and positive".into()));
+ }
+ let (nx, ny) = layout.cells(width, height);
+ let (sx, sy) = (starts(nx, p.tile, p.overlap), starts(ny, p.tile, p.overlap));
+ let (cols, rows) = (sx.len(), sy.len());
+ if cols == 0 || rows == 0 {
+ return Err(Error::Input("the picture needs too many tiles".into()));
+ }
+ let tiles: Vec<(usize, usize, bool, bool, bool, bool)> = sy
+ .iter()
+ .enumerate()
+ .flat_map(|(j, &y0)| sx.iter().enumerate().map(move |(i, &x0)| (x0, y0, i > 0, i + 1 < cols, j > 0, j + 1 < rows)))
+ .collect();
+ let total = tiles.len();
+
+ let mut acc = vec![[0f32; 3]; pixels];
+ let mut wsum = vec![0f32; nx * ny];
+ let group = p.parallel.clamp(1, 64);
+ let mut done = 0;
+ // where the time goes, printed under `LIGHTCRAFT_PROFILE`
+ let (mut t_wait, mut t_total) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
+ let began = std::time::Instant::now();
+
+ // One group of tiles at a time: pack, run the model, check and bring each to the input's scale, in parallel.
+ // Blending (sequential: the tiles add into one picture, in tile order so the result does not depend on timing) of
+ // the previous group runs while the next group is on the model, so neither the card nor the cores wait for it.
+ type Tile = (usize, usize, bool, bool, bool, bool);
+ let compute = |chunk: &[Tile]| -> Vec<Result<Vec<f32>, Error>> {
+ chunk
+ .par_iter()
+ .map(|&(x0, y0, bx, ax, by, ay)| {
+ let mut input = vec![0f32; 4 * p.tile * p.tile];
+ if !pack_tile(mosaic, width, height, layout, x0 as isize, y0 as isize, p.tile, &mut input) {
+ return Err(Error::Input("a tile could not be packed".into()));
+ }
+ let mut out = runner.run(&input)?;
+ let side = 2 * p.tile;
+ if out.len() != 3 * side * side {
+ return Err(Error::Model(format!("{} numbers for a tile, not the {} expected", out.len(), 3 * side * side)));
+ }
+ if out.iter().any(|v| !v.is_finite()) {
+ return Err(Error::Model("numbers that are not finite".into()));
+ }
+ finish_tile(&input, &mut out, p, (bx, ax, by, ay));
+ Ok(out)
+ })
+ .collect()
+ };
+ let mut blend = |chunk: &[Tile], outs: Vec<Result<Vec<f32>, Error>>| -> Result<(), Error> {
+ for (&(x0, y0, bx, ax, by, ay), r) in chunk.iter().zip(outs) {
+ blend_tile(&r?, &mut acc, &mut wsum, (width, height), (nx, ny), layout, (x0, y0), p, (bx, ax, by, ay));
+ done += 1;
+ if let Some(f) = ctl.progress {
+ f(done, total);
+ }
+ }
+ Ok(())
+ };
+ let mut previous: Option<(&[Tile], Vec<Result<Vec<f32>, Error>>)> = None;
+ for chunk in tiles.chunks(group) {
+ if ctl.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
+ return Err(Error::Cancelled);
+ }
+ let (outs, blended) = rayon::join(
+ || compute(chunk),
+ || match previous.take() {
+ Some((c, o)) => {
+ let started = std::time::Instant::now();
+ let r = blend(c, o);
+ (r, started.elapsed())
+ }
+ None => (Ok(()), std::time::Duration::ZERO),
+ },
+ );
+ blended.0?;
+ t_wait += blended.1;
+ previous = Some((chunk, outs));
+ }
+ if let Some((c, o)) = previous.take() {
+ let started = std::time::Instant::now();
+ blend(c, o)?;
+ t_wait += started.elapsed();
+ }
+ t_total += began.elapsed();
+ let started = std::time::Instant::now();
+ // normalise by the weights
+ let (ox, oy) = (layout.position(0, 0).0, layout.position(0, 0).1);
+ acc.par_chunks_mut(width).enumerate().for_each(|(y, row)| {
+ let cy = ((y as isize - oy) / 2) as usize;
+ for (x, px) in row.iter_mut().enumerate() {
+ let cx = ((x as isize - ox) / 2) as usize;
+ let w = wsum.get(cy * nx + cx).copied().unwrap_or(0.0);
+ if w > 0.0 {
+ *px = px.map(|v| v / w);
+ }
+ }
+ });
+ if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
+ eprintln!(
+ "[denoise] {total} tiles, {} at once: tiles {:.0} ms (blending, {:.0} ms of it, overlaps the model), normalise {:.0} ms",
+ group,
+ t_total.as_secs_f64() * 1e3,
+ t_wait.as_secs_f64() * 1e3,
+ started.elapsed().as_secs_f64() * 1e3
+ );
+ }
+ Ok(Rgb32f { width, height, data: acc })
+}
+
+/// Bring a tile's output to the input's scale and keep clipped highlights as they were (in place).
+fn finish_tile(input: &[f32], out: &mut [f32], p: &Params, edges: (bool, bool, bool, bool)) {
+ let t = p.tile;
+ let side = 2 * t;
+ if let Gain::MatchMean { nominal, max_deviation } = p.gain {
+ // the part of the tile whose scale is measured: away from the edges that blend into a neighbour
+ let m = p.overlap / 2;
+ let (i0, i1) = (if edges.0 { m } else { 0 }, t - if edges.1 { m } else { 0 });
+ let (j0, j1) = (if edges.2 { m } else { 0 }, t - if edges.3 { m } else { 0 });
+ let cells = ((i1 - i0) * (j1 - j0)) as f64;
+ let plane_sum = |pl: usize| -> f64 {
+ (j0..j1)
+ .map(|j| input.get(pl * t * t + j * t + i0..pl * t * t + j * t + i1).map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>()))
+ .sum()
+ };
+ let in_mean = [plane_sum(0) / cells, (plane_sum(1) + plane_sum(2)) / (2.0 * cells), plane_sum(3) / cells];
+ for (c, &mean_in) in in_mean.iter().enumerate() {
+ let sum_out: f64 = (2 * j0..2 * j1)
+ .map(|y| {
+ out.get(c * side * side + y * side + 2 * i0..c * side * side + y * side + 2 * i1)
+ .map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>())
+ })
+ .sum();
+ let mean_out = sum_out / (4.0 * cells);
+ let measured = if mean_in as f32 >= MIN_SIGNAL && mean_out > 0.0 { (mean_out / mean_in) as f32 } else { nominal };
+ let g = measured.clamp(nominal * (1.0 - max_deviation), nominal * (1.0 + max_deviation));
+ if g.is_finite() && g > 0.0 {
+ let inv = 1.0 / g;
+ if let Some(plane) = out.get_mut(c * side * side..(c + 1) * side * side) {
+ plane.iter_mut().for_each(|v| *v *= inv);
+ }
+ }
+ }
+ }
+ if let Some(clip) = p.clip {
+ protect_clipped(input, out, t, clip);
+ }
+}
+
+/// Where the mosaic is at (or within a cell of) white, the output becomes the plain cell values again, so a clipped
+/// area stays exactly as clipped as the camera recorded it (highlight recovery later depends on that).
+fn protect_clipped(input: &[f32], out: &mut [f32], t: usize, clip: f32) {
+ let side = 2 * t;
+ let plane = t * t;
+ let (Some(r), Some(g1), Some(g2), Some(b)) =
+ (input.get(0..plane), input.get(plane..2 * plane), input.get(2 * plane..3 * plane), input.get(3 * plane..4 * plane))
+ else {
+ return;
+ };
+ let mut m: Vec<f32> = (0..plane).map(|i| r[i].max(g1[i]).max(g2[i]).max(b[i])).collect();
+ if !m.iter().any(|&v| v >= clip) {
+ return;
+ }
+ // 3×3 maximum over cells (separable)
+ let dilate = |src: &[f32], horizontal: bool| -> Vec<f32> {
+ (0..plane)
+ .map(|i| {
+ let (x, y) = (i % t, i / t);
+ let mut best = src[i];
+ for d in [-1isize, 1] {
+ let (nx, ny) = if horizontal { (x as isize + d, y as isize) } else { (x as isize, y as isize + d) };
+ if nx >= 0 && ny >= 0 && (nx as usize) < t && (ny as usize) < t {
+ best = best.max(src[ny as usize * t + nx as usize]);
+ }
+ }
+ best
+ })
+ .collect()
+ };
+ m = dilate(&dilate(&m, true), false);
+ let lo = clip - 0.03;
+ for j in 0..t {
+ for i in 0..t {
+ let k = j * t + i;
+ let s = ((m[k] - lo) / (clip - lo)).clamp(0.0, 1.0);
+ if s <= 0.0 {
+ continue;
+ }
+ let s = s * s * (3.0 - 2.0 * s);
+ let plain = [r[k], 0.5 * (g1[k] + g2[k]), b[k]];
+ for (c, &pv) in plain.iter().enumerate() {
+ for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
+ if let Some(v) = out.get_mut(c * side * side + (2 * j + sy) * side + 2 * i + sx) {
+ *v += (pv - *v) * s;
+ }
+ }
+ }
+ }
+ }
+}
+
+/// Add a finished tile to the picture with its blend weights.
+#[allow(clippy::too_many_arguments)]
+fn blend_tile(
+ out: &[f32],
+ acc: &mut [[f32; 3]],
+ wsum: &mut [f32],
+ size: (usize, usize),
+ cells: (usize, usize),
+ layout: Layout,
+ at: (usize, usize),
+ p: &Params,
+ edges: (bool, bool, bool, bool),
+) {
+ let (width, height) = size;
+ let (nx, ny) = cells;
+ let (t, side) = (p.tile, 2 * p.tile);
+ let fade = p.overlap;
+ let wx: Vec<f32> = (0..t).map(|i| weight(i, t, fade, edges.0, edges.1)).collect();
+ let wy: Vec<f32> = (0..t).map(|j| weight(j, t, fade, edges.2, edges.3)).collect();
+ for j in 0..t {
+ let cy = at.1 + j;
+ for i in 0..t {
+ let cx = at.0 + i;
+ // cells past the picture's edge (a picture smaller than one tile is padded) belong to no pixel
+ if cx >= nx || cy >= ny {
+ continue;
+ }
+ let (Some(&a), Some(&b)) = (wx.get(i), wy.get(j)) else { continue };
+ let w = a * b;
+ if let Some(s) = wsum.get_mut(cy * nx + cx) {
+ *s += w;
+ }
+ let (px0, py0) = layout.position(cx as isize, cy as isize);
+ for (sy, sx) in [(0isize, 0isize), (0, 1), (1, 0), (1, 1)] {
+ let (x, y) = (px0 + sx, py0 + sy);
+ if x < 0 || y < 0 || x as usize >= width || y as usize >= height {
+ continue;
+ }
+ let o = (2 * j + sy as usize) * side + 2 * i + sx as usize;
+ let (Some(&r), Some(&g), Some(&bl)) = (out.get(o), out.get(side * side + o), out.get(2 * side * side + o)) else { continue };
+ if let Some(px) = acc.get_mut(y as usize * width + x as usize) {
+ px[0] += w * r;
+ px[1] += w * g;
+ px[2] += w * bl;
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// A stand-in model: the cell's samples as an RGB picture at twice the resolution (a plain 2 × 2 "demosaic"),
+ /// times `scale`, so the output has a scale of its own like the real model's.
+ struct Mock {
+ tile: usize,
+ scale: f32,
+ }
+ impl TileRunner for Mock {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
+ let (t, side) = (self.tile, 2 * self.tile);
+ let plane = t * t;
+ let mut out = vec![0.0; 3 * side * side];
+ for j in 0..t {
+ for i in 0..t {
+ let k = j * t + i;
+ let rgb = [input[k], 0.5 * (input[plane + k] + input[2 * plane + k]), input[3 * plane + k]];
+ for (c, v) in rgb.iter().enumerate() {
+ for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
+ out[c * side * side + (2 * j + sy) * side + 2 * i + sx] = v * self.scale;
+ }
+ }
+ }
+ }
+ Ok(out)
+ }
+ }
+
+ fn params(tile: usize, overlap: usize, gain: Gain) -> Params {
+ Params { tile, overlap, gain, clip: None, parallel: 2 }
+ }
+
+ #[test]
+ fn overflowing_tiles_and_nan_gain_are_errors_before_the_model_runs() {
+ let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();
+ let model = Mock { tile: 16, scale: 1.0 };
+ for (tile, overlap, gain) in [
+ (usize::MAX, 0, Gain::None),
+ (16, usize::MAX, Gain::None),
+ (16, 0, Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN }),
+ (16, 0, Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 }),
+ ] {
+ assert!(matches!(denoise_bayer(&[0.5], 1, 1, layout, &model, &params(tile, overlap, gain), &Control::default()), Err(Error::Input(_))));
+ }
+ }
+
+ /// A smooth picture whose colour at a sample is `base[colour] · (1 + shading)`.
+ fn mosaic(w: usize, h: usize, cell: [u8; 4]) -> Vec<f32> {
+ (0..w * h)
+ .map(|i| {
+ let (x, y) = (i % w, i / w);
+ let c = cell[(y % 2) * 2 + x % 2] as usize;
+ [0.5, 0.3, 0.2][c] * (0.6 + 0.4 * ((x as f32 / 23.0).sin() * (y as f32 / 31.0).cos()))
+ })
+ .collect()
+ }
+
+ fn reference(m: &[f32], w: usize, h: usize, layout: Layout) -> Rgb32f {
+ // what the mock gives for the whole picture in one tile
+ let (nx, ny) = layout.cells(w, h);
+ let t = nx.max(ny).div_ceil(16) * 16;
+ denoise_bayer(m, w, h, layout, &Mock { tile: t, scale: 1.0 }, &params(t, 16, Gain::None), &Control::default()).unwrap()
+ }
+
+ #[test]
+ fn tiles_blend_to_the_same_picture_as_one_tile_for_every_layout() {
+ for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
+ let layout = Layout::from_cell(cell).unwrap();
+ let (w, h) = (150usize, 110usize);
+ let m = mosaic(w, h, cell);
+ let want = reference(&m, w, h, layout);
+ let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &Control::default()).unwrap();
+ assert_eq!((got.width, got.height), (w, h));
+ for (a, b) in got.data.iter().zip(&want.data) {
+ for c in 0..3 {
+ assert!((a[c] - b[c]).abs() < 1e-5, "layout {cell:?}: {a:?} vs {b:?}");
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn the_picture_does_not_depend_on_how_many_tiles_run_at_once_or_how_long_each_takes() {
+ // tiles that finish in a different order every time: blending still adds them in tile order
+ struct Jittery(Mock);
+ impl TileRunner for Jittery {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
+ let wait = (input.iter().take(64).sum::<f32>() * 1000.0) as u64 % 7;
+ std::thread::sleep(std::time::Duration::from_millis(wait));
+ self.0.run(input)
+ }
+ }
+ let layout = Layout { dx: 0, dy: 1 };
+ let (w, h) = (150usize, 110usize);
+ let m = mosaic(w, h, [1, 0, 2, 1]);
+ let run = |parallel: usize| {
+ let mut p = params(32, 8, Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 });
+ p.parallel = parallel;
+ denoise_bayer(&m, w, h, layout, &Jittery(Mock { tile: 32, scale: 1.01e6 }), &p, &Control::default()).unwrap()
+ };
+ let one = run(1);
+ for parallel in [2, 3, 8] {
+ assert_eq!(run(parallel).data, one.data, "{parallel} at once");
+ }
+ }
+
+ #[test]
+ fn a_failing_tile_or_a_cancel_ends_the_picture_without_a_panic() {
+ struct Fails(Mock, std::sync::atomic::AtomicUsize);
+ impl TileRunner for Fails {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
+ if self.1.fetch_add(1, Ordering::Relaxed) == 4 {
+ return Err(Error::Runtime("boom".into()));
+ }
+ self.0.run(input)
+ }
+ }
+ let layout = Layout { dx: 0, dy: 0 };
+ let (w, h) = (150usize, 110usize);
+ let m = mosaic(w, h, [0, 1, 1, 2]);
+ let r = denoise_bayer(
+ &m,
+ w,
+ h,
+ layout,
+ &Fails(Mock { tile: 32, scale: 1.0 }, Default::default()),
+ &params(32, 8, Gain::None),
+ &Control::default(),
+ );
+ assert!(matches!(r, Err(Error::Runtime(_))), "{r:?}");
+ let stop = AtomicBool::new(true);
+ let ctl = Control { cancel: Some(&stop), progress: None };
+ let r = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &ctl);
+ assert!(matches!(r, Err(Error::Cancelled)));
+ }
+
+ #[test]
+ fn a_scale_of_its_own_is_taken_out_tile_by_tile() {
+ let layout = Layout { dx: 0, dy: 0 };
+ let (w, h) = (150usize, 110usize);
+ let m = mosaic(w, h, [0, 1, 1, 2]);
+ let want = reference(&m, w, h, layout);
+ let gain = Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 };
+ // the scale differs a little from nominal; each tile's own scale is measured and removed
+ let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.02e6 }, &params(32, 8, gain), &Control::default()).unwrap();
+ let mean = |img: &Rgb32f, c: usize| img.data.iter().map(|p| f64::from(p[c])).sum::<f64>() / img.data.len() as f64;
+ for c in 0..3 {
+ let (a, b) = (mean(&got, c), mean(&want, c));
+ assert!((a - b).abs() / b < 0.01, "channel {c}: {a} vs {b}");
+ }
+ // a scale far from nominal is clamped rather than trusted
+ let wild = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 3.0e6 }, &params(32, 8, gain), &Control::default()).unwrap();
+ assert!(mean(&wild, 1) > 2.0 * mean(&want, 1));
+ }
+
+ #[test]
+ fn a_picture_smaller_than_a_tile_is_padded_by_mirroring() {
+ let layout = Layout { dx: 1, dy: 1 };
+ let (w, h) = (20usize, 14usize);
+ let m = mosaic(w, h, [2, 1, 1, 0]);
+ let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 64, scale: 1.0 }, &params(64, 16, Gain::None), &Control::default()).unwrap();
+ assert_eq!((got.width, got.height), (w, h));
+ assert!(got.data.iter().all(|p| p.iter().all(|v| v.is_finite() && *v > 0.0)));
+ }
+
+ #[test]
+ fn clipped_highlights_stay_clipped() {
+ // a model that darkens everything by 10 % would pull a clipped plateau below white
+ struct Dim(Mock);
+ impl TileRunner for Dim {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
+ Ok(self.0.run(input)?.into_iter().map(|v| v * 0.9).collect())
+ }
+ }
+ let layout = Layout { dx: 0, dy: 0 };
+ let (w, h) = (96usize, 96usize);
+ let m: Vec<f32> = (0..w * h).map(|i| if i % w > 40 && i % w < 70 { 1.0 } else { 0.4 }).collect();
+ let mut p = params(32, 8, Gain::None);
+ p.clip = Some(0.99);
+ let got = denoise_bayer(&m, w, h, layout, &Dim(Mock { tile: 32, scale: 1.0 }), &p, &Control::default()).unwrap();
+ let mid = got.data[48 * w + 55];
+ assert!(mid.iter().all(|&v| v > 0.995), "plateau {mid:?}");
+ let dark = got.data[48 * w + 10];
+ assert!(dark.iter().all(|&v| (v - 0.36).abs() < 0.01), "unclipped area still goes through the model: {dark:?}");
+ }
+
+ #[test]
+ fn cancelling_and_progress_work() {
+ let layout = Layout { dx: 0, dy: 0 };
+ let (w, h) = (150usize, 110usize);
+ let m = mosaic(w, h, [0, 1, 1, 2]);
+ let cancel = AtomicBool::new(true);
+ let r = denoise_bayer(
+ &m,
+ w,
+ h,
+ layout,
+ &Mock { tile: 32, scale: 1.0 },
+ &params(32, 8, Gain::None),
+ &Control { cancel: Some(&cancel), progress: None },
+ );
+ assert!(matches!(r, Err(Error::Cancelled)));
+ let seen = std::sync::Mutex::new(Vec::new());
+ let f = |d: usize, n: usize| seen.lock().unwrap().push((d, n));
+ denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &Control { cancel: None, progress: Some(&f) })
+ .unwrap();
+ let seen = seen.into_inner().unwrap();
+ let total = tile_count(w, h, layout, 32, 8);
+ assert!(total > 1);
+ assert_eq!(seen.last(), Some(&(total, total)));
+ assert!(seen.windows(2).all(|p| p[1].0 == p[0].0 + 1));
+ }
+
+ #[test]
+ fn hostile_inputs_are_errors() {
+ let layout = Layout { dx: 0, dy: 0 };
+ let mock = Mock { tile: 32, scale: 1.0 };
+ let p = params(32, 8, Gain::None);
+ assert!(matches!(denoise_bayer(&[], 0, 0, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));
+ assert!(matches!(denoise_bayer(&[0.0; 10], 4, 4, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));
+ assert!(matches!(denoise_bayer(&[0.0; 16], 4, 4, layout, &mock, &params(32, 16, Gain::None), &Control::default()), Err(Error::Input(_))));
+ struct Bad(Vec<f32>);
+ impl TileRunner for Bad {
+ fn run(&self, _: &[f32]) -> Result<Vec<f32>, Error> {
+ Ok(self.0.clone())
+ }
+ }
+ assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_))));
+ assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_))));
+ }
+}
diff --git a/crates/denoise/src/runtime.rs b/crates/denoise/src/runtime.rs
new file mode 100644
index 000000000..4314537a5
--- /dev/null
+++ b/crates/denoise/src/runtime.rs
@@ -0,0 +1,423 @@
+//! Model loading, a shared installation self-test, and deterministic test tiles.
+//! `CpuRunner` is the pure-Rust executor; scalar operator tests live in `reference`.
+
+use std::path::Path;
+
+use web_time::Instant;
+
+use serde::Serialize;
+
+use crate::manifest::{DenoiserManifest, Gain};
+use crate::run::{Error, TileRunner};
+
+/// What `self_test` found.
+#[derive(Clone, Debug, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SelfTest {
+ pub ok: bool,
+ /// Milliseconds to load and optimise the model.
+ pub load_ms: f64,
+ /// Median milliseconds to run one tile.
+ pub tile_ms: f64,
+ /// Megapixels of picture one tile covers, for estimating a whole photo's time.
+ pub tile_megapixels: f64,
+ /// The model's output scale relative to its input on the test picture (what `Gain::MatchMean` takes out).
+ pub scale: f64,
+ /// Each check: what was checked and whether it held.
+ pub checks: Vec<(String, bool)>,
+}
+
+/// The default pure-Rust CPU model, with the same manifest and installation self-test as the reference.
+#[derive(Clone)]
+pub struct CpuRunner {
+ runner: crate::cpu::NetRunner,
+ manifest: DenoiserManifest,
+ load_ms: f64,
+}
+impl CpuRunner {
+ pub fn load(path: &Path, manifest: &DenoiserManifest) -> Result<Self, Error> {
+ crate::manifest::validate(manifest).map_err(|e| Error::Load(e.to_string()))?;
+ let started = Instant::now();
+ let net = crate::onnx::read(path).map_err(|e| Error::Load(e.to_string()))?;
+ let runner = crate::cpu::NetRunner::new(&net, manifest.tile as usize).map_err(|e| Error::Load(e.to_string()))?;
+ Ok(Self { runner, manifest: manifest.clone(), load_ms: started.elapsed().as_secs_f64() * 1e3 })
+ }
+ pub fn manifest(&self) -> &DenoiserManifest {
+ &self.manifest
+ }
+ pub fn net(&self) -> &crate::net::Net {
+ self.runner.net()
+ }
+ pub fn bytes_per_tile(&self) -> usize {
+ self.runner.bytes_per_tile()
+ }
+ pub fn self_test(&self) -> SelfTest {
+ self_test(self, &self.manifest, self.load_ms)
+ }
+}
+impl TileRunner for CpuRunner {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
+ self.runner.run(input)
+ }
+}
+fn self_test(runner: &dyn TileRunner, manifest: &DenoiserManifest, load_ms: f64) -> SelfTest {
+ let t = manifest.tile as usize;
+ let mut checks: Vec<(String, bool)> = Vec::new();
+ let (input, base) = test_tile(t);
+ let (mut times, mut scale) = (Vec::new(), 0.0f64);
+ let mut run = || {
+ let started = Instant::now();
+ let r = runner.run(&input);
+ times.push(started.elapsed().as_secs_f64() * 1000.0);
+ r
+ };
+ let (a, b) = (run(), run());
+ match (&a, &b) {
+ (Ok(a), Ok(b)) => {
+ checks.push((format!("gives {} finite numbers per tile", a.len()), true));
+ checks.push((
+ "gives the same answer for the same tile".into(),
+ a.iter().zip(b).all(|(x, y)| (x - y).abs() <= 1e-4 * x.abs().max(y.abs()).max(1e-9)),
+ ));
+ let side = 2 * t;
+ let mean_in = base.iter().map(|&v| f64::from(v)).sum::<f64>() / base.len().max(1) as f64;
+ let mean_out = a.iter().map(|&v| f64::from(v)).sum::<f64>() / a.len().max(1) as f64;
+ scale = if mean_in > 0.0 { mean_out / mean_in } else { 0.0 };
+ if let Gain::MatchMean { nominal, max_deviation } = manifest.gain {
+ let ratio = scale / f64::from(nominal);
+ let tolerance = f64::from(max_deviation).max(0.05) * 4.0;
+ checks.push((format!("its scale is the {nominal:.3e} the description says (found {scale:.3e})"), (ratio - 1.0).abs() <= tolerance));
+ }
+ // noise left after bringing the answer back to the input's scale, against the noise that went in
+ let g = if scale > 0.0 { 1.0 / scale } else { 1.0 };
+ let (mut noise_in, mut noise_out, mut n_in, mut n_out) = (0.0f64, 0.0f64, 0usize, 0usize);
+ for (i, (&v, &s)) in input.iter().zip(base.iter().cycle()).enumerate() {
+ // centre of each plane only: edges are the model's guess
+ let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t);
+ if x >= t / 4 && x < 3 * t / 4 && y >= t / 4 && y < 3 * t / 4 {
+ noise_in += f64::from(v - s).powi(2);
+ n_in += 1;
+ }
+ }
+ for c in 0..3 {
+ for y in side / 4..3 * side / 4 {
+ for x in side / 4..3 * side / 4 {
+ if let (Some(&o), Some(&s)) = (a.get(c * side * side + y * side + x), base.get((y / 2) * t + x / 2)) {
+ noise_out += (f64::from(o) * g - f64::from(s)).powi(2);
+ n_out += 1;
+ }
+ }
+ }
+ }
+ let (rms_in, rms_out) = ((noise_in / n_in.max(1) as f64).sqrt(), (noise_out / n_out.max(1) as f64).sqrt());
+ checks.push((format!("takes noise down (from {rms_in:.4} to {rms_out:.4})"), rms_out < 0.8 * rms_in));
+ }
+ (Err(e), _) | (_, Err(e)) => checks.push((format!("runs: {e}"), false)),
+ }
+ times.sort_by(|x, y| x.total_cmp(y));
+ SelfTest {
+ ok: !checks.is_empty() && checks.iter().all(|(_, ok)| *ok),
+ load_ms,
+ tile_ms: times.get(times.len() / 2).copied().unwrap_or(0.0),
+ tile_megapixels: (2 * t * 2 * t) as f64 / 1e6,
+ scale,
+ checks,
+ }
+}
+
+/// A deterministic test tile: four planes of a smooth gradient with noise on top, and the gradient alone for one
+/// plane (`tile × tile`).
+pub fn test_tile(t: usize) -> (Vec<f32>, Vec<f32>) {
+ if t == 0 || t > 2048 {
+ return (Vec::new(), Vec::new());
+ }
+ let mut seed = 0x2545_f491u32;
+ let mut next = move || {
+ seed ^= seed << 13;
+ seed ^= seed >> 17;
+ seed ^= seed << 5;
+ (seed >> 8) as f32 / (1u32 << 24) as f32
+ };
+ let base: Vec<f32> = (0..t * t).map(|i| 0.12 + 0.3 * (((i % t) + (i / t)) as f32 / (2 * t) as f32)).collect();
+ let input = (0..4 * t * t)
+ .map(|i| {
+ // sum of four uniforms: close to Gaussian, deviation 0.0577 × 2 before scaling
+ let n = next() + next() + next() + next() - 2.0;
+ base.get(i % (t * t)).copied().unwrap_or(0.0) + 0.05 * n
+ })
+ .collect();
+ (input, base)
+}
+
+/// A tile for checking that two runners of one model agree: the whole range a raw photo has rather than the test tile's
+/// mid-greys — a noisy ramp from black to white across the tile, a flat black corner and a clipped white one (four
+/// planes of `tile × tile`).
+pub fn check_tile(t: usize) -> Vec<f32> {
+ if t == 0 || t > 2048 {
+ return Vec::new();
+ }
+ let mut seed = 0x9e37_79b9u32;
+ let mut next = move || {
+ seed ^= seed << 13;
+ seed ^= seed >> 17;
+ seed ^= seed << 5;
+ (seed >> 8) as f32 / (1u32 << 24) as f32
+ };
+ let (edge, span) = ((t / 8).max(1), (2 * t).saturating_sub(2).max(1) as f32);
+ (0..4 * t * t)
+ .map(|i| {
+ let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t);
+ let n = next() + next() + next() + next() - 2.0;
+ if x < edge && y < edge {
+ 0.0
+ } else if x + edge >= t && y + edge >= t {
+ 1.0
+ } else {
+ ((x + y) as f32 / span + 0.05 * n).clamp(0.0, 1.0)
+ }
+ })
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::licence::Licence;
+ use crate::manifest::Domain;
+ use crate::synthetic_proto::{len_field, value_info_bytes};
+
+ fn varint(mut v: u64, out: &mut Vec<u8>) {
+ loop {
+ let b = (v & 0x7f) as u8;
+ v >>= 7;
+ if v == 0 {
+ out.push(b);
+ return;
+ }
+ out.push(b | 0x80);
+ }
+ }
+
+ fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
+ varint(n << 3, out);
+ varint(v, out);
+ }
+
+ fn node(op: &str, inputs: &[&str], output: &str, ints: &[(&str, i64)]) -> Vec<u8> {
+ let mut n = Vec::new();
+ for i in inputs {
+ len_field(1, i.as_bytes(), &mut n);
+ }
+ len_field(2, output.as_bytes(), &mut n);
+ len_field(4, op.as_bytes(), &mut n);
+ for (name, value) in ints {
+ let mut a = Vec::new();
+ len_field(1, name.as_bytes(), &mut a);
+ varint_field(3, *value as u64, &mut a);
+ varint_field(20, 2, &mut a);
+ len_field(5, &a, &mut n);
+ }
+ n
+ }
+
+ fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec<u8> {
+ let mut t = Vec::new();
+ for d in dims {
+ varint_field(1, *d, &mut t);
+ }
+ varint_field(2, 1, &mut t);
+ len_field(8, name.as_bytes(), &mut t);
+ let raw: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
+ len_field(9, &raw, &mut t);
+ t
+ }
+
+ /// A tiny working denoiser-shaped network: `[1, 4, t, t]` → a 1 × 1 convolution to 12 channels (each output
+ /// channel is `scale` times one input plane, so the scale is known) → depth-to-space → `[1, 3, 2t, 2t]`.
+ fn tiny_model(t: u64, scale: f32) -> Vec<u8> {
+ tiny_model_with(t, scale, 12)
+ }
+
+ /// The same with another number of convolution channels; only 12 can be arranged into the three colours.
+ fn tiny_model_with(t: u64, scale: f32, channels: u64) -> Vec<u8> {
+ let mut graph = Vec::new();
+ len_field(1, &node("Conv", &["data", "w"], if channels == 12 { "c" } else { "rgb" }, &[("group", 1)]), &mut graph);
+ if channels == 12 {
+ let mut depth_to_space = node("DepthToSpace", &["c"], "rgb", &[("blocksize", 2)]);
+ let mut mode = Vec::new();
+ len_field(1, b"mode", &mut mode);
+ len_field(4, b"CRD", &mut mode);
+ varint_field(20, 3, &mut mode);
+ len_field(5, &mode, &mut depth_to_space);
+ len_field(1, &depth_to_space, &mut graph);
+ }
+ // CRD output channel colour·4 + dy·2 + dx reads input plane [R, G1, B] of the tile.
+ let mut w = vec![0f32; channels as usize * 4];
+ for oc in 0..channels as usize {
+ let colour = (oc / 4).min(2);
+ let plane = [0, 1, 3][colour];
+ w[oc * 4 + plane] = scale;
+ }
+ len_field(5, &tensor("w", &[channels, 4, 1, 1], &w), &mut graph);
+ len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(t), Ok(t)]), &mut graph);
+ len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * t), Ok(2 * t)]), &mut graph);
+ let mut ops = Vec::new();
+ len_field(1, b"", &mut ops);
+ varint_field(2, 13, &mut ops);
+ let mut model = Vec::new();
+ varint_field(1, 8, &mut model);
+ len_field(7, &graph, &mut model);
+ len_field(8, &ops, &mut model);
+ model
+ }
+
+ fn manifest(tile: u32, nominal: f32) -> DenoiserManifest {
+ DenoiserManifest {
+ id: "tiny".into(),
+ name: "Tiny".into(),
+ version: "1".into(),
+ licence: Licence::default(),
+ source: None,
+ sha256: None,
+ size_bytes: None,
+ provenance: String::new(),
+ domain: Domain::BayerToRgb,
+ tile,
+ overlap: 16,
+ gain: Gain::MatchMean { nominal, max_deviation: 0.05 },
+ }
+ }
+
+ fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {
+ let p = std::env::temp_dir().join(format!("lc-denoise-runtime-{name}-{}.onnx", std::process::id()));
+ std::fs::write(&p, bytes).unwrap();
+ p
+ }
+
+ #[test]
+ fn a_working_model_loads_runs_and_has_the_scale_it_says() {
+ let p = temp_model("ok", &tiny_model(64, 1000.0));
+ let r = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap();
+ let input: Vec<f32> = (0..4 * 64 * 64).map(|i| (i % 97) as f32 / 97.0).collect();
+ let out = r.run(&input).unwrap();
+ assert_eq!(out.len(), 3 * 128 * 128);
+ // output (colour c, pixel (2y+dy, 2x+dx)) = scale × input plane [R, G1, B][c] at (y, x)
+ for (c, plane) in [0usize, 1, 3].into_iter().enumerate() {
+ let (y, x) = (5usize, 9usize);
+ assert!((out[c * 128 * 128 + (2 * y + 1) * 128 + 2 * x] - 1000.0 * input[plane * 64 * 64 + y * 64 + x]).abs() < 1e-2);
+ }
+ let _ = std::fs::remove_file(p);
+ }
+
+ #[test]
+ fn the_self_test_checks_scale_and_noise() {
+ // a model that only multiplies cannot take noise down: it must fail that check but pass the others
+ let p = temp_model("selftest", &tiny_model(64, 1000.0));
+ let t = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap().self_test();
+ assert!(!t.ok, "{t:?}");
+ assert!(t.checks.iter().any(|(c, ok)| c.contains("same answer") && *ok), "{t:?}");
+ assert!(t.checks.iter().any(|(c, ok)| c.contains("scale") && *ok), "{t:?}");
+ assert!(t.checks.iter().any(|(c, ok)| c.contains("noise") && !*ok), "{t:?}");
+ // and one that claims the wrong scale fails that check
+ let q = CpuRunner::load(&p, &manifest(64, 5.0)).unwrap().self_test();
+ assert!(q.checks.iter().any(|(c, ok)| c.contains("scale") && !*ok), "{q:?}");
+ let _ = std::fs::remove_file(p);
+ }
+
+ #[test]
+ fn wrong_sizes_and_broken_files_are_errors() {
+ let p = temp_model("sizes", &tiny_model(64, 1.0));
+ let r = CpuRunner::load(&p, &manifest(64, 1.0)).unwrap();
+ assert!(matches!(r.run(&[0.0; 10]), Err(Error::Input(_))));
+ // a fully convolutional model works at whatever tile size the manifest asks for
+ let wide = CpuRunner::load(&p, &manifest(128, 1.0)).unwrap();
+ assert_eq!(wide.run(&vec![0.5; 4 * 128 * 128]).unwrap().len(), 3 * 256 * 256);
+ // a model that gives another number of values than the manifest promises is an error
+ let q = temp_model("channels", &tiny_model_with(64, 1.0, 8));
+ assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "shape is refused before execution");
+ let _ = std::fs::remove_file(q);
+ for (name, bytes) in [("junk", b"nonsense".to_vec()), ("empty", Vec::new())] {
+ let q = temp_model(name, &bytes);
+ assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "{name}");
+ let _ = std::fs::remove_file(q);
+ }
+ assert!(CpuRunner::load(&std::env::temp_dir().join("does-not-exist.onnx"), &manifest(64, 1.0)).is_err());
+ let _ = std::fs::remove_file(p);
+ }
+
+ #[test]
+ fn the_check_tile_covers_black_to_white() {
+ for t in [1, 2, 7, 64] {
+ let c = check_tile(t);
+ assert_eq!(c.len(), 4 * t * t);
+ assert!(c.iter().all(|v| (0.0..=1.0).contains(v)), "{t}");
+ }
+ let c = check_tile(64);
+ let near = |lo: f32, hi: f32| c.iter().filter(|v| (lo..=hi).contains(*v)).count();
+ assert!(near(0.0, 0.0) >= 4 * 8 * 8 && near(1.0, 1.0) >= 4 * 8 * 8, "a flat black and a clipped white corner");
+ for band in 0..10 {
+ let lo = band as f32 / 10.0;
+ assert!(near(lo, lo + 0.1) > 100, "values around {lo}");
+ }
+ assert_eq!(check_tile(64), c, "the same tile every time");
+ }
+
+ #[test]
+ fn truncated_and_corrupted_models_never_panic() {
+ let good = tiny_model(64, 1.0);
+ for n in (0..good.len()).step_by(5) {
+ let q = temp_model("trunc", &good[..n]);
+ let _ = CpuRunner::load(&q, &manifest(64, 1.0));
+ let _ = std::fs::remove_file(q);
+ }
+ for at in (0..good.len()).step_by(7) {
+ let mut bad = good.clone();
+ if let Some(b) = bad.get_mut(at) {
+ *b = b.wrapping_add(0x55);
+ }
+ let q = temp_model("flip", &bad);
+ if let Ok(r) = CpuRunner::load(&q, &manifest(64, 1.0)) {
+ let _ = r.run(&vec![0.5; 4 * 64 * 64]);
+ }
+ let _ = std::fs::remove_file(q);
+ }
+ }
+
+ /// Opt-in: `LC_DENOISE_MODEL=<model_bayer.onnx> cargo test -p lightcraft-denoise --release
+ /// -- --ignored --nocapture` self-tests a real model (RawNIND's Bayer model: tile 512, scale about 1e6).
+ #[test]
+ #[ignore = "needs a real model file: set LC_DENOISE_MODEL"]
+ fn a_real_model_passes_the_self_test() {
+ let Some(path) = std::env::var_os("LC_DENOISE_MODEL") else { return };
+ let m = manifest(512, 1.0e6);
+ let r = CpuRunner::load(Path::new(&path), &m).unwrap();
+ let t = r.self_test();
+ println!("load {:.0} ms | {:.0} ms per tile | scale {:.4e} | {:?}", t.load_ms, t.tile_ms, t.scale, t.checks);
+ assert!(t.ok);
+ }
+}
+
+#[cfg(test)]
+mod cpu_tests {
+ use super::*;
+ #[test]
+ fn manifest_and_test_tile_sizes_are_checked_without_tract() {
+ for t in [0, 2049, usize::MAX] {
+ assert!(test_tile(t).0.is_empty());
+ assert!(check_tile(t).is_empty());
+ }
+ let path = std::env::temp_dir().join(format!("lc-cpu-load-{}.onnx", std::process::id()));
+ std::fs::write(&path, crate::synthetic::unet_onnx(64, 5, 1, 7)).unwrap();
+ let mut m = crate::known::find("rawnind-bayer").unwrap().manifest;
+ m.tile = 64;
+ m.overlap = 16;
+ m.gain = Gain::None;
+ let r = CpuRunner::load(&path, &m).unwrap();
+ let x = check_tile(64);
+ assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());
+ assert!(r.self_test().checks.iter().any(|(text, ok)| text.contains("same answer") && *ok));
+ m.tile = u32::MAX;
+ assert!(CpuRunner::load(&path, &m).is_err());
+ std::fs::remove_file(path).unwrap();
+ }
+}
diff --git a/crates/denoise/src/synthetic.rs b/crates/denoise/src/synthetic.rs
new file mode 100644
index 000000000..8ceafca77
--- /dev/null
+++ b/crates/denoise/src/synthetic.rs
@@ -0,0 +1,234 @@
+//! A small ONNX U-Net built in memory, for tests of the layers that read and run denoise networks (here, in the GPU
+//! runner and in the engine): the same layer kinds and wiring as the real denoisers, with deterministic random weights.
+//! Nothing here is a model anyone would use.
+
+use crate::synthetic_proto::{len_field, value_info_bytes};
+
+fn varint(mut v: u64, out: &mut Vec<u8>) {
+ loop {
+ let b = (v & 0x7f) as u8;
+ v >>= 7;
+ if v == 0 {
+ out.push(b);
+ return;
+ }
+ out.push(b | 0x80);
+ }
+}
+
+fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
+ varint(n << 3, out);
+ varint(v, out);
+}
+
+enum A<'a> {
+ Int(i64),
+ Ints(&'a [i64]),
+ Float(f32),
+ Str(&'a str),
+}
+
+fn node(op: &str, inputs: &[&str], output: &str, attrs: &[(&str, A)]) -> Vec<u8> {
+ let mut n = Vec::new();
+ for i in inputs {
+ len_field(1, i.as_bytes(), &mut n);
+ }
+ len_field(2, output.as_bytes(), &mut n);
+ len_field(4, op.as_bytes(), &mut n);
+ for (name, value) in attrs {
+ let mut a = Vec::new();
+ len_field(1, name.as_bytes(), &mut a);
+ match value {
+ A::Int(v) => {
+ varint_field(3, *v as u64, &mut a);
+ varint_field(20, 2, &mut a);
+ }
+ A::Ints(vs) => {
+ for v in *vs {
+ varint_field(8, *v as u64, &mut a);
+ }
+ varint_field(20, 7, &mut a);
+ }
+ A::Float(v) => {
+ varint((2 << 3) | 5, &mut a);
+ a.extend_from_slice(&v.to_le_bytes());
+ varint_field(20, 1, &mut a);
+ }
+ A::Str(s) => {
+ len_field(4, s.as_bytes(), &mut a);
+ varint_field(20, 3, &mut a);
+ }
+ }
+ len_field(5, &a, &mut n);
+ }
+ n
+}
+
+fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec<u8> {
+ let mut t = Vec::new();
+ for d in dims {
+ varint_field(1, *d, &mut t);
+ }
+ varint_field(2, 1, &mut t);
+ len_field(8, name.as_bytes(), &mut t);
+ let raw: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
+ len_field(9, &raw, &mut t);
+ t
+}
+
+struct Rng(u64);
+
+impl Rng {
+ fn next(&mut self) -> f32 {
+ self.0 ^= self.0 >> 12;
+ self.0 ^= self.0 << 25;
+ self.0 ^= self.0 >> 27;
+ ((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
+ }
+}
+
+struct Builder {
+ nodes: Vec<Vec<u8>>,
+ inits: Vec<Vec<u8>>,
+ rng: Rng,
+ n: usize,
+}
+
+impl Builder {
+ fn name(&mut self, kind: &str) -> String {
+ self.n += 1;
+ format!("{kind}{}", self.n)
+ }
+
+ /// Random weights scaled so activations keep a sane size through the layers.
+ fn weights(&mut self, name: &str, dims: &[u64], fan_in: usize) -> String {
+ let count: u64 = dims.iter().product();
+ let scale = (3.0 / fan_in as f32).sqrt();
+ let values: Vec<f32> = (0..count).map(|_| self.rng.next() * scale).collect();
+ self.inits.push(tensor(name, dims, &values));
+ name.to_string()
+ }
+
+ fn bias(&mut self, name: &str, n: u64) -> String {
+ let values: Vec<f32> = (0..n).map(|_| self.rng.next() * 0.1).collect();
+ self.inits.push(tensor(name, &[n], &values));
+ name.to_string()
+ }
+
+ /// `Conv` (+ `LeakyRelu` when `leaky`); gives the name of the tensor that results.
+ fn conv(&mut self, src: &str, cin: u64, cout: u64, k: u64, leaky: bool) -> String {
+ let (w, b, out) = (self.name("w"), self.name("b"), self.name("conv"));
+ let w = self.weights(&w, &[cout, cin, k, k], (cin * k * k) as usize);
+ let b = self.bias(&b, cout);
+ let pad = (k as i64 - 1) / 2;
+ self.nodes.push(node(
+ "Conv",
+ &[src, &w, &b],
+ &out,
+ &[("kernel_shape", A::Ints(&[k as i64, k as i64])), ("pads", A::Ints(&[pad; 4])), ("strides", A::Ints(&[1, 1])), ("group", A::Int(1))],
+ ));
+ if !leaky {
+ return out;
+ }
+ let act = self.name("act");
+ self.nodes.push(node("LeakyRelu", &[&out], &act, &[("alpha", A::Float(0.2))]));
+ act
+ }
+
+ fn pool(&mut self, src: &str) -> String {
+ let out = self.name("pool");
+ self.nodes.push(node(
+ "MaxPool",
+ &[src],
+ &out,
+ &[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))],
+ ));
+ out
+ }
+
+ fn up(&mut self, src: &str, cin: u64, cout: u64) -> String {
+ let (w, b, out) = (self.name("uw"), self.name("ub"), self.name("up"));
+ let w = self.weights(&w, &[cin, cout, 2, 2], (cin * 4) as usize);
+ let b = self.bias(&b, cout);
+ self.nodes.push(node(
+ "ConvTranspose",
+ &[src, &w, &b],
+ &out,
+ &[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))],
+ ));
+ out
+ }
+
+ fn concat(&mut self, a: &str, b: &str) -> String {
+ let out = self.name("cat");
+ self.nodes.push(node("Concat", &[a, b], &out, &[("axis", A::Int(1))]));
+ out
+ }
+}
+
+/// A denoiser-shaped U-Net as ONNX bytes: `[1, 4, tile, tile]` → `[1, 3, 2·tile, 2·tile]`, with `depth` poolings
+/// (so `tile` must be a multiple of `2^depth`) and `ch` channels on the first level, doubling on each (every count a
+/// multiple of 4). Each level is two 3 × 3 convolutions with a leaky ReLU, as in the real networks; the output is a
+/// 1 × 1 convolution to 12 channels and depth-to-space.
+pub fn unet_onnx(tile: u64, ch: u64, depth: usize, seed: u64) -> Vec<u8> {
+ let mut b = Builder { nodes: Vec::new(), inits: Vec::new(), rng: Rng(seed | 1), n: 0 };
+ let mut skips: Vec<(String, u64)> = Vec::new();
+ let (mut cur, mut c_in, mut c) = ("data".to_string(), 4u64, ch);
+ for level in 0..=depth {
+ let x = b.conv(&cur, c_in, c, 3, true);
+ let x = b.conv(&x, c, c, 3, true);
+ if level < depth {
+ skips.push((x.clone(), c));
+ cur = b.pool(&x);
+ c_in = c;
+ c *= 2;
+ } else {
+ cur = x;
+ }
+ }
+ for (skip, sc) in skips.into_iter().rev() {
+ let up = b.up(&cur, c, sc);
+ let joined = b.concat(&up, &skip);
+ let x = b.conv(&joined, 2 * sc, sc, 3, true);
+ cur = b.conv(&x, sc, sc, 3, true);
+ c = sc;
+ }
+ let rgb = b.conv(&cur, c, 12, 1, false);
+ b.nodes.push(node("DepthToSpace", &[&rgb], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]));
+
+ let mut graph = Vec::new();
+ for n in &b.nodes {
+ len_field(1, n, &mut graph);
+ }
+ for t in &b.inits {
+ len_field(5, t, &mut graph);
+ }
+ len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(tile), Ok(tile)]), &mut graph);
+ len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * tile), Ok(2 * tile)]), &mut graph);
+ let mut ops = Vec::new();
+ len_field(1, b"", &mut ops);
+ varint_field(2, 13, &mut ops);
+ let mut model = Vec::new();
+ varint_field(1, 8, &mut model);
+ len_field(7, &graph, &mut model);
+ len_field(8, &ops, &mut model);
+ model
+}
+
+/// A tiny smoothing network for installing a real CPU model in setup-flow tests.
+/// It averages a 3 × 3 neighborhood of the four Bayer planes and returns three identical channels.
+pub fn smoothing_onnx() -> Vec<u8> {
+ let mut graph = Vec::new();
+ len_field(1, &node("Conv", &["data", "weights"], "samples", &[("kernel_shape", A::Ints(&[3, 3])), ("pads", A::Ints(&[1, 1, 1, 1]))]), &mut graph);
+ len_field(1, &node("DepthToSpace", &["samples"], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]), &mut graph);
+ len_field(5, &tensor("weights", &[12, 4, 3, 3], &[1.0 / 36.0; 12 * 4 * 3 * 3]), &mut graph);
+ len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(64), Ok(64)]), &mut graph);
+ len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(128), Ok(128)]), &mut graph);
+ let mut model = Vec::new();
+ varint_field(1, 8, &mut model);
+ len_field(7, &graph, &mut model);
+ let mut ops = Vec::new();
+ varint_field(2, 13, &mut ops);
+ len_field(8, &ops, &mut model);
+ model
+}
diff --git a/crates/denoise/src/synthetic_proto.rs b/crates/denoise/src/synthetic_proto.rs
new file mode 100644
index 000000000..a78d586bd
--- /dev/null
+++ b/crates/denoise/src/synthetic_proto.rs
@@ -0,0 +1,44 @@
+pub(crate) fn varint(mut v: u64, out: &mut Vec<u8>) {
+ loop {
+ let b = (v & 0x7f) as u8;
+ v >>= 7;
+ if v == 0 {
+ out.push(b);
+ return;
+ }
+ out.push(b | 0x80);
+ }
+}
+
+pub fn len_field(n: u64, body: &[u8], out: &mut Vec<u8>) {
+ varint(n << 3 | 2, out);
+ varint(body.len() as u64, out);
+ out.extend_from_slice(body);
+}
+
+pub(crate) fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
+ varint(n << 3, out);
+ varint(v, out);
+}
+
+/// A graph input or output: `Ok(n)` is a fixed dimension, `Err(name)` a named (dynamic) one.
+pub fn value_info_bytes(name: &str, elem: u64, dims: &[std::result::Result<u64, &str>]) -> Vec<u8> {
+ let mut shape = Vec::new();
+ for d in dims {
+ let mut dim = Vec::new();
+ match d {
+ Ok(v) => varint_field(1, *v, &mut dim),
+ Err(p) => len_field(2, p.as_bytes(), &mut dim),
+ }
+ len_field(1, &dim, &mut shape);
+ }
+ let mut tensor = Vec::new();
+ varint_field(1, elem, &mut tensor);
+ len_field(2, &shape, &mut tensor);
+ let mut ty = Vec::new();
+ len_field(1, &tensor, &mut ty);
+ let mut vi = Vec::new();
+ len_field(1, name.as_bytes(), &mut vi);
+ len_field(2, &ty, &mut vi);
+ vi
+}
diff --git a/crates/denoise/src/tiles.rs b/crates/denoise/src/tiles.rs
new file mode 100644
index 000000000..59898c5a4
--- /dev/null
+++ b/crates/denoise/src/tiles.rs
@@ -0,0 +1,100 @@
+//! Cutting a picture into overlapping tiles and putting the results back together.
+//!
+//! A model runs on a fixed square, so a picture is covered by tiles spaced evenly along each axis, each sharing
+//! at least `overlap` cells with its neighbours. Each tile's influence fades towards its own edges (except at the
+//! picture's edge), and the result is the sum of tile × weight divided by the sum of weights, so any set of
+//! overlapping tiles blends consistently and a seam between two tiles of slightly different scale is a smooth ramp.
+
+/// Where the tiles start along one axis of `n` cells: `[0]` when the axis fits one tile (the tile is then padded by
+/// mirroring), else evenly spaced starts, the first at 0 and the last at `n − tile`, each tile sharing at least
+/// `overlap` cells with the next.
+pub fn starts(n: usize, tile: usize, overlap: usize) -> Vec<usize> {
+ if n <= tile || tile == 0 {
+ return vec![0];
+ }
+ let step = tile.saturating_sub(overlap).max(1);
+ // tiles needed so that start k·spacing with spacing ≤ step reaches n − tile
+ let Some(k) = (n - tile).div_ceil(step).checked_add(1).filter(|&k| k <= 1_000_000) else { return Vec::new() };
+ let last = n - tile;
+ (0..k).map(|i| if k == 1 { 0 } else { ((i as u128 * last as u128 + (k - 1) as u128 / 2) / (k - 1) as u128) as usize }).collect()
+}
+
+/// Weight of the cell at offset `p` (0-based, inside a tile of `tile` cells) along one axis: 1 in the middle, easing
+/// down to a sliver over `fade` cells at an edge that has a neighbour, 1 right up to an edge that is the picture's.
+pub fn weight(p: usize, tile: usize, fade: usize, has_before: bool, has_after: bool) -> f32 {
+ let fade = fade.max(1) as f32;
+ let ramp = |d: usize| {
+ let t = ((d as f32 + 0.5) / fade).min(1.0);
+ // smoothstep, never exactly 0 so that a cell is always covered
+ (t * t * (3.0 - 2.0 * t)).max(1e-3)
+ };
+ let before = if has_before { ramp(p) } else { 1.0 };
+ let after = if has_after { ramp(tile.saturating_sub(1).saturating_sub(p)) } else { 1.0 };
+ before.min(after)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn hostile_axes_have_bounded_tile_counts_and_checked_spacing() {
+ assert!(starts(usize::MAX, 1, 0).is_empty());
+ assert!(starts(usize::MAX, 2, usize::MAX).is_empty());
+ let s = starts(usize::MAX, usize::MAX / 2, 0);
+ assert_eq!(s.first(), Some(&0));
+ assert_eq!(s.last(), Some(&(usize::MAX - usize::MAX / 2)));
+ assert!(s.windows(2).all(|s| s[0] < s[1]));
+ }
+
+ #[test]
+ fn one_tile_when_it_fits() {
+ assert_eq!(starts(100, 512, 64), vec![0]);
+ assert_eq!(starts(512, 512, 64), vec![0]);
+ assert_eq!(starts(0, 512, 64), vec![0]);
+ assert_eq!(starts(10, 0, 0), vec![0]);
+ }
+
+ #[test]
+ fn tiles_cover_the_axis_and_overlap_enough() {
+ for n in [513usize, 600, 900, 1000, 1024, 1500, 3024, 3600, 5000, 12345] {
+ let s = starts(n, 512, 64);
+ assert_eq!(s[0], 0, "n={n}");
+ assert_eq!(*s.last().unwrap(), n - 512, "n={n}");
+ for w in s.windows(2) {
+ assert!(w[1] > w[0], "starts increase, n={n}");
+ assert!(w[0] + 512 >= w[1] + 64, "at least 64 shared cells between {} and {}, n={n}", w[0], w[1]);
+ }
+ }
+ }
+
+ #[test]
+ fn weights_fade_only_towards_neighbours() {
+ let (t, f) = (512, 64);
+ // a lone tile: full weight everywhere
+ assert!((0..t).all(|p| weight(p, t, f, false, false) == 1.0));
+ // the first tile fades on its far side only
+ assert_eq!(weight(0, t, f, false, true), 1.0);
+ assert!(weight(t - 1, t, f, false, true) < 0.01);
+ // the last tile fades on its near side only
+ assert!(weight(0, t, f, true, false) < 0.01);
+ assert_eq!(weight(t - 1, t, f, true, false), 1.0);
+ // a middle tile is 1 in the middle and everywhere positive
+ assert_eq!(weight(t / 2, t, f, true, true), 1.0);
+ assert!((0..t).all(|p| weight(p, t, f, true, true) > 0.0));
+ }
+
+ #[test]
+ fn two_overlapping_tiles_blend_to_a_partition_of_unity() {
+ // at every cell of the shared region the normalised weights are in [0, 1] and sum to 1
+ let (t, f) = (512usize, 64usize);
+ let s = starts(900, t, 64);
+ assert_eq!(s.len(), 2);
+ for cell in s[1]..s[0] + t {
+ let a = weight(cell - s[0], t, f, false, true);
+ let b = weight(cell - s[1], t, f, true, false);
+ let (na, nb) = (a / (a + b), b / (a + b));
+ assert!((na + nb - 1.0).abs() < 1e-6 && na >= 0.0 && nb >= 0.0);
+ }
+ }
+}
diff --git a/crates/denoise/tests/real_model.rs b/crates/denoise/tests/real_model.rs
new file mode 100644
index 000000000..bea7e712d
--- /dev/null
+++ b/crates/denoise/tests/real_model.rs
@@ -0,0 +1,248 @@
+//! Opt-in end-to-end check of the real RawNIND Bayer model on real mosaics with noise added:
+//!
+//! ```text
+//! LC_DENOISE_MODEL=<model_bayer.onnx> LC_DENOISE_MOSAICS=<folder> LC_DENOISE_OUT=<folder> \
+//! cargo test --release -p lightcraft-denoise --test real_model -- --ignored --nocapture
+//! ```
+//!
+//! `LC_DENOISE_MOSAICS` holds `<name>.mosaic.f32` (little-endian f32, normalised: black 0, white 1, not white
+//! balanced), `<name>.rgb.f32` (a clean demosaic of it, camera RGB, interleaved) and `<name>.txt` (`width`,
+//! `height`, `cfa`, `wb` lines). Noise is Poisson-Gaussian, made here. Optional: `LC_DENOISE_PARALLEL` (tiles at
+//! once, default 8), `LC_DENOISE_ONLY` (one name), `LC_DENOISE_ISO` (noise strength, default 1.0; 0 adds none),
+//! `LC_DENOISE_FULL` (the whole frame instead of a 2048 × 1536 crop).
+
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+use lightcraft_denoise::bayer::Layout;
+use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};
+use lightcraft_denoise::run::{Control, Params, denoise_bayer};
+use lightcraft_denoise::runtime::CpuRunner;
+
+struct Rng(u64);
+impl Rng {
+ fn next(&mut self) -> f64 {
+ self.0 ^= self.0 >> 12;
+ self.0 ^= self.0 << 25;
+ self.0 ^= self.0 >> 27;
+ ((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 11) as f64 + 0.5) / (1u64 << 53) as f64
+ }
+ fn gauss(&mut self) -> f64 {
+ (-2.0 * self.next().ln()).sqrt() * (std::f64::consts::TAU * self.next()).cos()
+ }
+}
+
+fn read_f32(path: &Path) -> Vec<f32> {
+ std::fs::read(path).unwrap().as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
+}
+
+struct Dump {
+ width: usize,
+ height: usize,
+ layout: Layout,
+ wb: [f32; 3],
+ mosaic: Vec<f32>,
+ rgb: Vec<f32>,
+}
+
+fn load(dir: &Path, name: &str) -> Dump {
+ let text = std::fs::read_to_string(dir.join(format!("{name}.txt"))).unwrap();
+ let field = |k: &str| text.lines().find_map(|l| l.strip_prefix(k).map(|v| v.trim().to_string())).unwrap();
+ let cell = match field("cfa").as_str() {
+ "RGGB" => [0, 1, 1, 2],
+ "BGGR" => [2, 1, 1, 0],
+ "GRBG" => [1, 0, 2, 1],
+ "GBRG" => [1, 2, 0, 1],
+ other => panic!("cfa {other}"),
+ };
+ let wb: Vec<f32> = field("wb").split_whitespace().map(|v| v.parse().unwrap()).collect();
+ Dump {
+ width: field("width").parse().unwrap(),
+ height: field("height").parse().unwrap(),
+ layout: Layout::from_cell(cell).unwrap(),
+ wb: [wb[0], wb[1], wb[2]],
+ mosaic: read_f32(&dir.join(format!("{name}.mosaic.f32"))),
+ rgb: read_f32(&dir.join(format!("{name}.rgb.f32"))),
+ }
+}
+
+/// Crop with an even origin so the Bayer phase is kept.
+fn crop(d: &Dump, x0: usize, y0: usize, w: usize, h: usize) -> (Vec<f32>, Vec<f32>) {
+ let mut m = Vec::with_capacity(w * h);
+ let mut c = Vec::with_capacity(w * h * 3);
+ for y in y0..y0 + h {
+ m.extend_from_slice(&d.mosaic[y * d.width + x0..y * d.width + x0 + w]);
+ c.extend_from_slice(&d.rgb[(y * d.width + x0) * 3..(y * d.width + x0 + w) * 3]);
+ }
+ (m, c)
+}
+
+/// 0 = red, 1 = green, 2 = blue at a mosaic position.
+fn site(layout: Layout, x: usize, y: usize) -> usize {
+ match ((x + layout.dx) % 2, (y + layout.dy) % 2) {
+ (0, 0) => 0,
+ (1, 1) => 2,
+ _ => 1,
+ }
+}
+
+/// The simplest demosaic (each missing colour is the mean of the same-coloured samples in the 3 × 3 around),
+/// as the baseline a user would get from the noisy picture without AI.
+fn bilinear(m: &[f32], w: usize, h: usize, layout: Layout) -> Vec<f32> {
+ let mut out = vec![0f32; w * h * 3];
+ for y in 0..h {
+ for x in 0..w {
+ let mut sum = [0f32; 3];
+ let mut n = [0f32; 3];
+ for yy in y.saturating_sub(1)..(y + 2).min(h) {
+ for xx in x.saturating_sub(1)..(x + 2).min(w) {
+ let c = site(layout, xx, yy);
+ sum[c] += m[yy * w + xx];
+ n[c] += 1.0;
+ }
+ }
+ for c in 0..3 {
+ out[(y * w + x) * 3 + c] = if n[c] > 0.0 { sum[c] / n[c] } else { 0.0 };
+ }
+ }
+ }
+ out
+}
+
+/// Signal-to-error ratio in dB of `got` against `want` (both interleaved RGB), ignoring a border.
+fn snr(got: &[f32], want: &[f32], w: usize, h: usize, border: usize) -> f64 {
+ let (mut sig, mut err) = (0f64, 0f64);
+ for y in border..h - border {
+ for x in border..w - border {
+ for c in 0..3 {
+ let (g, t) = (f64::from(got[(y * w + x) * 3 + c]), f64::from(want[(y * w + x) * 3 + c]));
+ sig += t * t;
+ err += (g - t).powi(2);
+ }
+ }
+ }
+ 10.0 * (sig / err.max(1e-30)).log10()
+}
+
+fn crc32(data: &[u8]) -> u32 {
+ let mut c = !0u32;
+ for &b in data {
+ c ^= u32::from(b);
+ for _ in 0..8 {
+ c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
+ }
+ }
+ !c
+}
+
+/// A viewable PNG of interleaved camera-RGB panels side by side: white balanced, one exposure for all, gamma.
+fn write_png(path: &Path, panels: &[&[f32]], w: usize, h: usize, wb: [f32; 3], exposure: f32) {
+ let total_w = w * panels.len();
+ let mut raw = Vec::with_capacity((total_w * 3 + 1) * h);
+ for y in 0..h {
+ raw.push(0u8);
+ for p in panels {
+ for x in 0..w {
+ for c in 0..3 {
+ let v = (p[(y * w + x) * 3 + c] * wb[c] * exposure).clamp(0.0, 1.0);
+ raw.push((v.powf(1.0 / 2.2) * 255.0 + 0.5) as u8);
+ }
+ }
+ }
+ }
+ let z = miniz_oxide::deflate::compress_to_vec_zlib(&raw, 6);
+ let mut png = b"\x89PNG\r\n\x1a\n".to_vec();
+ let mut chunk = |kind: &[u8; 4], body: &[u8]| {
+ png.extend_from_slice(&(body.len() as u32).to_be_bytes());
+ let mut k = kind.to_vec();
+ k.extend_from_slice(body);
+ png.extend_from_slice(&k);
+ png.extend_from_slice(&crc32(&k).to_be_bytes());
+ };
+ let mut ihdr = Vec::new();
+ ihdr.extend_from_slice(&(total_w as u32).to_be_bytes());
+ ihdr.extend_from_slice(&(h as u32).to_be_bytes());
+ ihdr.extend_from_slice(&[8, 2, 0, 0, 0]);
+ chunk(b"IHDR", &ihdr);
+ chunk(b"IDAT", &z);
+ chunk(b"IEND", &[]);
+ std::fs::write(path, png).unwrap();
+}
+
+#[test]
+#[ignore = "needs a real model and mosaics: see the module docs"]
+fn the_real_model_denoises_real_mosaics() {
+ let (Some(model), Some(dir)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_MOSAICS")) else { return };
+ let dir = PathBuf::from(dir);
+ let out_dir = std::env::var_os("LC_DENOISE_OUT").map(PathBuf::from);
+ let parallel: usize = std::env::var("LC_DENOISE_PARALLEL").ok().and_then(|v| v.parse().ok()).unwrap_or(8);
+ let iso: f64 = std::env::var("LC_DENOISE_ISO").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0);
+ let names: Vec<String> = match std::env::var("LC_DENOISE_ONLY") {
+ Ok(n) => vec![n],
+ Err(_) => ["arw-sony-a7m3-compressed", "cr2-canon-6d", "nef-nikon-d5100-lossless", "dng-google-pixel2xl", "pef-pentax-k3"]
+ .map(String::from)
+ .to_vec(),
+ };
+ let manifest = DenoiserManifest {
+ id: "rawnind-bayer".into(),
+ name: "RawNIND Bayer".into(),
+ version: "1".into(),
+ licence: Default::default(),
+ source: None,
+ sha256: None,
+ size_bytes: None,
+ provenance: String::new(),
+ domain: Domain::BayerToRgb,
+ tile: 512,
+ overlap: 64,
+ gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
+ };
+ let runner = CpuRunner::load(Path::new(&model), &manifest).unwrap();
+ let params = Params { tile: 512, overlap: 64, gain: manifest.gain, clip: Some(1.0), parallel };
+ for name in names {
+ let d = load(&dir, &name);
+ // a crop of about 2 × 1.5 tiles, away from the edges, even origin
+ let full = std::env::var_os("LC_DENOISE_FULL").is_some();
+ let (w, h) = if full { (d.width & !1, d.height & !1) } else { (2048usize, 1536usize) };
+ let (x0, y0) = (((d.width - w) / 3) & !1, ((d.height - h) / 2) & !1);
+ let (clean, reference) = crop(&d, x0, y0, w, h);
+ // Poisson-Gaussian noise: variance a·signal + b, as a sensor at high ISO
+ let (a, b) = (1.5e-3 * iso, (4.0e-3 * iso.sqrt()).powi(2));
+ let mut rng = Rng(0x9e37_79b9_7f4a_7c15 ^ name.len() as u64);
+ let noisy: Vec<f32> = clean
+ .iter()
+ .map(|&v| {
+ let v = f64::from(v);
+ let sigma = (a * v.max(0.0) + b).sqrt();
+ ((v + sigma * rng.gauss()) as f32).min(1.0)
+ })
+ .collect();
+ let started = Instant::now();
+ let done = std::sync::atomic::AtomicUsize::new(0);
+ let progress = |n: usize, total: usize| done.store(n * 1000 / total, std::sync::atomic::Ordering::Relaxed);
+ let denoised = denoise_bayer(&noisy, w, h, d.layout, &runner, &params, &Control { cancel: None, progress: Some(&progress) }).unwrap();
+ let secs = started.elapsed().as_secs_f64();
+ let baseline = bilinear(&noisy, w, h, d.layout);
+ let clean_bilinear = bilinear(&clean, w, h, d.layout);
+ let den: Vec<f32> = denoised.data.iter().flat_map(|p| p.iter().copied()).collect();
+ let (s_noisy, s_den) = (snr(&baseline, &reference, w, h, 64), snr(&den, &reference, w, h, 64));
+ let s_floor = snr(&clean_bilinear, &reference, w, h, 64);
+ println!(
+ "{name}: {w}×{h} crop, {} tiles, parallel {parallel}, {secs:.1} s | SNR vs clean demosaic: noisy bilinear {s_noisy:.2} dB, denoised {s_den:.2} dB (clean bilinear {s_floor:.2} dB)",
+ lightcraft_denoise::run::tile_count(w, h, d.layout, 512, 64)
+ );
+ if let Some(out) = &out_dir {
+ std::fs::create_dir_all(out).unwrap();
+ let mean = reference.iter().map(|&v| f64::from(v)).sum::<f64>() / reference.len() as f64;
+ let exposure = (0.18 / mean.max(1e-6)) as f32;
+ // three panels, a 512 × 384 window each, at 1:1
+ let (pw, ph, px, py) = (512usize, 384usize, 700usize, 500usize);
+ let window =
+ |buf: &[f32]| -> Vec<f32> { (0..ph).flat_map(|y| buf[((py + y) * w + px) * 3..((py + y) * w + px + pw) * 3].to_vec()).collect() };
+ write_png(&out.join(format!("{name}.png")), &[&window(&baseline), &window(&den), &window(&reference)], pw, ph, d.wb, exposure);
+ }
+ if iso >= 4.0 {
+ assert!(s_den > s_noisy + 2.0, "{name}: denoised {s_den:.2} dB is not clearly better than noisy {s_noisy:.2} dB");
+ }
+ }
+}
diff --git a/crates/develop/src/controls.rs b/crates/develop/src/controls.rs
index 5696d6706..c8a001b79 100644
--- a/crates/develop/src/controls.rs
+++ b/crates/develop/src/controls.rs
@@ -207,6 +207,7 @@ controls! {
"grain.amount" => grain.amount, "Grain", Grain, 0, 100, 0, 1, 0, Plain;
"grain.size" => grain.size, "Size", Grain, 0, 100, 25, 1, 0, Plain;
"grain.roughness" => grain.roughness, "Roughness", Grain, 0, 100, 50, 1, 0, Plain;
+ "enhance.denoise" => enhance.denoise, "Amount", Detail, 0, 100, 0, 1, 0, Plain;
"detail.sharpenAmount" => detail.sharpen_amount, "Sharpening", Detail, 0, 150, 0, 1, 0, Plain;
"detail.sharpenRadius" => detail.sharpen_radius, "Radius", Detail, 0.5, 3, 1, 0.1, 1, Plain;
"detail.sharpenDetail" => detail.sharpen_detail, "Detail", Detail, 0, 100, 25, 1, 0, Plain;
diff --git a/crates/develop/src/lib.rs b/crates/develop/src/lib.rs
index bfce1893e..b7c952511 100644
--- a/crates/develop/src/lib.rs
+++ b/crates/develop/src/lib.rs
@@ -94,6 +94,13 @@ impl DevelopSettings {
std::borrow::Cow::Owned(d)
}
+ /// How much of the AI-denoised picture to mix in, 0 (none) to 1 (all of it): the Denoise amount, unless the Detail
+ /// section is switched off or the value is not a number.
+ pub fn denoise_amount(&self) -> f32 {
+ let a = self.enhance.denoise;
+ if a.is_finite() && self.enhance.denoise_enabled() && self.section_enabled("detail") { (a / 100.0).clamp(0.0, 1.0) as f32 } else { 0.0 }
+ }
+
pub fn set_section_enabled(&mut self, section: &str, on: bool) {
self.disabled_sections.retain(|s| s != section);
if !on {
@@ -112,6 +119,7 @@ impl DevelopSettings {
self.curve = d;
}
Section::Color => self.wb.mode = WbMode::AsShot,
+ Section::Detail => self.enhance.denoise_on = None,
_ => {}
}
}
@@ -193,4 +201,40 @@ mod tests {
s.set_section_enabled("effects", true);
assert!(s.section_enabled("effects"));
}
+
+ #[test]
+ fn the_denoise_amount_is_a_slider_and_follows_the_detail_section() {
+ let mut s = DevelopSettings::default();
+ assert_eq!(s.denoise_amount(), 0.0);
+ assert!(controls::set(&mut s, "enhance.denoise", 60.0));
+ assert_eq!(controls::get(&s, "enhance.denoise"), Some(60.0));
+ assert!((s.denoise_amount() - 0.6).abs() < 1e-6);
+ // the slider's range is 0 to 100; values outside it are brought in
+ controls::set(&mut s, "enhance.denoise", 500.0);
+ assert_eq!(s.denoise_amount(), 1.0);
+ controls::set(&mut s, "enhance.denoise", -5.0);
+ assert_eq!(s.denoise_amount(), 0.0);
+ // switching the Detail section off switches the denoise off with it; a settings file with a bad number does nothing
+ controls::set(&mut s, "enhance.denoise", 80.0);
+ s.set_section_enabled("detail", false);
+ assert_eq!(s.denoise_amount(), 0.0);
+ s.set_section_enabled("detail", true);
+ s.enhance.denoise = f64::NAN;
+ assert_eq!(s.denoise_amount(), 0.0);
+ // it is part of Detail: a reset of the section clears it
+ controls::set(&mut s, "enhance.denoise", 40.0);
+ s.reset_section(Section::Detail);
+ assert_eq!(s.enhance.denoise, 0.0);
+ // The later independent switch preserves Amount when off, and can be on at zero.
+ controls::set(&mut s, "enhance.denoise", 75.0);
+ s.enhance.denoise_on = Some(false);
+ assert_eq!(s.denoise_amount(), 0.0);
+ assert_eq!(s.enhance.denoise, 75.0);
+ s.enhance.denoise_on = Some(true);
+ assert_eq!(s.denoise_amount(), 0.75);
+ controls::set(&mut s, "enhance.denoise", 0.0);
+ assert!(s.enhance.denoise_enabled());
+ s.reset_section(Section::Detail);
+ assert_eq!(s.enhance.denoise_on, None);
+ }
}
diff --git a/crates/develop/src/settings.rs b/crates/develop/src/settings.rs
index 910cd07dd..5f0db8ab7 100644
--- a/crates/develop/src/settings.rs
+++ b/crates/develop/src/settings.rs
@@ -915,6 +915,14 @@ pub struct LensBlur {
#[serde(default)]
pub struct Enhance {
pub denoise: f64,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub denoise_on: Option<bool>,
pub raw_details: bool,
pub super_resolution: bool,
}
+
+impl Enhance {
+ pub fn denoise_enabled(&self) -> bool {
+ self.denoise_on.unwrap_or(self.denoise > 0.0)
+ }
+}
diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml
index a916d1806..536df7160 100644
--- a/crates/engine/Cargo.toml
+++ b/crates/engine/Cargo.toml
@@ -21,6 +21,7 @@ web-time = { workspace = true }
lightcraft-preview = { workspace = true }
lightcraft-codecs = { workspace = true }
lightcraft-raw = { workspace = true }
+lightcraft-denoise = { workspace = true }
lightcraft-meta = { workspace = true }
miniz_oxide = { workspace = true }
lightcraft-merge = { workspace = true }
@@ -38,6 +39,10 @@ lightcraft-segment = { workspace = true, optional = true }
[features]
# AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds.
sam = ["dep:lightcraft-segment"]
+denoise = []
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+lightcraft-fetch = { workspace = true }
[dev-dependencies]
libc = { workspace = true }
diff --git a/crates/engine/src/cmd/denoise.rs b/crates/engine/src/cmd/denoise.rs
new file mode 100644
index 000000000..c51037917
--- /dev/null
+++ b/crates/engine/src/cmd/denoise.rs
@@ -0,0 +1,543 @@
+//! Denoise commands (`denoise.*`): the models (list, install, download, test, remove, choose), the settings, and the
+//! work (queue, cancel, status, pump, clear). The machinery they drive is [`crate::denoise`].
+//!
+//! Models live in the folder the host provides (`Session::denoise.models_dir`): one subfolder per model with
+//! `model.onnx`, `denoise-model.json` (its manifest) and `installed.json` (what the user accepted and the file's hash),
+//! plus `settings.json` beside them. Model files, archives and manifests are hostile input: sizes are capped, ids are
+//! validated before they become folder names, and nothing outside a model's own folder is ever touched.
+
+use std::path::{Path, PathBuf};
+use std::sync::PoisonError;
+
+use lightcraft_catalog::PhotoId;
+use lightcraft_denoise::archive;
+use lightcraft_denoise::hash::sha256_file;
+use lightcraft_denoise::known::{self, Known};
+use lightcraft_denoise::manifest::{self, DenoiserManifest, MAX_MANIFEST_BYTES, MAX_MODEL_BYTES};
+use serde_json::{Value, json};
+
+use super::{CommandSpec, always, bad, bool_or, cmd, str_param};
+use crate::denoise::{Pace, RunOn, Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings};
+use crate::model_download::{self, State};
+use crate::{EngineError, Result, Session};
+
+fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {
+ EngineError::Other(format!("{what}: {e}"))
+}
+
+fn models_dir(s: &Session, cmd: &str) -> Result<PathBuf> {
+ s.denoise.models_dir.clone().ok_or_else(|| bad(cmd, "this build has nowhere to keep denoise models (the desktop app does)"))
+}
+
+/// What a command says about a model.
+fn row(m: &DenoiserManifest, download_host: Option<String>, installed: bool, selected: bool, accepted: &Value) -> Value {
+ json!({
+ "id": m.id,
+ "name": m.name,
+ "version": m.version,
+ "licence": m.licence,
+ "provenance": m.provenance,
+ "source": m.source,
+ "sizeBytes": m.size_bytes,
+ "sha256": m.sha256,
+ "tile": m.tile,
+ "known": known::find(&m.id).is_some(),
+ // a pinned address LightCraft can fetch it from (the user presses Download), and which site that is
+ "downloadHost": download_host,
+ "installed": installed,
+ "selected": selected,
+ "accepted": accepted,
+ })
+}
+
+fn host_of(k: &Known) -> Option<String> {
+ k.download.as_ref().map(|d| lightcraft_denoise::known::host(&d.url).to_string())
+}
+
+fn list(s: &mut Session, _: &Value) -> Result<Value> {
+ s.denoise_refresh_active(true);
+ let dir = s.denoise.models_dir.clone();
+ let st = dir.as_deref().map(read_settings).unwrap_or_default();
+ let installed = dir.as_deref().map(installed_models).unwrap_or_default();
+ let on_disk = |id: &str| installed.iter().find(|i| i.manifest.id == id);
+ let chosen = |id: &str| st.model.as_deref() == Some(id);
+ let mut models: Vec<Value> = Vec::new();
+ for k in known::all() {
+ let found = on_disk(&k.manifest.id);
+ models.push(row(&k.manifest, host_of(&k), found.is_some(), chosen(&k.manifest.id), found.map_or(&Value::Null, |f| &f.accepted)));
+ }
+ for i in installed.iter().filter(|i| known::find(&i.manifest.id).is_none()) {
+ models.push(row(&i.manifest, None, true, chosen(&i.manifest.id), &i.accepted));
+ }
+ Ok(json!({
+ "dir": dir.as_ref().map(|d| d.display().to_string()),
+ "productsDir": s.denoise_products_dir().map(|d| d.display().to_string()),
+ "model": st.model,
+ // whether this build can run denoise models (the `denoise` feature: our pure-Rust CPU/GPU runners)
+ "runtime": cfg!(feature = "denoise"),
+ "auto": st.auto(),
+ "cacheGb": st.cache_bytes() >> 30,
+ "threads": st.threads,
+ "runOn": st.run_on().name(),
+ "models": models,
+ }))
+}
+
+/// What a file the user pointed at is, as far as installing it goes.
+struct Source {
+ manifest: DenoiserManifest,
+ /// The `.onnx` inside the archive; `None` when the file is the `.onnx`.
+ entry: Option<&'static str>,
+ file_name: String,
+}
+
+fn identify(path: &Path, c: &str, sha_known: Option<&str>) -> Result<Source> {
+ let meta = std::fs::metadata(path).map_err(|e| bad(c, format!("cannot read `{}`: {e}", path.display())))?;
+ if !meta.is_file() {
+ return Err(bad(c, format!("`{}` is not a file", path.display())));
+ }
+ if meta.len() == 0 || meta.len() > MAX_MODEL_BYTES {
+ return Err(bad(c, "a model file must be between 1 byte and 2 GiB"));
+ }
+ let sha256 = match sha_known {
+ Some(h) => h.to_string(),
+ None => sha256_file(path).map_err(|e| fail(&format!("could not read {}", path.display()), e))?,
+ };
+ let file_name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
+ // an archive LightCraft knows (the file darktable publishes the model in)
+ if let Some(k) = known::all().into_iter().find(|k| k.download.as_ref().is_some_and(|d| d.sha256 == sha256)) {
+ return Ok(Source { manifest: k.manifest, entry: k.entry, file_name });
+ }
+ // the `.onnx` of a known model
+ if let Some(m) = known::by_sha256(&sha256) {
+ return Ok(Source { manifest: m, entry: None, file_name });
+ }
+ // an `.onnx` with its manifest beside it
+ let sibling = path.with_file_name("denoise-model.json");
+ if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx"))
+ && let Some(bytes) = read_capped(&sibling, MAX_MANIFEST_BYTES)
+ {
+ let m = manifest::parse(&bytes).map_err(|e| bad(c, format!("{}: {e}", sibling.display())))?;
+ return Ok(Source { manifest: m, entry: None, file_name });
+ }
+ Err(bad(
+ c,
+ "LightCraft does not know this file. A denoise model is an .onnx file with a denoise-model.json beside it that says how to use it \
+ (see docs/denoise.md), or an archive LightCraft can fetch itself (`denoise.models.download`)",
+ ))
+}
+
+/// Install the model in `src`. A model that passes its self-test becomes the one in use (`activate`, the default): installing
+/// a model is how someone says they want it. `verified` is for a download whose hash was already checked.
+fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activate: bool, verified: bool) -> Result<Value> {
+ let dir = models_dir(s, c)?;
+ let ident = identify(src, c, None)?;
+ let m = ident.manifest.clone();
+ manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?;
+ if !acknowledged {
+ return Err(bad(
+ c,
+ "the licence has not been accepted: show the user its terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree",
+ ));
+ }
+ let home = dir.join(&m.id);
+ std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?;
+ let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx"));
+ let _ = std::fs::remove_file(&part);
+ let placed: Result<()> = match ident.entry {
+ Some(entry) => {
+ archive::extract(src, entry, &part, MAX_MODEL_BYTES).map(|_| ()).map_err(|e| fail("could not take the model out of the archive", e))
+ }
+ None if verified && std::fs::rename(src, &part).is_ok() => Ok(()),
+ None => std::fs::copy(src, &part).map(|_| ()).map_err(|e| fail("could not copy the model (is the disk full?)", e)),
+ };
+ let sha256 = placed.and_then(|_| sha256_file(&part).map_err(|e| fail("could not check the model", e))).and_then(|h| match &m.sha256 {
+ Some(want) if *want != h => Err(EngineError::Other("the model file is not the one its description names (its SHA-256 differs)".into())),
+ _ => Ok(h),
+ });
+ let sha256 = match sha256 {
+ Ok(h) => h,
+ Err(e) => {
+ let _ = std::fs::remove_file(&part);
+ return Err(e);
+ }
+ };
+ let _ = std::fs::remove_file(&final_path);
+ if let Err(e) = std::fs::rename(&part, &final_path) {
+ let _ = std::fs::remove_file(&part);
+ return Err(fail("could not finish the copy", e));
+ }
+ // a model that does not work is never left installed
+ let run_on = s.denoise.settings.run_on();
+ let test = (s.denoise.loader)(&final_path, &m).and_then(|model| model.self_test(run_on));
+ let test = match test {
+ Ok(t) => t,
+ Err(e) => {
+ let _ = std::fs::remove_file(&final_path);
+ let _ = std::fs::remove_file(home.join("denoise-model.json"));
+ let _ = std::fs::remove_dir(&home);
+ return Err(bad(c, e));
+ }
+ };
+ write_atomic(&home.join("denoise-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?).map_err(EngineError::Other)?;
+ let accepted = json!({
+ "acceptedAt": (s.clock)(),
+ "licence": m.licence.name,
+ "commercial": m.licence.commercial,
+ "fileName": ident.file_name,
+ "sha256": sha256,
+ "selfTest": test,
+ });
+ write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?).map_err(EngineError::Other)?;
+ // a file that was downloaded and waited for the user's acceptance has served: the installed copy is the model now
+ if src.parent() == Some(dir.join(model_download::STAGING).as_path()) {
+ let _ = std::fs::remove_file(src);
+ s.denoise.downloads.staged_file_used(src);
+ }
+ let mut st = read_settings(&dir);
+ if activate {
+ st.model = Some(m.id.clone());
+ write_settings(&dir, &st).map_err(EngineError::Other)?;
+ }
+ s.denoise.touch();
+ s.denoise_refresh_active(true);
+ let host = known::find(&m.id).and_then(|k| host_of(&k));
+ Ok(json!({"installed": row(&m, host, true, st.model.as_deref() == Some(m.id.as_str()), &accepted), "model": st.model}))
+}
+
+fn install(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.models.install";
+ let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
+ install_file(s, C, Path::new(path), bool_or(p, "acknowledged", false), bool_or(p, "activate", true), false)
+}
+
+/// What an error says, without the command it came from (for a line shown to the user).
+fn plain(e: &EngineError) -> String {
+ match e {
+ EngineError::BadParams { msg, .. } => msg.clone(),
+ other => other.to_string(),
+ }
+}
+
+/// Install every download that has arrived: each was accepted when it was started, so it is installed and chosen without
+/// another question. Called by `denoise.models.downloads` and by every frame's `denoise.pump`.
+pub(crate) fn finish_downloads(s: &mut Session) {
+ const C: &str = "denoise.models.download";
+ for (id, path, _sha) in s.denoise.downloads.finished() {
+ match install_file(s, C, &path, true, true, true) {
+ Ok(_) => s.denoise.downloads.set_outcome(&id, State::Installed),
+ Err(e) => {
+ // a file that cannot be installed is of no use to anyone: throw it away, say why
+ let _ = std::fs::remove_file(&path);
+ s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e)));
+ }
+ }
+ }
+}
+
+fn download(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.models.download";
+ let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
+ let dir = models_dir(s, C)?;
+ let k = known::find(id).ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;
+ let spec = k.download.clone().ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;
+ if !cfg!(feature = "denoise") {
+ return Err(bad(C, "this build cannot run denoise models, so there is nothing to download for it"));
+ }
+ if !bool_or(p, "acknowledged", false) {
+ return Err(bad(
+ C,
+ "the licence has not been accepted: show the user the model's terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree",
+ ));
+ }
+ if installed_models(&dir).iter().any(|i| i.manifest.id == spec.id) {
+ return Err(bad(C, "that model is already installed"));
+ }
+ let host = lightcraft_denoise::known::host(&spec.url).to_string();
+ s.denoise.downloads.start(spec, &dir).map_err(|e| bad(C, e))?;
+ Ok(json!({"started": id, "from": host}))
+}
+
+fn download_row(id: &str, state: &State) -> Value {
+ let from = known::find(id).and_then(|k| host_of(&k));
+ match state {
+ State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}),
+ State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}),
+ State::Installed => json!({"id": id, "state": "installed", "from": from}),
+ State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}),
+ State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}),
+ }
+}
+
+fn downloads(s: &mut Session, _: &Value) -> Result<Value> {
+ finish_downloads(s);
+ let all: Vec<Value> = s.denoise.downloads.snapshot().iter().map(|(id, st)| download_row(id, st)).collect();
+ Ok(json!({"running": s.denoise.downloads.running(), "downloads": all}))
+}
+
+fn download_cancel(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.models.downloadCancel";
+ let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
+ Ok(json!({"discarded": s.denoise.downloads.discard(id)}))
+}
+
+fn test(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.models.test";
+ let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
+ let dir = models_dir(s, C)?;
+ let found = installed_models(&dir).into_iter().find(|i| i.manifest.id == id).ok_or_else(|| bad(C, "that model is not installed"))?;
+ let run_on = s.denoise.settings.run_on();
+ let result = (s.denoise.loader)(&found.onnx, &found.manifest).and_then(|m| m.self_test(run_on));
+ let (ok, detail) = match &result {
+ Ok(v) => (true, v.clone()),
+ Err(e) => (false, json!(e)),
+ };
+ // keep the latest outcome with the model's record
+ let mut record = found.accepted;
+ if let Some(o) = record.as_object_mut() {
+ o.insert("selfTest".into(), if ok { detail.clone() } else { json!({"ok": false, "error": detail}) });
+ write_atomic(&found.onnx.with_file_name("installed.json"), &serde_json::to_vec_pretty(&record).map_err(|e| fail("record", e))?)
+ .map_err(EngineError::Other)?;
+ }
+ Ok(json!({"id": id, "ok": ok, "result": detail}))
+}
+
+fn remove(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.models.remove";
+ let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
+ let dir = models_dir(s, C)?;
+ if !lightcraft_denoise::licence::valid_id(id) {
+ return Err(bad(C, "not a model id"));
+ }
+ let home = dir.join(id);
+ // only a folder that is a model (it has a manifest) is ever deleted
+ if !home.join("denoise-model.json").is_file() {
+ return Err(bad(C, "that model is not installed"));
+ }
+ std::fs::remove_dir_all(&home).map_err(|e| fail("could not remove the model", e))?;
+ let mut st = read_settings(&dir);
+ if st.model.as_deref() == Some(id) {
+ st.model = installed_models(&dir).into_iter().map(|i| i.manifest.id).find(|m| m != id);
+ write_settings(&dir, &st).map_err(EngineError::Other)?;
+ }
+ s.denoise.touch();
+ s.denoise_refresh_active(true);
+ Ok(json!({"removed": id, "model": st.model}))
+}
+
+fn select(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.models.select";
+ let dir = models_dir(s, C)?;
+ let id = match p.get("id") {
+ None | Some(Value::Null) => None,
+ Some(v) => Some(v.as_str().ok_or_else(|| bad(C, "`id` must be a model id or null"))?.to_string()),
+ };
+ if let Some(id) = &id
+ && !installed_models(&dir).iter().any(|i| &i.manifest.id == id)
+ {
+ return Err(bad(C, "that denoise model is not installed"));
+ }
+ let mut st = read_settings(&dir);
+ st.model = id;
+ write_settings(&dir, &st).map_err(EngineError::Other)?;
+ s.denoise.touch();
+ s.denoise_refresh_active(true);
+ Ok(json!({"model": st.model}))
+}
+
+fn settings(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.settings";
+ let dir = models_dir(s, C)?;
+ let mut st: Settings = read_settings(&dir);
+ if let Some(v) = p.get("auto") {
+ st.auto = Some(v.as_bool().ok_or_else(|| bad(C, "`auto` must be true or false"))?);
+ }
+ if let Some(v) = p.get("cacheGb") {
+ let n =
+ v.as_u64().filter(|n| (1..=10_000).contains(n)).ok_or_else(|| bad(C, "`cacheGb` must be a whole number of gigabytes from 1 to 10000"))?;
+ st.cache_gb = Some(n as u32);
+ }
+ if let Some(v) = p.get("threads") {
+ st.threads = match v {
+ Value::Null => None,
+ v => Some(v.as_u64().filter(|n| (1..=64).contains(n)).ok_or_else(|| bad(C, "`threads` must be from 1 to 64, or null"))? as u32),
+ };
+ }
+ let run_on = match p.get("runOn") {
+ None => None,
+ Some(v) => Some(v.as_str().and_then(RunOn::parse).ok_or_else(|| bad(C, "`runOn` must be auto, gpu or cpu"))?),
+ };
+ if let Some(r) = run_on {
+ st.run_on = Some(r.name().to_string());
+ st.gpu = None;
+ }
+ write_settings(&dir, &st).map_err(EngineError::Other)?;
+ if run_on.is_some() {
+ // a choice of where denoise runs is also a go-ahead to try the card again: a set-up that crashed or gave up
+ // before is forgotten, and the loaded model sets the card up afresh for the next photo
+ forget_gpu_set_ups(&dir);
+ if let Some(a) = &s.denoise.active {
+ *a.model.lock().unwrap_or_else(PoisonError::into_inner) = None;
+ }
+ }
+ s.denoise.touch();
+ s.denoise_refresh_active(true);
+ // a smaller limit takes effect now
+ s.denoise_evict(None);
+ Ok(json!({"auto": st.auto(), "cacheGb": st.cache_bytes() >> 30, "threads": st.threads, "runOn": st.run_on().name()}))
+}
+
+fn ids_param(p: &Value) -> Option<Vec<PhotoId>> {
+ p.get("ids").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_u64).map(PhotoId).collect())
+}
+
+fn status_json(s: &Session, photo: Option<PhotoId>) -> Value {
+ let (files, bytes) = s.denoise_cache_usage();
+ let st = &s.denoise.settings;
+ let model = s.denoise.active.as_ref().map(|a| json!({"id": a.id, "name": a.manifest.name}));
+ let mut v = json!({
+ "enabled": s.denoise.active.is_some(),
+ "model": model,
+ "runtime": cfg!(feature = "denoise"),
+ "productsDir": s.denoise_products_dir().map(|d| d.display().to_string()),
+ "queued": s.denoise.queued(),
+ "running": s.denoise_running_json(),
+ "ready": s.media.denoise.len(),
+ "made": s.denoise_made(),
+ "auto": st.auto(),
+ "runOn": st.run_on().name(),
+ "device": s.denoise_device(),
+ "cache": {"files": files, "bytes": bytes, "limitBytes": st.cache_bytes()},
+ "generation": s.denoise.generation,
+ });
+ if let (Some(id), Some(o)) = (photo, v.as_object_mut()) {
+ o.insert("photo".into(), serde_json::to_value(s.denoise_photo_state(id)).unwrap_or(Value::Null));
+ }
+ v
+}
+
+/// A per-photo switch, independent of the selection and Auto Sync. Turning on starts at 50%.
+fn toggle(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.toggle";
+ let id = match p.get("id") {
+ Some(v) => PhotoId(v.as_u64().ok_or_else(|| bad(C, "`id` must be a photo id"))?),
+ None => s.active().ok_or_else(|| bad(C, "select a raw photo"))?,
+ };
+ let photo = s.catalog.photo(id).ok_or_else(|| bad(C, "no such photo"))?;
+ if !crate::denoise::eligible(photo) {
+ return Err(bad(C, "AI Denoise applies to raw photos developed from their sensor data"));
+ }
+ let mut d = (*s.develop_of(id).ok_or_else(|| bad(C, "no such photo"))?).clone();
+ let on = match p.get("enabled") {
+ Some(v) => v.as_bool().ok_or_else(|| bad(C, "`enabled` must be true or false"))?,
+ None => !d.enhance.denoise_enabled(),
+ };
+ if on {
+ if !d.enhance.denoise.is_finite() {
+ return Err(bad(C, "the photo has an invalid Denoise amount"));
+ }
+ s.denoise_refresh_active(true);
+ if s.denoise.active.is_none() {
+ return Err(bad(C, "AI Denoise needs a model: install one in Settings > AI Denoise"));
+ }
+ }
+ // The switch is its own setting: the Amount stays where it is (0 included) when it is flipped, so flipping compares
+ // with and without. A photo that never had an Amount starts at 50 the first time.
+ let amount = if on && d.enhance.denoise_on.is_none() && d.enhance.denoise <= 0.0 { 50.0 } else { d.enhance.denoise };
+ if d.enhance.denoise != amount || d.enhance.denoise_on != Some(on) {
+ s.end_interaction()?;
+ d.enhance.denoise = amount;
+ d.enhance.denoise_on = Some(on);
+ let op = s.develop_op(id, d, "AI Denoise").ok_or_else(|| bad(C, "no such photo"))?;
+ s.commit("AI Denoise", op)?;
+ }
+ if on {
+ s.denoise_enqueue(&[id], true, false);
+ } else {
+ s.denoise_cancel(Some(id), false);
+ }
+ Ok(json!({"id": id.0, "enabled": on, "amount": amount}))
+}
+
+fn queue(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "denoise.queue";
+ s.denoise_refresh_active(true);
+ if s.denoise.active.is_none() {
+ return Err(bad(C, "no denoise model is chosen: install one first (`denoise.models.download` or `denoise.models.install`)"));
+ }
+ if s.denoise_products_dir().is_none() {
+ return Err(bad(C, "denoise keeps its pictures in the library's folder: open a library on disk first"));
+ }
+ let ids = match ids_param(p) {
+ Some(ids) => ids,
+ None => match str_param(p, "scope").unwrap_or("selected") {
+ "selected" => s.selection.active.iter().chain(s.selection.ids.iter()).copied().collect(),
+ "visible" => s.visible_cloned().to_vec(),
+ "withAmount" => s.catalog.photos().filter(|ph| ph.develop.denoise_amount() > 0.0).map(|ph| ph.id).collect(),
+ other => return Err(bad(C, format!("unknown scope `{other}`: use selected, visible or withAmount"))),
+ },
+ };
+ let queued = s.denoise_enqueue(&ids, false, bool_or(p, "retry", false));
+ let mut v = status_json(s, None);
+ if let Some(o) = v.as_object_mut() {
+ o.insert("added".into(), json!(queued));
+ }
+ Ok(v)
+}
+
+fn cancel(s: &mut Session, p: &Value) -> Result<Value> {
+ let photo = p.get("id").and_then(Value::as_u64).map(PhotoId);
+ let n = s.denoise_cancel(photo, bool_or(p, "all", photo.is_none()));
+ Ok(json!({"cancelled": n}))
+}
+
+fn status(s: &mut Session, p: &Value) -> Result<Value> {
+ s.denoise_refresh_active(false);
+ Ok(status_json(s, p.get("id").and_then(Value::as_u64).map(PhotoId)))
+}
+
+/// `denoise.pump {pace?}`: cheap to call every frame. Takes in what the background job has finished and keeps one going.
+fn pump(s: &mut Session, p: &Value) -> Result<Value> {
+ s.denoise_pump(Pace::parse(p.get("pace").and_then(Value::as_str)));
+ Ok(json!({
+ "active": s.denoise.active.is_some(),
+ "running": s.denoise_running_json(),
+ "queued": s.denoise.queued(),
+ "ready": s.media.denoise.len(),
+ "generation": s.denoise.generation,
+ "downloading": s.denoise.downloads.running(),
+ }))
+}
+
+fn clear(s: &mut Session, _: &Value) -> Result<Value> {
+ let (files, bytes) = s.denoise_clear();
+ Ok(json!({"deleted": files, "bytes": bytes}))
+}
+
+pub fn specs() -> Vec<CommandSpec> {
+ vec![
+ cmd!(query "denoise.models.list", "Denoise Models", [], None, "{} → {dir, productsDir, model, runtime, auto, cacheGb, threads, runOn, models: [{id, name, version, licence{name, commercial, url, notice}, provenance, source, sizeBytes, sha256, tile, known, downloadHost, installed, selected, accepted}]}", always, list),
+ cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true} → {installed, model} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install),
+ cmd!(query "denoise.models.download", "Download Denoise Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a model LightCraft has a pinned address for (see `downloadHost` in the list) in the background over HTTPS with lightcraft-fetch. `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, installed and chosen by itself; `denoise.models.downloads` shows how far it is", always, download),
+ cmd!(query "denoise.models.downloads", "Denoise Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived", always, downloads),
+ cmd!(query "denoise.models.downloadCancel", "Cancel Denoise Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel),
+ cmd!(query "denoise.models.test", "Test Denoise Model", [], None, "{id} → {ok, result} — load an installed model and check it gives sensible pictures; needs the denoise runtime", always, test),
+ cmd!(query "denoise.models.remove", "Remove Denoise Model", [], None, "{id} → {removed, model} — delete an installed model (its cached pictures stay until they are evicted)", always, remove),
+ cmd!(query "denoise.models.select", "Choose Denoise Model", [], None, "{id: installed model | null} → {model} — null switches denoise off", always, select),
+ cmd!(query "denoise.settings", "Denoise Settings", [], None, "{auto?: bool, cacheGb?: 1..10000, threads?: 1..64 | null, runOn?: auto | gpu | cpu} → the settings — `auto`: make the picture of the photos being looked at that have a Denoise amount; `cacheGb`: how much the cached pictures may take; `threads`: most tiles run at once; `runOn`: where the model runs — `auto` (the default) the graphics card where it is faster than the processor (both timed on this computer when the card is set up), `gpu` the card whenever it can run the model, `cpu` the processor. Setting `runOn` also lets the card be tried again after a set-up that failed or closed LightCraft", always, settings),
+ cmd!(
+ "denoise.toggle",
+ "AI Denoise",
+ [],
+ None,
+ "{id?: photoId, enabled?: bool} → {id, enabled, amount} — turn AI Denoise on (50% initially) or off for one raw photo, independent of selection and Auto Sync; needs an installed model to turn on",
+ always,
+ toggle
+ ),
+ cmd!(query "denoise.queue", "Denoise Photos", [], None, "{ids?: [photoId], scope?: selected | visible | withAmount, retry?: false} → {added, …status} — make the denoised pictures of raw photos that have none, in the background. `retry` tries photos that failed before", always, queue),
+ cmd!(query "denoise.cancel", "Cancel Denoise", [], None, "{id?, all?} → {cancelled} — stop the photo being made and/or forget the ones waiting (all of them when `id` is omitted)", always, cancel),
+ cmd!(query "denoise.status", "Denoise Status", [], None, "{id?} → {enabled, model, runtime, productsDir, queued, running: {photo, done, total, seconds} | null, ready, made, auto, runOn, device: {kind: gpu | cpu | pending | none, adapter?, reason?, cardMs?, cpuMs?, fellBack?}, cache: {files, bytes, limitBytes}, generation, photo?: {state: notApplicable | noModel | ready | queued | running | failed | idle, …}}", always, status),
+ cmd!(query "denoise.pump", "Denoise Pump", [], None, "{pace?: pause | light | normal | full} → {active, running, queued, ready, generation, downloading} — called every frame by the app: takes in finished work and keeps one photo going", always, pump),
+ cmd!(query "denoise.clear", "Clear Denoise Cache", [], None, "{} → {deleted, bytes} — delete every cached denoised picture of this library (they are made again when wanted)", always, clear),
+ ]
+}
diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs
index 9fc831161..9f1d255c1 100644
--- a/crates/engine/src/cmd/mod.rs
+++ b/crates/engine/src/cmd/mod.rs
@@ -10,6 +10,7 @@ mod color;
pub(crate) mod convert;
mod cull;
pub mod curves;
+pub(crate) mod denoise;
mod develop;
mod edit;
mod export;
@@ -140,6 +141,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
v.extend(missing::specs());
v.extend(metadata::specs());
v.extend(filters::specs());
+ v.extend(denoise::specs());
v
})
}
diff --git a/crates/engine/src/config.rs b/crates/engine/src/config.rs
new file mode 100644
index 000000000..96632f7ac
--- /dev/null
+++ b/crates/engine/src/config.rs
@@ -0,0 +1,23 @@
+//! Shared, opt-in per-user denoise model storage for native hosts.
+
+use crate::Session;
+use std::path::PathBuf;
+
+pub fn default_denoise_models_dir() -> Option<PathBuf> {
+ std::env::var_os("LIGHTCRAFT_DENOISE_MODELS")
+ .filter(|v| !v.is_empty())
+ .map(PathBuf::from)
+ .or_else(|| crate::camera_profiles::config_dir().map(|d| d.join("denoise-models")))
+}
+
+impl Session {
+ pub fn with_default_denoise_models(mut self) -> Self {
+ self.set_denoise_models_dir(default_denoise_models_dir());
+ self
+ }
+
+ pub fn set_denoise_models_dir(&mut self, dir: Option<PathBuf>) {
+ self.denoise.models_dir = dir;
+ self.denoise.touch();
+ }
+}
diff --git a/crates/engine/src/denoise.rs b/crates/engine/src/denoise.rs
new file mode 100644
index 000000000..37d79dcd2
--- /dev/null
+++ b/crates/engine/src/denoise.rs
@@ -0,0 +1,1629 @@
+//! AI denoise, the session's side (`docs/denoise.md`): which photos have a denoised picture, making the missing ones
+//! in the background, and keeping the cache within its size.
+//!
+//! The denoised picture of a raw photo is *cached data*, never a file in the library: it is made from the untouched
+//! original by the model the user chose, kept in `<library>/denoise/<key>.lcdn`, and thrown away whenever space is
+//! needed (it is made again when it is next wanted). Its name is a hash of the model, the algorithm and the photo's
+//! content, so a changed file or another model simply does not find it. The Denoise *amount* is an ordinary develop
+//! setting (`enhance.denoise`): a render mixes the plain and the denoised picture by it ([`crate::media`]).
+//!
+//! What runs where: the session thread only decides (which photos, which model) and keeps an index of what is ready
+//! ([`crate::media::DenoiseIndex`], memory only). One background thread at a time decodes a photo, runs the model on
+//! its mosaic tile by tile and writes the product ([`make_product`]); an export that finds its photo's product missing
+//! makes it itself, on its own thread, so it never comes out without the denoise it was asked for.
+//!
+//! Failure is a message, never a panic: a file that is not a Bayer raw, a model that does not load, a full disk, a
+//! panic inside the runtime and a cancelled job all end as an [`Outcome`] the pump records.
+
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError};
+use std::time::{Duration, SystemTime};
+
+use lightcraft_catalog::{Photo, PhotoId, Source};
+use lightcraft_denoise::bayer::Layout;
+use lightcraft_denoise::manifest::DenoiserManifest;
+use lightcraft_denoise::run::{Control, Params, TileRunner, denoise_bayer};
+use lightcraft_denoise::{Error as RunError, product};
+use lightcraft_develop::DevelopSettings;
+use lightcraft_preview::Hasher128;
+use serde::{Deserialize, Serialize};
+use serde_json::{Value, json};
+
+use crate::media::{DenoiseSpec, MakeProduct, PairLoader};
+use crate::merge::ByteReader;
+use crate::model_download::Downloads;
+use crate::{Selection, Session};
+
+/// Extension of a product file.
+pub const EXT: &str = "lcdn";
+/// Where the products of a library are kept, inside the library's folder (safe to delete, like `thumbs/`).
+pub const DIR: &str = "denoise";
+/// What the cache may take up unless the user says otherwise (GB). A product is about as large as the raw file's
+/// decoded samples, compressed: ~100 MB for 24 megapixels.
+pub const DEFAULT_CACHE_GB: u32 = 20;
+/// Samples at or above this (white = 1) are clipped: the model leaves them as the camera recorded them.
+const CLIP: f32 = 0.99;
+/// Most photos waiting in the queue.
+const MAX_QUEUE: usize = 5000;
+/// Most selected photos the automatic queue looks at in one pump.
+const AUTO_LOOK: usize = 64;
+/// How long a look at the model settings is trusted.
+const SETTINGS_TTL: Duration = Duration::from_secs(2);
+/// Stack of the threads that run the model (the runtime recurses through its graph).
+const STACK: usize = 16 << 20;
+
+// ---------------------------------------------------------------------------------------------------------------------
+// the model
+
+/// Where the model runs (Settings ▸ AI Denoise ▸ Speed; `denoise.settings {runOn}`).
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum RunOn {
+ /// The graphics card where it is faster than the processor at the pace the work goes at (both are timed on this
+ /// computer when the card is set up), else the processor.
+ #[default]
+ Auto,
+ /// The graphics card whenever it can run the model, even where the processor would be faster.
+ Gpu,
+ /// Always the processor.
+ Cpu,
+}
+
+impl RunOn {
+ /// `auto`, `gpu` or `cpu`.
+ pub fn parse(name: &str) -> Option<RunOn> {
+ match name {
+ "auto" => Some(RunOn::Auto),
+ "gpu" => Some(RunOn::Gpu),
+ "cpu" => Some(RunOn::Cpu),
+ _ => None,
+ }
+ }
+
+ pub fn name(self) -> &'static str {
+ match self {
+ RunOn::Auto => "auto",
+ RunOn::Gpu => "gpu",
+ RunOn::Cpu => "cpu",
+ }
+ }
+}
+
+/// A loaded denoise model: what runs tiles, and a self-check.
+pub(crate) trait Model: Send + Sync {
+ /// What runs a photo's tiles, and how many to run at once, when `threads` may be used and the setting is `run_on`
+ /// (a graphics card is kept busy by a few). The first time the card may be used it is set up and checked here.
+ fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize);
+ /// Check it does something sensible: the test's result, or why not. Unless `run_on` is the processor the graphics
+ /// card is set up and checked too, and the result says where the work will run.
+ fn self_test(&self, run_on: RunOn) -> Result<Value, String>;
+ /// Where the tiles run: `{kind: "gpu", adapter}`, `{kind: "cpu", reason?}` or `{kind: "pending"}` (the card is not
+ /// set up yet), with the card's and the processor's time for the check tile (`cardMs`, `cpuMs`) once both were
+ /// timed. Never starts anything.
+ fn device(&self, _run_on: RunOn) -> Value {
+ json!({"kind": "cpu"})
+ }
+}
+
+/// Loads the model at a path as its manifest describes it.
+pub(crate) type Loader = Arc<dyn Fn(&Path, &DenoiserManifest) -> Result<Arc<dyn Model>, String> + Send + Sync>;
+
+/// Tiles run at once on a graphics card: enough to keep it fed while others are packed and blended.
+#[cfg(feature = "denoise")]
+const GPU_PARALLEL: usize = 4;
+/// How far the GPU's answer on the check tile may be from the CPU's (of the answer's largest value) before it is not used.
+#[cfg(feature = "denoise")]
+const GPU_AGREE: f32 = 1e-3;
+/// Longest the card may take to set up (build its kernels, run the check tile, time it) before the processor does the
+/// work instead.
+#[cfg(feature = "denoise")]
+const GPU_SETUP_LIMIT: Duration = Duration::from_secs(30);
+/// Tiles the card may get wrong (an error, numbers that are not finite) before it stops being used for the model.
+#[cfg(feature = "denoise")]
+const GPU_MAX_FALLBACKS: usize = 3;
+/// The file beside a model while the graphics card is set up for it (see [`set_up_card`]).
+pub(crate) const GPU_SETUP_MARKER: &str = "gpu-setup.marker";
+/// Why the card is not tried when a set-up marker is found.
+#[cfg(feature = "denoise")]
+const GPU_CRASHED: &str = "LightCraft closed while it was setting up the graphics card for AI Denoise last time";
+/// Why the card is not used when its set-up ran out of time (the start of it).
+#[cfg(feature = "denoise")]
+const GPU_SLOW: &str = "setting up the graphics card took longer than";
+/// Why the card is not used while a set-up that ran out of time is still going.
+#[cfg(feature = "denoise")]
+const GPU_STUCK: &str = "the graphics card is still setting up from before (it took too long)";
+
+/// A set-up that may work another time (a crash, the time limit), unlike one the card or the model cannot do.
+#[cfg(feature = "denoise")]
+fn worth_retrying(why: &str) -> bool {
+ why == GPU_CRASHED || why == GPU_STUCK || why.starts_with(GPU_SLOW)
+}
+
+/// The model on our pure-Rust CPU runner, and on the graphics card when there is one that can run it: the card is set up for the
+/// first photo that may use it, checked against the CPU's answer on a check tile and timed against it, and used while
+/// it keeps working; a tile it gets wrong is run on the CPU, so a photo is always finished.
+#[cfg(feature = "denoise")]
+struct Network {
+ cpu: lightcraft_denoise::runtime::CpuRunner,
+ path: PathBuf,
+ tile: usize,
+ gpu: OnceLock<Result<GpuSide, String>>,
+ /// Tiles the card got wrong (an error, numbers that are not finite) and the CPU ran.
+ fallbacks: AtomicUsize,
+ /// Tiles at once of the photo started last (0: none yet), to say where the work runs.
+ last_threads: AtomicUsize,
+}
+
+#[cfg(feature = "denoise")]
+struct GpuSide {
+ runner: Box<dyn TileRunner + Send>,
+ adapter: String,
+ /// The check tile on the card, the fastest of a few runs (ms).
+ card_ms: f64,
+ /// The check tile on the CPU, on one thread (ms).
+ cpu_ms: f64,
+}
+
+/// Whether the card (`card_ms` a tile) is faster than the CPU running `threads` tiles at once, from the CPU's time for
+/// one tile alone (`cpu_ms`). Tiles at once share the memory and the cores' second threads, so each one more counts as
+/// half a tile's speed: on a 16-core laptop one tile alone took 1.4 s and sixteen at once finished one every 0.2 s.
+#[cfg(feature = "denoise")]
+fn card_is_faster(card_ms: f64, cpu_ms: f64, threads: usize) -> bool {
+ let at_once = 1.0 + 0.5 * threads.saturating_sub(1) as f64;
+ card_ms.is_finite() && card_ms <= cpu_ms / at_once
+}
+
+/// Card set-ups of this process that have not finished, by marker (their markers are not a crash's).
+#[cfg(feature = "denoise")]
+static SETTING_UP: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
+
+/// Removes a set-up's marker when the set-up ends, however it ends — but not when the process dies inside the driver.
+#[cfg(feature = "denoise")]
+struct SetUpEnded(PathBuf);
+
+#[cfg(feature = "denoise")]
+impl Drop for SetUpEnded {
+ fn drop(&mut self) {
+ let _ = std::fs::remove_file(&self.0);
+ SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner).retain(|m| m != &self.0);
+ }
+}
+
+/// Set the graphics card up with `card` on a thread of its own while `meanwhile` runs here, and give up on the card when
+/// it takes longer than `limit` (a driver building kernels for minutes, or stuck). The `marker` file exists while `card`
+/// runs: one found before it starts was left by a LightCraft that closed during a set-up (a driver crash takes the app
+/// with it), so the card is not tried again until the user chooses where denoise runs (`denoise.settings {runOn}`).
+#[cfg(feature = "denoise")]
+fn set_up_card<T: Send + 'static, R>(
+ marker: &Path,
+ limit: Duration,
+ card: impl FnOnce() -> Result<T, String> + Send + 'static,
+ meanwhile: impl FnOnce() -> R,
+) -> Result<(T, R), String> {
+ let ended = {
+ let mut running = SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner);
+ if running.iter().any(|m| m == marker) {
+ return Err(GPU_STUCK.into());
+ }
+ if marker.exists() {
+ return Err(GPU_CRASHED.into());
+ }
+ let secs = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map_or(0, |d| d.as_secs());
+ // a marker that cannot be written only loses the crash guard
+ let _ = std::fs::write(marker, format!("setting up the graphics card for AI Denoise (unix time {secs})\n"));
+ running.push(marker.to_path_buf());
+ SetUpEnded(marker.to_path_buf())
+ };
+ let started = std::time::Instant::now();
+ let (tx, rx) = std::sync::mpsc::channel();
+ std::thread::Builder::new()
+ .name("denoise-gpu-setup".into())
+ .stack_size(STACK)
+ .spawn(move || {
+ let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(card)).unwrap_or_else(|_| Err("the GPU runner gave up".into()));
+ drop(ended);
+ let _ = tx.send(r);
+ })
+ .map_err(|e| format!("could not start setting up the graphics card: {e}"))?;
+ let here = meanwhile();
+ match rx.recv_timeout(limit.saturating_sub(started.elapsed())) {
+ Ok(r) => r.map(|t| (t, here)),
+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(format!("{GPU_SLOW} {} s, so it is not used", limit.as_secs_f64())),
+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err("the GPU runner gave up".into()),
+ }
+}
+
+/// The card's time for one tile once it is warm: the fastest of up to three runs, stopping after about two seconds (a
+/// slow card is slow enough to tell at once).
+#[cfg(feature = "denoise")]
+fn card_tile_ms(runner: &dyn TileRunner, input: &[f32]) -> f64 {
+ if runner.run(input).is_err() {
+ return f64::INFINITY;
+ }
+ let began = std::time::Instant::now();
+ let mut best = f64::INFINITY;
+ for _ in 0..3 {
+ let one = std::time::Instant::now();
+ if runner.run(input).is_err() {
+ break;
+ }
+ best = best.min(one.elapsed().as_secs_f64() * 1000.0);
+ if began.elapsed() > Duration::from_secs(2) {
+ break;
+ }
+ }
+ best
+}
+
+/// Set the card up for the model at `path`: kernels built, the check tile run on both and compared, both timed.
+#[cfg(feature = "denoise")]
+fn make_gpu(cpu: &lightcraft_denoise::runtime::CpuRunner, path: &Path, tile: usize) -> Result<GpuSide, String> {
+ let input = Arc::new(lightcraft_denoise::runtime::check_tile(tile));
+ let (on_card, net) = (input.clone(), cpu.net().clone());
+ let card = move || -> Result<_, String> {
+ let runner = lightcraft_gpu::nn::runner(&net, tile)?;
+ let got = runner.run(&on_card).map_err(|e| format!("the GPU could not run the check tile: {e}"))?;
+ let card_ms = card_tile_ms(&runner, &on_card);
+ Ok((runner, got, card_ms))
+ };
+ let on_cpu = || {
+ let started = std::time::Instant::now();
+ (cpu.run(&input), started.elapsed().as_secs_f64() * 1000.0)
+ };
+ let ((runner, got, card_ms), (want, cpu_ms)) = set_up_card(&path.with_file_name(GPU_SETUP_MARKER), GPU_SETUP_LIMIT, card, on_cpu)?;
+ let want = want.map_err(|e| format!("the CPU could not check the GPU: {e}"))?;
+ let scale = want.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-12);
+ let worst = want.iter().zip(&got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
+ if got.len() != want.len() || !got.iter().all(|v| v.is_finite()) || worst > GPU_AGREE {
+ return Err(format!("its answer on the check tile is not the CPU's (off by {worst:.1e} of the largest value): not used"));
+ }
+ Ok(GpuSide { adapter: runner.adapter(), runner: Box::new(runner), card_ms, cpu_ms })
+}
+
+#[cfg(feature = "denoise")]
+impl Network {
+ /// The card when it can be used now, else why not. `start` sets it up when that has not been tried yet; without it
+ /// nothing is started.
+ fn card(&self, start: bool) -> Result<&GpuSide, String> {
+ let side = if start {
+ Some(self.gpu.get_or_init(|| {
+ let made = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_gpu(&self.cpu, &self.path, self.tile)))
+ .unwrap_or_else(|_| Err("the GPU runner gave up".into()));
+ if let Err(why) = &made {
+ log::info!("denoise: running on the CPU ({why})");
+ }
+ made
+ }))
+ } else {
+ self.gpu.get()
+ };
+ let g = match side {
+ None => return Err("the graphics card is not set up yet".into()),
+ Some(Err(why)) => return Err(why.clone()),
+ Some(Ok(g)) => g,
+ };
+ if let Some(why) = lightcraft_gpu::nn::broken_reason() {
+ return Err(format!("the graphics card stopped working ({why})"));
+ }
+ if self.fallbacks.load(Ordering::Relaxed) >= GPU_MAX_FALLBACKS {
+ return Err(format!("{} got {GPU_MAX_FALLBACKS} tiles wrong", g.adapter));
+ }
+ Ok(g)
+ }
+
+ /// The card, when a photo made `threads` tiles at a time under `run_on` runs on it; else why the CPU does.
+ fn choose(&self, run_on: RunOn, threads: usize, start: bool) -> Result<&GpuSide, String> {
+ if run_on == RunOn::Cpu {
+ return Err("the processor is chosen in Settings".into());
+ }
+ let g = self.card(start)?;
+ if run_on == RunOn::Auto && !card_is_faster(g.card_ms, g.cpu_ms, threads) {
+ return Err(format!("it is faster than {} on this computer", g.adapter));
+ }
+ Ok(g)
+ }
+
+ fn fell_back(&self, why: &str) {
+ let n = self.fallbacks.fetch_add(1, Ordering::Relaxed) + 1;
+ if n == 1 {
+ log::warn!("denoise: a tile came back wrong from the GPU ({why}); the CPU runs it");
+ } else if n == GPU_MAX_FALLBACKS {
+ log::warn!("denoise: the GPU got {n} tiles wrong; the CPU does the rest");
+ }
+ }
+}
+
+/// The card, with the CPU running any tile it gets wrong. Handed out only for photos that run on the card.
+#[cfg(feature = "denoise")]
+impl TileRunner for Network {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, RunError> {
+ if let Ok(g) = self.card(false) {
+ match g.runner.run(input) {
+ Ok(out) if out.iter().all(|v| v.is_finite()) => return Ok(out),
+ Ok(_) => self.fell_back("numbers that are not finite"),
+ Err(e) => self.fell_back(&e.to_string()),
+ }
+ }
+ self.cpu.run(input)
+ }
+}
+
+#[cfg(feature = "denoise")]
+impl Model for Network {
+ fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
+ let threads = threads.max(1);
+ self.last_threads.store(threads, Ordering::Relaxed);
+ match self.choose(run_on, threads, true) {
+ Ok(_) => (self, threads.min(GPU_PARALLEL)),
+ Err(_) => (&self.cpu, threads),
+ }
+ }
+
+ fn self_test(&self, run_on: RunOn) -> Result<Value, String> {
+ let t = self.cpu.self_test();
+ if !t.ok {
+ let failed: Vec<&str> = t.checks.iter().filter(|(_, ok)| !*ok).map(|(c, _)| c.as_str()).collect();
+ return Err(format!("the model failed its self-test: it does not {}", failed.join(", and does not ")));
+ }
+ let mut v = serde_json::to_value(&t).map_err(|e| e.to_string())?;
+ // the card is set up and checked here, so the test says where the work will run
+ if run_on != RunOn::Cpu {
+ let _ = self.card(true);
+ }
+ if let Some(o) = v.as_object_mut() {
+ o.insert("device".into(), self.device(run_on));
+ }
+ Ok(v)
+ }
+
+ fn device(&self, run_on: RunOn) -> Value {
+ if run_on != RunOn::Cpu && self.gpu.get().is_none() {
+ return json!({"kind": "pending"});
+ }
+ let threads = match self.last_threads.load(Ordering::Relaxed) {
+ 0 => Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None),
+ n => n,
+ };
+ let mut v = match self.choose(run_on, threads, false) {
+ Ok(g) => json!({"kind": "gpu", "adapter": g.adapter}),
+ // a set-up that crashed or ran out of time can be tried again (`denoise.settings {runOn}`)
+ Err(why) => json!({"kind": "cpu", "retry": worth_retrying(&why), "reason": why}),
+ };
+ if let (Some(Ok(g)), Some(o)) = (self.gpu.get(), v.as_object_mut()) {
+ // measured on this computer, on the check tile: the card's best run, and one tile alone on the CPU
+ o.insert("cardMs".into(), json!(g.card_ms));
+ o.insert("cpuMs".into(), json!(g.cpu_ms));
+ o.insert("fellBack".into(), json!(self.fallbacks.load(Ordering::Relaxed)));
+ }
+ v
+ }
+}
+
+/// The loader of this build: our pure-Rust executor with the `denoise` feature, else one that says why nothing can run.
+pub(crate) fn default_loader() -> Loader {
+ #[cfg(feature = "denoise")]
+ {
+ Arc::new(|path, manifest| {
+ let cpu = lightcraft_denoise::runtime::CpuRunner::load(path, manifest).map_err(|e| e.to_string())?;
+ Ok(Arc::new(Network {
+ cpu,
+ path: path.to_path_buf(),
+ tile: manifest.tile as usize,
+ gpu: OnceLock::new(),
+ fallbacks: AtomicUsize::new(0),
+ last_threads: AtomicUsize::new(0),
+ }) as Arc<dyn Model>)
+ })
+ }
+ #[cfg(not(feature = "denoise"))]
+ {
+ Arc::new(|_, _| Err("this build cannot run denoise models".to_string()))
+ }
+}
+
+/// A model that is loaded on first use and kept, shared by the background job and the exports.
+pub(crate) type ModelSlot = Arc<Mutex<Option<Arc<dyn Model>>>>;
+
+// ---------------------------------------------------------------------------------------------------------------------
+// settings and installed models
+
+/// `settings.json` in the models folder.
+#[derive(Clone, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+pub(crate) struct Settings {
+ /// The installed model in use; `None`: denoise is off.
+ pub model: Option<String>,
+ /// Photos with a Denoise amount above 0 that are being looked at (active, selected) get their picture made without
+ /// being asked (`None`: yes).
+ pub auto: Option<bool>,
+ /// Cache size limit (GB; `None`: [`DEFAULT_CACHE_GB`]).
+ pub cache_gb: Option<u32>,
+ /// Most tiles run at once (`None`: by how hard the pump says to work).
+ pub threads: Option<u32>,
+ /// Where the model runs: `auto`, `gpu` or `cpu` (`None` or anything else: `auto`; see [`RunOn`]).
+ pub run_on: Option<String>,
+ /// What came before `runOn`: `false` was the processor only.
+ pub gpu: Option<bool>,
+}
+
+impl Settings {
+ pub fn auto(&self) -> bool {
+ self.auto.unwrap_or(true)
+ }
+
+ pub fn run_on(&self) -> RunOn {
+ match self.run_on.as_deref().and_then(RunOn::parse) {
+ Some(r) => r,
+ None if self.gpu == Some(false) => RunOn::Cpu,
+ None => RunOn::Auto,
+ }
+ }
+
+ pub fn cache_bytes(&self) -> u64 {
+ u64::from(self.cache_gb.unwrap_or(DEFAULT_CACHE_GB).clamp(1, 10_000)) << 30
+ }
+}
+
+pub(crate) fn read_capped(path: &Path, max: usize) -> Option<Vec<u8>> {
+ use std::io::Read;
+ let mut buf = Vec::new();
+ std::fs::File::open(path).ok()?.take(max as u64 + 1).read_to_end(&mut buf).ok()?;
+ (buf.len() <= max).then_some(buf)
+}
+
+pub(crate) fn read_settings(dir: &Path) -> Settings {
+ read_capped(&dir.join("settings.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
+}
+
+/// Write a file next to its final name and rename it into place.
+pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> {
+ let part = path.with_extension("part");
+ std::fs::write(&part, bytes).map_err(|e| format!("could not write {}: {e}", path.display()))?;
+ std::fs::rename(&part, path).map_err(|e| {
+ let _ = std::fs::remove_file(&part);
+ format!("could not write {}: {e}", path.display())
+ })
+}
+
+pub(crate) fn write_settings(dir: &Path, st: &Settings) -> Result<(), String> {
+ std::fs::create_dir_all(dir).map_err(|e| format!("could not create the denoise models folder: {e}"))?;
+ write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| e.to_string())?)
+}
+
+/// An installed model folder: `<id>/model.onnx`, `denoise-model.json`, `installed.json`.
+pub(crate) struct Installed {
+ pub manifest: DenoiserManifest,
+ pub onnx: PathBuf,
+ /// What was recorded when it was installed (terms accepted, hash of the file, self-test).
+ pub accepted: Value,
+}
+
+impl Installed {
+ /// Names the model file the cache is made with: a hash of the file as installed, so another file with the same id
+ /// never finds this one's products.
+ pub fn fingerprint(&self) -> String {
+ let sha = self.accepted.get("sha256").and_then(Value::as_str).or(self.manifest.sha256.as_deref()).unwrap_or("");
+ let len = std::fs::metadata(&self.onnx).map(|m| m.len()).unwrap_or(0);
+ format!("{}@{}:{len}", self.manifest.id, sha)
+ }
+}
+
+/// The models in the folder. A folder that is not a valid model (wrong name, no model file, a manifest that does not
+/// validate or does not match its folder) is ignored, never trusted.
+pub(crate) fn installed_models(dir: &Path) -> Vec<Installed> {
+ let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
+ let mut out = Vec::new();
+ for e in entries.flatten() {
+ let p = e.path();
+ let Some(name) = p.file_name().and_then(|n| n.to_str()).map(str::to_string) else { continue };
+ let onnx = p.join("model.onnx");
+ if !lightcraft_denoise::licence::valid_id(&name) || !p.is_dir() || !onnx.is_file() {
+ continue;
+ }
+ let Some(m) = read_capped(&p.join("denoise-model.json"), lightcraft_denoise::manifest::MAX_MANIFEST_BYTES)
+ .and_then(|b| lightcraft_denoise::manifest::parse(&b).ok())
+ else {
+ continue;
+ };
+ if m.id != name {
+ continue;
+ }
+ let accepted = read_capped(&p.join("installed.json"), 8192).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null);
+ out.push(Installed { manifest: m, onnx, accepted });
+ }
+ out.sort_by(|a, b| a.manifest.id.cmp(&b.manifest.id));
+ out
+}
+
+/// Forget that setting the graphics card up for a model closed LightCraft before (the markers [`set_up_card`] leaves):
+/// the user chose where denoise runs, so the card may be tried again.
+pub(crate) fn forget_gpu_set_ups(dir: &Path) {
+ for i in installed_models(dir) {
+ let _ = std::fs::remove_file(i.onnx.with_file_name(GPU_SETUP_MARKER));
+ }
+}
+
+/// The model in use.
+pub(crate) struct Active {
+ pub id: String,
+ pub manifest: DenoiserManifest,
+ pub onnx: PathBuf,
+ pub fingerprint: String,
+ pub model: ModelSlot,
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// products
+
+/// The photos whose files can be denoised at all: raw files developed from their sensor data and read from disk.
+pub(crate) fn eligible(p: &Photo) -> bool {
+ p.develops_raw() && matches!(p.source, Source::File { .. }) && !p.deleted
+}
+
+/// A product's name: what it was made from and with.
+pub(crate) fn product_key(fingerprint: &str, p: &Photo) -> String {
+ let mut h = Hasher128::new();
+ h.str("lcdn").u64(u64::from(product::ALGORITHM)).str(fingerprint).str(&crate::media::content_key(p));
+ h.finish().to_string()
+}
+
+pub(crate) fn product_path(dir: &Path, key: &str) -> PathBuf {
+ dir.join(format!("{key}.{EXT}"))
+}
+
+/// The `.lcdn` files in `dir`: path, size, last change.
+fn products_in(dir: &Path) -> Vec<(PathBuf, u64, SystemTime)> {
+ let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
+ entries
+ .flatten()
+ .filter(|e| e.path().extension().is_some_and(|x| x == EXT))
+ .filter_map(|e| {
+ let m = e.metadata().ok()?;
+ m.is_file().then(|| (e.path(), m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH)))
+ })
+ .collect()
+}
+
+/// Make sure only one thread writes a product at a time: a second one waits, then finds it made.
+fn making() -> &'static (Mutex<HashSet<PathBuf>>, Condvar) {
+ static MAKING: OnceLock<(Mutex<HashSet<PathBuf>>, Condvar)> = OnceLock::new();
+ MAKING.get_or_init(|| (Mutex::new(HashSet::new()), Condvar::new()))
+}
+
+struct Slot(PathBuf);
+
+impl Slot {
+ fn enter(path: &Path) -> Slot {
+ let (set, cv) = making();
+ let mut g = set.lock().unwrap_or_else(PoisonError::into_inner);
+ while g.contains(path) {
+ g = cv.wait(g).unwrap_or_else(PoisonError::into_inner);
+ }
+ g.insert(path.to_path_buf());
+ Slot(path.to_path_buf())
+ }
+}
+
+impl Drop for Slot {
+ fn drop(&mut self) {
+ let (set, cv) = making();
+ set.lock().unwrap_or_else(PoisonError::into_inner).remove(&self.0);
+ cv.notify_all();
+ }
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// making a product
+
+/// Everything a thread needs to make one product, detached from the session.
+#[derive(Clone)]
+pub(crate) struct JobSpec {
+ /// The original file.
+ pub path: String,
+ pub read: Option<ByteReader>,
+ pub product: PathBuf,
+ pub key: String,
+ pub manifest: DenoiserManifest,
+ pub onnx: PathBuf,
+ pub model: ModelSlot,
+ pub loader: Loader,
+ /// Tiles run at once.
+ pub parallel: usize,
+ /// Where the model runs (the setting).
+ pub run_on: RunOn,
+}
+
+/// Why a product was not made.
+#[derive(Debug, PartialEq)]
+pub(crate) enum MakeError {
+ /// Denoise does not apply to this file (not a Bayer raw): nothing is wrong.
+ Unsupported(String),
+ Failed(String),
+ Cancelled,
+}
+
+impl From<RunError> for MakeError {
+ fn from(e: RunError) -> Self {
+ match e {
+ RunError::Cancelled => MakeError::Cancelled,
+ other => MakeError::Failed(other.to_string()),
+ }
+ }
+}
+
+/// Progress and cancel of a running job.
+#[derive(Default)]
+pub(crate) struct Progress {
+ done: AtomicUsize,
+ total: AtomicUsize,
+ cancel: AtomicBool,
+}
+
+impl Progress {
+ /// Ask the job to stop at the next tile.
+ pub(crate) fn cancel_now(&self) {
+ self.cancel.store(true, Ordering::Relaxed);
+ }
+}
+
+/// The loaded model, loading it first when it is not (kept for the next photo).
+fn loaded(spec: &JobSpec) -> Result<Arc<dyn Model>, MakeError> {
+ if let Some(m) = spec.model.lock().unwrap_or_else(PoisonError::into_inner).clone() {
+ return Ok(m);
+ }
+ let m = (spec.loader)(&spec.onnx, &spec.manifest).map_err(|e| MakeError::Failed(format!("the denoise model could not be loaded: {e}")))?;
+ *spec.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(m.clone());
+ Ok(m)
+}
+
+/// The cell `[(0,0), (1,0), (0,1), (1,1)]` colours of a Bayer mosaic and so its layout.
+fn layout_of(cfa: &lightcraft_raw::Cfa) -> Option<Layout> {
+ Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)])
+}
+
+/// Make the product of `spec` unless it is there: decode the photo, run the model over its mosaic, write the result.
+/// `Ok(true)` when it was made now, `Ok(false)` when it already existed.
+pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Result<bool, MakeError> {
+ let _slot = Slot::enter(&spec.product);
+ if product::is_current(&spec.product, &spec.key) {
+ return Ok(false);
+ }
+ let began = std::time::Instant::now();
+ let cancelled = || progress.is_some_and(|p| p.cancel.load(Ordering::Relaxed));
+ if cancelled() {
+ return Err(MakeError::Cancelled);
+ }
+ let bytes = match &spec.read {
+ Some(r) => r(&spec.path).map_err(MakeError::Failed)?,
+ None => std::fs::read(&spec.path).map_err(|e| MakeError::Failed(format!("{}: {e}", spec.path)))?,
+ };
+ // the decode and the normalised mosaic wait their turn at the process-wide memory gate, as every decode does
+ let (mosaic, layout) = {
+ let _held = crate::memory::work_gate().acquire(bytes.len().saturating_mul(4));
+ if lightcraft_raw::probe(&bytes).is_none() {
+ return Err(MakeError::Unsupported("not a raw file".into()));
+ }
+ let raw = match lightcraft_raw::decode(&bytes) {
+ Ok(r) => r,
+ Err(lightcraft_raw::RawError::Unsupported(why)) => return Err(MakeError::Unsupported(why)),
+ Err(e) => return Err(MakeError::Failed(e.to_string())),
+ };
+ drop(bytes);
+ if raw.cpp != 1 || !raw.cfa.as_ref().is_some_and(lightcraft_raw::Cfa::is_bayer) {
+ return Err(MakeError::Unsupported(
+ "only Bayer raw files can be denoised (this one has another sensor layout or is already demosaiced)".into(),
+ ));
+ }
+ let n = raw.normalized().map_err(|e| MakeError::Failed(e.to_string()))?;
+ drop(raw);
+ let layout =
+ n.cfa.as_ref().and_then(layout_of).ok_or_else(|| MakeError::Unsupported("the colour filter layout is not a Bayer pattern".into()))?;
+ (n, layout)
+ };
+ if cancelled() {
+ return Err(MakeError::Cancelled);
+ }
+ let decoded = began.elapsed();
+ let model = loaded(spec)?;
+ let (runner, parallel) = model.runner(spec.run_on, spec.parallel.max(1));
+ let params = Params {
+ tile: spec.manifest.tile as usize,
+ overlap: spec.manifest.overlap as usize,
+ gain: spec.manifest.gain,
+ clip: Some(CLIP),
+ parallel: parallel.max(1),
+ };
+ let report = |done: usize, total: usize| {
+ if let Some(p) = progress {
+ p.total.store(total, Ordering::Relaxed);
+ p.done.store(done, Ordering::Relaxed);
+ }
+ };
+ let ctl = Control { cancel: progress.map(|p| &p.cancel), progress: Some(&report) };
+ // the tiles run on threads of their own, so a busy window's renders keep theirs
+ let pool = rayon::ThreadPoolBuilder::new()
+ .num_threads(params.parallel)
+ .stack_size(STACK)
+ .thread_name(|i| format!("denoise-{i}"))
+ .build()
+ .map_err(|e| MakeError::Failed(format!("could not start the worker threads: {e}")))?;
+ let rgb = pool.install(|| denoise_bayer(&mosaic.data, mosaic.width, mosaic.height, layout, runner, &params, &ctl))?;
+ drop(mosaic);
+ if cancelled() {
+ return Err(MakeError::Cancelled);
+ }
+ let ran = began.elapsed();
+ product::write(&spec.product, &rgb, &spec.key).map_err(|e| MakeError::Failed(format!("could not save the denoised picture: {e}")))?;
+ if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
+ eprintln!(
+ "[denoise] read + decode {:.0} ms, model (set-up, tiles, blend) {:.0} ms, write {:.0} ms",
+ decoded.as_secs_f64() * 1e3,
+ (ran - decoded).as_secs_f64() * 1e3,
+ (began.elapsed() - ran).as_secs_f64() * 1e3
+ );
+ }
+ Ok(true)
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// the session's state
+
+/// How hard the background work may go (the app picks it from what the user is doing, like the face scan).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Pace {
+ /// Start nothing new.
+ Pause,
+ /// Two tiles at once.
+ Light,
+ /// Half the machine.
+ Normal,
+ /// Most of the machine: the user is watching the progress.
+ Full,
+}
+
+impl Pace {
+ /// `pause`, `light`, `normal` or `full`; anything else is `normal`.
+ pub fn parse(name: Option<&str>) -> Pace {
+ match name {
+ Some("pause") => Pace::Pause,
+ Some("light") => Pace::Light,
+ Some("full") => Pace::Full,
+ _ => Pace::Normal,
+ }
+ }
+
+ /// Tiles to run at once on a machine with `cores` threads (`cap`: the user's own limit).
+ pub fn parallel(self, cores: usize, cap: Option<u32>) -> usize {
+ let n = match self {
+ Pace::Pause | Pace::Light => 2,
+ Pace::Normal => cores / 2,
+ Pace::Full => cores * 4 / 5,
+ }
+ .clamp(1, 16);
+ cap.map_or(n, |c| n.min(c.max(1) as usize))
+ }
+}
+
+/// How a photo stands, for the interface.
+#[derive(Clone, Debug, PartialEq, Serialize)]
+#[serde(tag = "state", rename_all = "camelCase")]
+pub enum PhotoState {
+ /// Not a photo denoise applies to (not a raw, offline, a demo).
+ NotApplicable,
+ /// No model is chosen: nothing can be made.
+ NoModel,
+ /// Its picture is made: the Amount slider works.
+ Ready,
+ Queued {
+ /// Photos ahead of it (0 = next).
+ ahead: usize,
+ },
+ Running {
+ done: usize,
+ total: usize,
+ },
+ /// Making it failed (`why`), or denoise does not apply to this file (`unsupported`).
+ Failed {
+ why: String,
+ unsupported: bool,
+ },
+ /// Nothing has asked for it yet.
+ Idle,
+}
+
+#[derive(Clone)]
+struct Failure {
+ key: String,
+ why: String,
+ unsupported: bool,
+}
+
+enum Outcome {
+ Done,
+ Failed(String, bool),
+ Cancelled,
+}
+
+struct Running {
+ photo: PhotoId,
+ key: String,
+ product: PathBuf,
+ progress: Arc<Progress>,
+ outcome: Arc<Mutex<Option<Outcome>>>,
+ started: web_time::Instant,
+}
+
+pub(crate) struct State {
+ /// Where the models are kept (`None` where there is no file system).
+ pub models_dir: Option<PathBuf>,
+ /// Models being downloaded at the user's request.
+ pub downloads: Downloads,
+ /// How a model is loaded (tests put another in).
+ pub loader: Loader,
+ pub(crate) active: Option<Active>,
+ active_seen: Option<web_time::Instant>,
+ /// `settings.json` as last read (with the model choice).
+ pub(crate) settings: Settings,
+ /// The index must be made again (the library, the model or the cache changed).
+ dirty: bool,
+ /// The catalog revision and time of the last index, to look again when photos changed.
+ reindexed: Option<(u64, web_time::Instant)>,
+ queue: VecDeque<PhotoId>,
+ running: Option<Running>,
+ failed: HashMap<PhotoId, Failure>,
+ /// Counts every change of what is ready, so the interface knows to draw again.
+ pub generation: u64,
+ /// Photos made since the session started.
+ made: u64,
+}
+
+impl Default for State {
+ fn default() -> Self {
+ State {
+ models_dir: None,
+ downloads: Downloads::default(),
+ loader: default_loader(),
+ active: None,
+ active_seen: None,
+ settings: Settings::default(),
+ dirty: true,
+ reindexed: None,
+ queue: VecDeque::new(),
+ running: None,
+ failed: HashMap::new(),
+ generation: 0,
+ made: 0,
+ }
+ }
+}
+
+impl Drop for State {
+ /// Quitting stops the job instead of leaving it to finish a photo nobody will see.
+ fn drop(&mut self) {
+ if let Some(r) = &self.running {
+ r.progress.cancel_now();
+ }
+ }
+}
+
+impl State {
+ /// The library was replaced: nothing carries over (photo ids are per library).
+ pub fn library_changed(&mut self) {
+ if let Some(r) = self.running.take() {
+ r.progress.cancel_now();
+ }
+ self.queue.clear();
+ self.failed.clear();
+ self.dirty = true;
+ self.generation += 1;
+ }
+
+ /// The model choice or the cache changed on disk: look again at the next pump.
+ pub fn touch(&mut self) {
+ self.active_seen = None;
+ self.dirty = true;
+ }
+
+ pub fn queued(&self) -> usize {
+ self.queue.len()
+ }
+}
+
+impl Session {
+ /// Where the active model's tiles run (see [`Model::device`]); `{kind: "none"}` while no model is loaded.
+ pub(crate) fn denoise_device(&self) -> Value {
+ let loaded = self.denoise.active.as_ref().and_then(|a| a.model.lock().unwrap_or_else(PoisonError::into_inner).clone());
+ match loaded {
+ Some(m) => m.device(self.denoise.settings.run_on()),
+ None => json!({"kind": "none"}),
+ }
+ }
+
+ /// The folder the products of the open library are kept in; `None` without a library on disk.
+ pub(crate) fn denoise_products_dir(&self) -> Option<PathBuf> {
+ self.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join(DIR))
+ }
+
+ /// Read which model is in use (at most every couple of seconds): the manifest and file of the model the settings
+ /// name, if it is installed.
+ pub(crate) fn denoise_refresh_active(&mut self, force: bool) {
+ let fresh = self.denoise.active_seen.is_some_and(|t| t.elapsed() < SETTINGS_TTL);
+ if fresh && !force {
+ return;
+ }
+ self.denoise.active_seen = Some(web_time::Instant::now());
+ self.denoise.settings = self.denoise.models_dir.as_deref().map(read_settings).unwrap_or_default();
+ let chosen = self.denoise.models_dir.as_deref().and_then(|dir| {
+ let id = self.denoise.settings.model.clone()?;
+ installed_models(dir).into_iter().find(|i| i.manifest.id == id)
+ });
+ let found = chosen.map(|i| (i.fingerprint(), i));
+ let same = match (&self.denoise.active, &found) {
+ (None, None) => true,
+ (Some(a), Some((fp, _))) => a.fingerprint == *fp,
+ _ => false,
+ };
+ if same {
+ return;
+ }
+ // another model: what was made with the old one is not this one's, and what failed may work now
+ self.denoise.active = found.map(|(fingerprint, i)| Active {
+ id: i.manifest.id.clone(),
+ manifest: i.manifest,
+ onnx: i.onnx,
+ fingerprint,
+ model: Arc::new(Mutex::new(None)),
+ });
+ if let Some(r) = self.denoise.running.take() {
+ r.progress.cancel_now();
+ }
+ self.denoise.queue.clear();
+ self.denoise.failed.clear();
+ self.denoise.dirty = true;
+ }
+
+ fn denoise_spec(&self, p: &Photo) -> Option<(DenoiseSpec, String)> {
+ let (dir, active) = (self.denoise_products_dir()?, self.denoise.active.as_ref()?);
+ let key = product_key(&active.fingerprint, p);
+ Some((DenoiseSpec { product: product_path(&dir, &key), key: key.clone(), make: None }, key))
+ }
+
+ /// Make the index of ready photos from the products on disk (a directory listing and a hash per photo: no file is
+ /// opened). Pictures that appeared or vanished drop the decoded sources they affect.
+ pub(crate) fn denoise_reindex(&mut self) {
+ self.denoise.dirty = false;
+ self.denoise.reindexed = Some((self.catalog.revision, web_time::Instant::now()));
+ let mut now: HashMap<PhotoId, DenoiseSpec> = HashMap::new();
+ if let (Some(dir), Some(active)) = (self.denoise_products_dir(), self.denoise.active.as_ref()) {
+ let names: HashSet<String> =
+ products_in(&dir).into_iter().filter_map(|(p, _, _)| p.file_stem().and_then(|s| s.to_str()).map(str::to_string)).collect();
+ if !names.is_empty() {
+ for p in self.catalog.photos().filter(|p| eligible(p)) {
+ let key = product_key(&active.fingerprint, p);
+ if names.contains(&key) {
+ now.insert(p.id, DenoiseSpec { product: product_path(&dir, &key), key, make: None });
+ }
+ }
+ }
+ }
+ let mut changed = false;
+ for id in self.media.denoise.ids() {
+ if !now.contains_key(&id) {
+ self.media.denoise.remove(id);
+ self.media.forget(id);
+ changed = true;
+ }
+ }
+ for (id, spec) in now {
+ if self.media.denoise.spec(id).is_none_or(|old| old.key != spec.key) {
+ self.media.denoise.set(id, spec);
+ self.media.forget(id);
+ changed = true;
+ }
+ }
+ if changed {
+ self.denoise.generation += 1;
+ }
+ }
+
+ /// A photo whose file changed (its content key is not the one its index entry was made for) has no picture.
+ fn denoise_check_photo(&mut self, id: PhotoId) {
+ let Some(p) = self.catalog.photo(id).cloned() else { return };
+ let Some((spec, _)) = self.denoise_spec(&p) else { return };
+ match self.media.denoise.spec(id) {
+ Some(have) if have.key != spec.key => {
+ self.media.denoise.remove(id);
+ self.media.forget(id);
+ self.denoise.generation += 1;
+ }
+ None if spec.product.is_file() && product::is_current(&spec.product, &spec.key) => {
+ // made by an export, or by another session
+ self.media.denoise.set(id, spec);
+ self.media.forget(id);
+ self.denoise.generation += 1;
+ }
+ _ => {}
+ }
+ }
+
+ /// How photo `id` stands.
+ pub fn denoise_photo_state(&self, id: PhotoId) -> PhotoState {
+ let Some(p) = self.catalog.photo(id) else { return PhotoState::NotApplicable };
+ if !eligible(p) {
+ return PhotoState::NotApplicable;
+ }
+ if self.denoise.active.is_none() {
+ // the model folder has not been looked at yet (just started): not "no model" until it has been
+ return if self.denoise.active_seen.is_none() { PhotoState::Idle } else { PhotoState::NoModel };
+ }
+ if self.denoise_products_dir().is_none() {
+ return PhotoState::Failed {
+ why: "AI Denoise keeps its pictures in the library folder, and this session has none.".into(),
+ unsupported: true,
+ };
+ }
+ if self.media.denoise.spec(id).is_some() {
+ return PhotoState::Ready;
+ }
+ if let Some(r) = self.denoise.running.as_ref().filter(|r| r.photo == id) {
+ return PhotoState::Running { done: r.progress.done.load(Ordering::Relaxed), total: r.progress.total.load(Ordering::Relaxed) };
+ }
+ if let Some(ahead) = self.denoise.queue.iter().position(|q| *q == id) {
+ return PhotoState::Queued { ahead: ahead + usize::from(self.denoise.running.is_some()) };
+ }
+ let key = self.denoise_spec(p).map(|s| s.1);
+ match self.denoise.failed.get(&id) {
+ Some(f) if Some(&f.key) == key.as_ref() => PhotoState::Failed { why: f.why.clone(), unsupported: f.unsupported },
+ _ => PhotoState::Idle,
+ }
+ }
+
+ /// Ask for photos' pictures to be made (the ones that are not made, queued or known to be impossible). With `retry`
+ /// a photo that failed before is tried again. A photo that is not raw is skipped. Returns how many were queued.
+ pub(crate) fn denoise_enqueue(&mut self, ids: &[PhotoId], front: bool, retry: bool) -> usize {
+ self.denoise_refresh_active(false);
+ if self.denoise.active.is_none() || self.denoise_products_dir().is_none() {
+ return 0;
+ }
+ let mut added = 0;
+ for id in ids {
+ let Some(p) = self.catalog.photo(*id).cloned() else { continue };
+ if !eligible(&p) || self.media.denoise.spec(*id).is_some() || self.denoise.queue.contains(id) {
+ continue;
+ }
+ if self.denoise.running.as_ref().is_some_and(|r| r.photo == *id) || self.denoise.queue.len() >= MAX_QUEUE {
+ continue;
+ }
+ let Some((_, key)) = self.denoise_spec(&p) else { continue };
+ if retry {
+ self.denoise.failed.remove(id);
+ } else if self.denoise.failed.get(id).is_some_and(|f| f.key == key) {
+ continue;
+ }
+ if front {
+ self.denoise.queue.push_front(*id);
+ } else {
+ self.denoise.queue.push_back(*id);
+ }
+ added += 1;
+ }
+ added
+ }
+
+ /// Stop the job that is running, and with `all` forget everything that waits.
+ pub(crate) fn denoise_cancel(&mut self, photo: Option<PhotoId>, all: bool) -> usize {
+ let mut n = 0;
+ if let Some(r) = &self.denoise.running
+ && photo.is_none_or(|p| p == r.photo)
+ {
+ r.progress.cancel_now();
+ n += 1;
+ }
+ let before = self.denoise.queue.len();
+ match photo {
+ Some(p) => self.denoise.queue.retain(|q| *q != p),
+ None if all => self.denoise.queue.clear(),
+ None => {}
+ }
+ n + before - self.denoise.queue.len()
+ }
+
+ /// The photos the automatic queue looks at: the one open and the ones selected, those with a Denoise amount.
+ fn denoise_wanted(&self) -> Vec<PhotoId> {
+ let Selection { active, ids, .. } = &self.selection;
+ active
+ .iter()
+ .chain(ids.iter().take(AUTO_LOOK))
+ .copied()
+ .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0))
+ .collect()
+ }
+
+ fn denoise_job_spec(&self, p: &Photo, parallel: usize) -> Option<JobSpec> {
+ let Source::File { path } = &p.source else { return None };
+ let (spec, _) = self.denoise_spec(p)?;
+ let active = self.denoise.active.as_ref()?;
+ Some(JobSpec {
+ path: path.clone(),
+ read: self.media.file_bytes.clone(),
+ product: spec.product,
+ key: spec.key,
+ manifest: active.manifest.clone(),
+ onnx: active.onnx.clone(),
+ model: active.model.clone(),
+ loader: self.denoise.loader.clone(),
+ parallel,
+ run_on: self.denoise.settings.run_on(),
+ })
+ }
+
+ /// Start the next queued photo on its own thread.
+ fn denoise_start_next(&mut self, parallel: usize) {
+ while let Some(id) = self.denoise.queue.pop_front() {
+ let Some(p) = self.catalog.photo(id).cloned() else { continue };
+ if !eligible(&p) || self.media.denoise.spec(id).is_some() {
+ continue;
+ }
+ let Some(spec) = self.denoise_job_spec(&p, parallel) else { continue };
+ // another session or an export may have made it since it was queued
+ if product::is_current(&spec.product, &spec.key) {
+ self.denoise_check_photo(id);
+ continue;
+ }
+ let progress = Arc::new(Progress::default());
+ let outcome: Arc<Mutex<Option<Outcome>>> = Arc::new(Mutex::new(None));
+ let (key, product) = (spec.key.clone(), spec.product.clone());
+ let (worker_progress, worker_outcome) = (progress.clone(), outcome.clone());
+ let started = std::thread::Builder::new().name("denoise".into()).stack_size(STACK).spawn(move || {
+ let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_product(&spec, Some(&worker_progress))));
+ let o = match result {
+ Ok(Ok(_)) => Outcome::Done,
+ Ok(Err(MakeError::Cancelled)) => Outcome::Cancelled,
+ Ok(Err(MakeError::Unsupported(why))) => Outcome::Failed(why, true),
+ Ok(Err(MakeError::Failed(why))) => Outcome::Failed(why, false),
+ Err(_) => Outcome::Failed("the denoiser stopped unexpectedly".into(), false),
+ };
+ *worker_outcome.lock().unwrap_or_else(PoisonError::into_inner) = Some(o);
+ });
+ match started {
+ Ok(_) => {
+ self.denoise.running = Some(Running { photo: id, key, product, progress, outcome, started: web_time::Instant::now() });
+ }
+ Err(e) => {
+ self.denoise.failed.insert(id, Failure { key, why: format!("could not start the denoiser: {e}"), unsupported: false });
+ continue;
+ }
+ }
+ return;
+ }
+ }
+
+ /// Take in the running job's result.
+ fn denoise_poll(&mut self) {
+ let Some(r) = &self.denoise.running else { return };
+ let Some(outcome) = r.outcome.lock().unwrap_or_else(PoisonError::into_inner).take() else { return };
+ let Some(r) = self.denoise.running.take() else { return };
+ match outcome {
+ Outcome::Done => {
+ self.denoise.made += 1;
+ self.denoise_check_photo(r.photo);
+ self.denoise_evict(Some(&r.product));
+ }
+ Outcome::Failed(why, unsupported) => {
+ self.denoise.failed.insert(r.photo, Failure { key: r.key, why, unsupported });
+ }
+ Outcome::Cancelled => {}
+ }
+ // the picture's state changed: the interface draws it again
+ self.denoise.generation += 1;
+ }
+
+ /// Keep the cache within its limit: the pictures nobody asks for go first, the oldest first; `keep` (the one just
+ /// made) never goes. The index forgets what was deleted.
+ pub(crate) fn denoise_evict(&mut self, keep: Option<&Path>) {
+ let Some(models) = self.denoise.models_dir.clone() else { return };
+ self.denoise_evict_to(read_settings(&models).cache_bytes(), keep);
+ }
+
+ /// [`Self::denoise_evict`] for a limit of `limit` bytes.
+ pub(crate) fn denoise_evict_to(&mut self, limit: u64, keep: Option<&Path>) {
+ let Some(dir) = self.denoise_products_dir() else { return };
+ let mut files = products_in(&dir);
+ let mut total: u64 = files.iter().map(|f| f.1).sum();
+ if total <= limit {
+ return;
+ }
+ // the pictures photos use now (a Denoise amount above 0) are the last to go
+ let wanted: HashSet<PathBuf> = self
+ .media
+ .denoise
+ .ids()
+ .into_iter()
+ .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0))
+ .filter_map(|id| self.media.denoise.spec(id).map(|s| s.product.clone()))
+ .collect();
+ files.sort_by_key(|(path, _, modified)| (wanted.contains(path), *modified));
+ let mut gone: HashSet<PathBuf> = HashSet::new();
+ for (path, size, _) in files {
+ if total <= limit {
+ break;
+ }
+ if keep == Some(path.as_path()) {
+ continue;
+ }
+ if std::fs::remove_file(&path).is_ok() {
+ total = total.saturating_sub(size);
+ gone.insert(path);
+ }
+ }
+ if gone.is_empty() {
+ return;
+ }
+ for id in self.media.denoise.ids() {
+ if self.media.denoise.spec(id).is_some_and(|s| gone.contains(&s.product)) {
+ self.media.denoise.remove(id);
+ self.media.forget(id);
+ }
+ }
+ self.denoise.generation += 1;
+ }
+
+ /// Delete every product of this library (they are made again when wanted).
+ pub(crate) fn denoise_clear(&mut self) -> (usize, u64) {
+ self.denoise_cancel(None, true);
+ let (mut files, mut bytes) = (0, 0);
+ if let Some(dir) = self.denoise_products_dir() {
+ for (path, size, _) in products_in(&dir) {
+ if std::fs::remove_file(&path).is_ok() {
+ files += 1;
+ bytes += size;
+ }
+ }
+ }
+ self.denoise.failed.clear();
+ self.denoise.touch();
+ (files, bytes)
+ }
+
+ /// Per frame: take in what finished, look at what the user is looking at, and keep one job going. Cheap when
+ /// nothing is happening (no file is opened unless a photo with a Denoise amount lacks its picture).
+ pub(crate) fn denoise_pump(&mut self, pace: Pace) {
+ crate::cmd::denoise::finish_downloads(self);
+ self.denoise_refresh_active(false);
+ // photos that changed (a replaced file has another key) must not keep the picture of what they were
+ let stale = !self.media.denoise.is_empty()
+ && self.denoise.reindexed.is_none_or(|(rev, at)| rev != self.catalog.revision && at.elapsed() > SETTINGS_TTL);
+ if self.denoise.dirty || stale {
+ self.denoise_reindex();
+ }
+ self.denoise_poll();
+ if self.denoise.active.is_none() || self.denoise_products_dir().is_none() {
+ return;
+ }
+ let (cap, auto) = (self.denoise.settings.threads, self.denoise.settings.auto());
+ let wanted = self.denoise_wanted();
+ for id in &wanted {
+ self.denoise_check_photo(*id);
+ }
+ if auto {
+ // the photo being edited goes first
+ let (open, others): (Vec<PhotoId>, Vec<PhotoId>) = wanted.into_iter().partition(|id| self.selection.active == Some(*id));
+ self.denoise_enqueue(&others, false, false);
+ self.denoise_enqueue(&open, true, false);
+ }
+ if self.denoise.running.is_none() && pace != Pace::Pause {
+ let cores = std::thread::available_parallelism().map_or(1, |n| n.get());
+ self.denoise_start_next(pace.parallel(cores, cap));
+ }
+ }
+
+ /// The statistics `denoise.status` reports besides the queue (reads the cache folder).
+ pub(crate) fn denoise_cache_usage(&self) -> (usize, u64) {
+ match self.denoise_products_dir() {
+ Some(dir) => {
+ let files = products_in(&dir);
+ (files.len(), files.iter().map(|f| f.1).sum())
+ }
+ None => (0, 0),
+ }
+ }
+
+ /// Are pictures made without being asked for photos being looked at (Settings ▸ AI Denoise)?
+ pub fn denoise_auto(&self) -> bool {
+ self.denoise.settings.auto()
+ }
+
+ /// Is a picture being made, or waiting for its turn with a model to make it? (Headless runs wait for this.)
+ pub fn denoise_busy(&self) -> bool {
+ self.denoise.running.is_some() || (!self.denoise.queue.is_empty() && self.denoise.active.is_some())
+ }
+
+ /// Seconds the running job has been going, and its progress.
+ pub(crate) fn denoise_running_json(&self) -> Value {
+ match &self.denoise.running {
+ Some(r) => json!({
+ "photo": r.photo.0,
+ "done": r.progress.done.load(Ordering::Relaxed),
+ "total": r.progress.total.load(Ordering::Relaxed),
+ "seconds": r.started.elapsed().as_secs_f64(),
+ }),
+ None => Value::Null,
+ }
+ }
+
+ pub(crate) fn denoise_made(&self) -> u64 {
+ self.denoise.made
+ }
+
+ /// What an export of photo `id` with `settings` needs for its Denoise amount: its denoised picture, made now when the
+ /// queue has not got to it. `Ok(None)` when denoise does not apply (amount 0, not a Bayer raw), `Err` when it should
+ /// have and cannot (no model chosen, nowhere to keep the picture, the model failing).
+ pub(crate) fn denoise_for_export(&mut self, id: PhotoId, settings: &DevelopSettings) -> Result<Option<(DenoiseSpec, PairLoader)>, String> {
+ if settings.denoise_amount() <= 0.0 {
+ return Ok(None);
+ }
+ let Some(p) = self.catalog.photo(id).cloned() else { return Ok(None) };
+ let Some(loader) = self.media.denoise.loader.clone() else { return Ok(None) };
+ if !eligible(&p) {
+ return Ok(None);
+ }
+ self.denoise_refresh_active(false);
+ let name = &p.file_name;
+ if self.denoise.active.is_none() {
+ return Err(format!("{name}: Denoise is on for this photo but no denoise model is chosen; choose one in Settings, or set Denoise to 0"));
+ }
+ if self.denoise_products_dir().is_none() {
+ return Err(format!("{name}: Denoise needs a library on disk to keep its pictures in"));
+ }
+ self.denoise_check_photo(id);
+ let Some(job) = self.denoise_job_spec(&p, Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None)) else {
+ return Ok(None);
+ };
+ let label = name.clone();
+ let spec = job.clone();
+ let make: MakeProduct = Arc::new(move || match make_product(&spec, None) {
+ Ok(_) => Ok(true),
+ Err(MakeError::Unsupported(_)) => Ok(false),
+ Err(MakeError::Failed(why)) => Err(format!("{label}: could not denoise: {why}")),
+ Err(MakeError::Cancelled) => Err(format!("{label}: denoise was cancelled")),
+ });
+ Ok(Some((DenoiseSpec { product: job.product, key: job.key, make: Some(make) }, loader)))
+ }
+}
+
+#[cfg(all(test, feature = "denoise"))]
+mod gpu_tests {
+ use super::*;
+ use lightcraft_denoise::manifest::{Domain, Gain};
+
+ /// A folder of the test's own: the card's set-up leaves its marker beside the model.
+ fn folder(name: &str) -> PathBuf {
+ let dir = std::env::temp_dir().join(format!("lc-engine-gpu-{}-{name}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ dir
+ }
+
+ /// The model file is read again when the card is set up (for the first photo), so it stays until the test ends.
+ fn model(name: &str, tile: u64) -> (Arc<dyn Model>, lightcraft_denoise::runtime::CpuRunner, PathBuf) {
+ let path = folder(name).join("model.onnx");
+ std::fs::write(&path, lightcraft_denoise::synthetic::unet_onnx(tile, 8, 2, 7)).unwrap();
+ let manifest = DenoiserManifest {
+ id: "synthetic".into(),
+ name: "Synthetic".into(),
+ version: "1".into(),
+ licence: Default::default(),
+ source: None,
+ sha256: None,
+ size_bytes: None,
+ provenance: String::new(),
+ domain: Domain::BayerToRgb,
+ tile: tile as u32,
+ overlap: 16,
+ gain: Gain::MatchMean { nominal: 1.0, max_deviation: 0.05 },
+ };
+ let loaded = default_loader()(&path, &manifest).unwrap();
+ let cpu = lightcraft_denoise::runtime::CpuRunner::load(&path, &manifest).unwrap();
+ (loaded, cpu, path)
+ }
+
+ fn worst(want: &[f32], got: &[f32]) -> f32 {
+ assert_eq!(want.len(), got.len());
+ let scale = want.iter().fold(1e-9f32, |m, v| m.max(v.abs()));
+ want.iter().zip(got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale
+ }
+
+ #[test]
+ fn a_tile_on_the_card_is_the_cpus_tile() {
+ let (model, cpu, path) = model("same", 64);
+ let (input, _) = lightcraft_denoise::runtime::test_tile(64);
+ let want = cpu.run(&input).unwrap();
+ let (runner, at_once) = model.runner(RunOn::Gpu, 16);
+ assert!(worst(&want, &runner.run(&input).unwrap()) < 1e-3);
+ let device = model.device(RunOn::Gpu);
+ eprintln!("device: {device}");
+ match device["kind"].as_str() {
+ Some("gpu") => {
+ assert!(device["adapter"].as_str().is_some_and(|a| !a.is_empty()));
+ assert!(at_once <= GPU_PARALLEL, "a card is fed by a few tiles at once");
+ assert!(device["cardMs"].as_f64().is_some_and(|ms| ms > 0.0) && device["cpuMs"].as_f64().is_some_and(|ms| ms > 0.0), "{device}");
+ }
+ Some("cpu") => {
+ assert!(device["reason"].as_str().is_some_and(|r| !r.is_empty()), "the CPU is only used for a reason: {device}");
+ assert_eq!(at_once, 16);
+ }
+ other => panic!("the first photo decides where the work runs, not {other:?}"),
+ }
+ assert!(!path.with_file_name(GPU_SETUP_MARKER).exists(), "a set-up that ended leaves no marker");
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ }
+
+ #[test]
+ fn with_the_processor_chosen_everything_runs_on_the_cpu() {
+ let (model, cpu, path) = model("cpu", 64);
+ let (input, _) = lightcraft_denoise::runtime::test_tile(64);
+ let (runner, at_once) = model.runner(RunOn::Cpu, 16);
+ assert_eq!(runner.run(&input).unwrap(), cpu.run(&input).unwrap(), "the CPU path is the CPU runner, bit for bit");
+ assert_eq!(at_once, 16, "the threads asked for are the threads used");
+ assert_eq!(model.device(RunOn::Cpu)["kind"], "cpu");
+ assert_eq!(model.device(RunOn::Auto)["kind"], "pending", "the card was not even set up");
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ }
+
+ #[test]
+ fn the_card_is_used_where_it_is_faster_than_the_processor() {
+ // measured: a tile alone on one CPU thread 1.4 s; an RTX 4090 Laptop 13 ms; a Radeon 610M 490 ms
+ for threads in [1, 2, 8, 16, 64] {
+ assert!(card_is_faster(13.0, 1400.0, threads), "a big card wins at {threads}");
+ }
+ assert!(card_is_faster(490.0, 1400.0, 2), "a small card beats the two threads background work gets");
+ assert!(!card_is_faster(490.0, 1400.0, 16), "and loses to sixteen");
+ assert!(!card_is_faster(f64::INFINITY, 1400.0, 1) && !card_is_faster(f64::NAN, 1400.0, 1), "a card that never ran is never faster");
+ assert!(!card_is_faster(10.0, 0.0, 1));
+ }
+
+ /// A card that gives numbers that are not finite.
+ struct Garbage(AtomicUsize);
+
+ impl TileRunner for Garbage {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, RunError> {
+ self.0.fetch_add(1, Ordering::Relaxed);
+ let n = input.len() / 4 * 12;
+ Ok(vec![f32::NAN; n])
+ }
+ }
+
+ #[test]
+ fn a_tile_the_card_gets_wrong_is_run_on_the_processor_and_three_stop_the_card() {
+ let (_, cpu, path) = model("garbage", 64);
+ let card = GpuSide { runner: Box::new(Garbage(AtomicUsize::new(0))), adapter: "Test card".into(), card_ms: 1.0, cpu_ms: 1000.0 };
+ let model = Network {
+ cpu: cpu.clone(),
+ path: path.clone(),
+ tile: 32,
+ gpu: OnceLock::from(Ok(card)),
+ fallbacks: AtomicUsize::new(0),
+ last_threads: AtomicUsize::new(0),
+ };
+ let (input, _) = lightcraft_denoise::runtime::test_tile(64);
+ let want = cpu.run(&input).unwrap();
+ let (runner, at_once) = model.runner(RunOn::Auto, 8);
+ assert_eq!(at_once, GPU_PARALLEL, "the (fast) card is chosen");
+ for _ in 0..GPU_MAX_FALLBACKS {
+ assert_eq!(runner.run(&input).unwrap(), want, "a wrong tile is the CPU's tile");
+ }
+ let device = model.device(RunOn::Auto);
+ assert_eq!(device["kind"], "cpu", "{device}");
+ assert!(device["reason"].as_str().unwrap().contains("tiles wrong"), "{device}");
+ assert_eq!(device["fellBack"], GPU_MAX_FALLBACKS);
+ let (_, at_once) = model.runner(RunOn::Gpu, 8);
+ assert_eq!(at_once, 8, "the next photo runs on the CPU, even with the card chosen");
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ }
+
+ #[test]
+ fn a_slow_card_is_left_out_unless_it_is_chosen() {
+ let (_, cpu, path) = model("slow", 64);
+ let slow = GpuSide { runner: Box::new(cpu.clone()), adapter: "Slow card".into(), card_ms: 490.0, cpu_ms: 1400.0 };
+ let model = Network {
+ cpu,
+ path: path.clone(),
+ tile: 32,
+ gpu: OnceLock::from(Ok(slow)),
+ fallbacks: AtomicUsize::new(0),
+ last_threads: AtomicUsize::new(0),
+ };
+ assert_eq!(model.runner(RunOn::Auto, 16).1, 16, "sixteen threads beat it");
+ let device = model.device(RunOn::Auto);
+ assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some("it is faster than Slow card on this computer")));
+ assert_eq!(device["cardMs"], 490.0, "the times measured are reported");
+ assert_eq!(device["retry"], false, "trying again would not make it faster");
+ assert_eq!(model.runner(RunOn::Auto, 2).1, GPU_PARALLEL.min(2), "two threads do not");
+ assert_eq!(model.device(RunOn::Auto)["kind"], "gpu");
+ assert_eq!(model.runner(RunOn::Gpu, 16).1, GPU_PARALLEL, "chosen, it is used anyway");
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ }
+
+ #[test]
+ fn a_set_up_marker_left_behind_keeps_the_card_off() {
+ let (model, _, path) = model("crashed", 64);
+ let marker = path.with_file_name(GPU_SETUP_MARKER);
+ std::fs::write(&marker, "setting up the graphics card for AI Denoise (unix time 0)").unwrap();
+ let (_, at_once) = model.runner(RunOn::Gpu, 4);
+ assert_eq!(at_once, 4, "the CPU does the work");
+ let device = model.device(RunOn::Gpu);
+ assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some(GPU_CRASHED)), "{device}");
+ assert_eq!(device["retry"], true, "the interface offers to try again");
+ assert!(marker.exists(), "it stays until the user chooses where denoise runs");
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ }
+
+ #[test]
+ fn a_set_up_runs_beside_the_cpu_and_removes_its_marker() {
+ let dir = folder("setup");
+ let marker = dir.join(GPU_SETUP_MARKER);
+ let seen = Arc::new(AtomicBool::new(false));
+ let (m, s) = (marker.clone(), seen.clone());
+ let r = set_up_card(
+ &marker,
+ Duration::from_secs(10),
+ move || {
+ s.store(m.exists(), Ordering::Relaxed);
+ Ok(7)
+ },
+ || "cpu",
+ );
+ assert_eq!(r, Ok((7, "cpu")));
+ assert!(seen.load(Ordering::Relaxed), "the marker is there while the card is set up");
+ assert!(!marker.exists(), "and gone after");
+ // a set-up that fails or panics removes it too
+ assert_eq!(set_up_card(&marker, Duration::from_secs(10), || Err::<(), _>("no card".into()), || ()), Err("no card".into()));
+ assert!(!marker.exists());
+ let r = set_up_card(&marker, Duration::from_secs(10), || -> Result<(), String> { panic!("driver") }, || ());
+ assert!(r.is_err() && !marker.exists(), "{r:?}");
+ // a marker found before the set-up starts means the last one took LightCraft down: the card is not touched
+ std::fs::write(&marker, "left behind").unwrap();
+ let touched = Arc::new(AtomicBool::new(false));
+ let t = touched.clone();
+ let r = set_up_card(
+ &marker,
+ Duration::from_secs(10),
+ move || {
+ t.store(true, Ordering::Relaxed);
+ Ok(())
+ },
+ || (),
+ );
+ assert_eq!(r, Err(GPU_CRASHED.to_string()));
+ assert!(!touched.load(Ordering::Relaxed) && marker.exists());
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn a_set_up_that_takes_too_long_is_given_up_on() {
+ let dir = folder("slowsetup");
+ let marker = dir.join(GPU_SETUP_MARKER);
+ let started = std::time::Instant::now();
+ let slow = || -> Result<(), String> {
+ std::thread::sleep(Duration::from_millis(600));
+ Ok(())
+ };
+ let r = set_up_card(&marker, Duration::from_millis(50), slow, || ());
+ assert!(r.as_ref().is_err_and(|e| e.contains("took longer") && worth_retrying(e)), "{r:?}");
+ assert!(started.elapsed() < Duration::from_millis(500), "the caller does not wait for the card");
+ // while the first is still stuck, another set-up of the same model does not start a second one
+ let again = set_up_card(&marker, Duration::from_secs(5), || Ok(()), || ());
+ assert!(again.is_err_and(|e| e.contains("still setting up") && worth_retrying(&e)));
+ // once it ends its marker goes, and the card can be set up again
+ while marker.exists() {
+ assert!(started.elapsed() < Duration::from_secs(10), "the stuck set-up ended");
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ while SETTING_UP.lock().unwrap().iter().any(|m| m == &marker) {
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ assert_eq!(set_up_card(&marker, Duration::from_secs(5), || Ok(1), || 2), Ok((1, 2)));
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+}
diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs
index c620be928..5ef823dbf 100644
--- a/crates/engine/src/files.rs
+++ b/crates/engine/src/files.rs
@@ -11,7 +11,7 @@ use lightcraft_pipeline::SourceInfo;
use lightcraft_raster::Rgb32f;
use lightcraft_raster::resample::{Filter, fit};
-use crate::media::{FileLoader, FileProbe, PreviewLoader, ProbeInfo};
+use crate::media::{DenoiseSpec, FileLoader, FileProbe, PairLoader, PreviewLoader, ProbeInfo};
fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option<String>) {
let shutter = m.exposure_time.map(|t| if t >= 1.0 { format!("{t:.0}") } else { format!("1/{:.0}", 1.0 / t) }).unwrap_or_default();
@@ -220,22 +220,56 @@ pub fn bin_factor(raw: &lightcraft_raw::RawImage, max_edge: usize) -> Option<usi
/// instead of each one waking the pool from outside and waiting for it (which costs more than the
/// loops themselves when the machine is busy).
pub fn load_bytes(bytes: &[u8], max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
- rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge))
+ rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge, None).map(|(img, _, info)| (img, info)))
}
/// [`load_bytes`] taking the file's bytes: a raw file's bytes are freed as soon as it is decoded
/// (less memory held while it is developed).
pub fn load_vec(bytes: Vec<u8>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
- rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge))
+ rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, None).map(|(img, _, info)| (img, info)))
}
-fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
+/// [`load_vec`] that also develops the photo's cached denoised picture (`spec`) from the same decode: the plain
+/// picture, the denoised one (`None` when it cannot be read or does not fit this raw: the plain one is then all
+/// there is) and the source facts. Both pictures are the same size.
+pub fn load_vec_pair(bytes: Vec<u8>, max_edge: usize, spec: &DenoiseSpec) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> {
+ rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, Some(spec)))
+}
+
+/// The denoised camera RGB of `raw` from its cached product, as the plain development would have it: the default
+/// crop, and `factor` × `factor` blocks averaged into one pixel when the plain one was binned. `None` when the product
+/// is missing, damaged, made from something else or not the size of this raw's active area.
+fn denoised_camera_rgb(raw: &lightcraft_raw::RawImage, spec: &DenoiseSpec, factor: usize) -> Option<Rgb32f> {
+ use lightcraft_denoise::product::{self, Window};
+ let a = raw.active_area;
+ let header = product::read_header_at(&spec.product).ok()?;
+ if header.key != spec.key || (header.width, header.height) != (a.width, a.height) {
+ return None;
+ }
+ if raw.opcodes.list3.is_empty() {
+ let c = raw.develop_crop(a.width, a.height);
+ let window = Window { x: c.x, y: c.y, width: c.width, height: c.height };
+ product::read_window(&spec.product, &spec.key, Some(window), factor, Some(HIGHLIGHT_CLIP)).ok()
+ } else {
+ // (the plain development is not binned either when the file has opcodes of this kind: `factor` is 1)
+ let whole = product::read(&spec.product, &spec.key, 1).ok()?;
+ Some(raw.finish_demosaiced(whole))
+ }
+}
+
+fn load_bytes_now(
+ bytes: std::borrow::Cow<'_, [u8]>,
+ max_edge: usize,
+ denoise: Option<&DenoiseSpec>,
+) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> {
if lightcraft_raw::probe(&bytes).is_some() {
let mut raw = match lightcraft_raw::decode(&bytes).map_err(|e| preview_reason(&bytes, e)) {
Ok(r) => r,
Err(Ok(why)) => {
// show the camera's embedded JPEG (rendered, not raw) until the variant is supported
- return load_embedded_preview(&bytes, max_edge).ok_or(format!("unsupported raw ({why}) without an embedded preview"));
+ return load_embedded_preview(&bytes, max_edge)
+ .map(|(img, info)| (img, None, info))
+ .ok_or(format!("unsupported raw ({why}) without an embedded preview"));
}
Err(Err(e)) => return Err(e),
};
@@ -250,71 +284,72 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
// Previews and thumbnails bin the mosaic straight to (about) the size they need; only
// larger levels (exports, 1:1) demosaic the whole sensor.
let t0 = web_time::Instant::now();
- let binned = match bin_factor(&raw, max_edge) {
+ let bin = bin_factor(&raw, max_edge);
+ let binned = match bin {
Some(k) => raw.develop_binned(k, HIGHLIGHT_CLIP).map_err(|e| e.to_string())?,
None => None,
};
- let mut img = match binned {
+ let factor = if binned.is_some() { bin.unwrap_or(1) } else { 1 };
+ let img = match binned {
Some(img) => img,
None => {
let method = if max_edge <= 600 { lightcraft_raw::Method::Bilinear } else { lightcraft_raw::Method::Ahd };
raw.develop(method).map_err(|e| e.to_string())?
}
};
+ // the same development from the denoised mosaic, when the photo has been denoised
+ let twin = denoise.and_then(|spec| denoised_camera_rgb(&raw, spec, factor)).filter(|d| d.width == img.width && d.height == img.height);
// the samples aren't needed any more (the colour model below reads only the tags)
raw.data = lightcraft_raw::RawData::U16(Vec::new());
- let mut stages = vec![("develop", t0.elapsed())];
- stages.push(("transform", t0.elapsed()));
- lightcraft_raw::highlight::reconstruct(&mut img, t.wb, HIGHLIGHT_CLIP);
- stages.push(("highlights", t0.elapsed()));
+ let wb = t.wb;
let m = camera_look.as_ref().map(|p| p.matrix.mul(&t.matrix)).unwrap_or(t.matrix).to_f32();
let hue_sat = camera_look.as_ref().and_then(|p| p.hue_sat.as_ref()).and_then(crate::camera_preview::HueSat::new);
let gain = 2f32.powf(t.baseline_exposure as f32);
- let wb = t.wb;
// A DNG's own profile look (hue/saturation map, look table), DNG spec chapter 6.
let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy));
- img.map_in_place(|p| {
- let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];
- let rgb = [
- m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],
- m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],
- m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],
- ];
- let rgb = match &tables {
- Some(tables) => tables.apply(rgb, gain),
- None => rgb.map(|v| v * gain),
- };
- // after the baseline exposure, as when it was fitted
- hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))
- });
- stages.push(("colour", t0.elapsed()));
- let img = fit(&img, max_edge, max_edge, Filter::Box);
- stages.push(("fit", t0.elapsed()));
- let img = img.into_oriented(raw.orientation);
- stages.push(("orient", t0.elapsed()));
+ // camera RGB → what the pipeline takes: clipped highlights rebuilt, white balance and the colour model applied,
+ // fitted to `max_edge` and upright (the same for the plain and the denoised picture)
+ let finish = |mut img: Rgb32f| -> Rgb32f {
+ lightcraft_raw::highlight::reconstruct(&mut img, wb, HIGHLIGHT_CLIP);
+ img.map_in_place(|p| {
+ let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];
+ let rgb = [
+ m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],
+ m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],
+ m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],
+ ];
+ let rgb = match &tables {
+ Some(tables) => tables.apply(rgb, gain),
+ None => rgb.map(|v| v * gain),
+ };
+ hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))
+ });
+ let img = fit(&img, max_edge, max_edge, Filter::Box);
+ img.into_oriented(raw.orientation)
+ };
+ let develop_ms = t0.elapsed();
+ let img = finish(img);
+ let twin = twin.map(finish);
if lightcraft_pipeline::profiling() {
- let mut prev = std::time::Duration::ZERO;
- let parts: Vec<String> = stages
- .iter()
- .map(|(n, t)| {
- let d = *t - prev;
- prev = *t;
- format!("{n} {:.1}", d.as_secs_f64() * 1e3)
- })
- .collect();
- eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", "));
+ let finish_ms = (t0.elapsed() - develop_ms).as_secs_f64() * 1e3;
+ eprintln!(
+ "[profile] raw source {}×{} (max {max_edge}): develop {:.1} ms, finish {finish_ms:.1} ms (after decode)",
+ img.width,
+ img.height,
+ develop_ms.as_secs_f64() * 1e3
+ );
}
let (temp, tint) = xy_to_temp_tint(xy);
let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback;
let camera_tone = camera_look.as_ref().map(|p| p.tone).or_else(|| raw.color.profile.tone_curve.as_ref().and_then(dng_tone_curve));
let (temp, tint) = if relative { (6500.0, 0.0) } else { (temp.round(), tint.round()) };
- return Ok((img, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone }));
+ return Ok((img, twin, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone }));
}
let d = lightcraft_codecs::decode(&bytes, fit_box(max_edge)).map_err(|e| e.to_string())?;
drop(bytes);
let img = d.to_working();
let img = if img.width.max(img.height) > max_edge { fit(&img, max_edge, max_edge, Filter::Mitchell) } else { img };
- Ok((img.into_oriented(Orientation::from_exif(d.orientation)), SourceInfo::default()))
+ Ok((img.into_oriented(Orientation::from_exif(d.orientation)), None, SourceInfo::default()))
}
/// A DNG `ProfileToneCurve` (linear in, linear out, 1.0 = white after exposure compensation) as the
@@ -408,6 +443,21 @@ pub fn fs_preview_loader() -> PreviewLoader {
Arc::new(|path: &str, max_edge: usize| embedded_preview_srgb(&std::fs::read(path).ok()?, max_edge))
}
+/// The filesystem-backed [`PairLoader`]: the file is read and decoded once, and developed as usual and from the photo's
+/// denoised picture. It holds two pictures at once, so it counts for more at the memory gate than a plain load.
+pub fn fs_pair_loader() -> PairLoader {
+ Arc::new(|path: &str, max_edge: usize, spec: &DenoiseSpec| {
+ let len = std::fs::metadata(path).map(|m| m.len() as usize).unwrap_or(0);
+ let weight = len * if max_edge <= crate::media::SourceLevel::Thumb.max_edge() { 4 } else { 10 };
+ let gate = crate::memory::work_gate();
+ let _permit = if crate::memory::is_background() { gate.acquire(weight) } else { gate.acquire_urgent(weight) };
+ let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?;
+ let r = load_vec_pair(bytes, max_edge, spec);
+ crate::memory::release();
+ r
+ })
+}
+
/// Filesystem-backed hooks (native). On the web the host installs bytes-based hooks instead.
///
/// Their working memory is bounded by [`crate::memory::work_gate`]: background loads (grid
@@ -443,6 +493,7 @@ impl crate::Session {
self.media.file_loader = Some(l);
self.media.file_probe = Some(p);
self.media.preview_loader = Some(fs_preview_loader());
+ self.media.denoise.loader = Some(fs_pair_loader());
self
}
}
@@ -483,6 +534,37 @@ mod tests {
assert!(probe_bytes("x.jpg", b"not an image").is_err());
}
+ #[test]
+ fn cached_denoise_uses_the_same_white_balance_profile_and_orientation_as_plain_raw() {
+ use lightcraft_denoise::product;
+ use lightcraft_raw::{DngWriteOptions, Method, Orientation, profile::HsvTable, write_dng};
+ let dir = crate::tests_xmp::temp_dir("denoise-color-finish");
+ let fixture = crate::tests_xmp::synthetic_dng_with(None, Default::default());
+ let mut raw = lightcraft_raw::decode(&fixture).unwrap();
+ raw.orientation = Orientation::Rotate90;
+ raw.color.baseline_exposure = 0.5;
+ raw.color.profile.look_table =
+ Some(HsvTable { hue_divisions: 1, sat_divisions: 2, val_divisions: 1, data: vec![[23.0, 0.65, 0.8]; 2], srgb_value: false });
+ let bytes = write_dng(&raw, &DngWriteOptions::default()).unwrap();
+ let decoded = lightcraft_raw::decode(&bytes).unwrap();
+ assert!(decoded.color.profile.look_table.is_some());
+ // A cache containing the ordinary demosaic must undergo the identical downstream finish.
+ let camera_rgb = decoded.develop(Method::Ahd).unwrap();
+ let key = "synthetic-color-finish";
+ let path = dir.join("same.lcdn");
+ product::write(&path, &camera_rgb, key).unwrap();
+ let spec = DenoiseSpec { product: path, key: key.into(), make: None };
+ let (ordinary, _) = load_bytes(&bytes, 2048).unwrap();
+ let (plain, cached, _) = load_vec_pair(bytes, 2048, &spec).unwrap();
+ assert_eq!(plain.data, ordinary.data, "requesting a twin must not change the ordinary path");
+ let cached = cached.unwrap();
+ assert_eq!((plain.width, plain.height), (24, 32));
+ assert_eq!((cached.width, cached.height), (plain.width, plain.height));
+ let worst = plain.data.iter().zip(&cached.data).flat_map(|(a, b)| a.iter().zip(b).map(|(a, b)| (a - b).abs())).fold(0.0f32, f32::max);
+ assert!(worst < 0.002, "color finish differs beyond cache half-float rounding: {worst}");
+ let _ = std::fs::remove_dir_all(dir);
+ }
+
/// Luminance (from linear RGB) minus its 9 × 9 local mean: tone differences between a render and a camera JPEG
/// mostly cancel, edges remain.
fn detail(w: usize, h: usize, lum: impl Fn(usize, usize) -> f32) -> Vec<f32> {
diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs
index f26b4ca7a..5f298de4e 100644
--- a/crates/engine/src/lib.rs
+++ b/crates/engine/src/lib.rs
@@ -14,9 +14,11 @@ pub mod availability;
mod camera_preview;
pub mod camera_profiles;
pub mod cmd;
+pub mod config;
pub mod crs;
pub mod crs_masks;
pub mod demo;
+pub mod denoise;
pub mod devices;
pub mod export;
pub mod files;
@@ -33,6 +35,7 @@ pub mod logging;
pub mod media;
pub mod memory;
pub mod merge;
+mod model_download;
pub mod originals;
pub mod preset_import;
pub mod preset_luminar;
@@ -132,8 +135,10 @@ pub struct Interaction {
/// Source of [`Session::visible_shared`] generations (process-wide, so two sessions never share one).
static VISIBLE_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
+static LIBRARY_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
pub struct Session {
+ library_generation: u64,
/// Auto Sync: edits to the active photo also change the other selected photos (the settings
/// that changed, nothing else).
pub auto_sync: bool,
@@ -157,6 +162,8 @@ pub struct Session {
/// Set by a command whose change must not rewrite the photo's XMP sidecar even with auto-write on
/// (a catalog-only edit of data the sidecar writer does not emit); consumed when the command ends.
pub(crate) skip_auto_write: bool,
+ /// AI denoise: the model in use, the photos that have their picture and the work in progress.
+ pub(crate) denoise: denoise::State,
/// Copied develop settings (partial JSON) for Paste.
pub clipboard: Option<Value>,
/// The folder on disk the [`LibrarySource::Folder`] view browses.
@@ -247,8 +254,13 @@ impl Default for Session {
}
impl Session {
+ pub fn library_generation(&self) -> u64 {
+ self.library_generation
+ }
+
pub fn new() -> Session {
Session {
+ library_generation: LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
auto_sync: false,
catalog: Catalog::new(),
source: LibrarySource::All,
@@ -263,6 +275,7 @@ impl Session {
redo: Vec::new(),
interaction: None,
skip_auto_write: false,
+ denoise: Default::default(),
clipboard: None,
meta_clipboard: None,
browse: None,
@@ -783,6 +796,8 @@ mod tests;
#[cfg(test)]
mod tests_color;
#[cfg(test)]
+mod tests_denoise;
+#[cfg(test)]
mod tests_export;
#[cfg(test)]
mod tests_folders;
diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs
index c1b91c4cd..384d90ea3 100644
--- a/crates/engine/src/library.rs
+++ b/crates/engine/src/library.rs
@@ -297,6 +297,9 @@ impl Session {
{
log::error!("library: {e}");
}
+ // Photo ids belong to one library; denoise work must not carry into the next.
+ self.denoise.library_changed();
+ self.media.denoise.clear();
let (mut journal, mut catalog, report) = Journal::open(catalog)?;
// A loaded catalog counts revisions from 0, like every other one. Caches (sidebar counts,
// keyword tree, the grid's list…) are keyed on the revision, so give each library loaded
@@ -305,6 +308,7 @@ impl Session {
static LOADS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
catalog.revision = LOADS.fetch_add(1, std::sync::atomic::Ordering::Relaxed).wrapping_add(1) << 32;
self.catalog = catalog;
+ self.library_generation = crate::LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
self.undo.clear();
self.redo.clear();
self.interaction = None;
diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs
index 1ed89e473..479831c98 100644
--- a/crates/engine/src/media.rs
+++ b/crates/engine/src/media.rs
@@ -125,6 +125,44 @@ pub type FileLoader = Arc<dyn Fn(&str, usize) -> Result<(Rgb32f, SourceInfo), St
/// `max_edge` (set by the app). `None`: no usable preview.
pub type PreviewLoader = Arc<dyn Fn(&str, usize) -> Option<Rgba8> + Send + Sync>;
+/// Makes a photo's denoised picture when it is not cached yet (an export does not wait for the background queue):
+/// `Ok(true)` once it is there, `Ok(false)` when denoise does not apply to this photo (render it plain), `Err` when it
+/// should have worked and did not.
+pub type MakeProduct = Arc<dyn Fn() -> Result<bool, String> + Send + Sync>;
+
+/// A raw photo's cached AI-denoised picture (see [`crate::denoise`]): where the file is and the key it must have been
+/// made with. A file that is missing, damaged or made for something else is a cache miss, never an error.
+#[derive(Clone)]
+pub struct DenoiseSpec {
+ pub product: std::path::PathBuf,
+ pub key: String,
+ /// Run before the picture is read, by a job that must not render without it (an export).
+ pub make: Option<MakeProduct>,
+}
+
+/// Decodes a raw file once and develops it twice: as usual, and from the denoised picture in `spec` when that can be
+/// read (`None` when it cannot). Both are at most `max_edge` long and the same size.
+pub type PairLoader = Arc<dyn Fn(&str, usize, &DenoiseSpec) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> + Send + Sync>;
+
+/// A photo's denoised picture next to its plain one, and the last mix of the two that was asked for.
+pub struct Twin {
+ pub image: Arc<Rgb32f>,
+ /// The Amount (in thousandths) and the picture mixed for it, so dragging other sliders keeps one source picture.
+ mix: std::sync::Mutex<Option<(u32, Arc<Rgb32f>)>>,
+}
+
+impl Twin {
+ pub fn new(image: Arc<Rgb32f>) -> Twin {
+ Twin { image, mix: std::sync::Mutex::new(None) }
+ }
+
+ /// Bytes held, the last mix included.
+ fn bytes(&self) -> usize {
+ let mixed = self.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref().map_or(0, |m| m.1.data.len() * 12);
+ self.image.data.len() * 12 + mixed
+ }
+}
+
/// Pixels and the source interpretation learned while decoding; they must be cached together.
#[derive(Clone)]
pub struct DecodedSource {
@@ -133,11 +171,45 @@ pub struct DecodedSource {
/// A smart preview's stored camera tone curve: the one decoder fact its pixels need that the
/// catalog's header facts lack (used when `info` is `None`).
pub camera_tone: Option<lightcraft_pipeline::tone::CameraTone>,
+ /// The same picture developed from the AI-denoised mosaic, when the photo has one cached.
+ pub denoised: Option<Arc<Twin>>,
}
impl DecodedSource {
pub fn new(image: Arc<Rgb32f>, info: Option<SourceInfo>) -> Self {
- DecodedSource { image, info, camera_tone: None }
+ DecodedSource { image, info, camera_tone: None, denoised: None }
+ }
+
+ /// The picture to develop for a Denoise `amount` (0 to 1, see [`DevelopSettings::denoise_amount`]): the plain one,
+ /// the denoised one, or a mix of the two. A mix is made once per amount and kept, so the stages after it are
+ /// reused until the amount changes.
+ pub fn image_for(&self, amount: f32) -> Arc<Rgb32f> {
+ let Some(twin) = &self.denoised else { return self.image.clone() };
+ if amount.is_nan() || amount <= 0.0 || twin.image.width != self.image.width || twin.image.height != self.image.height {
+ return self.image.clone();
+ }
+ if amount >= 1.0 {
+ return twin.image.clone();
+ }
+ let thousandths = (amount * 1000.0).round().clamp(1.0, 999.0) as u32;
+ let mut memo = twin.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+ if let Some((t, img)) = memo.as_ref()
+ && *t == thousandths
+ {
+ return img.clone();
+ }
+ let mut mixed = (*self.image).clone();
+ if !lightcraft_denoise::product::blend(&mut mixed, &twin.image, thousandths as f32 / 1000.0) {
+ return self.image.clone();
+ }
+ let mixed = Arc::new(mixed);
+ *memo = Some((thousandths, mixed.clone()));
+ mixed
+ }
+
+ /// Bytes held: the picture, its denoised twin and the last mix.
+ pub fn bytes(&self) -> usize {
+ self.image.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>() + self.denoised.as_ref().map_or(0, |t| t.bytes())
}
/// What to render these pixels against: the decoder's facts, else `header` (the catalog's)
@@ -161,6 +233,8 @@ pub enum SourceRef {
/// The photo's smart preview, used when the original can't be read (offline drive). The
/// UI thread never checks whether the original is there; the render worker finds out.
fallback: Option<std::path::PathBuf>,
+ /// The photo has a cached denoised picture: load it with the plain one (see [`PairLoader`]).
+ denoise: Option<(DenoiseSpec, PairLoader)>,
},
/// A smart preview standing in for a missing original.
Smart {
@@ -177,15 +251,45 @@ impl SourceRef {
let image = match self {
SourceRef::Loaded(a) => return Ok((**a).clone()),
SourceRef::Demo { scene, max_edge } => Arc::new(scene.render_fit(*max_edge)),
- SourceRef::File { path, max_edge, loader, fallback } => {
- let r = match loader {
+ SourceRef::File { path, max_edge, loader, fallback, denoise } => {
+ let plain = || match loader {
Some(l) => l(path, *max_edge).map(|(image, info)| DecodedSource::new(Arc::new(image), Some(info))),
None => Err(format!("no decoder available for {path}")),
};
+ // the denoised picture comes with the plain one from a single decode; when it cannot be read the
+ // pair loader gives the plain picture alone (the cached file is never a reason to fail a render)
+ let r = match denoise {
+ Some((spec, pair)) => {
+ // a job that needs the denoised picture has it made first; if that fails the job fails (no smart
+ // preview stands in: an export must not quietly come out without the denoise it was asked for)
+ let made = match &spec.make {
+ Some(make) => make()?,
+ None => true,
+ };
+ if made {
+ pair(path, *max_edge, spec).and_then(|(image, twin, info)| {
+ if spec.make.is_some() && twin.is_none() {
+ return Err("the cached AI Denoise picture could not be read; clear the denoise cache and retry".into());
+ }
+ Ok(DecodedSource {
+ image: Arc::new(image),
+ info: Some(info),
+ camera_tone: None,
+ denoised: twin.map(|t| Arc::new(Twin::new(Arc::new(t)))),
+ })
+ })
+ } else {
+ plain()
+ }
+ }
+ None => plain(),
+ };
// An offline original renders from its smart preview (no decoder facts: header ones).
return match (r, fallback) {
#[cfg(not(target_arch = "wasm32"))]
- (Err(e), Some(sp)) if crate::smart::is_valid(sp) => crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")),
+ (Err(e), Some(sp)) if denoise.as_ref().is_none_or(|(spec, _)| spec.make.is_none()) && crate::smart::is_valid(sp) => {
+ crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}"))
+ }
(r, _) => r,
};
}
@@ -224,6 +328,54 @@ pub struct MediaCache {
scenes: Vec<lightcraft_scenes::Scene>,
/// Rendered thumbnails (memory, plus disk once a library is attached).
pub rendered: Arc<PreviewCache>,
+ /// Which photos have a cached denoised picture, and how to read it.
+ pub denoise: DenoiseIndex,
+}
+
+/// The photos that have a cached denoised picture for the denoise model in use. Only memory: nothing here touches the
+/// file system, so it can be asked on the UI thread.
+#[derive(Default)]
+pub struct DenoiseIndex {
+ ready: std::collections::HashMap<PhotoId, DenoiseSpec>,
+ /// Reads the pictures (`None`: this build cannot, and nothing is ever ready).
+ pub loader: Option<PairLoader>,
+}
+
+impl DenoiseIndex {
+ /// What to load for `id` besides its plain picture, and how.
+ pub fn get(&self, id: PhotoId) -> Option<(&DenoiseSpec, &PairLoader)> {
+ self.ready.get(&id).zip(self.loader.as_ref())
+ }
+
+ pub fn spec(&self, id: PhotoId) -> Option<&DenoiseSpec> {
+ self.ready.get(&id)
+ }
+
+ pub fn set(&mut self, id: PhotoId, spec: DenoiseSpec) {
+ self.ready.insert(id, spec);
+ }
+
+ pub fn remove(&mut self, id: PhotoId) -> bool {
+ self.ready.remove(&id).is_some()
+ }
+
+ pub fn clear(&mut self) {
+ self.ready.clear();
+ }
+
+ pub fn len(&self) -> usize {
+ self.ready.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.ready.is_empty()
+ }
+
+ pub fn ids(&self) -> Vec<PhotoId> {
+ let mut v: Vec<PhotoId> = self.ready.keys().copied().collect();
+ v.sort();
+ v
+ }
}
impl Default for MediaCache {
@@ -244,6 +396,7 @@ impl Default for MediaCache {
availability: Default::default(),
scenes: Vec::new(),
rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))),
+ denoise: DenoiseIndex::default(),
}
}
}
@@ -253,10 +406,6 @@ fn rendered_budget(share: usize) -> usize {
RENDERED_MEM_BYTES.min(share / 4)
}
-fn source_bytes(img: &Rgb32f) -> usize {
- img.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>()
-}
-
impl MediaCache {
/// Keep rendered thumbnails on disk in `dir` as well. The cache this replaces is retired
/// (its files stay): render jobs still holding it can't write through it any more, even
@@ -319,8 +468,8 @@ impl MediaCache {
let Some((tick, which)) = candidates.into_iter().flatten().min() else { break };
let freed = match which {
0 => self.thumbs.pop_oldest(),
- 1 => self.previews.iter().position(|e| e.2 == tick).map(|i| source_bytes(&self.previews.remove(i).1.image)),
- 2 => self.full.take().map(|e| source_bytes(&e.1.image)),
+ 1 => self.previews.iter().position(|e| e.2 == tick).map(|i| self.previews.remove(i).1.bytes()),
+ 2 => self.full.take().map(|e| e.1.bytes()),
_ => self.rendered.evict_oldest(),
};
match freed {
@@ -372,7 +521,7 @@ impl MediaCache {
let tick = lightcraft_preview::next_tick();
match level {
SourceLevel::Thumb => {
- let cost = source_bytes(&img.image);
+ let cost = img.bytes();
self.thumbs.insert(id, img, cost);
}
SourceLevel::Preview => {
@@ -415,13 +564,42 @@ impl MediaCache {
/// Decoded sources held: (thumbnail level, preview level, full size).
pub fn usage(&self) -> (crate::memory::Usage, crate::memory::Usage, crate::memory::Usage) {
use crate::memory::Usage;
- let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| source_bytes(&e.1.image)).sum());
- let full = self.full.as_ref().map(|e| Usage::new(1, source_bytes(&e.1.image))).unwrap_or_default();
+ let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| e.1.bytes()).sum());
+ let full = self.full.as_ref().map(|e| Usage::new(1, e.1.bytes())).unwrap_or_default();
(Usage::new(self.thumbs.len(), self.thumbs.cost()), previews, full)
}
+ /// What a render of photo `id` with settings `s` adds to its identity for the denoised picture it will use: 0 unless
+ /// it uses one (the Denoise amount is above 0 and the photo has a cached denoised picture).
+ pub fn denoise_salt(&self, id: PhotoId, s: &DevelopSettings) -> u64 {
+ match self.denoise.spec(id) {
+ Some(spec) if s.denoise_amount() > 0.0 => Hasher128::new().str(&spec.key).finish().0 as u64 | 1,
+ _ => 0,
+ }
+ }
+
+ /// How to load photo `p` at `level`: from the cache when its source is there, else from its file (with its denoised
+ /// picture beside it when it has one), its smart preview or the demo scene.
pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
- if let Some(a) = self.get_source(p.id, level) {
+ self.source_ref_with(p, level, true)
+ }
+
+ /// [`Self::source_ref`] that never takes a decoded source from memory: for a job that needs something the cached
+ /// one lacks (an export, whose denoised picture is made when it runs).
+ pub fn source_ref_uncached(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
+ self.source_ref_with(p, level, false)
+ }
+
+ fn source_ref_with(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {
+ let mut r = self.source_ref_plain(p, level, cached);
+ if let (SourceRef::File { denoise, .. }, Some((spec, loader))) = (&mut r, self.denoise.get(p.id)) {
+ *denoise = Some((spec.clone(), loader.clone()));
+ }
+ r
+ }
+
+ fn source_ref_plain(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {
+ if cached && let Some(a) = self.get_source(p.id, level) {
return SourceRef::Loaded(Box::new(a));
}
// Procedural scenes have a nominal size: "full" is that size, not unbounded.
@@ -459,10 +637,12 @@ impl MediaCache {
}
match self.scenes.iter().find(|s| s.id == *scene) {
Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge },
- None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None },
+ None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None, denoise: None },
}
}
- Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None },
+ Source::File { path } => {
+ SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None, denoise: None }
+ }
}
}
}
@@ -573,7 +753,8 @@ impl RenderJob {
let was_loaded = matches!(self.source, SourceRef::Loaded(_));
match self.source.load_source() {
Ok(source) => {
- let src = &source.image;
+ // the plain picture, or (Denoise amount above 0 and a denoised picture cached) the mix the amount asks for
+ let src = &source.image_for(self.settings.denoise_amount());
let info = source.info_or(self.info);
// Thumbnails (many small jobs side by side) stay on the CPU; views and exports use
// the GPU when there is one.
@@ -757,22 +938,23 @@ impl crate::Session {
let content = source_key.0 as u64;
// Temporary "before" settings must not replace the photo's cached develop hash.
let settings_hash = if before { settings.hash64() } else { self.media.settings_hashes.get(id, &settings) };
+ let salt = self.media.denoise_salt(id, &settings);
let key = settings_hash
^ ((max_w as u64) << 40)
^ ((max_h as u64) << 20)
^ (apply_crop as u64)
^ (level as u64) << 60
^ id.0.wrapping_mul(0x9e37_79b9_7f4a_7c15)
- ^ content.rotate_left(17);
+ ^ content.rotate_left(17)
+ ^ salt;
let cache = thumb_bucket.map(|b| {
- let k = Hasher128::new()
- .str(&content_key)
- .u64(settings_hash)
- .u64(b as u64)
- .u64(RENDER_CACHE_VERSION)
- .u64(crate::camera_profiles::cache_key())
- .finish();
- (self.media.rendered.clone(), k)
+ let mut k = Hasher128::new();
+ k.str(&content_key).u64(settings_hash).u64(b as u64).u64(RENDER_CACHE_VERSION).u64(crate::camera_profiles::cache_key());
+ // (only a render that uses a denoised picture has it in its key, so every other thumbnail stays valid)
+ if salt != 0 {
+ k.u64(salt);
+ }
+ (self.media.rendered.clone(), k.finish())
});
Some(RenderJob {
request_id: 0,
@@ -1052,6 +1234,15 @@ impl crate::Session {
depth: lightcraft_pipeline::OutputDepth,
) -> Result<RenderJob, String> {
let mut job = self.render_job(id, max_w, max_h, false, true).ok_or("no such photo")?;
+ // An export is made from the denoised picture its Denoise amount asks for, made when the job runs if the
+ // background queue has not got to it. (A source kept in memory may lack the picture, so the job reads the file.)
+ if let Some(wanted) = self.denoise_for_export(id, &job.settings)? {
+ let p = self.catalog.photo(id).ok_or("no such photo")?.clone();
+ job.source = self.media.source_ref_uncached(&p, job.level);
+ if let SourceRef::File { denoise, .. } = &mut job.source {
+ *denoise = Some(wanted);
+ }
+ }
job.request.space = space;
job.request.depth = depth;
job.key ^= (space as u64 + 1).wrapping_mul(0xa076_1d64_78bd_642f) ^ (depth as u64 + 1).wrapping_mul(0xe703_7ed1_a0b4_28db);
@@ -1219,6 +1410,7 @@ mod tests {
Ok((image, info))
})),
fallback: None,
+ denoise: None,
};
job.info = SourceInfo::default(); // Header facts cannot override decoder facts.
job.settings = Arc::new(DevelopSettings::default());
diff --git a/crates/engine/src/model_download.rs b/crates/engine/src/model_download.rs
new file mode 100644
index 000000000..f07c373d2
--- /dev/null
+++ b/crates/engine/src/model_download.rs
@@ -0,0 +1,363 @@
+//! Downloading a denoise model explicitly requested after showing its terms.
+//!
+//! The transfer is [`lightcraft_fetch`] (pure-Rust HTTPS with resume, size and SHA-256 checks), on a background thread,
+//! so the UI never waits for the network. Only a model listed in [`lightcraft_denoise::known::Download`] can be fetched:
+//! the address and hash come from our pinned catalog or the user's model catalog, and no token is sent with it. What arrives
+//! matches the model's recorded size and SHA-256 or it is thrown away; it then waits in a staging folder until the model
+//! is installed (`denoise.models.install`, or by itself once the user has accepted the model's terms in the download).
+//!
+//! Failure is a message, never a panic: no network, a stalled or truncated transfer, a full disk, a file that is not the
+//! one expected, and a panic inside the thread itself all end as [`State::Failed`].
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, PoisonError};
+use std::time::Duration;
+
+use lightcraft_denoise::known::Download;
+
+/// The folder inside the models folder where a downloaded file waits to be installed. Its name starts with a dot, so it
+/// can never be a model's own folder (ids start with a letter or digit).
+pub const STAGING: &str = ".downloads";
+
+/// Where a download stands.
+#[derive(Clone, Debug, PartialEq)]
+pub enum State {
+ /// Fetching (`bytes` of `total` so far); the file is checked as the last step.
+ Running {
+ bytes: u64,
+ total: u64,
+ },
+ /// Fetched and verified (its SHA-256 is `sha256`), waiting in the staging folder at `path` to be installed.
+ Done {
+ path: PathBuf,
+ sha256: String,
+ },
+ /// Installed and in use: nothing more to do but tell the user.
+ Installed,
+ Failed(String),
+ Cancelled,
+}
+
+struct Shared {
+ state: Mutex<State>,
+ cancel: AtomicBool,
+}
+
+impl Shared {
+ fn new(total: u64) -> Self {
+ Shared { state: Mutex::new(State::Running { bytes: 0, total }), cancel: AtomicBool::new(false) }
+ }
+
+ fn set(&self, s: State) {
+ *self.state.lock().unwrap_or_else(PoisonError::into_inner) = s;
+ }
+
+ fn get(&self) -> State {
+ self.state.lock().unwrap_or_else(PoisonError::into_inner).clone()
+ }
+}
+
+/// The downloads of one session, by model id.
+#[derive(Default)]
+pub struct Downloads {
+ jobs: HashMap<String, Arc<Shared>>,
+}
+
+impl Downloads {
+ /// Start fetching `spec` into `<models dir>/.downloads/`. Different models can download side by side; the same
+ /// one is started once.
+ pub fn start(&mut self, spec: Download, models_dir: &Path) -> Result<(), String> {
+ if self.jobs.get(&spec.id).is_some_and(|j| matches!(j.get(), State::Running { .. })) {
+ return Err("that model is already downloading".into());
+ }
+ let staging = models_dir.join(STAGING);
+ std::fs::create_dir_all(&staging).map_err(|e| format!("could not create the download folder: {e}"))?;
+ let shared = Arc::new(Shared::new(spec.size_bytes));
+ let worker = shared.clone();
+ let id = spec.id.clone();
+ std::thread::Builder::new()
+ .name("denoise-model-download".into())
+ .spawn(move || {
+ let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&spec, &spec.url, &staging, &worker)));
+ if outcome.is_err() {
+ worker.set(State::Failed("the download stopped unexpectedly".into()));
+ }
+ })
+ .map_err(|e| format!("could not start the download: {e}"))?;
+ self.jobs.insert(id, shared);
+ Ok(())
+ }
+
+ /// Every download this session has started and not discarded, by model id.
+ pub fn snapshot(&self) -> Vec<(String, State)> {
+ let mut all: Vec<_> = self.jobs.iter().map(|(id, j)| (id.clone(), j.get())).filter(|(_, s)| *s != State::Cancelled).collect();
+ all.sort_by(|a, b| a.0.cmp(&b.0));
+ all
+ }
+
+ pub fn running(&self) -> bool {
+ self.jobs.values().any(|j| matches!(j.get(), State::Running { .. }))
+ }
+
+ /// Stop a running download, or discard a finished one (its staged file is deleted). `true` if there was one.
+ pub fn discard(&mut self, id: &str) -> bool {
+ let Some(job) = self.jobs.get(id) else { return false };
+ job.cancel.store(true, Ordering::Relaxed);
+ match job.get() {
+ // the thread notices, deletes the partial file and marks itself cancelled
+ State::Running { .. } => {}
+ State::Done { path, .. } => {
+ let _ = std::fs::remove_file(path);
+ self.jobs.remove(id);
+ }
+ State::Installed | State::Failed(_) | State::Cancelled => {
+ self.jobs.remove(id);
+ }
+ }
+ true
+ }
+
+ /// The staged file at `path` has been installed (its copy is the model now) or is no longer wanted.
+ pub fn staged_file_used(&mut self, path: &Path) {
+ self.jobs.retain(|_, j| !matches!(j.get(), State::Done { path: p, .. } if p == path));
+ }
+
+ /// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256).
+ pub fn finished(&self) -> Vec<(String, PathBuf, String)> {
+ let mut out: Vec<_> = self
+ .jobs
+ .iter()
+ .filter_map(|(id, j)| match j.get() {
+ State::Done { path, sha256 } => Some((id.clone(), path, sha256)),
+ _ => None,
+ })
+ .collect();
+ out.sort_by(|a, b| a.0.cmp(&b.0));
+ out
+ }
+
+ /// Record how installing a finished download went (`Installed` or `Failed`).
+ pub fn set_outcome(&mut self, id: &str, state: State) {
+ match self.jobs.get(id) {
+ Some(j) => j.set(state),
+ None => {
+ let j = Shared::new(0);
+ j.set(state);
+ self.jobs.insert(id.to_string(), Arc::new(j));
+ }
+ }
+ }
+}
+
+impl Drop for Downloads {
+ /// Quitting while a download runs stops it instead of leaving it behind.
+ fn drop(&mut self) {
+ for job in self.jobs.values() {
+ job.cancel.store(true, Ordering::Relaxed);
+ }
+ let started = std::time::Instant::now();
+ while self.running() && started.elapsed() < Duration::from_secs(1) {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ }
+}
+
+/// What went wrong, in words the user can act on.
+#[cfg(not(target_arch = "wasm32"))]
+fn explain(e: &lightcraft_fetch::DownloadError) -> String {
+ use lightcraft_fetch::DownloadError;
+ match e {
+ DownloadError::AllMirrorsFailed { errors, .. } => {
+ let detail: String = errors.join("; ").chars().take(300).collect();
+ format!("could not download the model ({detail}): check the internet connection, or get the file from its page and add it yourself")
+ }
+ other => other.to_string(),
+ }
+}
+
+/// Fetch `spec` from its exact URL into `staging`, check it and leave it there.
+#[cfg(not(target_arch = "wasm32"))]
+fn fetch(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) -> Result<(), lightcraft_fetch::DownloadError> {
+ let file = lightcraft_fetch::FileSpec { name: &spec.file_name, size: Some(spec.size_bytes), sha256: Some(&spec.sha256), max: spec.size_bytes };
+ lightcraft_fetch::download_url(&file, mirror, staging, &lightcraft_fetch::Options::default(), &shared.cancel, &mut |p| {
+ shared.set(State::Running { bytes: p.done.min(spec.size_bytes), total: spec.size_bytes });
+ })
+}
+
+#[cfg(target_arch = "wasm32")]
+fn fetch(_: &Download, _: &str, _: &Path, _: &Shared) -> Result<(), String> {
+ Err("the web build downloads no models".into())
+}
+
+/// The whole job: reuse a verified earlier copy, else fetch, check and stage. Always ends in a final [`State`].
+fn run(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) {
+ let target = staging.join(&spec.file_name);
+ let part = staging.join(format!("{}.part", spec.file_name));
+ match fetch(spec, mirror, staging, shared) {
+ Ok(()) => shared.set(State::Done { path: target, sha256: spec.sha256.to_string() }),
+ #[cfg(not(target_arch = "wasm32"))]
+ Err(lightcraft_fetch::DownloadError::Cancelled) => {
+ let _ = std::fs::remove_file(&part);
+ shared.set(State::Cancelled);
+ }
+ #[cfg(not(target_arch = "wasm32"))]
+ Err(e) => {
+ shared.set(State::Failed(explain(&e)));
+ }
+ #[cfg(target_arch = "wasm32")]
+ Err(e) => {
+ let _ = std::fs::remove_file(&part);
+ shared.set(State::Failed(e));
+ }
+ }
+}
+
+#[cfg(all(test, not(target_arch = "wasm32")))]
+mod tests {
+ use super::*;
+ use lightcraft_denoise::hash::sha256_hex;
+ use std::io::{BufRead, BufReader, Write};
+ use std::net::TcpListener;
+
+ fn scratch(name: &str) -> PathBuf {
+ let d = std::env::temp_dir().join(format!("lc-face-dl-{name}-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&d);
+ std::fs::create_dir_all(&d).unwrap();
+ d
+ }
+
+ fn spec_for(bytes: &[u8]) -> Download {
+ Download {
+ id: "test-model".into(),
+ url: "https://example.invalid/m/test-model.onnx".into(),
+ file_name: "test-model.onnx".into(),
+ size_bytes: bytes.len() as u64,
+ sha256: sha256_hex(bytes),
+ }
+ }
+
+ /// A local server that answers every request with `status` and `body` (the whole file, no ranges).
+ fn serve(status: &'static str, body: Vec<u8>) -> String {
+ let l = TcpListener::bind("127.0.0.1:0").unwrap();
+ let base = format!("http://{}/m", l.local_addr().unwrap());
+ std::thread::spawn(move || {
+ for mut s in l.incoming().flatten() {
+ let mut r = BufReader::new(s.try_clone().unwrap());
+ loop {
+ let mut line = String::new();
+ if r.read_line(&mut line).unwrap_or(0) == 0 || line.trim().is_empty() {
+ break;
+ }
+ }
+ let _ = s.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes());
+ let _ = s.write_all(&body);
+ }
+ });
+ base
+ }
+
+ #[test]
+ fn fetches_checks_and_stages_a_file() {
+ let dir = scratch("ok");
+ let bytes: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();
+ let (staging, spec) = (dir.join(STAGING), spec_for(&bytes));
+ std::fs::create_dir_all(&staging).unwrap();
+ let mirror = serve("200 OK", bytes.clone());
+ let shared = Shared::new(spec.size_bytes);
+ run(&spec, &mirror, &staging, &shared);
+ let State::Done { path, sha256 } = shared.get() else { panic!("{:?}", shared.get()) };
+ assert_eq!((std::fs::read(&path).unwrap(), sha256), (bytes, spec.sha256.to_string()));
+ assert!(!staging.join("test-model.onnx.part").exists());
+ // asked again with the copy still staged: no fetch needed (nothing listens there)
+ let again = Shared::new(spec.size_bytes);
+ run(&spec, "http://127.0.0.1:1/m", &staging, &again);
+ assert!(matches!(again.get(), State::Done { .. }), "{:?}", again.get());
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn a_file_that_does_not_match_is_thrown_away() {
+ let dir = scratch("bad");
+ let staging = dir.join(STAGING);
+ std::fs::create_dir_all(&staging).unwrap();
+ // the right size and the wrong content
+ let spec = spec_for(&[7u8; 5000]);
+ let shared = Shared::new(spec.size_bytes);
+ run(&spec, &serve("200 OK", vec![8u8; 5000]), &staging, &shared);
+ assert!(matches!(shared.get(), State::Failed(_)), "{:?}", shared.get());
+ // nothing is left behind: not the file, not a partial one
+ assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn an_unreachable_or_refusing_server_is_a_message() {
+ let dir = scratch("down");
+ let staging = dir.join(STAGING);
+ std::fs::create_dir_all(&staging).unwrap();
+ let spec = spec_for(b"abc");
+ for mirror in ["http://127.0.0.1:1/m".to_string(), serve("404 Not Found", b"not found".to_vec())] {
+ let shared = Shared::new(3);
+ run(&spec, &mirror, &staging, &shared);
+ let State::Failed(why) = shared.get() else { panic!("{:?}", shared.get()) };
+ assert!(why.contains("internet connection"), "{why}");
+ }
+ assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn a_cancelled_download_leaves_nothing() {
+ let dir = scratch("cancel");
+ let staging = dir.join(STAGING);
+ std::fs::create_dir_all(&staging).unwrap();
+ let spec = spec_for(b"abc");
+ let shared = Shared::new(3);
+ shared.cancel.store(true, Ordering::Relaxed);
+ run(&spec, &serve("200 OK", b"abc".to_vec()), &staging, &shared);
+ assert_eq!(shared.get(), State::Cancelled);
+ assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn dropping_the_session_stops_a_running_download() {
+ let shared = Arc::new(Shared::new(10));
+ // a stand-in for the download thread: notices the cancel flag the way the downloader does
+ let worker = shared.clone();
+ let handle = std::thread::spawn(move || {
+ while !worker.cancel.load(Ordering::Relaxed) {
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ worker.set(State::Cancelled);
+ });
+ let mut d = Downloads::default();
+ d.jobs.insert("m".into(), shared.clone());
+ assert!(d.running());
+ let started = std::time::Instant::now();
+ drop(d);
+ assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish");
+ assert_eq!(shared.get(), State::Cancelled);
+ handle.join().unwrap();
+ }
+
+ #[test]
+ fn discarding_a_finished_download_deletes_its_file() {
+ let dir = scratch("discard");
+ let file = dir.join("staged.onnx");
+ std::fs::write(&file, b"x").unwrap();
+ let mut d = Downloads::default();
+ let shared = Arc::new(Shared::new(1));
+ shared.set(State::Done { path: file.clone(), sha256: String::new() });
+ d.jobs.insert("m".into(), shared);
+ assert_eq!(d.snapshot().len(), 1);
+ assert!(!d.running());
+ assert!(d.discard("m"));
+ assert!(!file.exists());
+ assert!(d.snapshot().is_empty());
+ assert!(!d.discard("m"), "nothing left to discard");
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+}
diff --git a/crates/engine/src/smart.rs b/crates/engine/src/smart.rs
index 8deff7bde..a48fcd040 100644
--- a/crates/engine/src/smart.rs
+++ b/crates/engine/src/smart.rs
@@ -190,7 +190,7 @@ pub fn is_valid(path: &Path) -> bool {
/// Load the proxy at `path`.
pub fn load(path: &Path) -> Result<crate::media::DecodedSource, String> {
let b = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?;
- decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone })
+ decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone, denoised: None })
}
#[cfg(test)]
diff --git a/crates/engine/src/tests_denoise.rs b/crates/engine/src/tests_denoise.rs
new file mode 100644
index 000000000..172f90824
--- /dev/null
+++ b/crates/engine/src/tests_denoise.rs
@@ -0,0 +1,699 @@
+//! AI denoise through the engine: installing a model, making a photo's denoised picture in the background and for an
+//! export, the Amount slider mixing it in, and the cache. A tiny stand-in model (it halves every sample) takes the
+//! place of a real one, which is enough to see what the machinery does with its answer; how well real models denoise is
+//! measured separately (`crates/denoise/tests/real_model.rs`).
+
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};
+use lightcraft_denoise::run::TileRunner;
+use serde_json::{Value, json};
+
+use crate::Session;
+use crate::denoise::{Loader, Model, PhotoState, RunOn};
+use crate::export::{ExportOptions, export_photo};
+use crate::tests_xmp::{synthetic_dng_of, temp_dir};
+
+/// A model that makes every output pixel half of its cell's colour: a picture that is plainly not the plain one.
+struct Dim {
+ runs: Arc<AtomicUsize>,
+}
+
+impl TileRunner for Dim {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, lightcraft_denoise::Error> {
+ self.runs.fetch_add(1, Ordering::Relaxed);
+ let t = ((input.len() / 4) as f64).sqrt() as usize;
+ let side = 2 * t;
+ let mut out = vec![0f32; 3 * side * side];
+ for y in 0..side {
+ for x in 0..side {
+ let i = (y / 2) * t + x / 2;
+ let (r, g, b) = (input[i], (input[t * t + i] + input[2 * t * t + i]) / 2.0, input[3 * t * t + i]);
+ out[y * side + x] = r * 0.5;
+ out[side * side + y * side + x] = g * 0.5;
+ out[2 * side * side + y * side + x] = b * 0.5;
+ }
+ }
+ Ok(out)
+ }
+}
+
+impl Model for Dim {
+ fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
+ (self, threads)
+ }
+
+ fn self_test(&self, _: RunOn) -> Result<Value, String> {
+ Ok(json!({"ok": true}))
+ }
+}
+
+fn dim_loader(runs: &Arc<AtomicUsize>) -> Loader {
+ let runs = runs.clone();
+ Arc::new(move |_, _| Ok(Arc::new(Dim { runs: runs.clone() }) as Arc<dyn Model>))
+}
+
+fn manifest(id: &str) -> DenoiserManifest {
+ DenoiserManifest {
+ id: id.into(),
+ name: "Stand-in".into(),
+ version: "1".into(),
+ licence: Default::default(),
+ source: None,
+ sha256: None,
+ size_bytes: None,
+ provenance: "made up for a test".into(),
+ domain: Domain::BayerToRgb,
+ tile: 64,
+ overlap: 16,
+ gain: Gain::None,
+ }
+}
+
+/// The model's `.onnx` (not a real one: the stand-in loader never reads it) with its manifest beside it.
+fn model_files(dir: &Path, id: &str) -> PathBuf {
+ let src = dir.join(format!("src-{id}"));
+ std::fs::create_dir_all(&src).unwrap();
+ std::fs::write(src.join("denoise-model.json"), serde_json::to_vec(&manifest(id)).unwrap()).unwrap();
+ std::fs::write(src.join("model.onnx"), format!("not an onnx file, only a stand-in for {id}")).unwrap();
+ src.join("model.onnx")
+}
+
+/// A Bayer DNG that is a smooth ramp, big enough for a 64-cell tile and a second one.
+fn ramp_dng(w: usize, h: usize, cfa: &str) -> Vec<u8> {
+ let data: Vec<u16> = (0..w * h).map(|i| 600 + ((i % w) * 30 + (i / w) * 35) as u16).collect();
+ synthetic_dng_of(w, h, cfa, data, None, Default::default())
+}
+
+struct Setup {
+ dir: PathBuf,
+ s: Session,
+ runs: Arc<AtomicUsize>,
+}
+
+/// A session with a library on disk holding one raw photo, and the stand-in model installed and chosen.
+fn setup(tag: &str, with_model: bool) -> Setup {
+ let dir = temp_dir(tag);
+ let originals = dir.join("originals");
+ std::fs::create_dir_all(&originals).unwrap();
+ std::fs::write(originals.join("ramp.dng"), ramp_dng(192, 160, "RGGB")).unwrap();
+ let runs = Arc::new(AtomicUsize::new(0));
+ let mut s = Session::new().with_fs();
+ s.open_library(dir.join("library"), false).unwrap();
+ s.execute("library.import", &json!({"paths": [originals.join("ramp.dng").to_string_lossy()]})).unwrap();
+ let id = s.catalog.photos().next().unwrap().id;
+ s.execute("library.select", &json!({"ids": [id.0]})).unwrap();
+ s.set_denoise_models_dir(Some(dir.join("denoise-models")));
+ s.denoise.loader = dim_loader(&runs);
+ if with_model {
+ let onnx = model_files(&dir, "stand-in");
+ s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
+ }
+ Setup { dir, s, runs }
+}
+
+fn photo(s: &Session) -> lightcraft_catalog::PhotoId {
+ s.catalog.photos().next().unwrap().id
+}
+
+/// Pump until `n` photos are ready (the job runs on its own thread).
+fn pump_until_ready(s: &mut Session, n: u64) -> Value {
+ for _ in 0..600 {
+ let r = s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
+ if r["ready"].as_u64() >= Some(n) {
+ return r;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(50));
+ }
+ panic!("not ready: {}", s.execute("denoise.status", &json!({})).unwrap());
+}
+
+fn mean(img: &lightcraft_raster::Rgba8) -> f64 {
+ let sum: u64 = img.data.iter().map(|p| u64::from(p[0]) + u64::from(p[1]) + u64::from(p[2])).sum();
+ sum as f64 / (img.data.len() * 3) as f64
+}
+
+fn rendered_mean(s: &mut Session, id: lightcraft_catalog::PhotoId) -> f64 {
+ mean(&s.render_now(id, 160, 160).unwrap().image)
+}
+
+fn set_amount(s: &mut Session, amount: f64) {
+ s.execute("develop.set", &json!({"control": "enhance.denoise", "value": amount})).unwrap();
+}
+
+#[test]
+fn a_model_is_installed_chosen_and_removed_with_the_terms_accepted() {
+ let mut x = setup("model", false);
+ let onnx = model_files(&x.dir, "stand-in");
+ // the terms come first
+ let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy()})).unwrap_err().to_string();
+ assert!(e.contains("licence has not been accepted"), "{e}");
+ let r = x.s.execute("denoise.models.list", &json!({})).unwrap();
+ assert_eq!(r["model"], Value::Null);
+ assert_eq!(r["runtime"], cfg!(feature = "denoise"));
+ // a file with no manifest is not guessed at
+ let stray = x.dir.join("stray.onnx");
+ std::fs::write(&stray, b"who knows").unwrap();
+ let e = x.s.execute("denoise.models.install", &json!({"path": stray.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
+ assert!(e.contains("does not know this file"), "{e}");
+ // installed, it is the model in use
+ let r = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
+ assert_eq!((r["installed"]["id"].clone(), r["model"].clone()), (json!("stand-in"), json!("stand-in")), "{r}");
+ assert!(r["installed"]["accepted"]["sha256"].as_str().is_some_and(|h| h.len() == 64), "the file's hash is recorded: {r}");
+ assert!(x.s.denoise.active.is_some());
+ assert_eq!(x.s.execute("denoise.status", &json!({})).unwrap()["model"]["id"], "stand-in");
+ // it can be checked again, switched off and on, and removed
+ assert_eq!(x.s.execute("denoise.models.test", &json!({"id": "stand-in"})).unwrap()["ok"], true);
+ assert_eq!(x.s.execute("denoise.models.select", &json!({"id": null})).unwrap()["model"], Value::Null);
+ assert!(x.s.denoise.active.is_none());
+ assert!(x.s.execute("denoise.models.select", &json!({"id": "nothing"})).is_err());
+ x.s.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap();
+ assert!(x.s.denoise.active.is_some());
+ let r = x.s.execute("denoise.models.remove", &json!({"id": "stand-in"})).unwrap();
+ assert_eq!(r["model"], Value::Null);
+ assert!(x.s.denoise.active.is_none());
+ assert!(!x.dir.join("denoise-models").join("stand-in").exists());
+ // ids that are not folder names, and folders that are not models, are never touched
+ for bad in ["../library", "", "a/b", "settings.json"] {
+ assert!(x.s.execute("denoise.models.remove", &json!({"id": bad})).is_err(), "{bad}");
+ }
+ assert!(x.dir.join("library").is_dir());
+ // a model that fails its self-test is not left installed
+ x.s.denoise.loader = Arc::new(|_, _| Err("it does not load".into()));
+ let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
+ assert!(e.contains("it does not load"), "{e}");
+ assert!(!x.dir.join("denoise-models").join("stand-in").exists());
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn a_manifest_that_does_not_validate_or_name_the_file_is_refused() {
+ let mut x = setup("manifest", false);
+ let onnx = model_files(&x.dir, "stand-in");
+ let json_path = onnx.with_file_name("denoise-model.json");
+ // a tile the model cannot run on
+ let mut m = manifest("stand-in");
+ m.tile = 50;
+ std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
+ let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
+ assert!(e.contains("tile"), "{e}");
+ // a hash that is not the file's
+ let mut m = manifest("stand-in");
+ m.sha256 = Some("0".repeat(64));
+ std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
+ let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
+ assert!(e.contains("SHA-256"), "{e}");
+ assert!(!x.dir.join("denoise-models").join("stand-in").join("model.onnx").exists(), "nothing half-installed is left");
+ // an id that would leave the folder
+ let mut m = manifest("../escape");
+ m.id = "../escape".into();
+ std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
+ assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err());
+ assert!(!x.dir.join("escape").exists());
+ // garbage instead of a manifest
+ std::fs::write(&json_path, b"{ not json").unwrap();
+ assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err());
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn a_photo_is_denoised_in_the_background_and_the_amount_mixes_it_in() {
+ let mut x = setup("background", true);
+ let id = photo(&x.s);
+ // nothing is made until a photo has an amount
+ let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
+ assert_eq!((r["queued"].clone(), r["running"].clone(), r["ready"].clone()), (json!(0), Value::Null, json!(0)), "{r}");
+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
+ let plain = rendered_mean(&mut x.s, id);
+ let key_before = x.s.render_job(id, 160, 160, false, true).unwrap().key;
+
+ set_amount(&mut x.s, 100.0);
+ // the photo being edited is picked up by itself, and made on another thread
+ let r = pump_until_ready(&mut x.s, 1);
+ assert_eq!(r["queued"], 0, "{r}");
+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
+ assert!(x.runs.load(Ordering::Relaxed) >= 4, "a 192 × 160 mosaic is four 64-cell tiles: {}", x.runs.load(Ordering::Relaxed));
+ let products: Vec<_> = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().collect();
+ assert_eq!(products.len(), 1, "one cached picture, no DNG or anything else in the library");
+ assert!(products[0].path().extension().is_some_and(|e| e == "lcdn"));
+ let originals: Vec<_> = std::fs::read_dir(x.dir.join("originals")).unwrap().flatten().collect();
+ assert_eq!(originals.len(), 1, "the original is untouched and alone");
+
+ // the render now comes from the denoised picture: darker at 100, in between at 50, the plain one at 0
+ let full = rendered_mean(&mut x.s, id);
+ assert!(full < plain - 3.0, "denoised {full} vs plain {plain}");
+ set_amount(&mut x.s, 50.0);
+ let half = rendered_mean(&mut x.s, id);
+ assert!(half < plain - 1.0 && half > full + 1.0, "{full} < {half} < {plain}");
+ set_amount(&mut x.s, 0.0);
+ assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01, "amount 0 is the plain picture again");
+ // …and a render that uses it has another identity than one that does not
+ assert_ne!(key_before, {
+ set_amount(&mut x.s, 100.0);
+ x.s.render_job(id, 160, 160, false, true).unwrap().key
+ });
+ // switching the Detail section off switches the denoise off with it
+ set_amount(&mut x.s, 100.0);
+ x.s.execute("develop.sectionEnabled", &json!({"section": "detail", "enabled": false})).unwrap();
+ assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01);
+ assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0);
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn what_was_made_is_found_again_by_a_new_session_and_a_new_model_starts_over() {
+ let mut x = setup("again", true);
+ let id = photo(&x.s);
+ set_amount(&mut x.s, 100.0);
+ pump_until_ready(&mut x.s, 1);
+ let made = x.runs.load(Ordering::Relaxed);
+ x.s.close_library().unwrap();
+ drop(x.s);
+
+ // reopened: ready without running the model again
+ let mut s2 = Session::new().with_fs();
+ s2.set_denoise_models_dir(Some(x.dir.join("denoise-models")));
+ s2.denoise.loader = dim_loader(&x.runs);
+ s2.open_library(x.dir.join("library"), false).unwrap();
+ s2.denoise_pump(crate::denoise::Pace::Full);
+ assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready);
+ assert_eq!(x.runs.load(Ordering::Relaxed), made, "nothing was made again");
+
+ // another model's pictures are different pictures
+ let onnx = model_files(&x.dir, "other");
+ s2.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
+ s2.denoise_pump(crate::denoise::Pace::Pause);
+ assert!(
+ matches!(s2.denoise_photo_state(id), PhotoState::Queued { .. }),
+ "the first model's picture is not this model's: it is to be made: {:?}",
+ s2.denoise_photo_state(id)
+ );
+ // and choosing the first again finds its picture where it was
+ s2.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap();
+ s2.denoise_pump(crate::denoise::Pace::Pause);
+ assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready);
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn an_export_makes_its_picture_when_the_queue_has_not_and_never_comes_out_without_it() {
+ let mut x = setup("export", true);
+ let id = photo(&x.s);
+ let o = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160}));
+ let decode = |bytes: &[u8]| {
+ let d = lightcraft_codecs::decode(bytes, lightcraft_codecs::DecodeOptions::fit(160, 160)).unwrap();
+ let w = d.to_working();
+ w.data.iter().map(|p| f64::from(p[0] + p[1] + p[2])).sum::<f64>() / (w.data.len() * 3) as f64
+ };
+ let plain = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
+ assert_eq!(x.runs.load(Ordering::Relaxed), 0, "no amount, no denoise");
+ set_amount(&mut x.s, 100.0);
+ // no pump: the export makes the picture itself
+ let denoised = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
+ assert!(x.runs.load(Ordering::Relaxed) >= 4);
+ assert!(denoised < plain * 0.9, "denoised {denoised} vs plain {plain}");
+ let products = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().count();
+ assert_eq!(products, 1);
+ // a second export finds it
+ let runs = x.runs.load(Ordering::Relaxed);
+ let again = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
+ assert_eq!(x.runs.load(Ordering::Relaxed), runs);
+ assert!((again - denoised).abs() < 1e-6);
+ // the pump takes in what the export made
+ x.s.denoise_pump(crate::denoise::Pace::Pause);
+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
+
+ // no model: the export says so instead of leaving the denoise out
+ x.s.execute("denoise.models.select", &json!({"id": null})).unwrap();
+ let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();
+ assert!(e.contains("no denoise model is chosen") && e.contains("ramp.dng"), "{e}");
+ set_amount(&mut x.s, 0.0);
+ assert!(export_photo(&mut x.s, id, &o, 1).is_ok(), "without an amount nothing is asked of the model");
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn a_model_that_fails_fails_the_export_and_is_not_retried_by_the_queue() {
+ let mut x = setup("fails", true);
+ let id = photo(&x.s);
+ x.s.denoise.loader = Arc::new(|_, _| Err("the model went away".into()));
+ // (the model chosen is read afresh: it was loaded by the install's self-test only)
+ set_amount(&mut x.s, 80.0);
+ let o = ExportOptions::from_json(&json!({"format": "png", "width": 100, "height": 100}));
+ let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();
+ assert!(e.contains("ramp.dng") && e.contains("the model went away"), "{e}");
+ // the background job fails too, says why once and does not try again every frame
+ for _ in 0..200 {
+ x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
+ if matches!(x.s.denoise_photo_state(id), PhotoState::Failed { .. }) {
+ break;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(20));
+ }
+ match x.s.denoise_photo_state(id) {
+ PhotoState::Failed { why, unsupported } => assert!(why.contains("the model went away") && !unsupported, "{why}"),
+ other => panic!("{other:?}"),
+ }
+ let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
+ assert_eq!((r["queued"].clone(), r["running"].clone()), (json!(0), Value::Null), "{r}");
+ // asking again by hand does retry
+ x.s.denoise.loader = dim_loader(&x.runs);
+ let r = x.s.execute("denoise.queue", &json!({"retry": true})).unwrap();
+ assert_eq!(r["added"], 1, "{r}");
+ pump_until_ready(&mut x.s, 1);
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn a_corrupt_cache_with_a_current_header_never_silently_exports_the_plain_photo() {
+ let mut x = setup("corrupt-export", true);
+ let id = photo(&x.s);
+ set_amount(&mut x.s, 100.0);
+ let options = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160}));
+ export_photo(&mut x.s, id, &options, 1).unwrap();
+ x.s.denoise_pump(crate::denoise::Pace::Pause);
+ let spec = x.s.media.denoise.spec(id).unwrap().clone();
+ let mut bytes = std::fs::read(&spec.product).unwrap();
+ for byte in bytes.iter_mut().rev().take(64) {
+ *byte ^= 0xff;
+ }
+ std::fs::write(&spec.product, bytes).unwrap();
+ assert!(lightcraft_denoise::product::is_current(&spec.product, &spec.key), "header/length still look current");
+ x.s.media.forget(id);
+ let error = export_photo(&mut x.s, id, &options, 1).err().unwrap();
+ assert!(error.contains("AI Denoise picture could not be read"), "{error}");
+ let _ = std::fs::remove_dir_all(x.dir);
+}
+
+#[test]
+fn only_raw_photos_on_disk_are_denoised_and_nothing_is_queued_without_a_model() {
+ let dir = temp_dir("eligible");
+ let mut demo = Session::with_demo();
+ demo.set_denoise_models_dir(Some(dir.join("models")));
+ let id = demo.active().unwrap();
+ assert_eq!(demo.denoise_photo_state(id), PhotoState::NotApplicable, "a generated photo has no sensor data");
+ let e = demo.execute("denoise.queue", &json!({})).unwrap_err().to_string();
+ assert!(e.contains("no denoise model is chosen"), "{e}");
+ // no models folder at all (the web build): the commands say so, nothing panics
+ let mut web = Session::with_demo();
+ for cmd in ["denoise.models.install", "denoise.models.select", "denoise.models.remove", "denoise.settings"] {
+ assert!(web.execute(cmd, &json!({"path": "x", "acknowledged": true, "id": "x"})).is_err(), "{cmd}");
+ }
+ assert_eq!(web.execute("denoise.pump", &json!({})).unwrap()["active"], false);
+ assert_eq!(web.execute("denoise.status", &json!({})).unwrap()["enabled"], false);
+ // no library open: nowhere to keep pictures, and the queue says so
+ let x = setup("eligible2", true);
+ let mut bare = Session::new().with_fs();
+ bare.set_denoise_models_dir(Some(x.dir.join("denoise-models")));
+ assert!(bare.denoise_products_dir().is_none());
+ let e = bare.execute("denoise.queue", &json!({})).unwrap_err().to_string();
+ assert!(e.contains("open a library on disk first"), "{e}");
+ assert_eq!(bare.execute("denoise.pump", &json!({})).unwrap()["queued"], 0);
+ let _ = std::fs::remove_dir_all(&dir);
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn files_that_cannot_be_denoised_fail_with_a_reason_and_never_a_panic() {
+ let dir = temp_dir("makefails");
+ let spec = |path: &Path| crate::denoise::JobSpec {
+ path: path.to_string_lossy().into_owned(),
+ read: None,
+ product: dir.join("p.lcdn"),
+ key: "k".into(),
+ manifest: manifest("stand-in"),
+ onnx: dir.join("model.onnx"),
+ model: Default::default(),
+ loader: dim_loader(&Arc::new(AtomicUsize::new(0))),
+ parallel: 2,
+ run_on: RunOn::Auto,
+ };
+ use crate::denoise::{MakeError, make_product};
+ // not a file
+ assert!(matches!(make_product(&spec(&dir.join("missing.dng")), None), Err(MakeError::Failed(_))));
+ // not a raw
+ let junk = dir.join("junk.dng");
+ std::fs::write(&junk, b"this is not a raw file at all").unwrap();
+ assert!(matches!(make_product(&spec(&junk), None), Err(MakeError::Unsupported(_))));
+ // a raw cut short
+ let whole = ramp_dng(192, 160, "RGGB");
+ let cut = dir.join("cut.dng");
+ std::fs::write(&cut, &whole[..whole.len() / 3]).unwrap();
+ assert!(make_product(&spec(&cut), None).is_err());
+ // a model that gives the wrong number of values is an error from the model, not a crash
+ struct Wrong;
+ impl TileRunner for Wrong {
+ fn run(&self, _: &[f32]) -> Result<Vec<f32>, lightcraft_denoise::Error> {
+ Ok(vec![0.0; 7])
+ }
+ }
+ impl Model for Wrong {
+ fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
+ (self, threads)
+ }
+ fn self_test(&self, _: RunOn) -> Result<Value, String> {
+ Err("no".into())
+ }
+ }
+ let good = dir.join("good.dng");
+ std::fs::write(&good, &whole).unwrap();
+ let mut s = spec(&good);
+ s.loader = Arc::new(|_, _| Ok(Arc::new(Wrong) as Arc<dyn Model>));
+ match make_product(&s, None) {
+ Err(MakeError::Failed(why)) => assert!(why.contains("7 numbers"), "{why}"),
+ other => panic!("{other:?}"),
+ }
+ assert!(!dir.join("p.lcdn").exists(), "a failed job leaves no product");
+ // a job told to stop stops before doing anything
+ let progress = crate::denoise::Progress::default();
+ progress.cancel_now();
+ assert_eq!(make_product(&spec(&good), Some(&progress)), Err(MakeError::Cancelled));
+ // a Bayer layout of the other kinds is denoised too, and a product that exists is not made again
+ for cfa in ["BGGR", "GRBG", "GBRG"] {
+ let f = dir.join(format!("{cfa}.dng"));
+ std::fs::write(&f, ramp_dng(192, 160, cfa)).unwrap();
+ let mut s = spec(&f);
+ s.product = dir.join(format!("{cfa}.lcdn"));
+ assert_eq!(make_product(&s, None), Ok(true), "{cfa}");
+ assert_eq!(make_product(&s, None), Ok(false), "{cfa} is already there");
+ }
+ let _ = std::fs::remove_dir_all(&dir);
+}
+
+#[test]
+fn two_threads_making_the_same_picture_make_it_once() {
+ let dir = temp_dir("twice");
+ let f = dir.join("a.dng");
+ std::fs::write(&f, ramp_dng(192, 160, "RGGB")).unwrap();
+ let runs = Arc::new(AtomicUsize::new(0));
+ let spec = crate::denoise::JobSpec {
+ path: f.to_string_lossy().into_owned(),
+ read: None,
+ product: dir.join("p.lcdn"),
+ key: "k".into(),
+ manifest: manifest("stand-in"),
+ onnx: dir.join("model.onnx"),
+ model: Default::default(),
+ loader: dim_loader(&runs),
+ parallel: 2,
+ run_on: RunOn::Auto,
+ };
+ let made: Vec<bool> = std::thread::scope(|sc| {
+ let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| crate::denoise::make_product(&spec, None).unwrap())).collect();
+ hs.into_iter().map(|h| h.join().unwrap()).collect()
+ });
+ assert_eq!(made.iter().filter(|m| **m).count(), 1, "{made:?}");
+ assert_eq!(runs.load(Ordering::Relaxed), 4, "one photo's four tiles, once");
+ assert!(lightcraft_denoise::product::is_current(&spec.product, "k"));
+ let _ = std::fs::remove_dir_all(&dir);
+}
+
+#[test]
+fn the_cache_is_kept_within_its_limit_and_what_photos_use_goes_last() {
+ let mut x = setup("evict", true);
+ let id = photo(&x.s);
+ set_amount(&mut x.s, 100.0);
+ pump_until_ready(&mut x.s, 1);
+ let used = x.dir.join("library").join("denoise");
+ // two older pictures nobody uses
+ for (i, name) in ["aaaa", "bbbb"].iter().enumerate() {
+ let f = used.join(format!("{name}.lcdn"));
+ std::fs::write(&f, vec![7u8; 1000]).unwrap();
+ let t = std::time::SystemTime::now() - std::time::Duration::from_secs(1000 - i as u64);
+ std::fs::File::options().write(true).open(&f).unwrap().set_modified(t).unwrap();
+ }
+ let size_of = |p: &Path| std::fs::metadata(p).unwrap().len();
+ let mine: PathBuf = std::fs::read_dir(&used).unwrap().flatten().map(|e| e.path()).find(|p| p.file_stem().is_some_and(|s| s.len() == 32)).unwrap();
+ // room for the photo's own picture and one stray: the oldest stray goes, then the other
+ x.s.denoise_evict_to(size_of(&mine) + 1500, None);
+ assert!(!used.join("aaaa.lcdn").exists() && used.join("bbbb.lcdn").exists() && mine.exists());
+ x.s.denoise_evict_to(size_of(&mine) + 10, None);
+ assert!(!used.join("bbbb.lcdn").exists() && mine.exists());
+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
+ // too small for even that: the photo's own picture is the last thing to go, and the photo knows
+ x.s.denoise_evict_to(10, None);
+ assert!(!mine.exists());
+ assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready);
+ // …and it is made again, because the photo still has its amount
+ pump_until_ready(&mut x.s, 1);
+ assert!(mine.exists());
+ // clearing deletes it all
+ let r = x.s.execute("denoise.clear", &json!({})).unwrap();
+ assert_eq!(r["deleted"], 1, "{r}");
+ assert_eq!(std::fs::read_dir(&used).unwrap().flatten().count(), 0);
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn settings_are_checked_saved_and_a_paused_pump_starts_nothing() {
+ let mut x = setup("settings", true);
+ let r = x.s.execute("denoise.settings", &json!({"auto": false, "cacheGb": 5, "threads": 3})).unwrap();
+ assert_eq!((r["auto"].clone(), r["cacheGb"].clone(), r["threads"].clone()), (json!(false), json!(5), json!(3)), "{r}");
+ for bad in [
+ json!({"cacheGb": 0}),
+ json!({"cacheGb": "big"}),
+ json!({"threads": 0}),
+ json!({"threads": 1000}),
+ json!({"auto": "yes"}),
+ json!({"runOn": "fastest"}),
+ json!({"runOn": true}),
+ ] {
+ assert!(x.s.execute("denoise.settings", &bad).is_err(), "{bad}");
+ }
+ assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["cacheGb"], 5, "a refused change changes nothing");
+ // the faster of the card and the processor unless the user says otherwise, and the interface can read the choice back
+ assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "auto");
+ assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap()["runOn"], "cpu");
+ assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu");
+ let status = x.s.execute("denoise.status", &json!({})).unwrap();
+ assert_eq!((status["runOn"].clone(), status["device"]["kind"].clone()), (json!("cpu"), json!("none")), "no model is loaded yet: {status}");
+ // the setting before `runOn` still reads as what it meant
+ let models = x.s.denoise.models_dir.clone().unwrap();
+ let mut st = crate::denoise::read_settings(&models);
+ (st.run_on, st.gpu) = (None, Some(false));
+ crate::denoise::write_settings(&models, &st).unwrap();
+ assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu");
+ // choosing where it runs lets the card be tried again after a set-up that closed LightCraft
+ let onnx = crate::denoise::installed_models(&models).first().unwrap().onnx.clone();
+ let marker = onnx.with_file_name(crate::denoise::GPU_SETUP_MARKER);
+ std::fs::write(&marker, "setting up").unwrap();
+ assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "auto"})).unwrap()["runOn"], "auto");
+ assert!(!marker.exists(), "the crash is forgotten");
+ assert_eq!(crate::denoise::read_settings(&models).gpu, None, "the old setting is not kept beside the new one");
+ // with auto off nothing is started by itself, and the photo is made when asked
+ let id = photo(&x.s);
+ set_amount(&mut x.s, 100.0);
+ for _ in 0..3 {
+ x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
+ }
+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
+ x.s.execute("denoise.settings", &json!({"auto": true})).unwrap();
+ // paused: it is queued but nothing starts
+ for _ in 0..3 {
+ let r = x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap();
+ assert_eq!(r["running"], Value::Null);
+ }
+ assert!(matches!(x.s.denoise_photo_state(id), PhotoState::Queued { .. }), "{:?}", x.s.denoise_photo_state(id));
+ // …and cancelling forgets it
+ assert_eq!(x.s.execute("denoise.cancel", &json!({})).unwrap()["cancelled"], 1);
+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
+ // when the pace allows, it is made
+ x.s.execute("denoise.queue", &json!({"scope": "withAmount"})).unwrap();
+ pump_until_ready(&mut x.s, 1);
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn a_photo_whose_file_changed_has_no_picture_until_it_is_made_again() {
+ let mut x = setup("changed", true);
+ let id = photo(&x.s);
+ set_amount(&mut x.s, 100.0);
+ pump_until_ready(&mut x.s, 1);
+ // the photo's content is something else now (a replaced file): its old picture is not its picture
+ let p = x.s.catalog.photo(id).unwrap().clone();
+ x.s.commit(
+ "test",
+ lightcraft_catalog::Op::SetContent {
+ id,
+ width: p.width,
+ height: p.height,
+ file_size: p.file_size + 1,
+ content_hash: Some("ffffffffffffffffffffffffffffffff".into()),
+ preview_only: None,
+ },
+ )
+ .unwrap();
+ x.s.denoise_pump(crate::denoise::Pace::Pause);
+ assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready);
+ // the old picture is not used by a render either
+ assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0);
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn a_photo_waits_for_its_turn_unless_pictures_are_not_made_on_their_own_or_have_nowhere_to_go() {
+ let mut x = setup("waits", true);
+ let id = photo(&x.s);
+ set_amount(&mut x.s, 80.0);
+ // automatic off: the open photo is left alone until it is asked for
+ x.s.execute("denoise.settings", &json!({"auto": false})).unwrap();
+ assert!(!x.s.denoise_auto());
+ x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
+ assert!(!x.s.denoise_busy());
+ x.s.execute("denoise.queue", &json!({"ids": [id.0]})).unwrap();
+ assert!(x.s.denoise_busy(), "queued with a model: the headless runs wait for it");
+ pump_until_ready(&mut x.s, 1);
+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
+ assert!(!x.s.denoise_busy());
+ // a session whose library is not on disk has nowhere to keep it: the photo says so instead of waiting for ever
+ x.s.denoise.settings.auto = Some(true);
+ if let Some(l) = x.s.library.as_mut() {
+ l.on_disk = false;
+ }
+ match x.s.denoise_photo_state(id) {
+ PhotoState::Failed { why, unsupported } => assert!(unsupported && why.contains("library folder"), "{why}"),
+ other => panic!("{other:?}"),
+ }
+ let _ = std::fs::remove_dir_all(&x.dir);
+}
+
+#[test]
+fn the_denoise_switch_validates_arguments_and_accepts_explicit_targets_without_selection() {
+ let mut x = setup("toggle", false);
+ let id = photo(&x.s);
+ for params in [json!({"id": "bad"}), json!({"id": u64::MAX}), json!({"id": id.0, "enabled": "yes"})] {
+ assert!(x.s.execute("denoise.toggle", &params).is_err());
+ }
+ assert!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).is_err());
+ assert_eq!(x.s.develop_of(id).unwrap().enhance.denoise, 0.0);
+ let model = model_files(&x.dir, "toggle-model");
+ x.s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();
+ x.s.selection = Default::default();
+ let undo = x.s.undo.len();
+ let r = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap();
+ assert_eq!(r["amount"], 50.0);
+ assert_eq!(x.s.undo.len(), undo + 1);
+ x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap();
+ assert_eq!(x.s.develop_of(id).unwrap().denoise_amount(), 0.0);
+ // the switch is its own setting: flipping it keeps the Amount (so it compares with and without), and an Amount of 0
+ // does not switch it off
+ x.s.execute("library.select", &json!({"ids": [id.0]})).unwrap();
+ x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap();
+ x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 80.0})).unwrap();
+ let off = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap();
+ assert_eq!(off["amount"], 80.0);
+ let d = x.s.develop_of(id).unwrap();
+ assert!(!d.enhance.denoise_enabled() && d.denoise_amount() == 0.0);
+ assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 80.0);
+ assert!((x.s.develop_of(id).unwrap().denoise_amount() - 0.8).abs() < 1e-6);
+ x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0.0})).unwrap();
+ assert!(x.s.develop_of(id).unwrap().enhance.denoise_enabled(), "sliding to 0 leaves it on");
+ assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 0.0);
+ // with no `enabled`, the toggle flips what the switch says
+ assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0})).unwrap()["enabled"], false);
+ drop(x.s);
+ let _ = std::fs::remove_dir_all(x.dir);
+}
diff --git a/crates/engine/src/tests_xmp.rs b/crates/engine/src/tests_xmp.rs
index fede1fec0..ab801e237 100644
--- a/crates/engine/src/tests_xmp.rs
+++ b/crates/engine/src/tests_xmp.rs
@@ -335,10 +335,15 @@ fn synthetic_dng(xmp: &str) -> Vec<u8> {
/// A tiny Bayer DNG written by our own DNG writer (optional embedded XMP, camera metadata).
pub(crate) fn synthetic_dng_with(xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec<u8> {
- use lightcraft_raw::*;
let (w, h) = (32usize, 24usize);
- let cfa = Cfa::bayer("RGGB").unwrap();
let data: Vec<u16> = (0..w * h).map(|i| 256 + ((i % w) * 300 + (i / w) * 200) as u16).collect();
+ synthetic_dng_of(w, h, "RGGB", data, xmp, metadata)
+}
+
+/// A synthetic Bayer DNG for denoise and cache tests.
+pub(crate) fn synthetic_dng_of(w: usize, h: usize, cfa: &str, data: Vec<u16>, xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec<u8> {
+ use lightcraft_raw::*;
+ let cfa = Cfa::bayer(cfa).unwrap();
let raw = RawImage {
format: RawFormat::Dng,
width: w,
diff --git a/crates/engine/tests/denoise_cpu.rs b/crates/engine/tests/denoise_cpu.rs
new file mode 100644
index 000000000..d01b7c5a5
--- /dev/null
+++ b/crates/engine/tests/denoise_cpu.rs
@@ -0,0 +1,101 @@
+//! Opt-in same-machine denoise model-stage benchmark on one public Bayer raw.
+//! LC_DENOISE_MODEL=<onnx> LC_DENOISE_RAW=<CC0 corpus/raw> cargo test --release
+//! -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture
+//! LC_DENOISE_ONLY selects a file (default arw-sony-a7m3-compressed.arw).
+//! The whole model stage includes packing, scale/clip protection, blending and normalization;
+//! file decode, model loading and GPU setup are outside the timer. It uses the application's tile pool convention.
+#![cfg(feature = "denoise")]
+
+use lightcraft_denoise::{
+ TileRunner,
+ bayer::Layout,
+ run::{Control, Params, denoise_bayer},
+ runtime::{CpuRunner, check_tile},
+};
+use lightcraft_raster::Rgb32f;
+use std::{path::Path, time::Instant};
+
+fn agree(a: &[f32], b: &[f32]) -> f32 {
+ assert_eq!(a.len(), b.len());
+ assert!(a.iter().chain(b).all(|v| v.is_finite()));
+ let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs()));
+ let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
+ assert!(error <= 1e-3, "maximum relative error {error:e}");
+ error
+}
+fn tile_time(name: &str, r: &dyn TileRunner, x: &[f32]) {
+ r.run(x).unwrap();
+ let mut times: Vec<_> = (0..5)
+ .map(|_| {
+ let t = Instant::now();
+ r.run(x).unwrap();
+ t.elapsed().as_secs_f64() * 1000.0
+ })
+ .collect();
+ times.sort_by(f64::total_cmp);
+ println!("{name}, one 512-cell tile, one thread: {:.1} ms median ({:.1}..{:.1})", times[2], times[0], times[4]);
+}
+fn photo_time(name: &str, r: &dyn TileRunner, raw: &lightcraft_raw::Normalized, layout: Layout, p: &Params) -> Rgb32f {
+ let pool = rayon::ThreadPoolBuilder::new().num_threads(p.parallel).build().unwrap();
+ let run = || pool.install(|| denoise_bayer(&raw.data, raw.width, raw.height, layout, r, p, &Control::default())).unwrap();
+ drop(run()); // warm the workers and their workspace cache
+ let mut times = Vec::new();
+ let mut answer = Rgb32f { width: 0, height: 0, data: Vec::new() };
+ for i in 0..3 {
+ let t = Instant::now();
+ answer = run();
+ let secs = t.elapsed().as_secs_f64();
+ times.push(secs);
+ println!("{name}, whole model stage, {} workers, run {i}: {secs:.3} s", p.parallel);
+ }
+ times.sort_by(f64::total_cmp);
+ println!("{name}, whole model stage median {:.3} s", times[1]);
+ answer
+}
+#[test]
+#[ignore = "needs LC_DENOISE_MODEL and LC_DENOISE_RAW; release timing and real-model accuracy"]
+fn denoise_stage_compares_cpu_and_gpu() {
+ let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL");
+ let folder = std::env::var_os("LC_DENOISE_RAW").expect("set LC_DENOISE_RAW to public CC0 corpus/raw");
+ let name = std::env::var("LC_DENOISE_ONLY").unwrap_or_else(|_| "arw-sony-a7m3-compressed.arw".into());
+ let bytes = std::fs::read(Path::new(&folder).join(&name)).unwrap();
+ let raw = lightcraft_raw::decode(&bytes).unwrap().normalized().unwrap();
+ let cfa = raw.cfa.as_ref().unwrap();
+ assert!(cfa.is_bayer());
+ let layout = Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)]).unwrap();
+ let m = lightcraft_denoise::known::find("rawnind-bayer").unwrap().manifest;
+ let cpu = CpuRunner::load(Path::new(&model), &m).unwrap();
+ let threads = std::env::var("LC_DENOISE_PARALLEL")
+ .ok()
+ .and_then(|v| v.parse::<usize>().ok())
+ .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get()).clamp(1, 16));
+ let p = Params { tile: 512, overlap: 64, gain: m.gain, clip: Some(1.0), parallel: threads };
+ println!(
+ "{name}: {}x{}, {:.2} MP, {} tiles, {} CPU workers, {:.1} MB/workspace, {:.1} GMAC/tile",
+ raw.width,
+ raw.height,
+ raw.width as f64 * raw.height as f64 / 1e6,
+ lightcraft_denoise::run::tile_count(raw.width, raw.height, layout, 512, 64),
+ threads,
+ cpu.bytes_per_tile() as f64 / 1e6,
+ cpu.net().macs(512) as f64 / 1e9
+ );
+ let x = check_tile(512);
+ let want = cpu.run(&x).unwrap();
+ tile_time("pure Rust", &cpu, &x);
+ let done = photo_time("pure Rust", &cpu, &raw, layout, &p);
+
+ match lightcraft_gpu::nn::runner(cpu.net(), 512) {
+ Ok(gpu) => {
+ println!("GPU: {}", gpu.adapter());
+ println!("CPU/check tile vs GPU: {:e}", agree(&want, &gpu.run(&x).unwrap()));
+ tile_time("GPU", &gpu, &x);
+ let gp = Params { parallel: 4, ..p };
+ let answer = photo_time("GPU", &gpu, &raw, layout, &gp);
+ let a: Vec<_> = done.data.iter().flatten().copied().collect();
+ let b: Vec<_> = answer.data.iter().flatten().copied().collect();
+ println!("CPU/whole photo vs GPU: {:e}", agree(&a, &b));
+ }
+ Err(e) => println!("GPU unavailable: {e}"),
+ }
+}
diff --git a/crates/engine/tests/denoise_real.rs b/crates/engine/tests/denoise_real.rs
new file mode 100644
index 000000000..8d8581753
--- /dev/null
+++ b/crates/engine/tests/denoise_real.rs
@@ -0,0 +1,117 @@
+//! Opt-in end-to-end check of AI denoise with the real RawNIND Bayer model, through the engine: install it, make a real
+//! raw photo's picture in the background, render and export it with the Amount slider, and see what a file that is not
+//! Bayer says.
+//!
+//! ```text
+//! LC_DENOISE_MODEL=<model_bayer.onnx> LC_DENOISE_RAW=<folder with raw files> \
+//! cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture
+//! ```
+//!
+//! `LC_DENOISE_RAW` is a folder of CC0 raw files (`cargo xtask corpus --download` → `corpus/raw`); every file in it is
+//! tried. Optional: `LC_DENOISE_ONLY` (a file name to try alone), `LC_DENOISE_RUN_ON=cpu|gpu|auto` (where the model runs; `cpu` to compare with the card).
+#![cfg(feature = "denoise")]
+
+use std::path::{Path, PathBuf};
+use std::time::{Duration, Instant};
+
+use lightcraft_engine::Session;
+use serde_json::{Value, json};
+
+fn temp(tag: &str) -> PathBuf {
+ let d = std::env::temp_dir().join(format!("lc-denoise-real-{tag}-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&d);
+ std::fs::create_dir_all(&d).unwrap();
+ d
+}
+
+/// Mean absolute second difference of the luma: the fine detail and the noise of a picture.
+fn roughness(img: &lightcraft_raster::Rgba8) -> f64 {
+ let (w, h) = (img.width, img.height);
+ let luma = |x: usize, y: usize| {
+ let p = img.data[y * w + x];
+ 0.299 * f64::from(p[0]) + 0.587 * f64::from(p[1]) + 0.114 * f64::from(p[2])
+ };
+ let (mut sum, mut n) = (0.0, 0usize);
+ for y in 1..h - 1 {
+ for x in 1..w - 1 {
+ sum += (4.0 * luma(x, y) - luma(x - 1, y) - luma(x + 1, y) - luma(x, y - 1) - luma(x, y + 1)).abs();
+ n += 1;
+ }
+ }
+ sum / n.max(1) as f64
+}
+
+fn files(dir: &Path) -> Vec<PathBuf> {
+ let only = std::env::var("LC_DENOISE_ONLY").ok();
+ let mut v: Vec<PathBuf> = std::fs::read_dir(dir)
+ .unwrap()
+ .flatten()
+ .map(|e| e.path())
+ .filter(|p| p.is_file() && only.as_deref().is_none_or(|o| p.file_name().is_some_and(|n| n.to_string_lossy() == o)))
+ .collect();
+ v.sort();
+ v
+}
+
+#[test]
+#[ignore = "needs the real model and raw files (see the file's header)"]
+fn the_real_model_denoises_real_raws_through_the_engine() {
+ let (Some(model), Some(raws)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_RAW")) else {
+ panic!("set LC_DENOISE_MODEL and LC_DENOISE_RAW");
+ };
+ let dir = temp("e2e");
+ let mut s = Session::new().with_fs();
+ s.open_library(dir.join("library"), false).unwrap();
+ s.set_denoise_models_dir(Some(dir.join("models")));
+ let t = Instant::now();
+ let r = s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();
+ println!("installed {} in {:.1} s: self-test {}", r["installed"]["id"], t.elapsed().as_secs_f64(), r["installed"]["accepted"]["selfTest"]);
+ assert_eq!(r["installed"]["id"], "rawnind-bayer");
+ if let Ok(run_on) = std::env::var("LC_DENOISE_RUN_ON") {
+ s.execute("denoise.settings", &json!({"runOn": run_on})).unwrap();
+ }
+
+ for file in files(Path::new(&raws)) {
+ let name = file.file_name().unwrap().to_string_lossy().into_owned();
+ let imported = s.execute("library.import", &json!({"paths": [file.to_string_lossy()]})).unwrap();
+ let Some(id) = imported["imported"].as_array().and_then(|a| a.first()).and_then(Value::as_u64) else {
+ println!("{name}: not imported ({imported})");
+ continue;
+ };
+ s.execute("library.select", &json!({"ids": [id]})).unwrap();
+ s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap();
+ let photo = lightcraft_engine::catalog::PhotoId(id);
+ let started = Instant::now();
+ let mut state = Value::Null;
+ while started.elapsed() < Duration::from_secs(600) {
+ s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
+ state = s.execute("denoise.status", &json!({"id": id})).unwrap()["photo"].clone();
+ if matches!(state["state"].as_str(), Some("ready" | "failed")) {
+ break;
+ }
+ std::thread::sleep(Duration::from_millis(250));
+ }
+ println!("{name}: {} in {:.1} s", state, started.elapsed().as_secs_f64());
+ if state["state"] == "failed" {
+ // a file that is not a Bayer raw says why, and its export is the plain one
+ assert_eq!(state["unsupported"], true, "{name}: {state}");
+ continue;
+ }
+ assert_eq!(state["state"], "ready", "{name}: {state}");
+ let denoised = s.render_now(photo, 1600, 1600).unwrap().image;
+ s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0})).unwrap();
+ let plain = s.render_now(photo, 1600, 1600).unwrap().image;
+ let (a, b) = (roughness(&plain), roughness(&denoised));
+ println!("{name}: roughness at 1600 px: plain {a:.3}, denoised {b:.3} ({:+.1} %)", 100.0 * (b - a) / a);
+ assert!(b < a, "{name}: the denoised picture is not smoother");
+ // the export is made from the same picture, without asking the queue
+ s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap();
+ let o = lightcraft_engine::export::ExportOptions::from_json(&json!({"format": "jpeg", "width": 1200, "height": 1200}));
+ let e = lightcraft_engine::export::export_photo(&mut s, photo, &o, 1).unwrap();
+ assert!(e.bytes.len() > 1000);
+ }
+ let status = s.execute("denoise.status", &json!({})).unwrap();
+ println!("device: {}", status["device"]);
+ println!("cache: {}", status["cache"]);
+ let _ = std::fs::remove_dir_all(&dir);
+}
diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml
index 765e4c6d7..314f69c83 100644
--- a/crates/gpu/Cargo.toml
+++ b/crates/gpu/Cargo.toml
@@ -9,6 +9,7 @@ description = "LightCraft GPU develop pipeline: wgpu compute (WGSL) twin of the
workspace = true
[dependencies]
+lightcraft-denoise = { workspace = true }
lightcraft-geom = { workspace = true }
lightcraft-color = { workspace = true }
lightcraft-raster = { workspace = true }
diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs
index 312d42635..b4e32ba9c 100644
--- a/crates/gpu/src/lib.rs
+++ b/crates/gpu/src/lib.rs
@@ -30,6 +30,8 @@ pub mod backend;
#[cfg(not(target_arch = "wasm32"))]
mod ctx;
#[cfg(not(target_arch = "wasm32"))]
+pub mod nn;
+#[cfg(not(target_arch = "wasm32"))]
mod params;
#[cfg(not(target_arch = "wasm32"))]
mod render;
@@ -145,6 +147,16 @@ pub fn enabled() -> bool {
ENABLED.load(Ordering::Relaxed) && !BROKEN.load(Ordering::Relaxed) && !env_disabled()
}
+/// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the
+/// environment switches and the rendering preference, but not a render-device failure (the devices are separate).
+#[cfg(not(target_arch = "wasm32"))]
+pub(crate) fn switched_off() -> Option<String> {
+ if env_disabled() {
+ return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());
+ }
+ (!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into())
+}
+
/// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process.
fn env_disabled() -> bool {
static OFF: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
diff --git a/crates/gpu/src/nn.rs b/crates/gpu/src/nn.rs
new file mode 100644
index 000000000..4231dae8c
--- /dev/null
+++ b/crates/gpu/src/nn.rs
@@ -0,0 +1,1048 @@
+//! Denoise networks on the GPU: a [`NetRunner`] executes a [`Net`] (the plain description of a U-Net's layers) with
+//! WGSL compute kernels, one tile at a time, behind the same [`TileRunner`] interface the CPU runner has.
+//!
+//! The runner has a device of its own, so a denoise job and the interactive renders are separate contexts that the
+//! driver time-slices, and a failure of one never takes the other down. It respects the same switches as GPU rendering
+//! (`LIGHTCRAFT_GPU`, `LIGHTCRAFT_GPU_BACKEND`, the GPU preference) and the same crash sentinel around device creation.
+//! `LIGHTCRAFT_GPU_ADAPTER` picks an adapter by (part of) its name, e.g. to try another card.
+//!
+//! Every failure is an `Err` (never a panic), and a device that errs or is lost stops being used: callers then run the
+//! CPU runner. The kernels (`wgsl/nn_conv.wgsl`, `wgsl/nn_pool.wgsl`) are tested against the plain-loop reference
+//! interpreter in `lightcraft_denoise::reference` and, with the real model, against the pure-Rust CPU runner.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Condvar, Mutex, OnceLock};
+use std::time::Duration;
+
+use bytemuck::{Pod, Zeroable};
+use lightcraft_denoise::net::{Net, Op};
+use lightcraft_denoise::run::{Error, TileRunner};
+use wgpu::util::DeviceExt;
+
+/// Longest wait for one tile's GPU work before the device is given up on.
+const TIMEOUT: Duration = Duration::from_secs(60);
+/// The largest tile side (in input cells) the dispatch sizes and 32-bit indexing are checked for.
+const MAX_TILE: usize = 1024;
+/// The largest single buffer used: indices in the kernels are 32-bit element offsets.
+const MAX_BUFFER: u64 = 2 << 30;
+/// Tiles that can be on the GPU at once (each has buffers of its own).
+const WORKSPACES: usize = 2;
+
+// ---------------------------------------------------------------------------------------------------------------------
+// the device
+
+struct Dev {
+ device: wgpu::Device,
+ queue: wgpu::Queue,
+ info: wgpu::AdapterInfo,
+ limits: wgpu::Limits,
+}
+
+static DEV: OnceLock<Result<Dev, String>> = OnceLock::new();
+static BROKEN: AtomicBool = AtomicBool::new(false);
+static BROKEN_WHY: Mutex<Option<String>> = Mutex::new(None);
+
+fn mark_broken(why: &str) {
+ let mut w = BROKEN_WHY.lock().unwrap_or_else(|e| e.into_inner());
+ if w.is_none() {
+ *w = Some(why.to_string());
+ }
+ BROKEN.store(true, Ordering::Relaxed);
+}
+
+/// Why the GPU stopped being used for denoise (a device error), if it did.
+pub fn broken_reason() -> Option<String> {
+ BROKEN.load(Ordering::Relaxed).then(|| BROKEN_WHY.lock().unwrap_or_else(|e| e.into_inner()).clone().unwrap_or_else(|| "device error".into()))
+}
+
+fn dev() -> Result<&'static Dev, String> {
+ if let Some(why) = broken_reason() {
+ return Err(format!("stopped after a GPU failure: {why}"));
+ }
+ if let Some(why) = crate::switched_off() {
+ return Err(why);
+ }
+ DEV.get_or_init(create_device).as_ref().map_err(|e| e.clone())
+}
+
+fn create_device() -> Result<Dev, String> {
+ let Some(backends) = crate::backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) };
+ crate::backend::with_init_marker(backends, || {
+ std::panic::catch_unwind(|| make_device(backends)).unwrap_or_else(|_| Err("device creation panicked".into()))
+ })
+}
+
+fn make_device(backends: wgpu::Backends) -> Result<Dev, String> {
+ let mut desc = wgpu::InstanceDescriptor::new_without_display_handle();
+ desc.backends = backends;
+ let instance = wgpu::Instance::new(desc);
+ let wanted = std::env::var("LIGHTCRAFT_GPU_ADAPTER").ok().map(|s| s.trim().to_lowercase()).filter(|s| !s.is_empty());
+ let adapter = match &wanted {
+ Some(w) => {
+ let all = pollster::block_on(instance.enumerate_adapters(backends));
+ let names: Vec<String> = all.iter().map(|a| a.get_info().name).collect();
+ all.into_iter()
+ .find(|a| {
+ let i = a.get_info();
+ i.device_type != wgpu::DeviceType::Cpu && i.name.to_lowercase().contains(w.as_str())
+ })
+ .ok_or_else(|| format!("no GPU adapter named like `{w}` (found: {})", names.join(", ")))?
+ }
+ None => pollster::block_on(
+ instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }),
+ )
+ .map_err(|e| format!("no GPU adapter among {backends:?} ({e})"))?,
+ };
+ let info = adapter.get_info();
+ if info.device_type == wgpu::DeviceType::Cpu {
+ return Err(format!("software adapter ({}) skipped: the CPU is faster", info.name));
+ }
+ let limits = adapter.limits();
+ if limits.max_storage_buffers_per_shader_stage < 6
+ || limits.max_compute_invocations_per_workgroup < 256
+ || limits.max_compute_workgroup_size_x < 256
+ {
+ return Err(format!("{} is too limited for the denoise kernels", info.name));
+ }
+ if limits.max_compute_workgroup_storage_size < 16 * 1024 {
+ return Err(format!("{} has too little workgroup memory for the denoise kernels", info.name));
+ }
+ let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {
+ label: Some("lightcraft-denoise"),
+ required_limits: limits.clone(),
+ ..Default::default()
+ }))
+ .map_err(|e| format!("{}: device creation failed ({e})", info.name))?;
+ device.on_uncaptured_error(std::sync::Arc::new(|e| {
+ log::error!("gpu (denoise): {e}");
+ mark_broken(&format!("device error: {e}"));
+ }));
+ device.set_device_lost_callback(|reason, msg| {
+ log::error!("gpu (denoise): device lost ({reason:?}): {msg}");
+ mark_broken(&format!("device lost ({reason:?}): {msg}"));
+ });
+ log::info!("gpu (denoise): {} ({:?})", info.name, info.backend);
+ Ok(Dev { device, queue, info, limits })
+}
+
+/// The adapter denoise would use ("NVIDIA GeForce RTX 4090 Laptop GPU (Dx12)"), or why it cannot use one. Creates the
+/// device on first use.
+pub fn adapter() -> Result<String, String> {
+ dev().map(|d| format!("{} ({:?})", d.info.name, d.info.backend))
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// the plan: what to run, in what buffers
+
+#[repr(C)]
+#[derive(Clone, Copy, Pod, Zeroable)]
+struct ConvParams {
+ h: u32,
+ w: u32,
+ c0: u32,
+ c1: u32,
+ n: u32,
+ npad: u32,
+ k: u32,
+ mode: u32,
+ cout: u32,
+ leaky: u32,
+ alpha: f32,
+ pad: u32,
+}
+
+#[repr(C)]
+#[derive(Clone, Copy, Pod, Zeroable)]
+struct PoolParams {
+ h: u32,
+ w: u32,
+ c4: u32,
+ total: u32,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
+enum Kernel {
+ Conv { bm: u32, bn: u32, bk: u32 },
+ Pool,
+}
+
+/// Where a step's tensor lives.
+#[derive(Clone, Copy, Debug)]
+enum Place {
+ Input,
+ Slot(usize),
+ Output,
+}
+
+struct Step {
+ kernel: Kernel,
+ groups: [u32; 3],
+ src: Place,
+ src2: Option<Place>,
+ dst: Place,
+ /// Index into the per-layer GPU data (weights, bias), for the convolutions.
+ layer: Option<usize>,
+ params: Vec<u8>,
+}
+
+struct Layout {
+ steps: Vec<Step>,
+ /// Bytes of each activation buffer.
+ slots: Vec<u64>,
+ in_floats: usize,
+ out_floats: usize,
+}
+
+/// Weights arranged for the kernel: `[k][npad]` rows with `k = tap · cin + ci` (columns are output channels, zero past
+/// `n`), and the bias padded the same.
+struct Layer {
+ weight: Vec<f32>,
+ bias: Vec<f32>,
+}
+
+fn unsupported<T>(why: impl Into<String>) -> Result<T, String> {
+ Err(why.into())
+}
+
+/// The kernel shape for `n` output columns: 64 × 64 blocks when `n` is a multiple of 64, else 128 × 32.
+fn blocks(n: usize, cin: usize) -> (u32, u32, u32) {
+ let bn = if n.is_multiple_of(64) { 64 } else { 32 };
+ let bk = if cin.is_multiple_of(16) {
+ 16
+ } else if cin.is_multiple_of(8) {
+ 8
+ } else {
+ 4
+ };
+ (4096 / bn, bn, bk)
+}
+
+fn round_up(v: usize, to: usize) -> usize {
+ v.div_ceil(to) * to
+}
+
+/// Work out the steps, the buffers and the arranged weights for `net` on `tile × tile` input cells.
+fn plan(net: &Net, tile: usize, max_binding: u64) -> Result<(Layout, Vec<Layer>), String> {
+ if !net.fits_tile(tile) {
+ return unsupported("the tile does not fit the network");
+ }
+ if tile > MAX_TILE {
+ return unsupported("the tile is larger than the kernels' dispatches reach");
+ }
+ let side = |t: usize| -> usize {
+ let level = net.shape(t).map_or(0, |s| s.level);
+ if level >= 0 { tile >> level } else { tile << (-level) }
+ };
+ let chan = |t: usize| net.shape(t).map_or(0, |s| s.channels);
+ if !net.in_channels.is_multiple_of(4) {
+ return unsupported("the input channel count is not a multiple of 4");
+ }
+ let ops = net.ops();
+ // the network must end in a 1 × 1 convolution and a depth-to-space (they run as one kernel)
+ let n_ops = ops.len();
+ let ends_right = matches!(
+ (n_ops.checked_sub(2).and_then(|i| ops.get(i)), ops.last()),
+ (Some(Op::Conv(c)), Some(Op::DepthToSpace2 { src })) if c.k == 1 && c.src2.is_none() && *src == n_ops - 1 && c.cout == 12 && c.leaky.is_none()
+ );
+ if !ends_right {
+ return unsupported("the network does not end in a 1 × 1 convolution to 12 channels and depth-to-space");
+ }
+ // the last layer that reads each tensor
+ let mut last_use = vec![0usize; n_ops + 1];
+ for (i, op) in ops.iter().enumerate() {
+ let reads: Vec<usize> = match op {
+ Op::Conv(c) => std::iter::once(c.src).chain(c.src2).collect(),
+ Op::ConvT2(c) => vec![c.src],
+ Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => vec![*src],
+ };
+ for t in reads {
+ if let Some(l) = last_use.get_mut(t) {
+ *l = i;
+ }
+ }
+ }
+
+ use Place::{Input, Output, Slot};
+ let mut slots: Vec<u64> = Vec::new();
+ let mut free: Vec<usize> = Vec::new();
+ let mut place: Vec<Option<Place>> = vec![None; n_ops + 1];
+ if let Some(p) = place.first_mut() {
+ *p = Some(Input);
+ }
+ let mut steps: Vec<Step> = Vec::new();
+ let mut layers: Vec<Layer> = Vec::new();
+ /// A buffer of at least `bytes`: the smallest free one that is big enough, else the biggest free one grown, else new.
+ fn alloc(bytes: u64, slots: &mut Vec<u64>, free: &mut Vec<usize>) -> usize {
+ let size = |s: usize| slots.get(s).copied().unwrap_or(0);
+ let fits = free.iter().enumerate().filter(|&(_, &s)| size(s) >= bytes).min_by_key(|&(_, &s)| size(s)).map(|(i, _)| i);
+ let pick = fits.or_else(|| free.iter().enumerate().max_by_key(|&(_, &s)| size(s)).map(|(i, _)| i));
+ match pick {
+ Some(i) => {
+ let s = free.swap_remove(i);
+ if let Some(b) = slots.get_mut(s) {
+ *b = (*b).max(bytes);
+ }
+ s
+ }
+ None => {
+ slots.push(bytes);
+ slots.len() - 1
+ }
+ }
+ }
+ let get = |place: &[Option<Place>], t: usize| place.get(t).copied().flatten().ok_or_else(|| format!("tensor {t} has no buffer"));
+
+ for (i, op) in ops.iter().enumerate() {
+ let out_t = i + 1;
+ match op {
+ Op::Conv(c) => {
+ if !c.cin.is_multiple_of(4) || !c.cout.is_multiple_of(4) {
+ return unsupported("a convolution whose channel counts are not multiples of 4");
+ }
+ let s = side(c.src);
+ let last = i + 2 == n_ops;
+ let (n, mode) = (c.cout, if last { 2u32 } else { 0u32 });
+ let (bm, bn, bk) = blocks(n, c.cin);
+ let npad = round_up(n, bn as usize);
+ // arranged weights: row (tap · cin + ci), column co
+ let taps = c.k * c.k;
+ let mut weight = vec![0f32; taps * c.cin * npad];
+ for co in 0..c.cout {
+ for ci in 0..c.cin {
+ for ky in 0..c.k {
+ for kx in 0..c.k {
+ let v = c.weight.get(((co * c.cin + ci) * c.k + ky) * c.k + kx).copied().unwrap_or(0.0);
+ if let Some(w) = weight.get_mut(((ky * c.k + kx) * c.cin + ci) * npad + co) {
+ *w = v;
+ }
+ }
+ }
+ }
+ }
+ let mut bias = vec![0f32; npad];
+ if let Some(b) = bias.get_mut(..c.cout) {
+ b.copy_from_slice(&c.bias);
+ }
+ let m = s * s;
+ let params = ConvParams {
+ h: s as u32,
+ w: s as u32,
+ c0: chan(c.src) as u32,
+ c1: c.src2.map_or(0, chan) as u32,
+ n: n as u32,
+ npad: npad as u32,
+ k: c.k as u32,
+ mode,
+ cout: c.cout as u32,
+ leaky: u32::from(c.leaky.is_some()),
+ alpha: c.leaky.unwrap_or(0.0),
+ pad: 0,
+ };
+ let src = get(&place, c.src)?;
+ let src2 = c.src2.map(|t| get(&place, t)).transpose()?;
+ let dst = if last {
+ Output
+ } else {
+ let slot = alloc((m * c.cout * 4) as u64, &mut slots, &mut free);
+ Slot(slot)
+ };
+ if let Some(p) = place.get_mut(out_t) {
+ *p = Some(dst);
+ }
+ layers.push(Layer { weight, bias });
+ steps.push(Step {
+ kernel: Kernel::Conv { bm, bn, bk },
+ groups: [m.div_ceil(bm as usize) as u32, (npad as u32) / bn, 1],
+ src,
+ src2,
+ dst,
+ layer: Some(layers.len() - 1),
+ params: bytemuck::bytes_of(&params).to_vec(),
+ });
+ }
+ Op::ConvT2(c) => {
+ if !c.cin.is_multiple_of(4) || !c.cout.is_multiple_of(4) {
+ return unsupported("a transposed convolution whose channel counts are not multiples of 4");
+ }
+ let s = side(c.src);
+ let n = 4 * c.cout;
+ let (bm, bn, bk) = blocks(n, c.cin);
+ let npad = round_up(n, bn as usize);
+ // arranged weights: row ci, column (ky · 2 + kx) · cout + co
+ let mut weight = vec![0f32; c.cin * npad];
+ for ci in 0..c.cin {
+ for co in 0..c.cout {
+ for ky in 0..2 {
+ for kx in 0..2 {
+ let v = c.weight.get(((ci * c.cout + co) * 2 + ky) * 2 + kx).copied().unwrap_or(0.0);
+ if let Some(w) = weight.get_mut(ci * npad + (ky * 2 + kx) * c.cout + co) {
+ *w = v;
+ }
+ }
+ }
+ }
+ }
+ let mut bias = vec![0f32; npad];
+ for q in 0..4 {
+ for co in 0..c.cout {
+ if let (Some(b), Some(v)) = (bias.get_mut(q * c.cout + co), c.bias.get(co)) {
+ *b = *v;
+ }
+ }
+ }
+ let m = s * s;
+ let params = ConvParams {
+ h: s as u32,
+ w: s as u32,
+ c0: c.cin as u32,
+ c1: 0,
+ n: n as u32,
+ npad: npad as u32,
+ k: 1,
+ mode: 1,
+ cout: c.cout as u32,
+ leaky: u32::from(c.leaky.is_some()),
+ alpha: c.leaky.unwrap_or(0.0),
+ pad: 0,
+ };
+ let src = get(&place, c.src)?;
+ let slot = alloc((4 * m * c.cout * 4) as u64, &mut slots, &mut free);
+ if let Some(p) = place.get_mut(out_t) {
+ *p = Some(Slot(slot));
+ }
+ layers.push(Layer { weight, bias });
+ steps.push(Step {
+ kernel: Kernel::Conv { bm, bn, bk },
+ groups: [m.div_ceil(bm as usize) as u32, (npad as u32) / bn, 1],
+ src,
+ src2: None,
+ dst: Slot(slot),
+ layer: Some(layers.len() - 1),
+ params: bytemuck::bytes_of(&params).to_vec(),
+ });
+ }
+ Op::MaxPool2 { src } => {
+ let (s, c) = (side(*src), chan(*src));
+ if !c.is_multiple_of(4) {
+ return unsupported("a max-pool over a channel count that is not a multiple of 4");
+ }
+ let total = (s / 2) * (s / 2) * (c / 4);
+ let params = PoolParams { h: s as u32, w: s as u32, c4: (c / 4) as u32, total: total as u32 };
+ let from = get(&place, *src)?;
+ let slot = alloc((total * 16) as u64, &mut slots, &mut free);
+ if let Some(p) = place.get_mut(out_t) {
+ *p = Some(Slot(slot));
+ }
+ let groups = total.div_ceil(64);
+ steps.push(Step {
+ kernel: Kernel::Pool,
+ groups: [groups.min(8192) as u32, groups.div_ceil(8192) as u32, 1],
+ src: from,
+ src2: None,
+ dst: Slot(slot),
+ layer: None,
+ params: bytemuck::bytes_of(&params).to_vec(),
+ });
+ }
+ // run as part of the 1 × 1 convolution before it
+ Op::DepthToSpace2 { .. } => {}
+ }
+ // buffers whose last reader was this layer are free for the next layer's result
+ for t in 0..=i {
+ if last_use.get(t).copied() == Some(i)
+ && let Some(Some(Slot(s))) = place.get(t)
+ && !free.contains(s)
+ {
+ free.push(*s);
+ }
+ }
+ }
+ if slots.iter().any(|&b| b > max_binding) {
+ return unsupported("an activation buffer is larger than this GPU allows");
+ }
+ if layers.iter().any(|l| (l.weight.len() as u64).saturating_mul(4) > max_binding) {
+ return unsupported("a weight buffer is larger than this GPU allows");
+ }
+ Ok((Layout { steps, slots, in_floats: net.in_channels * tile * tile, out_floats: 3 * (2 * tile) * (2 * tile) }, layers))
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// the GPU objects
+
+struct Pipelines {
+ conv: HashMap<(u32, u32, u32), wgpu::ComputePipeline>,
+ conv_layout: wgpu::BindGroupLayout,
+ pool: wgpu::ComputePipeline,
+ pool_layout: wgpu::BindGroupLayout,
+ /// How long each kernel took to build (the driver compiles them), for the log and the tests.
+ built: Vec<(String, f64)>,
+}
+
+fn entry(binding: u32, ty: wgpu::BufferBindingType) -> wgpu::BindGroupLayoutEntry {
+ wgpu::BindGroupLayoutEntry {
+ binding,
+ visibility: wgpu::ShaderStages::COMPUTE,
+ ty: wgpu::BindingType::Buffer { ty, has_dynamic_offset: false, min_binding_size: None },
+ count: None,
+ }
+}
+
+fn pipeline(d: &Dev, label: &str, src: &str, layout: &wgpu::BindGroupLayout) -> wgpu::ComputePipeline {
+ let module = d.device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some(label), source: wgpu::ShaderSource::Wgsl(src.into()) });
+ let pl = d.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: None, bind_group_layouts: &[Some(layout)], immediate_size: 0 });
+ d.device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
+ label: Some(label),
+ layout: Some(&pl),
+ module: &module,
+ entry_point: Some("main"),
+ // The kernels write all of their workgroup memory before reading any of it, so the zeroing wgpu adds by default
+ // is not needed; on DX12 it is written out as one assignment per element, which makes the shader compiler take
+ // a minute over a 2k-element array.
+ compilation_options: wgpu::PipelineCompilationOptions { zero_initialize_workgroup_memory: false, ..Default::default() },
+ cache: None,
+ })
+}
+
+fn conv_source(bm: u32, bn: u32, bk: u32) -> String {
+ include_str!("wgsl/nn_conv.wgsl")
+ .replace("{{BM}}", &bm.to_string())
+ .replace("{{BN}}", &bn.to_string())
+ .replace("{{BK}}", &bk.to_string())
+ .replace("{{A_VECS}}", &(bk * bm / 4).to_string())
+ .replace("{{B_VECS}}", &(bk * bn / 4).to_string())
+}
+
+fn conv_bgl(d: &Dev) -> wgpu::BindGroupLayout {
+ use wgpu::BufferBindingType::{Storage, Uniform};
+ d.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+ label: None,
+ entries: &[
+ entry(0, Uniform),
+ entry(1, Storage { read_only: true }),
+ entry(2, Storage { read_only: true }),
+ entry(3, Storage { read_only: true }),
+ entry(4, Storage { read_only: true }),
+ entry(5, Storage { read_only: false }),
+ ],
+ })
+}
+
+fn pool_bgl(d: &Dev) -> wgpu::BindGroupLayout {
+ use wgpu::BufferBindingType::{Storage, Uniform};
+ d.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+ label: None,
+ entries: &[entry(0, Uniform), entry(1, Storage { read_only: true }), entry(2, Storage { read_only: false })],
+ })
+}
+
+fn build_pipelines(d: &Dev, layout: &Layout) -> Pipelines {
+ let (conv_layout, pool_layout) = (conv_bgl(d), pool_bgl(d));
+ let mut conv = HashMap::new();
+ let mut built = Vec::new();
+ for s in &layout.steps {
+ if let Kernel::Conv { bm, bn, bk } = s.kernel {
+ conv.entry((bm, bn, bk)).or_insert_with(|| {
+ let started = std::time::Instant::now();
+ let p = pipeline(d, "nn_conv", &conv_source(bm, bn, bk), &conv_layout);
+ built.push((format!("conv {bm}x{bn}x{bk}"), started.elapsed().as_secs_f64() * 1000.0));
+ p
+ });
+ }
+ }
+ let started = std::time::Instant::now();
+ let pool = pipeline(d, "nn_pool", include_str!("wgsl/nn_pool.wgsl"), &pool_layout);
+ built.push(("pool".into(), started.elapsed().as_secs_f64() * 1000.0));
+ Pipelines { conv, conv_layout, pool, pool_layout, built }
+}
+
+/// One tile's buffers and the bound steps.
+struct Workspace {
+ input: wgpu::Buffer,
+ output: wgpu::Buffer,
+ staging: wgpu::Buffer,
+ bound: Vec<(Kernel, [u32; 3], wgpu::BindGroup)>,
+ _slots: Vec<wgpu::Buffer>,
+}
+
+struct Shared {
+ dev: &'static Dev,
+ layout: Layout,
+ pipes: Pipelines,
+ weights: Vec<(wgpu::Buffer, wgpu::Buffer)>,
+ uniforms: Vec<wgpu::Buffer>,
+ dummy: wgpu::Buffer,
+}
+
+fn storage(d: &Dev, label: &str, bytes: u64, extra: wgpu::BufferUsages) -> wgpu::Buffer {
+ d.device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some(label),
+ size: bytes.max(16),
+ usage: wgpu::BufferUsages::STORAGE | extra,
+ mapped_at_creation: false,
+ })
+}
+
+impl Shared {
+ fn workspace(&self) -> Workspace {
+ let d = self.dev;
+ let slots: Vec<wgpu::Buffer> = self.layout.slots.iter().map(|&b| storage(d, "activations", b, wgpu::BufferUsages::empty())).collect();
+ let input = storage(d, "input", (self.layout.in_floats * 4) as u64, wgpu::BufferUsages::COPY_DST);
+ let output = storage(d, "output", (self.layout.out_floats * 4) as u64, wgpu::BufferUsages::COPY_SRC);
+ let staging = d.device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("readback"),
+ size: (self.layout.out_floats * 4) as u64,
+ usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
+ mapped_at_creation: false,
+ });
+ let buf = |p: Place| -> &wgpu::Buffer {
+ match p {
+ Place::Input => &input,
+ Place::Output => &output,
+ Place::Slot(i) => slots.get(i).unwrap_or(&self.dummy),
+ }
+ };
+ let mut bound = Vec::new();
+ for (i, s) in self.layout.steps.iter().enumerate() {
+ let uniform = self.uniforms.get(i).unwrap_or(&self.dummy);
+ let bg = match s.kernel {
+ Kernel::Conv { .. } => {
+ let (w, b) = s.layer.and_then(|l| self.weights.get(l)).map_or((&self.dummy, &self.dummy), |(w, b)| (w, b));
+ d.device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: None,
+ layout: &self.pipes.conv_layout,
+ entries: &[
+ wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 1, resource: buf(s.src).as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 2, resource: s.src2.map_or(&self.dummy, buf).as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 3, resource: w.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 4, resource: b.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 5, resource: buf(s.dst).as_entire_binding() },
+ ],
+ })
+ }
+ Kernel::Pool => d.device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: None,
+ layout: &self.pipes.pool_layout,
+ entries: &[
+ wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 1, resource: buf(s.src).as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 2, resource: buf(s.dst).as_entire_binding() },
+ ],
+ }),
+ };
+ bound.push((s.kernel, s.groups, bg));
+ }
+ Workspace { input, output, staging, bound, _slots: slots }
+ }
+}
+
+/// A network on the GPU for one tile size. Cheap to share between threads: each call takes a free workspace.
+pub struct NetRunner {
+ shared: Shared,
+ tile: usize,
+ in_channels: usize,
+ free: Mutex<Vec<Workspace>>,
+ /// Workspaces made so far (they are made when a call finds none free, up to [`WORKSPACES`]).
+ made: Mutex<usize>,
+ wake: Condvar,
+ bytes: u64,
+}
+
+/// Put `net` on the GPU for `tile × tile` input cells, or say why it cannot be (no GPU, a layer or a size the kernels
+/// do not do, the device refusing the buffers). Creates the device on first use and compiles the kernels.
+pub fn runner(net: &Net, tile: usize) -> Result<NetRunner, String> {
+ let d = dev()?;
+ let max_binding = d.limits.max_storage_buffer_binding_size.min(d.limits.max_buffer_size).min(MAX_BUFFER);
+ let (layout, layers) = plan(net, tile, max_binding)?;
+ let scope = [
+ d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory),
+ d.device.push_error_scope(wgpu::ErrorFilter::Validation),
+ d.device.push_error_scope(wgpu::ErrorFilter::Internal),
+ ];
+ let pipes = build_pipelines(d, &layout);
+ let weights: Vec<(wgpu::Buffer, wgpu::Buffer)> = layers
+ .iter()
+ .map(|l| {
+ let make = |v: &[f32]| {
+ d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("weights"),
+ contents: bytemuck::cast_slice(v),
+ usage: wgpu::BufferUsages::STORAGE,
+ })
+ };
+ (make(&l.weight), make(&l.bias))
+ })
+ .collect();
+ let uniforms: Vec<wgpu::Buffer> = layout
+ .steps
+ .iter()
+ .map(|s| {
+ d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("params"),
+ contents: &s.params,
+ usage: wgpu::BufferUsages::UNIFORM,
+ })
+ })
+ .collect();
+ let dummy = storage(d, "dummy", 16, wgpu::BufferUsages::empty());
+ let bytes: u64 = layout.slots.iter().sum::<u64>() + ((layout.in_floats + layout.out_floats) * 8) as u64;
+ let shared = Shared { dev: d, layout, pipes, weights, uniforms, dummy };
+ let first = shared.workspace();
+ let [oom, validation, internal] = scope;
+ let errors = [pollster::block_on(internal.pop()), pollster::block_on(validation.pop()), pollster::block_on(oom.pop())];
+ if let Some(e) = errors.into_iter().flatten().next() {
+ return Err(format!("{}: the denoise kernels could not be set up ({e})", d.info.name));
+ }
+ Ok(NetRunner { shared, tile, in_channels: net.in_channels, free: Mutex::new(vec![first]), made: Mutex::new(1), wake: Condvar::new(), bytes })
+}
+
+impl NetRunner {
+ /// The adapter this runner uses.
+ pub fn adapter(&self) -> String {
+ format!("{} ({:?})", self.shared.dev.info.name, self.shared.dev.info.backend)
+ }
+
+ /// How long each kernel took to build, e.g. `conv 64x64x16: 812 ms`.
+ pub fn build_report(&self) -> String {
+ self.shared.pipes.built.iter().map(|(k, ms)| format!("{k}: {ms:.0} ms")).collect::<Vec<_>>().join(", ")
+ }
+
+ /// Device memory one tile in flight uses (bytes), without the weights.
+ pub fn bytes_per_tile(&self) -> u64 {
+ self.bytes
+ }
+
+ /// A free workspace, a new one when fewer than [`WORKSPACES`] exist, else wait for one to come back.
+ fn take(&self) -> Result<Workspace, Error> {
+ let mut free = self.free.lock().unwrap_or_else(|e| e.into_inner());
+ loop {
+ if let Some(w) = free.pop() {
+ return Ok(w);
+ }
+ {
+ let mut made = self.made.lock().unwrap_or_else(|e| e.into_inner());
+ if *made < WORKSPACES {
+ *made += 1;
+ drop(made);
+ drop(free);
+ let d = self.shared.dev;
+ let scopes =
+ [d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), d.device.push_error_scope(wgpu::ErrorFilter::Validation)];
+ let w = self.shared.workspace();
+ let [oom, validation] = scopes;
+ let (validation, oom) = (pollster::block_on(validation.pop()), pollster::block_on(oom.pop()));
+ if let Some(e) = oom.or(validation) {
+ self.forget();
+ return Err(Error::Runtime(format!("the GPU could not make room for another tile: {e}")));
+ }
+ return Ok(w);
+ }
+ }
+ free = self.wake.wait(free).unwrap_or_else(|e| e.into_inner());
+ }
+ }
+
+ fn give(&self, w: Workspace) {
+ self.free.lock().unwrap_or_else(|e| e.into_inner()).push(w);
+ self.wake.notify_one();
+ }
+
+ /// A workspace is gone (dropped after a failure): room for a new one, and a waiting caller may make it.
+ fn forget(&self) {
+ let mut made = self.made.lock().unwrap_or_else(|e| e.into_inner());
+ *made = made.saturating_sub(1);
+ drop(made);
+ self.wake.notify_one();
+ }
+
+ fn execute(&self, w: &Workspace, input: &[f32]) -> Result<Vec<f32>, Error> {
+ let d = self.shared.dev;
+ let (t, c) = (self.tile, self.in_channels);
+ // planar → channels innermost
+ let mut nhwc = vec![0f32; input.len()];
+ for (ch, plane) in input.chunks_exact(t * t).enumerate() {
+ for (i, v) in plane.iter().enumerate() {
+ if let Some(o) = nhwc.get_mut(i * c + ch) {
+ *o = *v;
+ }
+ }
+ }
+ let mut calls = Vec::with_capacity(w.bound.len());
+ for (kernel, groups, bg) in &w.bound {
+ let pipe = match kernel {
+ Kernel::Conv { bm, bn, bk } => self.shared.pipes.conv.get(&(*bm, *bn, *bk)),
+ Kernel::Pool => Some(&self.shared.pipes.pool),
+ };
+ let Some(pipe) = pipe else { return Err(Error::Runtime("a kernel is missing".into())) };
+ calls.push((pipe, groups, bg));
+ }
+ let scopes = [
+ d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory),
+ d.device.push_error_scope(wgpu::ErrorFilter::Validation),
+ d.device.push_error_scope(wgpu::ErrorFilter::Internal),
+ ];
+ d.queue.write_buffer(&w.input, 0, bytemuck::cast_slice(&nhwc));
+ let mut enc = d.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None });
+ {
+ let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("denoise tile"), timestamp_writes: None });
+ for (pipe, groups, bg) in calls {
+ pass.set_pipeline(pipe);
+ pass.set_bind_group(0, bg, &[]);
+ pass.dispatch_workgroups(groups[0], groups[1], groups[2]);
+ }
+ }
+ let bytes = (self.shared.layout.out_floats * 4) as u64;
+ enc.copy_buffer_to_buffer(&w.output, 0, &w.staging, 0, bytes);
+ d.queue.submit([enc.finish()]);
+ let slice = w.staging.slice(..bytes);
+ let (tx, rx) = std::sync::mpsc::channel();
+ slice.map_async(wgpu::MapMode::Read, move |r| {
+ let _ = tx.send(r);
+ });
+ let waited = d.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: Some(TIMEOUT) });
+ let [oom, validation, internal] = scopes;
+ let errors = [pollster::block_on(internal.pop()), pollster::block_on(validation.pop()), pollster::block_on(oom.pop())];
+ let [internal, validation, oom] = errors;
+ if let Some(e) = oom {
+ // not a broken device: this tile runs on the CPU, and the next may fit
+ return Err(Error::Runtime(format!("the GPU ran out of memory: {e}")));
+ }
+ if let Some(e) = internal.or(validation) {
+ mark_broken(&format!("device error: {e}"));
+ return Err(Error::Runtime(format!("the GPU reported an error: {e}")));
+ }
+ if let Err(e) = waited {
+ mark_broken(&format!("the GPU did not finish a tile: {e}"));
+ return Err(Error::Runtime(format!("the GPU did not finish: {e}")));
+ }
+ match rx.recv_timeout(Duration::from_secs(10)) {
+ Ok(Ok(())) => {}
+ Ok(Err(e)) => return Err(Error::Runtime(format!("reading the result back failed: {e}"))),
+ Err(e) => return Err(Error::Runtime(format!("reading the result back failed: {e}"))),
+ }
+ let out = {
+ let data = slice.get_mapped_range().map_err(|e| Error::Runtime(format!("reading the result back failed: {e:?}")))?;
+ data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect::<Vec<f32>>()
+ };
+ w.staging.unmap();
+ Ok(out)
+ }
+}
+
+impl TileRunner for NetRunner {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
+ if let Some(why) = broken_reason() {
+ return Err(Error::Runtime(format!("the GPU is not used any more: {why}")));
+ }
+ if input.len() != self.shared.layout.in_floats {
+ return Err(Error::Input("the tile is not the size the model wants".into()));
+ }
+ let w = self.take()?;
+ let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.execute(&w, input)));
+ match r {
+ Ok(Ok(out)) => {
+ self.give(w);
+ Ok(out)
+ }
+ Ok(Err(e)) => {
+ // its readback buffer may be left half-mapped: a fresh workspace is made when one is next needed
+ drop(w);
+ self.forget();
+ Err(e)
+ }
+ Err(_) => {
+ // a panic inside wgpu: this workspace is not given back, and the GPU is not used again
+ mark_broken("a denoise tile panicked");
+ drop(w);
+ self.forget();
+ Err(Error::Runtime("the GPU runner gave up".into()))
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use lightcraft_denoise::{onnx, reference, synthetic};
+
+ fn net(tile: u64, ch: u64, depth: usize, seed: u64) -> Net {
+ let path = std::env::temp_dir().join(format!("lc-nn-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id()));
+ std::fs::write(&path, synthetic::unet_onnx(tile, ch, depth, seed)).unwrap();
+ let n = onnx::read(&path).unwrap();
+ let _ = std::fs::remove_file(&path);
+ n
+ }
+
+ fn picture(len: usize, seed: u64) -> Vec<f32> {
+ let mut s = seed | 1;
+ (0..len)
+ .map(|_| {
+ s ^= s >> 12;
+ s ^= s << 25;
+ s ^= s >> 27;
+ (s.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32
+ })
+ .collect()
+ }
+
+ /// The runner, or `None` (with a note) on a machine with no usable GPU.
+ fn gpu(net: &Net, tile: usize) -> Option<NetRunner> {
+ match runner(net, tile) {
+ Ok(r) => Some(r),
+ Err(e) => {
+ eprintln!("skipped: no GPU runner here ({e})");
+ None
+ }
+ }
+ }
+
+ fn worst(a: &[f32], b: &[f32]) -> f32 {
+ assert_eq!(a.len(), b.len());
+ let scale = b.iter().fold(1e-3f32, |m, v| m.max(v.abs()));
+ a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) / scale
+ }
+
+ #[test]
+ fn the_gpu_gives_what_the_reference_gives() {
+ // (tile, channels, depth): a tile that is not a multiple of the block size, odd channel counts, a deeper net
+ for (tile, ch, depth, seed) in
+ [(1u64, 4u64, 0usize, 1u64), (3, 4, 0, 8), (6, 4, 1, 9), (20, 12, 2, 2), (32, 8, 2, 3), (48, 12, 2, 4), (64, 16, 3, 5)]
+ {
+ let n = net(tile, ch, depth, seed);
+ let Some(g) = gpu(&n, tile as usize) else { return };
+ let x = picture(4 * (tile * tile) as usize, seed);
+ let want = reference::run(&n, tile as usize, &x).unwrap();
+ let got = g.run(&x).unwrap();
+ let cpu = lightcraft_denoise::cpu::NetRunner::new(&n, tile as usize).unwrap().run(&x).unwrap();
+ assert!(worst(&got, &cpu) < 1e-3, "GPU and pure-Rust CPU differ");
+ let e = worst(&got, &want);
+ eprintln!("{}: tile {tile}, {ch} channels, depth {depth}: off by {e:e} of the output's size", g.adapter());
+ assert!(e < 2e-4, "tile {tile}, {ch} channels, depth {depth}: the GPU is off by {e:e} of the output's size");
+ }
+ }
+
+ #[test]
+ fn a_runner_serves_many_tiles_from_many_threads() {
+ let n = net(32, 8, 2, 7);
+ let Some(g) = gpu(&n, 32) else { return };
+ let inputs: Vec<Vec<f32>> = (0..12).map(|i| picture(4 * 32 * 32, 100 + i)).collect();
+ let wants: Vec<Vec<f32>> = inputs.iter().map(|x| reference::run(&n, 32, x).unwrap()).collect();
+ std::thread::scope(|s| {
+ for chunk in inputs.chunks(3).zip(wants.chunks(3)) {
+ let g = &g;
+ s.spawn(move || {
+ for (x, want) in chunk.0.iter().zip(chunk.1) {
+ let e = worst(&g.run(x).unwrap(), want);
+ assert!(e < 2e-4, "off by {e:e}");
+ }
+ });
+ }
+ });
+ }
+
+ /// The real model on the GPU against the pure-Rust CPU runner, with timings (the numbers in docs/denoise.md come from here):
+ /// `LC_DENOISE_MODEL=<model_bayer.onnx> cargo test --release -p lightcraft-gpu --lib real_model -- --ignored --nocapture`
+ #[test]
+ #[ignore = "needs the real model: see the doc comment"]
+ fn real_model_on_the_gpu_matches_cpu() {
+ use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};
+ use lightcraft_denoise::runtime::CpuRunner;
+ use std::time::Instant;
+ let Some(model) = std::env::var_os("LC_DENOISE_MODEL") else { return };
+ let net = onnx::read(std::path::Path::new(&model)).unwrap();
+ let tile = 512usize;
+ let started = Instant::now();
+ let g = runner(&net, tile).unwrap();
+ println!("{}: set up in {:.2} s, {:.0} MB per tile in flight", g.adapter(), started.elapsed().as_secs_f64(), g.bytes_per_tile() as f64 / 1e6);
+ println!("kernels: {}", g.build_report());
+ let manifest = DenoiserManifest {
+ id: "rawnind-bayer".into(),
+ name: "RawNIND Bayer".into(),
+ version: "1".into(),
+ licence: Default::default(),
+ source: None,
+ sha256: None,
+ size_bytes: None,
+ provenance: String::new(),
+ domain: Domain::BayerToRgb,
+ tile: 512,
+ overlap: 64,
+ gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
+ };
+ let cpu = CpuRunner::load(std::path::Path::new(&model), &manifest).unwrap();
+ // a gradient with noise, in the model's input range
+ let noise = picture(4 * tile * tile, 42);
+ let input: Vec<f32> =
+ (0..4 * tile * tile).map(|i| (0.2 + 0.3 * ((i % tile) as f32 / tile as f32) + 0.1 * (noise[i] - 0.5)).clamp(0.0, 1.0)).collect();
+ let t = Instant::now();
+ let want = cpu.run(&input).unwrap();
+ println!("CPU: {:.0} ms for one tile (one call runs on one core)", t.elapsed().as_secs_f64() * 1000.0);
+ let t = Instant::now();
+ let got = g.run(&input).unwrap();
+ println!("gpu, first tile: {:.0} ms", t.elapsed().as_secs_f64() * 1000.0);
+ let e = worst(&got, &want);
+ println!("worst difference {e:e} of the output's size");
+ assert!(e < 1e-3, "the GPU does not give the CPU answer ({e:e})");
+ let mut times: Vec<f64> = (0..8)
+ .map(|_| {
+ let t = Instant::now();
+ g.run(&input).unwrap();
+ t.elapsed().as_secs_f64() * 1000.0
+ })
+ .collect();
+ times.sort_by(|a, b| a.total_cmp(b));
+ println!("gpu, one tile at a time: median {:.0} ms (best {:.0}, worst {:.0})", times[times.len() / 2], times[0], times[times.len() - 1]);
+ // two threads, as the denoiser feeds it
+ let t = Instant::now();
+ std::thread::scope(|s| {
+ for _ in 0..2 {
+ s.spawn(|| {
+ for _ in 0..8 {
+ g.run(&input).unwrap();
+ }
+ });
+ }
+ });
+ println!("gpu, 16 tiles from 2 threads: {:.0} ms per tile", t.elapsed().as_secs_f64() * 1000.0 / 16.0);
+ }
+
+ /// How long the driver takes to build one conv kernel variant: `LC_SHADER=<file.wgsl> LC_BLOCKS=128,32,16`.
+ #[test]
+ #[ignore = "a probe for shader compile times"]
+ fn time_a_shader_build() {
+ let Some(path) = std::env::var_os("LC_SHADER") else { return };
+ let blocks: Vec<u32> = std::env::var("LC_BLOCKS").unwrap_or_else(|_| "128,32,16".into()).split(',').filter_map(|v| v.parse().ok()).collect();
+ let (bm, bn, bk) = (blocks[0], blocks[1], blocks[2]);
+ let src = std::fs::read_to_string(path)
+ .unwrap()
+ .replace("{{BM}}", &bm.to_string())
+ .replace("{{BN}}", &bn.to_string())
+ .replace("{{BK}}", &bk.to_string())
+ .replace("{{A_VECS}}", &(bk * bm / 4).to_string())
+ .replace("{{B_VECS}}", &(bk * bn / 4).to_string());
+ let d = dev().unwrap();
+ let layout = conv_bgl(d);
+ let started = std::time::Instant::now();
+ let _ = pipeline(d, "probe", &src, &layout);
+ println!("{} {bm}x{bn}x{bk}: built in {:.0} ms", d.info.backend.to_str(), started.elapsed().as_secs_f64() * 1000.0);
+ }
+
+ #[test]
+ fn a_tile_of_the_wrong_size_is_an_error_not_a_crash() {
+ let n = net(16, 4, 1, 9);
+ let Some(g) = gpu(&n, 16) else { return };
+ assert!(g.run(&[0.0; 10]).is_err());
+ assert!(g.run(&[]).is_err());
+ }
+
+ #[test]
+ fn networks_the_kernels_cannot_run_are_refused() {
+ let n = net(16, 4, 1, 11);
+ // a tile the network does not fit
+ assert!(plan(&n, 15, u64::MAX).is_err());
+ // a buffer limit smaller than the activations
+ assert!(plan(&n, 16, 64).is_err());
+ }
+}
diff --git a/crates/gpu/src/wgsl/nn_conv.wgsl b/crates/gpu/src/wgsl/nn_conv.wgsl
new file mode 100644
index 000000000..d991e9e76
--- /dev/null
+++ b/crates/gpu/src/wgsl/nn_conv.wgsl
@@ -0,0 +1,162 @@
+// Convolution for the denoise networks, as an implicit matrix product: every output pixel is a row, every input
+// channel of every kernel tap is a column of K, and the output channels are the columns of N.
+//
+// out[m][n] = bias[n] + sum_k A[m][k] * W[k][n] A[m][k] = input pixel (m + tap offset), channel ci
+//
+// Tensors are NHWC, channels innermost, read four at a time (every channel count is a multiple of 4). A workgroup of
+// 256 threads makes a BM x BN block of the output, 4 x 4 values per thread, and walks K in steps of BK through
+// workgroup memory. The source can be two tensors whose channels follow each other (a skip connection's join, so it
+// is never made), and a leaky ReLU is applied on the way out. Specialised by the host: BM, BN, BK (BM * BN = 4096).
+//
+// Output modes:
+// 0 NHWC at the input's size.
+// 1 a 2x2 stride-2 transposed convolution: the columns are (dy, dx, co) and each lands in the output pixel
+// (2y + dy, 2x + dx) of an output twice the size.
+// 2 the last 1x1 convolution followed by depth-to-space (CRD): columns 4c + 2dy + dx land in plane c at
+// (2y + dy, 2x + dx) of a planar (CHW) output twice the size.
+
+struct P {
+ h: u32,
+ w: u32,
+ c0: u32,
+ c1: u32,
+ n: u32,
+ npad: u32,
+ k: u32,
+ mode: u32,
+ cout: u32,
+ leaky: u32,
+ alpha: f32,
+ pad: u32,
+}
+
+@group(0) @binding(0) var<uniform> p: P;
+@group(0) @binding(1) var<storage, read> src0: array<vec4<f32>>;
+@group(0) @binding(2) var<storage, read> src1: array<vec4<f32>>;
+@group(0) @binding(3) var<storage, read> wgt: array<vec4<f32>>;
+@group(0) @binding(4) var<storage, read> bias: array<vec4<f32>>;
+@group(0) @binding(5) var<storage, read_write> dst: array<f32>;
+
+const BM: u32 = {{BM}}u;
+const BN: u32 = {{BN}}u;
+const BK: u32 = {{BK}}u;
+const TX: u32 = BN / 4u;
+const A_ITEMS: u32 = BM * (BK / 4u);
+const B_ITEMS: u32 = BK * (BN / 4u);
+
+// A[k][m] four pixels to a vec4, B[k][n] four channels to a vec4
+var<workgroup> As: array<vec4<f32>, {{A_VECS}}>;
+var<workgroup> Bs: array<vec4<f32>, {{B_VECS}}>;
+
+@compute @workgroup_size(256)
+fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
+ let m0 = wg.x * BM;
+ let n0 = wg.y * BN;
+ let tx = tid % TX;
+ let ty = tid / TX;
+ let hw = p.h * p.w;
+ let cin = p.c0 + p.c1;
+ let taps = p.k * p.k;
+ let pad = (p.k - 1u) / 2u;
+ let chunks = cin / BK;
+
+ var acc0 = vec4<f32>(0.0);
+ var acc1 = vec4<f32>(0.0);
+ var acc2 = vec4<f32>(0.0);
+ var acc3 = vec4<f32>(0.0);
+
+ for (var tap = 0u; tap < taps; tap++) {
+ let ty_off = i32(tap / p.k) - i32(pad);
+ let tx_off = i32(tap % p.k) - i32(pad);
+ for (var chunk = 0u; chunk < chunks; chunk++) {
+ let ci0 = chunk * BK;
+ for (var item = tid; item < A_ITEMS; item += 256u) {
+ let ml = item / (BK / 4u);
+ let q = item % (BK / 4u);
+ let m = m0 + ml;
+ var v = vec4<f32>(0.0);
+ if (m < hw) {
+ let y = i32(m / p.w) + ty_off;
+ let x = i32(m % p.w) + tx_off;
+ if (y >= 0 && y < i32(p.h) && x >= 0 && x < i32(p.w)) {
+ let pix = u32(y) * p.w + u32(x);
+ let ci = ci0 + q * 4u;
+ if (ci < p.c0) {
+ v = src0[pix * (p.c0 / 4u) + ci / 4u];
+ } else {
+ v = src1[pix * (p.c1 / 4u) + (ci - p.c0) / 4u];
+ }
+ }
+ }
+ let base = (q * 4u) * (BM / 4u) + ml / 4u;
+ let lane = ml % 4u;
+ As[base][lane] = v.x;
+ As[base + BM / 4u][lane] = v.y;
+ As[base + 2u * (BM / 4u)][lane] = v.z;
+ As[base + 3u * (BM / 4u)][lane] = v.w;
+ }
+ for (var item = tid; item < B_ITEMS; item += 256u) {
+ let r = item / (BN / 4u);
+ let c4 = item % (BN / 4u);
+ let krow = tap * cin + ci0 + r;
+ Bs[item] = wgt[krow * (p.npad / 4u) + n0 / 4u + c4];
+ }
+ workgroupBarrier();
+ for (var kk = 0u; kk < BK; kk++) {
+ let a = As[kk * (BM / 4u) + ty];
+ let b = Bs[kk * TX + tx];
+ acc0 += a.x * b;
+ acc1 += a.y * b;
+ acc2 += a.z * b;
+ acc3 += a.w * b;
+ }
+ workgroupBarrier();
+ }
+ }
+
+ let n = n0 + tx * 4u;
+ if (n >= p.n) {
+ return;
+ }
+ let bv = bias[n / 4u];
+ for (var i = 0u; i < 4u; i++) {
+ let m = m0 + ty * 4u + i;
+ if (m >= hw) {
+ continue;
+ }
+ var v = bv;
+ if (i == 0u) { v += acc0; } else if (i == 1u) { v += acc1; } else if (i == 2u) { v += acc2; } else { v += acc3; }
+ if (p.leaky != 0u) {
+ v = select(v * p.alpha, v, v >= vec4<f32>(0.0));
+ }
+ if (p.mode == 0u) {
+ let o = m * p.n + n;
+ dst[o] = v.x;
+ dst[o + 1u] = v.y;
+ dst[o + 2u] = v.z;
+ dst[o + 3u] = v.w;
+ } else if (p.mode == 1u) {
+ let q = n / p.cout;
+ let co = n % p.cout;
+ let y = m / p.w;
+ let x = m % p.w;
+ let opix = (2u * y + q / 2u) * (2u * p.w) + 2u * x + q % 2u;
+ let o = opix * p.cout + co;
+ dst[o] = v.x;
+ dst[o + 1u] = v.y;
+ dst[o + 2u] = v.z;
+ dst[o + 3u] = v.w;
+ } else {
+ let c = n / 4u;
+ let y = m / p.w;
+ let x = m % p.w;
+ let ow = 2u * p.w;
+ let plane = c * (2u * p.h) * ow;
+ let o = plane + (2u * y) * ow + 2u * x;
+ dst[o] = v.x;
+ dst[o + 1u] = v.y;
+ dst[o + ow] = v.z;
+ dst[o + ow + 1u] = v.w;
+ }
+ }
+}
diff --git a/crates/gpu/src/wgsl/nn_pool.wgsl b/crates/gpu/src/wgsl/nn_pool.wgsl
new file mode 100644
index 000000000..6117eb069
--- /dev/null
+++ b/crates/gpu/src/wgsl/nn_pool.wgsl
@@ -0,0 +1,28 @@
+// 2x2 maximum with stride 2 over an NHWC tensor read and written four channels at a time.
+
+struct P {
+ h: u32,
+ w: u32,
+ c4: u32,
+ total: u32,
+}
+
+@group(0) @binding(0) var<uniform> p: P;
+@group(0) @binding(1) var<storage, read> src: array<vec4<f32>>;
+@group(0) @binding(2) var<storage, read_write> dst: array<vec4<f32>>;
+
+@compute @workgroup_size(64)
+fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
+ let o = gid.x + gid.y * nwg.x * 64u;
+ if (o >= p.total) {
+ return;
+ }
+ let c = o % p.c4;
+ let pix = o / p.c4;
+ let ow = p.w / 2u;
+ let oy = pix / ow;
+ let ox = pix % ow;
+ let i00 = ((2u * oy) * p.w + 2u * ox) * p.c4 + c;
+ let i10 = i00 + p.w * p.c4;
+ dst[o] = max(max(src[i00], src[i00 + p.c4]), max(src[i10], src[i10 + p.c4]));
+}
diff --git a/crates/mcp/src/headless.rs b/crates/mcp/src/headless.rs
index bdebec60a..7d2d929c7 100644
--- a/crates/mcp/src/headless.rs
+++ b/crates/mcp/src/headless.rs
@@ -31,7 +31,7 @@ impl Drop for Headless {
impl Default for Headless {
fn default() -> Self {
- Self::new(Session::new().with_fs())
+ Self::new(Session::new().with_fs().with_default_denoise_models())
}
}
@@ -42,7 +42,7 @@ impl Headless {
/// A headless session with the procedurally generated demo library.
pub fn demo() -> Self {
- Self::new(Session::with_demo().with_fs())
+ Self::new(Session::with_demo().with_fs().with_default_denoise_models())
}
fn photo_or_active(&self, p: &Value) -> Result<PhotoId, String> {
diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs
index d63e52544..ec52b3ddf 100644
--- a/crates/raw/src/lib.rs
+++ b/crates/raw/src/lib.rs
@@ -677,14 +677,27 @@ impl RawImage {
/// The result is camera RGB (not white balanced), white level = 1.0, not oriented.
pub fn develop(&self, method: Method) -> Result<Rgb32f> {
let n = self.normalized()?;
- let mut rgb = demosaic(&n, method);
+ let rgb = demosaic(&n, method);
drop(n);
+ Ok(self.finish_demosaiced(rgb))
+ }
+
+ /// The last steps of [`Self::develop`] for camera RGB made some other way (a denoiser that demosaics too) from
+ /// [`Self::normalized`]'s mosaic: apply `OpcodeList3`, then the default crop.
+ pub fn finish_demosaiced(&self, mut rgb: Rgb32f) -> Rgb32f {
opcodes::apply_list3(&self.opcodes.list3, &mut rgb);
- let c = self.crop.clipped(rgb.width, rgb.height);
- if c.width == 0 || c.height == 0 || (c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height) {
- return Ok(rgb);
+ let c = self.develop_crop(rgb.width, rgb.height);
+ if c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height {
+ return rgb;
}
- Ok(rgb.into_crop(c.x, c.y, c.width, c.height))
+ rgb.into_crop(c.x, c.y, c.width, c.height)
+ }
+
+ /// The part of a `width × height` demosaiced picture (the active area) that [`Self::develop`] keeps: the default
+ /// crop, or all of it when there is none.
+ pub fn develop_crop(&self, width: usize, height: usize) -> Rect {
+ let c = self.crop.clipped(width, height);
+ if c.width == 0 || c.height == 0 { Rect::new(0, 0, width, height) } else { c }
}
}
@@ -717,6 +730,43 @@ mod tests {
}
}
+ #[test]
+ fn a_picture_made_elsewhere_is_finished_like_develop_does() {
+ let (w, h) = (12usize, 10usize);
+ let raw = RawImage {
+ format: RawFormat::Dng,
+ width: w,
+ height: h,
+ cpp: 1,
+ data: RawData::U16((0..w * h).map(|i| (100 + (i * 37) % 900) as u16).collect()),
+ cfa: Cfa::bayer("RGGB"),
+ bits: 12,
+ black: BlackLevel::uniform(64.0),
+ white: vec![1023.0],
+ active_area: Rect::new(2, 2, 8, 6),
+ crop: Rect::new(1, 1, 5, 4),
+ orientation: Orientation::Normal,
+ color: ColorData::default(),
+ wb_multipliers: None,
+ linearized: false,
+ opcodes: OpcodeLists::default(),
+ metadata: Metadata::default(),
+ };
+ let direct = raw.develop(Method::Bilinear).unwrap();
+ let by_hand = raw.finish_demosaiced(demosaic(&raw.normalized().unwrap(), Method::Bilinear));
+ assert_eq!((direct.width, direct.height), (5, 4));
+ assert_eq!(direct, by_hand);
+ // the crop is what develop keeps; no usable crop keeps everything
+ assert_eq!(raw.develop_crop(8, 6), Rect::new(1, 1, 5, 4));
+ let mut whole = raw.clone();
+ whole.crop = Rect::default();
+ assert_eq!(whole.develop_crop(8, 6), Rect::new(0, 0, 8, 6));
+ assert_eq!(whole.develop(Method::Bilinear).unwrap().width, 8);
+ // a crop that pokes out is clipped to the picture
+ whole.crop = Rect::new(6, 4, 10, 10);
+ assert_eq!(whole.develop_crop(8, 6), Rect::new(6, 4, 2, 2));
+ }
+
#[test]
fn black_levels() {
let b = BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![1.0, 2.0, 3.0, 4.0], delta_h: vec![0.5, 0.0], delta_v: vec![0.0, 10.0] };
diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml
index 49bdfc901..7836d96c5 100644
--- a/crates/ui-egui/Cargo.toml
+++ b/crates/ui-egui/Cargo.toml
@@ -26,6 +26,7 @@ log = { workspace = true }
[dev-dependencies]
lightcraft-codecs = { workspace = true }
+lightcraft-denoise = { workspace = true }
[build-dependencies]
serde_json = { workspace = true }
diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs
index 206d75b18..48688b6a8 100644
--- a/crates/ui-egui/src/headless.rs
+++ b/crates/ui-egui/src/headless.rs
@@ -194,6 +194,7 @@ impl Headless {
|| self.app.scan.is_some()
|| self.app.import.is_some()
|| self.app.export.is_some()
+ || self.app.session.denoise_busy()
|| !self.app.tasks.is_empty()
|| !self.app.synthetic.is_empty()
|| !self.events.is_empty()
@@ -291,6 +292,61 @@ impl Headless {
#[cfg(test)]
mod tests {
+ #[test]
+ fn detail_offers_ai_denoise_for_raw_photos_and_leads_to_its_settings() {
+ use crate::state::Dialog;
+ use lightcraft_catalog::{MediaKind, Op, Photo, PhotoId, Source};
+ let mut h = demo([1400.0, 900.0]);
+ let t = Duration::from_secs(10);
+ let dir = std::env::temp_dir().join(format!("lc-ui-denoise-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ h.app.session.set_denoise_models_dir(Some(dir.join("denoise-models")));
+ // a raw photo (its file need not exist: nothing here reads it)
+ let id = PhotoId(h.app.session.catalog.photos().map(|p| p.id.0).max().unwrap_or(0) + 1);
+ let mut raw = Photo::new(
+ id,
+ Source::File { path: dir.join("IMG_1.dng").to_string_lossy().into_owned() },
+ "IMG_1.dng",
+ "DNG",
+ 6000,
+ 4000,
+ "2026-10-01T00:00:00",
+ );
+ raw.kind = MediaKind::Raw;
+ h.app.session.commit("setup", Op::AddPhoto { photo: Box::new(raw) }).unwrap();
+ let widgets = |h: &mut Headless| h.request("ui.widgets", json!({}), t).to_string();
+ h.request("engine.execute", json!({"command": "library.select", "params": {"ids": [id.0]}}), t);
+ h.request("ui.set", json!({"view": "detail"}), t);
+ h.app.ui.open_sections = vec!["detail".to_string()];
+ h.settle(SETTLE);
+ h.step();
+ h.step();
+ let w = widgets(&mut h);
+ assert!(w.contains("\"denoise:setup\""), "a raw photo with no model is offered the setup: {w}");
+ // one click lands in Settings ▸ AI Denoise, where the model LightCraft knows is listed
+ let r = h.request("ui.clickWidget", json!({"id": "denoise:setup"}), t);
+ assert_eq!(r["ok"], true, "{r}");
+ h.step();
+ h.step();
+ assert_eq!(h.app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() }));
+ h.settle(SETTLE);
+ let listed = h.app.session.execute("denoise.models.list", &json!({})).unwrap();
+ assert!(listed["models"].as_array().unwrap().iter().any(|m| m["id"] == "rawnind-bayer" && m["installed"] == false), "{listed}");
+ h.app.ui.dialog = None;
+ // a photo that is not raw: no offer
+ h.request(
+ "engine.execute",
+ json!({"command": "library.select", "params": {"ids": [h.app.session.catalog.photos().find(|p| p.id != id).unwrap().id.0]}}),
+ t,
+ );
+ h.settle(SETTLE);
+ h.step();
+ h.step();
+ assert!(!widgets(&mut h).contains("\"denoise:setup\""), "nothing is offered for a photo denoise does not apply to");
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
use super::*;
/// Generous: renders are slow when the machine is loaded (parallel builds), and a timed-out
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index aeab2794b..be2095fcd 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -19,6 +19,7 @@ pub mod menu_level;
pub mod menubar;
pub mod menus;
pub mod merge;
+mod model_setup;
pub mod panels;
pub mod pick;
pub mod region;
@@ -157,6 +158,7 @@ pub struct Perf {
}
pub struct LightcraftApp {
+ pub(crate) model_setup: model_setup::Pending,
/// Per-catalog-revision caches of library-wide results the panels show every frame
/// (expensive on big libraries).
pub caches: Caches,
@@ -312,6 +314,7 @@ impl LightcraftApp {
gpu_applied: None,
memory_applied: None,
library_problem: None,
+ model_setup: Default::default(),
}
}
@@ -324,6 +327,9 @@ impl LightcraftApp {
/// Run a UI or engine command by id. The single entry point for every frontend path.
pub fn run(&mut self, id: &str, params: Value) -> Result<Value, String> {
+ if let Some(result) = model_setup::intercept(self, id, &params) {
+ return result;
+ }
if let Some(r) = menus::run_ui_command(self, id, &params) {
return r;
}
@@ -751,6 +757,8 @@ impl LightcraftApp {
ctx.request_repaint_after(std::time::Duration::from_secs(3));
}
self.session.persist_if_dirty();
+ panels::denoise::pump(self, ctx);
+ model_setup::pump(self, ctx);
self.collect_screenshots(ctx);
self.issue_screenshots(ctx);
if self.fonts_ready {
@@ -988,6 +996,11 @@ pub struct Caches {
album_counts: Option<(u64, std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>>)>,
/// How often the album counts were recomputed (tests check that unchanged frames don't).
pub album_count_scans: usize,
+ /// AI denoise: what the pump last saw, the model list and the downloads being watched.
+ pub denoise: panels::denoise::Ui,
+ /// When the user last dragged, typed or scrolled, and when they last moved the pointer (egui time).
+ pub last_input: f64,
+ pub last_move: f64,
/// The grid's date runs, layout and indexes (by the visible list's generation).
pub grid: panels::grid::GridCache,
/// What the grid did on its frames (benchmarks and tests check unchanged frames stay cheap).
@@ -1213,3 +1226,6 @@ mod cache_tests {
assert_eq!(c.album_count_scans, 1);
}
}
+
+#[cfg(test)]
+mod tests_model_setup;
diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs
index 5ca01cb40..96fc1dc97 100644
--- a/crates/ui-egui/src/menus.rs
+++ b/crates/ui-egui/src/menus.rs
@@ -47,6 +47,7 @@ pub fn language_from_command(id: &str) -> Option<crate::i18n::Locale> {
}
pub const UI_COMMANDS: &[UiCommand] = &[
+ ("modelSetup.cancel", "Cancel Pending Photo Action", None, ""),
("view.photoGrid", "Photo Grid", None, "View"),
("view.squareGrid", "Square Grid", None, "View"),
// G: Photo Grid ↔ Square Grid (from other views: the photo grid)
diff --git a/crates/ui-egui/src/model_setup.rs b/crates/ui-egui/src/model_setup.rs
new file mode 100644
index 000000000..238a4afea
--- /dev/null
+++ b/crates/ui-egui/src/model_setup.rs
@@ -0,0 +1,133 @@
+//! Resume a denoise request after model setup, bound to its original photo and library.
+
+use crate::{LightcraftApp, state::Dialog};
+use lightcraft_catalog::{PhotoId, Source};
+use lightcraft_engine::denoise::PhotoState;
+use serde_json::{Value, json};
+
+struct Request {
+ library: u64,
+ photo: PhotoId,
+ source: Source,
+}
+
+impl Request {
+ fn capture(app: &LightcraftApp, id: PhotoId) -> Result<Self, String> {
+ let photo = app.session.catalog.photo(id).filter(|p| !p.deleted).ok_or("the requested photo is no longer available")?;
+ Ok(Self { library: app.session.library_generation(), photo: id, source: photo.source.clone() })
+ }
+
+ fn valid(&self, app: &LightcraftApp) -> bool {
+ self.library == app.session.library_generation()
+ && app.session.catalog.photo(self.photo).is_some_and(|p| !p.deleted && p.source == self.source)
+ }
+}
+
+#[derive(Default)]
+pub(crate) struct Pending {
+ denoise: Option<Request>,
+ next_check: f64,
+}
+
+pub(crate) fn denoise_requested(app: &LightcraftApp, id: PhotoId) -> bool {
+ app.model_setup.denoise.as_ref().is_some_and(|r| r.photo == id && r.valid(app))
+}
+
+pub(crate) fn intercept(app: &mut LightcraftApp, command: &str, params: &Value) -> Option<Result<Value, String>> {
+ if matches!(command, "app.openLibrary" | "file.restoreLibrary") {
+ app.model_setup = Pending::default();
+ return None;
+ }
+ if command == "modelSetup.cancel" {
+ if params.get("kind").and_then(Value::as_str) != Some("denoise") {
+ return Some(Err("modelSetup.cancel: use kind denoise".into()));
+ }
+ app.model_setup.denoise = None;
+ app.ui.status = "Pending photo action cancelled".into();
+ return Some(Ok(json!({"cancelled": true})));
+ }
+ if command == "denoise.models.downloadCancel" {
+ app.model_setup.denoise = None;
+ }
+ if command != "denoise.toggle" {
+ return None;
+ }
+ let id = match params.get("id") {
+ Some(v) => match v.as_u64() {
+ Some(id) => PhotoId(id),
+ None => return Some(Err("denoise.toggle: id must be a photo id".into())),
+ },
+ None => match app.session.active() {
+ Some(id) => id,
+ None => return Some(Err("select a raw photo".into())),
+ },
+ };
+ let enabled = app.session.develop_of(id).is_some_and(|d| d.enhance.denoise_enabled());
+ let on = match params.get("enabled") {
+ Some(v) => match v.as_bool() {
+ Some(on) => on,
+ None => return Some(Err("denoise.toggle: enabled must be true or false".into())),
+ },
+ None => !enabled && !denoise_requested(app, id),
+ };
+ if !on {
+ app.model_setup.denoise = None;
+ return Some(app.session.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).map_err(|e| e.to_string()));
+ }
+ let _ = app.session.execute("denoise.status", &json!({"id": id.0}));
+ if !matches!(app.session.denoise_photo_state(id), PhotoState::NoModel) {
+ return None;
+ }
+ if app.session.execute("denoise.models.list", &json!({})).ok().is_none_or(|v| v["dir"].is_null()) {
+ return None;
+ }
+ Some(Request::capture(app, id).map(|request| {
+ app.model_setup.denoise = Some(request);
+ app.model_setup.next_check = 0.0;
+ app.ui.dialog = Some(Dialog::Settings { tab: "denoise".into() });
+ app.ui.status = "Install a denoise model to enable AI Denoise on this photo".into();
+ json!({"setupRequired": true, "photo": id.0})
+ }))
+}
+
+pub(crate) fn notice(app: &mut LightcraftApp, ui: &mut egui::Ui, kind: &str) {
+ if kind != "denoise" || !app.model_setup.denoise.as_ref().is_some_and(|r| r.valid(app)) {
+ return;
+ }
+ ui.add(egui::Label::new("After installation, AI Denoise will turn on for the requesting photo.").wrap());
+ let r = ui.small_button("Cancel pending action");
+ crate::widgets::register(ui.ctx(), "modelSetup:cancel:denoise", r.rect);
+ if r.clicked() {
+ let _ = app.run("modelSetup.cancel", json!({"kind": "denoise"}));
+ }
+ ui.add_space(6.0);
+}
+
+pub(crate) fn pump(app: &mut LightcraftApp, ctx: &egui::Context) {
+ if app.model_setup.denoise.is_none() {
+ return;
+ }
+ if app.session.interaction.is_some() {
+ ctx.request_repaint_after(std::time::Duration::from_millis(250));
+ return;
+ }
+ let now = ctx.input(|i| i.time);
+ if now < app.model_setup.next_check {
+ return;
+ }
+ app.model_setup.next_check = now + 0.25;
+ let Some(request) = app.model_setup.denoise.take() else { return };
+ if !request.valid(app) {
+ app.ui.status = "Pending photo action cancelled because its library or photo changed".into();
+ return;
+ }
+ if matches!(app.session.denoise_photo_state(request.photo), PhotoState::NoModel | PhotoState::NotApplicable) {
+ app.model_setup.denoise = Some(request);
+ ctx.request_repaint_after(std::time::Duration::from_millis(250));
+ return;
+ }
+ match app.run("denoise.toggle", json!({"id": request.photo.0, "enabled": true})) {
+ Ok(_) => app.ui.status = "AI Denoise is enabled on the requesting photo".into(),
+ Err(error) => app.ui.status = error,
+ }
+}
diff --git a/crates/ui-egui/src/panels/denoise.rs b/crates/ui-egui/src/panels/denoise.rs
new file mode 100644
index 000000000..bd69b0cfa
--- /dev/null
+++ b/crates/ui-egui/src/panels/denoise.rs
@@ -0,0 +1,647 @@
+//! AI denoise in the interface: the line under the Detail panel's Denoise slider (what the picture is doing), the
+//! Settings ▸ Denoise tab (the model and the cache) and the per-frame pump that keeps the engine's background work going.
+//!
+//! The Amount slider is an ordinary develop control (`enhance.denoise`); everything here is about the cleaned picture the
+//! slider mixes in. It is cached data made in the background for the photos being looked at, and never a file in the
+//! library. A model is fetched only when the user presses Download and accepts its terms.
+
+use egui::RichText;
+use lightcraft_catalog::PhotoId;
+use lightcraft_engine::denoise::PhotoState;
+use serde_json::{Value, json};
+
+use super::settings::{check, choices, heading, hint};
+use crate::LightcraftApp;
+use crate::theme::Tokens;
+use crate::widgets::register;
+
+/// How long a read of the model list is reused (it is a few small files, but not for every frame).
+const REFRESH_SECS: f64 = 1.5;
+/// Cache limits offered (GB).
+const CACHE_LIMITS: [u32; 6] = [5, 10, 20, 50, 100, 250];
+
+/// What the interface keeps between frames.
+#[derive(Default)]
+pub struct Ui {
+ next_pump: f64,
+ generation: u64,
+ active: bool,
+ /// (tiles done, tiles in all) of the photo being made.
+ running: Option<(u64, u64)>,
+ queued: u64,
+ /// Bumped when a model is installed, removed or chosen, so Settings re-reads the list at once.
+ pub epoch: u64,
+ /// Models the user pressed Download for that have not been installed yet.
+ dl_watch: Vec<String>,
+ list: Option<(u64, f64, Value)>,
+}
+
+/// Called every frame: keeps the engine's denoise work going and asks for a redraw when a photo's picture appears (the
+/// loupe and the thumbnails then render from it) or while one is being made (the progress).
+pub fn pump(app: &mut LightcraftApp, ctx: &egui::Context) {
+ watch_downloads(app, ctx);
+ let now = ctx.input(|i| i.time);
+ let (working, moving) = ctx.input(|i| {
+ let keys = i.events.iter().any(|e| matches!(e, egui::Event::Key { .. } | egui::Event::Text(_) | egui::Event::MouseWheel { .. }));
+ (i.pointer.any_down() || keys, i.pointer.is_moving())
+ });
+ if working {
+ app.caches.last_input = now;
+ }
+ if moving || working {
+ app.caches.last_move = now;
+ }
+ if now < app.caches.denoise.next_pump {
+ return;
+ }
+ let watching = matches!(&app.ui.dialog, Some(crate::state::Dialog::Settings { tab }) if tab == "denoise");
+ let pace = window_pace(app, ctx, now, watching);
+ let Ok(v) = app.session.execute("denoise.pump", &json!({"pace": pace})) else { return };
+ let c = &mut app.caches.denoise;
+ let generation = v["generation"].as_u64().unwrap_or(0);
+ if generation != c.generation {
+ c.generation = generation;
+ ctx.request_repaint();
+ }
+ c.active = v["active"] == true;
+ c.queued = v["queued"].as_u64().unwrap_or(0);
+ c.running = v["running"].is_object().then(|| (v["running"]["done"].as_u64().unwrap_or(0), v["running"]["total"].as_u64().unwrap_or(0)));
+ let busy = c.running.is_some() || c.queued > 0;
+ c.next_pump = now
+ + if busy {
+ 0.1
+ } else if c.active {
+ 0.5
+ } else {
+ 1.0
+ };
+ // frames are drawn only when something asks for one: while a photo is being made, ask again for its progress
+ if busy {
+ ctx.request_repaint_after(std::time::Duration::from_millis(100));
+ }
+}
+
+/// The engine's model list, re-read at most every [`REFRESH_SECS`], or at once after an action changed it.
+fn models(app: &mut LightcraftApp, ctx: &egui::Context) -> Value {
+ let now = ctx.input(|i| i.time);
+ let epoch = app.caches.denoise.epoch;
+ if let Some((e, at, v)) = &app.caches.denoise.list
+ && *e == epoch
+ && now - at < REFRESH_SECS
+ {
+ return v.clone();
+ }
+ let v = app.run("denoise.models.list", json!({})).unwrap_or(Value::Null);
+ app.caches.denoise.list = Some((epoch, now, v.clone()));
+ v
+}
+
+/// Follows the downloads the user started: keeps redrawing while one runs (the progress bar), and says so when a
+/// model has been installed and is in use (the engine does all of that by itself).
+fn watch_downloads(app: &mut LightcraftApp, ctx: &egui::Context) {
+ if app.caches.denoise.dl_watch.is_empty() {
+ return;
+ }
+ let Ok(v) = app.session.execute("denoise.models.downloads", &json!({})) else {
+ app.caches.denoise.dl_watch.clear();
+ return;
+ };
+ let rows: Vec<Value> = v["downloads"].as_array().cloned().unwrap_or_default();
+ let (mut keep, mut installed) = (Vec::new(), Vec::new());
+ for id in std::mem::take(&mut app.caches.denoise.dl_watch) {
+ match rows.iter().find(|r| r["id"] == id.as_str()).and_then(|r| r["state"].as_str()) {
+ Some("running" | "done") => keep.push(id),
+ Some("installed") => installed.push(id),
+ _ => {}
+ }
+ }
+ app.caches.denoise.dl_watch = keep;
+ for id in installed {
+ let _ = app.session.execute("denoise.models.downloadCancel", &json!({"id": id}));
+ app.caches.denoise.epoch += 1;
+ let text = "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works";
+ app.ui.status = text.into();
+ app.toast(ctx, text);
+ }
+ ctx.request_repaint_after(std::time::Duration::from_millis(120));
+}
+
+/// Per-photo AI Denoise switch, including a request waiting for model installation.
+pub fn detail_toggle(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, on: bool, applicable: bool) {
+ let mut enabled = on || crate::model_setup::denoise_requested(app, id);
+ egui::Frame::NONE.inner_margin(egui::Margin { left: 24, right: 22, top: 6, bottom: 2 }).show(ui, |ui| {
+ let r = ui.add_enabled(applicable, egui::Checkbox::new(&mut enabled, crate::i18n::tr("AI Denoise")));
+ register(ui.ctx(), "denoise:enabled", r.rect);
+ if r.changed() {
+ let _ = app.run("denoise.toggle", json!({"id": id.0, "enabled": enabled}));
+ }
+ });
+}
+
+/// The line under the Denoise slider for the photo `id`, whose Amount is `amount`.
+pub fn detail_status(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, amount: f64) {
+ let t = Tokens::get(ui.ctx());
+ let state = app.session.denoise_photo_state(id);
+ let pad = egui::Margin { left: 24, right: 22, top: 0, bottom: 6 };
+ let note = |ui: &mut egui::Ui, text: &str, color: egui::Color32| {
+ ui.add(egui::Label::new(RichText::new(crate::i18n::tr(text)).size(11.0).color(color)).wrap());
+ };
+ match state {
+ PhotoState::NotApplicable | PhotoState::Ready => {}
+ PhotoState::NoModel => {
+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {
+ note(ui, "AI Denoise needs a model: a one-time download, in Settings.", t.text_dim);
+ ui.add_space(2.0);
+ let r = ui.small_button(crate::i18n::tr("Set up AI Denoise…"));
+ register(ui.ctx(), "denoise:setup", r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.toggle", json!({"id": id.0, "enabled": true}));
+ }
+ });
+ }
+ PhotoState::Idle => {
+ if amount > 0.0 && app.session.denoise_auto() {
+ // the pump has not met this photo yet: let it come round soon instead of waiting out its pause
+ let now = ui.ctx().input(|i| i.time);
+ app.caches.denoise.next_pump = app.caches.denoise.next_pump.min(now + 0.05);
+ ui.ctx().request_repaint_after(std::time::Duration::from_millis(60));
+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| note(ui, "Waiting to start…", t.text_dim));
+ } else if amount > 0.0 {
+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {
+ note(ui, "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", t.text_dim);
+ ui.add_space(2.0);
+ let r = ui.small_button(crate::i18n::tr("Make it now"));
+ register(ui.ctx(), "denoise:make", r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.queue", json!({"ids": [id.0]}));
+ }
+ });
+ }
+ }
+ PhotoState::Queued { ahead } => {
+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {
+ let text = if ahead == 0 { "Next up for AI Denoise…".to_string() } else { format!("In line for AI Denoise ({ahead} ahead)…") };
+ note(ui, &text, t.text_dim);
+ });
+ }
+ PhotoState::Running { done, total } => {
+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {
+ let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 };
+ let text =
+ if total > 0 { format!("Making the denoised picture… {}%", (frac * 100.0).round() as u32) } else { "Starting…".to_string() };
+ let bar = ui.add(egui::ProgressBar::new(frac).desired_width(ui.available_width().min(240.0)).text(RichText::new(text).size(11.0)));
+ register(ui.ctx(), "denoise:progress", bar.rect);
+ });
+ }
+ PhotoState::Failed { why, unsupported } => {
+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {
+ let color = if unsupported { t.text_dim } else { t.caution };
+ note(ui, &sentence(&why), color);
+ if !unsupported {
+ let r = ui.small_button(crate::i18n::tr("Try again"));
+ register(ui.ctx(), "denoise:retry", r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.queue", json!({"ids": [id.0], "retry": true}));
+ }
+ }
+ });
+ }
+ }
+}
+
+/// The Settings ▸ Denoise tab.
+pub fn settings_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) {
+ let list = models(app, ui.ctx());
+ heading(ui, t, "AI Denoise");
+ crate::model_setup::notice(app, ui, "denoise");
+ if list["dir"].is_null() {
+ hint(ui, t, "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)");
+ return;
+ }
+ hint(
+ ui,
+ t,
+ "Cleans noise out of raw photos with an AI model, before the picture is built: the AI Denoise Amount slider under Detail sets how much of it you see. \
+ Your files are never changed and no extra files are made in your library: the cleaned picture is cached data that is made again when needed.",
+ );
+ let runtime = list["runtime"].as_bool() == Some(true);
+ if !runtime {
+ hint(ui, t, "This build cannot run denoise models.");
+ }
+ let all: Vec<Value> = list["models"].as_array().cloned().unwrap_or_default();
+ let downloads = app
+ .session
+ .execute("denoise.models.downloads", &json!({}))
+ .map(|v| v["downloads"].as_array().cloned().unwrap_or_default())
+ .unwrap_or_default();
+ heading(ui, t, "Models");
+ for m in &all {
+ let dl = downloads.iter().find(|d| d["id"] == m["id"]);
+ model_row(app, ui, t, m, dl, runtime);
+ }
+ if all.iter().all(|m| m["selected"] != true) && all.iter().any(|m| m["installed"] == true) {
+ hint(ui, t, "AI Denoise is off: press Use on a model.");
+ }
+ hint(ui, t, "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.");
+ work_section(app, ui, t, &list);
+}
+
+/// What the cache holds, its limit, and what is being made.
+fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &Value) {
+ let status = app.session.execute("denoise.status", &json!({})).unwrap_or(Value::Null);
+ if status["enabled"] != true {
+ return;
+ }
+ heading(ui, t, "Speed");
+ let current = list["runOn"].as_str().unwrap_or("auto").to_string();
+ let mut run_on = current.as_str();
+ let options = [("auto", "Automatic"), ("gpu", "Graphics card"), ("cpu", "Processor")];
+ if ui.horizontal(|ui| choices(ui, "denoiseRunOn", &options, &mut run_on)).inner {
+ let _ = app.run("denoise.settings", json!({"runOn": run_on}));
+ app.caches.denoise.epoch += 1;
+ }
+ hint(
+ ui,
+ t,
+ "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.",
+ );
+ hint(ui, t, &device_line(&status["device"], run_on));
+ if run_on != "cpu" && status["device"]["retry"] == true {
+ let r = ui.small_button(crate::i18n::tr("Try the graphics card again"));
+ register(ui.ctx(), "denoise:retryGpu", r.rect);
+ if r.clicked() {
+ // choosing where it runs, even the same again, forgets the failed set-up
+ let _ = app.run("denoise.settings", json!({"runOn": run_on}));
+ app.caches.denoise.epoch += 1;
+ }
+ }
+ heading(ui, t, "Photos");
+ let mut auto = list["auto"].as_bool().unwrap_or(true);
+ if check(ui, "settings.denoiseAuto", &mut auto, "Make the denoised picture of the photos I am looking at") {
+ let _ = app.run("denoise.settings", json!({"auto": auto}));
+ app.caches.denoise.epoch += 1;
+ }
+ hint(ui, t, "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.");
+ let (running, queued) = (app.caches.denoise.running, app.caches.denoise.queued);
+ match running {
+ Some((done, total)) => {
+ let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 };
+ let name = status["running"]["photo"]
+ .as_u64()
+ .and_then(|p| app.session.catalog.photo(PhotoId(p)).map(|p| p.file_name.clone()))
+ .unwrap_or_default();
+ let text =
+ format!("Denoising {name}: {done} of {total} tiles{}", if queued > 0 { format!(" · {queued} waiting") } else { String::new() });
+ let bar = ui.add(egui::ProgressBar::new(frac).desired_width(380.0).text(RichText::new(text).size(11.5)));
+ register(ui.ctx(), "denoise:settingsProgress", bar.rect);
+ }
+ None if queued > 0 => hint(ui, t, &count(queued, "photo waiting", "photos waiting")),
+ None => {
+ let ready = status["ready"].as_u64().unwrap_or(0);
+ hint(ui, t, &count(ready, "photo has its denoised picture.", "photos have their denoised picture."))
+ }
+ }
+ ui.horizontal(|ui| {
+ let r = ui.button("Denoise all photos with an Amount");
+ register(ui.ctx(), "denoise:queueAll", r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.queue", json!({"scope": "withAmount"}));
+ }
+ if running.is_some() || queued > 0 {
+ let r = ui.button("Stop");
+ register(ui.ctx(), "denoise:stop", r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.cancel", json!({}));
+ }
+ }
+ });
+ heading(ui, t, "Cache");
+ let (files, bytes) = (status["cache"]["files"].as_u64().unwrap_or(0), status["cache"]["bytes"].as_u64().unwrap_or(0));
+ hint(
+ ui,
+ t,
+ &format!(
+ "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.",
+ count(files, "cached picture", "cached pictures"),
+ mb(Some(bytes))
+ ),
+ );
+ ui.horizontal(|ui| {
+ ui.label(RichText::new(crate::i18n::tr("Limit")).color(t.text_label));
+ let current = list["cacheGb"].as_u64().unwrap_or(20) as u32;
+ let mut choice = current;
+ egui::ComboBox::from_id_salt("denoise-cache-limit").selected_text(format!("{current} GB")).show_ui(ui, |ui| {
+ for gb in CACHE_LIMITS {
+ ui.selectable_value(&mut choice, gb, format!("{gb} GB"));
+ }
+ });
+ if choice != current {
+ let _ = app.run("denoise.settings", json!({"cacheGb": choice}));
+ app.caches.denoise.epoch += 1;
+ }
+ let r = ui.button("Clear cache");
+ register(ui.ctx(), "denoise:clear", r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.clear", json!({}));
+ }
+ });
+}
+
+/// Where the work runs, in a sentence (`denoise.status` → `device`), for the setting `run_on`. No timings: they are this
+/// computer's, and only the comparison matters.
+fn device_line(d: &Value, run_on: &str) -> String {
+ match d["kind"].as_str() {
+ _ if run_on == "cpu" => "The processor does the work.".into(),
+ Some("gpu") => format!("Running on {}.", d["adapter"].as_str().unwrap_or("the graphics card")),
+ Some("cpu") => {
+ format!("Running on the processor: {}.", d["reason"].as_str().unwrap_or("the graphics card is not used").trim_end_matches('.'))
+ }
+ _ => "The graphics card is set up when the next photo is made.".into(),
+ }
+}
+
+fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, dl: Option<&Value>, can_run: bool) {
+ let id = m["id"].as_str().unwrap_or("").to_string();
+ let (installed, selected) = (m["installed"].as_bool() == Some(true), m["selected"].as_bool() == Some(true));
+ let dl_state = if installed { None } else { dl.and_then(|d| d["state"].as_str()) };
+ let host = if can_run { m["downloadHost"].as_str().unwrap_or("").to_string() } else { String::new() };
+ egui::Frame::NONE.inner_margin(egui::Margin::symmetric(0, 3)).show(ui, |ui| {
+ ui.horizontal(|ui| {
+ ui.vertical(|ui| {
+ ui.horizontal(|ui| {
+ ui.label(RichText::new(m["name"].as_str().unwrap_or("")).color(t.text));
+ let size = mb(m["sizeBytes"].as_u64());
+ if !size.is_empty() {
+ ui.label(RichText::new(size).color(t.text_dim));
+ }
+ if selected {
+ ui.label(RichText::new("in use").color(t.accent));
+ } else if installed {
+ ui.label(RichText::new("installed").color(t.text_dim));
+ }
+ });
+ ui.label(RichText::new(licence_line(m)).font(t.font(11.5)).color(if m["licence"]["commercial"] == "yes" {
+ t.text_dim
+ } else {
+ t.caution
+ }));
+ match m["accepted"]["selfTest"].as_object() {
+ Some(test) if test.get("ok").and_then(Value::as_bool) == Some(true) => {
+ ui.label(RichText::new("Tested and works on this computer").font(t.font(11.5)).color(t.text_dim));
+ }
+ Some(_) => {
+ ui.label(RichText::new("Failed its last test").font(t.font(11.5)).color(t.caution));
+ }
+ None => {}
+ }
+ match (dl_state, dl) {
+ (Some("running"), Some(d)) => {
+ let (got, total) = (d["bytes"].as_u64().unwrap_or(0), d["total"].as_u64().unwrap_or(0));
+ let frac = if total > 0 { (got as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 };
+ let text = if total > 0 && got >= total {
+ "Checking…".to_string()
+ } else {
+ format!("{} of {} from {host}", mb(Some(got)), mb(Some(total)))
+ };
+ let bar = ui.add(egui::ProgressBar::new(frac).desired_width(260.0).text(RichText::new(text).font(t.font(11.5))));
+ register(ui.ctx(), format!("denoise:progress:{id}"), bar.rect);
+ }
+ (Some("done"), _) => {
+ ui.label(RichText::new("Downloaded and checked. Installing…").font(t.font(11.5)).color(t.text_dim));
+ }
+ (Some("failed"), Some(d)) => {
+ let why = sentence(d["error"].as_str().unwrap_or("The download failed"));
+ ui.add(egui::Label::new(RichText::new(why).font(t.font(11.5)).color(t.caution)).wrap());
+ }
+ _ => {}
+ }
+ });
+ ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
+ if installed {
+ let r = ui.button("Remove");
+ register(ui.ctx(), format!("denoise:remove:{id}"), r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.models.remove", json!({"id": id}));
+ app.caches.denoise.epoch += 1;
+ }
+ if selected {
+ let r = ui.button("Turn off");
+ register(ui.ctx(), format!("denoise:off:{id}"), r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.models.select", json!({"id": null}));
+ app.caches.denoise.epoch += 1;
+ }
+ } else {
+ let r = ui.button("Use");
+ register(ui.ctx(), format!("denoise:use:{id}"), r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.models.select", json!({"id": id}));
+ app.caches.denoise.epoch += 1;
+ }
+ }
+ } else {
+ match dl_state {
+ Some("running") => {
+ let r = ui.button("Cancel");
+ register(ui.ctx(), format!("denoise:cancelDownload:{id}"), r.rect);
+ if r.clicked() {
+ let _ = app.run("denoise.models.downloadCancel", json!({"id": id}));
+ app.caches.denoise.dl_watch.retain(|w| w != &id);
+ }
+ }
+ // the engine installs it within a moment; nothing to press
+ Some("done") => {}
+ _ => {
+ if let Some(url) = m["source"].as_str() {
+ let r = ui
+ .button("Open page")
+ .on_hover_text("Opens the model's own page in your browser, to read about it or get the file yourself.");
+ register(ui.ctx(), format!("denoise:get:{id}"), r.rect);
+ if r.clicked() {
+ open_page(app, url);
+ }
+ }
+ if !host.is_empty() {
+ let label = if dl_state == Some("failed") { "Try again" } else { "Download" };
+ let tip = format!(
+ "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.",
+ mb(m["sizeBytes"].as_u64())
+ );
+ let r = ui.button(label).on_hover_text(tip);
+ register(ui.ctx(), format!("denoise:download:{id}"), r.rect);
+ if r.clicked() {
+ open_download_dialog(app, m);
+ }
+ }
+ }
+ }
+ }
+ });
+ });
+ });
+}
+
+/// Pressing Download: the model's terms first, in the licence dialog. Accepting them starts the download;
+/// nothing is fetched before.
+fn open_download_dialog(app: &mut LightcraftApp, m: &Value) {
+ let info = json!({
+ "kind": "known",
+ "domain": "denoise",
+ "fileName": m["id"],
+ "sizeBytes": m["sizeBytes"],
+ "model": m,
+ "alreadyInstalled": false,
+ "download": m["id"],
+ "host": m["downloadHost"],
+ "assumptions": [],
+ "reason": null,
+ });
+ app.ui.dialog = Some(crate::state::Dialog::DenoiseModel { info, accepted: false });
+}
+
+/// The dialog's OK for a denoise model: start the download and watch it.
+pub fn install(app: &mut LightcraftApp, info: &Value, accepted: bool) -> Result<Value, String> {
+ if !accepted {
+ return Err("Tick the box to accept the model's terms first".into());
+ }
+ app.caches.denoise.epoch += 1;
+ let id = info["download"].as_str().ok_or("There is no download for this model")?;
+ let r = app.run("denoise.models.download", json!({"id": id, "acknowledged": true}));
+ if r.is_ok() {
+ app.caches.denoise.dl_watch.push(id.to_string());
+ // the progress is in Settings: stay there
+ app.ui.dialog = Some(crate::state::Dialog::Settings { tab: "denoise".into() });
+ }
+ r
+}
+
+/// "1 photo" / "3 photos": the count with the right noun.
+fn count(n: u64, one: &str, many: &str) -> String {
+ format!("{n} {}", if n == 1 { one } else { many })
+}
+
+pub(super) fn window_pace(app: &LightcraftApp, ctx: &egui::Context, now: f64, watching: bool) -> &'static str {
+ let (focused, minimized) = ctx.input(|i| (i.focused, i.raw.viewports.get(&i.raw.viewport_id).and_then(|v| v.minimized).unwrap_or(false)));
+ pace_for(focused, minimized, now - app.caches.last_input, now - app.caches.last_move, watching)
+}
+
+fn pace_for(focused: bool, minimized: bool, worked: f64, moved: f64, watching: bool) -> &'static str {
+ if minimized {
+ "light"
+ } else if !focused {
+ // still on screen, but the user is working in another app: no input here to get in the way of
+ "normal"
+ } else if worked < 0.4 {
+ "pause"
+ } else if moved < 3.0 {
+ "light"
+ } else if watching {
+ "full"
+ } else {
+ "normal"
+ }
+}
+
+pub(super) fn mb(bytes: Option<u64>) -> String {
+ match bytes {
+ Some(b) if b >= 10_000_000 => format!("{} MB", (b as f64 / 1e6).round() as u64),
+ Some(b) if b >= 1_000_000 => format!("{:.1} MB", b as f64 / 1e6),
+ Some(b) => format!("{} KB", (b as f64 / 1e3).round().max(1.0) as u64),
+ None => String::new(),
+ }
+}
+
+pub(super) fn sentence(s: &str) -> String {
+ let s = s.trim();
+ let mut c = s.chars();
+ let mut out: String = c.next().map(|f| f.to_uppercase().collect()).unwrap_or_default();
+ out.push_str(c.as_str());
+ if !out.is_empty() && !out.ends_with(['.', '!', '?']) {
+ out.push('.');
+ }
+ out
+}
+
+pub(super) fn licence_line(m: &Value) -> String {
+ let name = m["licence"]["name"].as_str().filter(|s| !s.is_empty()).unwrap_or("Unknown licence");
+ let terms = match m["licence"]["commercial"].as_str() {
+ Some("yes") => "commercial use allowed",
+ Some("no") => "non-commercial use only",
+ _ => "terms unclear",
+ };
+ format!("{name} · {terms}")
+}
+
+pub(super) fn open_page(app: &mut LightcraftApp, url: &str) {
+ if let Some(f) = app.services.open_url.as_mut() {
+ let _ = f(url);
+ }
+}
+
+pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info: &Value, accepted: &mut bool) {
+ let kind = info["kind"].as_str().unwrap_or("unsupported");
+ let file = info["fileName"].as_str().unwrap_or("");
+ if kind == "unsupported" {
+ ui.label(RichText::new("This file cannot be used yet").font(t.semibold(13.5)).color(t.caution));
+ ui.add(
+ egui::Label::new(
+ RichText::new(sentence(info["reason"].as_str().unwrap_or("It is not a denoise model LightCraft understands"))).color(t.text_label),
+ )
+ .wrap(),
+ );
+ ui.label(RichText::new(format!("{file} · {}", mb(info["sizeBytes"].as_u64()))).color(t.text_dim));
+ return;
+ }
+ let m = &info["model"];
+ ui.horizontal(|ui| {
+ ui.label(RichText::new(m["name"].as_str().unwrap_or(file)).font(t.semibold(14.0)).color(t.text));
+ ui.label(RichText::new(mb(info["sizeBytes"].as_u64())).color(t.text_dim));
+ });
+ if info["alreadyInstalled"] == true {
+ ui.label(RichText::new("Already installed. Installing again is harmless.").color(t.text_dim));
+ }
+ let download = info["download"].is_string();
+ if download {
+ let host = info["host"].as_str().unwrap_or("its own repository");
+ let after = if info["domain"] == "denoise" { "starts using it for AI Denoise" } else { "makes AI Denoise available" };
+ ui.add(
+ egui::Label::new(
+ RichText::new(format!("Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}."))
+ .color(t.text_label),
+ )
+ .wrap(),
+ );
+ }
+ let commercial = m["licence"]["commercial"].as_str().unwrap_or("unknown");
+ ui.label(RichText::new(licence_line(m)).font(t.semibold(12.5)).color(if commercial == "yes" { t.text } else { t.caution }));
+ let notice = m["licence"]["notice"].as_str().unwrap_or("");
+ if !notice.is_empty() {
+ ui.add(egui::Label::new(RichText::new(notice).color(t.text_label)).wrap());
+ }
+ let provenance = m["provenance"].as_str().unwrap_or("");
+ if !provenance.is_empty() {
+ ui.add(egui::Label::new(RichText::new(format!("Trained on: {provenance}")).color(t.text_dim)).wrap());
+ }
+ if kind == "draft" {
+ ui.add_space(2.0);
+ ui.label(RichText::new("LightCraft does not know this model, so it assumed:").color(t.text_label));
+ for a in info["assumptions"].as_array().into_iter().flatten().filter_map(Value::as_str) {
+ ui.add(egui::Label::new(RichText::new(format!("• {a}")).font(t.font(12.0)).color(t.text_dim)).wrap());
+ }
+ }
+ if let Some(url) = m["source"].as_str().or(m["licence"]["url"].as_str()) {
+ let r = ui.link("Open the model's page");
+ register(ui.ctx(), "denoiseModel:page", r.rect);
+ if r.clicked() {
+ open_page(app, url);
+ }
+ }
+ ui.add_space(4.0);
+ check(ui, "denoiseModel.accept", accepted, "I have read these terms and accept them for my own use");
+ let closing = if download {
+ "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it."
+ } else {
+ "The model is kept on this computer only. LightCraft never uploads or shares it."
+ };
+ ui.label(RichText::new(closing).font(t.font(11.5)).color(t.text_dim));
+}
diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs
index cd087776d..da400bc2b 100644
--- a/crates/ui-egui/src/panels/dialogs.rs
+++ b/crates/ui-egui/src/panels/dialogs.rs
@@ -85,6 +85,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
Dialog::NewSmartAlbum { .. } => "Create Smart Album",
Dialog::AllMetadata { .. } => "All Metadata",
Dialog::SystemInfo { .. } => "System Info",
+ Dialog::DenoiseModel { .. } => "AI Denoise Model",
Dialog::WhatsNew => "What's New",
Dialog::Cull { .. } => "Assisted Culling",
Dialog::SmartRules { id: None, .. } => "New Smart Album",
@@ -174,6 +175,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
}
});
}
+ Dialog::DenoiseModel { info, accepted } => crate::panels::denoise::model_dialog(app, ui, &t, info, accepted),
Dialog::SystemInfo { rows } => {
egui::Grid::new("sysinfo").num_columns(2).spacing([16.0, 4.0]).striped(true).show(ui, |ui| {
for (k, v) in rows.iter() {
@@ -784,7 +786,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
}
ui.add_space(4.0);
ui.horizontal(|ui| {
- let informational = matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. });
+ let unusable_model = matches!(&dlg, Dialog::DenoiseModel { info, .. } if info["kind"] == "unsupported");
+ let informational = unusable_model || matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. });
let sam = &app.session.segmenter;
let (sam_installed, sam_running, sam_failed) = (sam.installed(), sam.download_status().running, sam.download_status().error.is_some());
// no download location in this build: nothing to offer but the manual install
@@ -810,6 +813,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
add_label.as_str()
}
Dialog::Merge { .. } => "Merge",
+ Dialog::DenoiseModel { info, .. } if info["download"].is_string() => "Accept & Download",
+ Dialog::DenoiseModel { .. } => "Install",
Dialog::ConfirmDelete { .. } => "Delete",
Dialog::RemoveFolder { .. } => "Remove",
Dialog::SamModel { then: Some(_), .. } if sam_installed => "Continue",
@@ -819,7 +824,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
_ if informational => "Close",
_ => "OK",
};
- let r = (!ok.is_empty()).then(|| ui.button(crate::i18n::tr(ok)));
+ let can_confirm = informational || !matches!(&dlg, Dialog::DenoiseModel { accepted: false, .. });
+ let r = (!ok.is_empty()).then(|| ui.add_enabled(can_confirm, egui::Button::new(crate::i18n::tr(ok))));
if let Some(r) = &r {
crate::widgets::register(ui.ctx(), "button:dialogOk", r.rect);
}
@@ -1025,6 +1031,7 @@ pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result<serde_jso
Dialog::RenameKeyword { from, to } => app.run("keyword.rename", json!({"from": from, "to": to})),
Dialog::MergeKeywords { from, into } => app.run("keyword.merge", json!({"from": from, "into": into})),
Dialog::AutoStack { gap } => app.run("stack.auto", json!({"gap": gap})),
+ Dialog::DenoiseModel { info, accepted } => crate::panels::denoise::install(app, info, *accepted),
Dialog::AllMetadata { .. } | Dialog::SystemInfo { .. } | Dialog::WhatsNew => Ok(serde_json::Value::Null),
Dialog::Cull { reject_below, pick_best } => {
let mut p = json!({"pickBest": pick_best});
diff --git a/crates/ui-egui/src/panels/edit.rs b/crates/ui-egui/src/panels/edit.rs
index 2188afa66..51a54bc93 100644
--- a/crates/ui-egui/src/panels/edit.rs
+++ b/crates/ui-egui/src/panels/edit.rs
@@ -272,7 +272,15 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
for c in ["detail.sharpenAmount", "detail.sharpenRadius", "detail.sharpenDetail", "detail.sharpenMasking"] {
control(app, ui, d, c, c == "detail.sharpenAmount" || d.detail.sharpen_amount > 0.0);
}
- sub_title(ui, crate::i18n::tr("Noise Reduction"));
+ // AI Denoise is its own tool, apart from the manual noise reduction sliders below it
+ sub_title(ui, crate::i18n::tr("AI Denoise"));
+ // AI Denoise: how much of the photo's cleaned picture to mix in (raw files only), and what that picture is doing
+ let applicable = !matches!(app.session.denoise_photo_state(id), lightcraft_engine::denoise::PhotoState::NotApplicable);
+ let on = d.enhance.denoise_enabled();
+ super::denoise::detail_toggle(app, ui, id, on, applicable);
+ control(app, ui, d, "enhance.denoise", applicable && on);
+ super::denoise::detail_status(app, ui, id, if on { d.enhance.denoise } else { 0.0 });
+ sub_title(ui, crate::i18n::tr("Manual Noise Reduction"));
for c in ["detail.nrLuminance", "detail.nrDetail", "detail.nrContrast", "detail.nrColor", "detail.nrColorDetail", "detail.nrColorSmoothness"]
{
control(app, ui, d, c, true);
diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs
index 300756485..1d2285385 100644
--- a/crates/ui-egui/src/panels/mod.rs
+++ b/crates/ui-egui/src/panels/mod.rs
@@ -4,6 +4,7 @@ pub mod bottombar;
pub mod chips;
pub mod compare;
pub mod crop_overlay;
+pub mod denoise;
pub mod detail;
pub mod dialogs;
pub mod edit;
diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs
index db5ca5a1a..112857767 100644
--- a/crates/ui-egui/src/panels/settings.rs
+++ b/crates/ui-egui/src/panels/settings.rs
@@ -1,4 +1,4 @@
-//! The Settings dialog (⌘,): General, Import, Performance, Interface.
+//! The Settings dialog (⌘,): General, Import, Performance, Interface, AI Denoise.
//!
//! Changes apply immediately (no OK/Cancel). Where they are stored:
//! - **app settings** ([`crate::state::AppSettings`]: startup view, delete confirmation, GPU,
@@ -17,7 +17,8 @@ use crate::theme::Tokens;
use crate::widgets::register;
/// (id, label) of the tabs, in order.
-pub const TABS: &[(&str, &str)] = &[("general", "General"), ("import", "Import"), ("performance", "Performance"), ("interface", "Interface")];
+pub const TABS: &[(&str, &str)] =
+ &[("general", "General"), ("import", "Import"), ("performance", "Performance"), ("interface", "Interface"), ("denoise", "AI Denoise")];
/// Thumbnail cache sizes offered (MB).
const CACHE_SIZES: [u32; 5] = [512, 1024, 2048, 4096, 8192];
@@ -43,11 +44,12 @@ pub fn body(app: &mut LightcraftApp, ui: &mut egui::Ui, tab: &mut String) {
"import" => import_tab(app, ui, &t),
"performance" => performance_tab(app, ui, &t),
"interface" => interface_tab(app, ui, &t),
+ "denoise" => super::denoise::settings_tab(app, ui, &t),
_ => general_tab(app, ui, &t),
}
}
-fn heading(ui: &mut egui::Ui, t: &Tokens, text: &str) {
+pub(super) fn heading(ui: &mut egui::Ui, t: &Tokens, text: &str) {
ui.add_space(4.0);
ui.label(RichText::new(crate::i18n::tr(text)).font(t.semibold(12.5)).color(t.text));
}
@@ -63,19 +65,19 @@ fn row<R>(ui: &mut egui::Ui, t: &Tokens, label: &str, add: impl FnOnce(&mut egui
.inner
}
-fn hint(ui: &mut egui::Ui, t: &Tokens, text: &str) {
+pub(super) fn hint(ui: &mut egui::Ui, t: &Tokens, text: &str) {
ui.label(RichText::new(crate::i18n::tr(text)).size(11.0).color(t.text_dim));
}
/// A checkbox addressable as `check:{id}`; true when toggled.
-fn check(ui: &mut egui::Ui, id: &str, value: &mut bool, label: &str) -> bool {
+pub(super) fn check(ui: &mut egui::Ui, id: &str, value: &mut bool, label: &str) -> bool {
let r = ui.checkbox(value, crate::i18n::tr(label));
register(ui.ctx(), format!("check:{id}"), r.rect);
r.changed()
}
/// Mutually exclusive buttons (`button:{id}-{index}`).
-fn choices<V: PartialEq + Copy>(ui: &mut egui::Ui, id: &str, options: &[(V, &str)], value: &mut V) -> bool {
+pub(super) fn choices<V: PartialEq + Copy>(ui: &mut egui::Ui, id: &str, options: &[(V, &str)], value: &mut V) -> bool {
let mut changed = false;
ui.spacing_mut().item_spacing.x = 4.0;
for (i, (v, l)) in options.iter().enumerate() {
diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs
index 17f5c58b7..45949bb8c 100644
--- a/crates/ui-egui/src/state.rs
+++ b/crates/ui-egui/src/state.rs
@@ -535,6 +535,10 @@ pub enum Dialog {
pick_best: bool,
},
/// Help ▸ System Info: (label, value) rows.
+ DenoiseModel {
+ info: serde_json::Value,
+ accepted: bool,
+ },
SystemInfo {
rows: Vec<(String, String)>,
},
diff --git a/crates/ui-egui/src/tests_model_setup.rs b/crates/ui-egui/src/tests_model_setup.rs
new file mode 100644
index 000000000..892df22db
--- /dev/null
+++ b/crates/ui-egui/src/tests_model_setup.rs
@@ -0,0 +1,171 @@
+//! Synthetic setup flows: model installation must resume the requested action on its original targets.
+use std::path::{Path, PathBuf};
+
+use lightcraft_catalog::{MediaKind, Op, Photo, PhotoId, Source};
+use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};
+use lightcraft_engine::Session;
+use serde_json::json;
+
+use crate::{LightcraftApp, Services, model_setup, state::Dialog};
+
+fn setup(name: &str) -> (LightcraftApp, PathBuf, PhotoId) {
+ let dir = std::env::temp_dir().join(format!("lc-model-flow-{name}-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ let mut s = Session::new().with_fs();
+ s.open_library(dir.join("library"), true).unwrap();
+ s.set_denoise_models_dir(Some(dir.join("denoise")));
+ s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap();
+ let id = PhotoId(s.catalog.photos().map(|p| p.id.0).max().unwrap() + 1);
+ let mut raw = Photo::new(
+ id,
+ Source::File { path: dir.join("synthetic.dng").to_string_lossy().into_owned() },
+ "synthetic.dng",
+ "DNG",
+ 128,
+ 128,
+ "2026-10-07",
+ );
+ raw.kind = MediaKind::Raw;
+ s.commit("fixture", Op::AddPhoto { photo: Box::new(raw) }).unwrap();
+ (LightcraftApp::new(s, Services::default()), dir, id)
+}
+
+fn install_denoise(app: &mut LightcraftApp, dir: &Path) {
+ let source = dir.join("source");
+ std::fs::create_dir_all(&source).unwrap();
+ let bytes = lightcraft_denoise::synthetic::smoothing_onnx();
+ std::fs::write(source.join("model.onnx"), &bytes).unwrap();
+ let m = DenoiserManifest {
+ id: "test-smoothing".into(),
+ name: "Synthetic smoothing".into(),
+ version: "1".into(),
+ licence: Default::default(),
+ source: None,
+ sha256: None,
+ size_bytes: Some(bytes.len() as u64),
+ provenance: "Synthetic tests".into(),
+ domain: Domain::BayerToRgb,
+ tile: 64,
+ overlap: 16,
+ gain: Gain::None,
+ };
+ std::fs::write(source.join("denoise-model.json"), serde_json::to_vec(&m).unwrap()).unwrap();
+ app.run("denoise.models.install", json!({"path": source.join("model.onnx").to_string_lossy(), "acknowledged": true})).unwrap();
+}
+
+fn amount(app: &LightcraftApp, id: PhotoId) -> f64 {
+ app.session.develop_of(id).unwrap().enhance.denoise
+}
+fn resume(app: &mut LightcraftApp) {
+ model_setup::pump(app, &egui::Context::default());
+}
+
+#[test]
+fn denoise_installation_enables_only_the_requesting_photo_once_and_is_undoable() {
+ let (mut app, dir, raw) = setup("denoise");
+ let other = app.session.active().unwrap();
+ app.session.auto_sync = true;
+ let r = app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();
+ assert_eq!(r["setupRequired"], true);
+ assert_eq!(app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() }));
+ assert_eq!(amount(&app, raw), 0.0, "setup alone never changes the photo");
+ app.run("library.select", json!({"ids": [other.0]})).unwrap();
+ let undo = app.session.undo.len();
+ install_denoise(&mut app, &dir);
+ resume(&mut app);
+ assert_eq!(amount(&app, raw), 50.0);
+ assert_eq!(amount(&app, other), 0.0);
+ assert_eq!(app.session.active(), Some(other));
+ assert_eq!(app.session.undo.len(), undo + 1);
+ resume(&mut app);
+ assert_eq!(app.session.undo.len(), undo + 1, "a later frame cannot apply it again");
+ app.run("edit.undo", json!({})).unwrap();
+ assert_eq!(amount(&app, raw), 0.0);
+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();
+ app.run("develop.set", json!({"ids": [raw.0], "control": "enhance.denoise", "value": 75})).unwrap();
+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();
+ assert_eq!(amount(&app, raw), 75.0, "enabling an already chosen amount keeps it");
+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": false})).unwrap();
+ assert_eq!(amount(&app, raw), 75.0, "switching off keeps the Amount");
+ assert_eq!(app.session.develop_of(raw).unwrap().denoise_amount(), 0.0, "but mixes nothing in");
+ drop(app);
+ let _ = std::fs::remove_dir_all(dir);
+}
+
+#[test]
+fn cancelled_denoise_requests_do_not_apply_after_installation() {
+ for (name, command, params) in [
+ ("cancel-action", "modelSetup.cancel", json!({"kind": "denoise"})),
+ ("cancel-toggle", "denoise.toggle", json!({"enabled": false})),
+ ("cancel-download", "denoise.models.downloadCancel", json!({"id": "rawnind-bayer"})),
+ ] {
+ let (mut app, dir, raw) = setup(name);
+ app.run("library.select", json!({"ids": [raw.0]})).unwrap();
+ app.run("denoise.toggle", json!({"enabled": true})).unwrap();
+ app.run(command, params).unwrap();
+ install_denoise(&mut app, &dir);
+ resume(&mut app);
+ assert_eq!(amount(&app, raw), 0.0, "{name}");
+ drop(app);
+ let _ = std::fs::remove_dir_all(dir);
+ }
+}
+
+#[test]
+fn pending_actions_do_not_cross_libraries_or_relinked_sources() {
+ for changed in ["library", "source"] {
+ let (mut app, dir, raw) = setup(changed);
+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();
+ if changed == "library" {
+ app.session.open_library(dir.join("other-library"), true).unwrap();
+ } else {
+ app.session
+ .commit(
+ "relink",
+ Op::Relink {
+ id: raw,
+ file_name: "another.dng".into(),
+ source: Source::File { path: dir.join("another.dng").to_string_lossy().into_owned() },
+ format: Some("DNG".into()),
+ },
+ )
+ .unwrap();
+ }
+ install_denoise(&mut app, &dir);
+ resume(&mut app);
+ assert!(!model_setup::denoise_requested(&app, raw));
+ assert!(app.session.catalog.photos().all(|p| p.develop.enhance.denoise == 0.0));
+ assert!(app.ui.status.contains("cancelled"));
+ drop(app);
+ let _ = std::fs::remove_dir_all(dir);
+ }
+}
+
+#[test]
+fn hostile_or_inapplicable_toggle_arguments_are_errors_without_changes() {
+ let (mut app, dir, raw) = setup("hostile");
+ let jpeg = app.session.active().unwrap();
+ for params in [json!({"id": "wrong"}), json!({"id": u64::MAX}), json!({"id": raw.0, "enabled": 1}), json!({"id": jpeg.0, "enabled": true})] {
+ assert!(app.run("denoise.toggle", params).is_err());
+ assert_eq!(amount(&app, raw), 0.0);
+ }
+ assert!(app.run("modelSetup.cancel", json!({"kind": "nonsense"})).is_err());
+ drop(app);
+ let _ = std::fs::remove_dir_all(dir);
+}
+
+#[test]
+fn model_installation_waits_for_an_active_slider_gesture_to_finish() {
+ let (mut app, dir, raw) = setup("interaction");
+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();
+ app.session.begin_interaction("Exposure").unwrap();
+ install_denoise(&mut app, &dir);
+ resume(&mut app);
+ assert_eq!(amount(&app, raw), 0.0);
+ assert!(app.session.interaction.is_some());
+ app.session.end_interaction().unwrap();
+ resume(&mut app);
+ assert_eq!(amount(&app, raw), 50.0);
+ drop(app);
+ let _ = std::fs::remove_dir_all(dir);
+}
diff --git a/docs/denoise.md b/docs/denoise.md
new file mode 100644
index 000000000..b8fd2901e
--- /dev/null
+++ b/docs/denoise.md
@@ -0,0 +1,118 @@
+# AI Denoise
+
+Edit ▸ Detail ▸ **AI Denoise** is a per-photo switch above **Amount** (0–100).
+Turning it on starts at 50, or preserves an already positive Amount. Turning it off
+preserves Amount and mixes none of the cleaned picture. It affects only the requesting
+photo, independently of Auto Sync, and is undoable. Manual luminance and colour noise
+reduction remain separate controls.
+
+If no model is selected, turning it on opens **Settings ▸ AI Denoise** without changing
+the photo. After installation, the pending request enables the original photo even if
+the selection changed. Cancel the pending action in Settings. Cancelled downloads,
+deleted or relinked photos, switching libraries and quitting cancel the request.
+An active slider gesture finishes before the pending action resumes.
+
+## Models and terms
+
+**No model weights are bundled.** Models live in the per-user `denoise-models` folder
+(override with `LIGHTCRAFT_DENOISE_MODELS`). Settings offers a pinned, verified download
+of RawNIND UtNet2 for Bayer sensors, or installation of a user-supplied ONNX file with
+a `denoise-model.json` beside it. Terms must be acknowledged before installation or
+download. Downloads use the shared pure-Rust `lightcraft-fetch` client, with resume,
+progress, cancellation, size limits and SHA-256 verification.
+
+RawNIND's offered ONNX weights are **GPL-3.0**, as recorded in the pinned model catalog.
+They are downloaded separately after consent and read by our own Rust executor.
+Whether releases should offer this download remains a maintainer policy decision;
+permissively licensed or independently trained weights remain a gap. This is not a
+claim that optional downloading settles licensing questions. No GPL decoder or neural
+runtime code is included. Model metadata is in `crates/denoise/src/known.rs`.
+
+Installation checks the manifest, model hash and size, then runs a synthetic self-test:
+finite output of the promised shape, repeatability, the declared scale and reduced
+noise. Unsupported operators, malformed files and a failed self-test return errors.
+
+The current contract is `bayerToRgb`: normalized, black-subtracted, non-white-balanced
+RGGB planes `[1, 4, tile, tile]` produce camera RGB `[1, 3, 2*tile, 2*tile]`.
+RawNIND uses a 512-cell tile, 64-cell overlap and a mean-matched nominal gain of 1e6.
+See `DenoiserManifest` for the validated JSON schema.
+
+## Photos, previews and exports
+
+Original RAW files stay untouched. A cleaned picture is cached under
+`<library>/denoise/<key>.lcdn`, keyed by source content, model and algorithm version.
+The compressed half-float cache has checked headers, bounded windows and checksums.
+It can be cleared or recreated; it is not a new catalog photo or a duplicate DNG.
+
+Plain and cleaned camera RGB go through the same highlight reconstruction, white
+balance, colour matrix, DNG profile tables, camera hue/saturation look, crop and
+orientation. Amount blends the two before the remaining develop pipeline, so edits
+and exports use the same data. At zero Amount or with Detail disabled, the ordinary
+render path remains in use.
+
+The queue prioritizes the open and selected photos, pauses after input, and adjusts
+its worker count while the user works. Progress, cancellation and retry are available
+in Settings and through commands. The default cache limit is 20 GB; unused products
+are evicted oldest first. A failed file is not continuously retried.
+
+An export with a nonzero effective Amount creates a missing product synchronously.
+It returns a clear error if it cannot produce the denoised result; it never silently
+exports the plain image instead. Unsupported sensor layouts retain manual noise
+reduction and report their limitation.
+
+## Pure-Rust CPU and GPU execution
+
+The CPU executor reads a bounded subset of ONNX into checked network data. Convolutions
+use bounded im2col blocks and `faer-core 0.17.1` with its Rust-intrinsic GEMM kernels,
+without BLAS, generated assembly or an inner thread pool. Tiles share the outer worker
+pool. A scalar implementation checks the optimized operators on synthetic networks.
+Tract is not a workspace dependency or a product feature.
+
+The GPU executor uses our WGSL kernels through wgpu. It checks a real model's output
+against the CPU and falls back on errors, unsupported networks or a slower device.
+Setup has a time limit and a crash marker. Automatic selection estimates throughput
+from a check tile; this estimate can choose imperfectly on unmeasured hardware.
+Settings offers Automatic, Graphics card and Processor. GPU tests cover supported
+synthetic networks against the scalar executor; a real-model test compares CPU/GPU.
+
+### Historical performance, measured 2026-10-07
+
+These are the contributor's measurements **before the current upstream integration**,
+on a Ryzen 9 7945HX and RTX 4090 Laptop GPU (DX12), release build, model SHA-256
+`da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901`.
+The 24.34 MP Sony a7 III photo is 35 tiles. Numbers exclude model loading, source
+decode, GPU setup, cache encoding and export, but include packing and blending.
+
+| Work | Pure-Rust CPU | Prior Tract reference | GPU |
+|---|---:|---:|---:|
+| One warm 512-cell tile, one caller | 1148 ms | 1463 ms | 15.1 ms |
+| Whole photo, 16 CPU / 4 GPU feeding workers | 5.185 s | 9.534 s | 0.582 s |
+
+The largest relative differences were 1.68e-6 CPU/reference and 1.57e-6 CPU/GPU for
+the check tile, below the required 1e-3 tolerance. Reference comparisons were made
+on the older contributor branch; that reference runtime is intentionally excluded
+from the current workspace. These numbers are not a fresh benchmark of this rebase.
+
+Fresh checks can be run using an already installed model and CC0 raw samples:
+
+```text
+LC_DENOISE_MODEL=<model.onnx> cargo test --release -p lightcraft-gpu --lib real_model_on_the_gpu_matches_cpu -- --ignored --nocapture
+LC_DENOISE_MODEL=<model.onnx> LC_DENOISE_RAW=<corpus/raw> cargo test --release -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture
+LC_DENOISE_MODEL=<model.onnx> LC_DENOISE_RAW=<corpus/raw> cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture
+```
+
+## Commands and limits
+
+UI, CLI, control channel and MCP dispatch `denoise.toggle`, `denoise.settings`,
+`denoise.queue`, `.cancel`, `.status`, `.pump`, `.clear`, and `denoise.models.*`
+(`list`, `install`, `download`, `downloads`, `downloadCancel`, `test`, `remove`,
+`select`). The develop control is `enhance.denoise`.
+
+Native apps enable inference through their `denoise` feature. A build without it and
+the web build report inference unavailable. Manifest/cache primitives still compile
+for WASM. Settings and models remain opt-in; nothing is downloaded on launch.
+
+This implementation supports **Bayer RAW only**. X-Trans, Foveon, demosaiced DNG and
+non-RAW images remain unsupported by this model contract. No measured quality parity
+with Lightroom is claimed. Apple/Metal and wider hardware verification, model policy,
+a linear-RGB contract and a side-by-side fidelity suite remain future work.
diff --git a/docs/parity.md b/docs/parity.md
index 90dc8a748..a47189493 100644
--- a/docs/parity.md
+++ b/docs/parity.md
@@ -79,7 +79,7 @@ Take the first one nobody is working on.
Object and Describe masks run SAM 3 in pure Rust (`crates/segment`); the weights (SAM License) are never bundled:
the app offers a consented, verified download, but **LightCraft's CDN mirrors are not configured yet** (the
built-in list is empty, so today users need their own mirror or a manual install). Subject / Sky / People could
- use the same model with fixed prompts; denoise / super-resolution models remain a maintainer decision.
+ use the same model with fixed prompts. Bayer RAW denoise now runs on pure-Rust CPU/GPU with separately installed weights; its opt-in GPL model policy, linear RGB and quality comparison remain open. Super-resolution models remain a maintainer decision.
7. **HDR** (Q. HDR, LR-EXP-HDR), **video** (R. Video), **Classic output modules** (Map view, Book, Slideshow module,
Print, publish): large, well understood, lower priority than 1–5.
8. **LR-VIEW-ZOOM** (P0): pinch zoom and two-finger pan now work, with stable image cursors between gesture events.
@@ -262,7 +262,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
| LR-EDIT-DETAIL-SHARPEN | Sharpening | P0 | ✅ | `ctl:detail.sharpenAmount`, `ctl:detail.sharpenRadius`, `ctl:detail.sharpenDetail`, `ctl:detail.sharpenMasking` | no Alt-drag mask preview |
| LR-EDIT-DETAIL-NR | Luminance noise reduction | P0 | ✅ | `ctl:detail.nrLuminance`, `ctl:detail.nrDetail`, `ctl:detail.nrContrast` | |
| LR-EDIT-DETAIL-CNR | Colour noise reduction | P0 | ✅ | `ctl:detail.nrColor`, `ctl:detail.nrColorDetail`, `ctl:detail.nrColorSmoothness` | |
-| LR-EDIT-DETAIL-DENOISE | AI denoise | P2 | ⬜ | | settings field reserved, not rendered |
+| LR-EDIT-DETAIL-DENOISE | AI denoise | P2 | 🟡 | cmd:denoise.toggle, cmd:denoise.models.install, cmd:denoise.queue, ctl:enhance.denoise, `crates/denoise/`, `docs/denoise.md` | non-destructive Bayer RAW, opt-in weights, pure-Rust CPU/GPU; X-Trans/linear RGB, model policy and Lightroom fidelity remain |
| LR-EDIT-DETAIL-RAWDETAILS | Improved demosaic toggle | P2 | ⬜ | | |
| LR-EDIT-DETAIL-SUPERRES | Super resolution | P2 | ⬜ | | |
| LR-EDIT-DETAIL-AISHARPEN | AI sharpen | OOS | 🚫 | | |
diff --git a/docs/whats-new.md b/docs/whats-new.md
index f9b295bde..b172faaae 100644
--- a/docs/whats-new.md
+++ b/docs/whats-new.md
@@ -2,6 +2,10 @@
## October 2026
+### AI RAW denoise
+- Detail has a per-photo AI Denoise switch and Amount. Bayer RAW photos use a disposable cache, with matching previews and exports and pure-Rust CPU/GPU inference.
+- Models are installed separately after accepting their terms; no weights are bundled. See [setup and limitations](denoise.md).
+
### Library keyboard culling
- Colour labels tint thumbnail surrounds in Square Grid and the Detail filmstrip, and a translucent footer along the bottom of the photo in Photo Grid. Label confirmations use a matching pale colour.
- Setting or clearing a colour label shows a brief bottom confirmation, like rating a photo; custom label names appear in the message too.
diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs
index 28d52677c..e330a8e1b 100644
--- a/xtask/src/layers.rs
+++ b/xtask/src/layers.rs
@@ -39,6 +39,7 @@ pub const TABLE: &[(&str, Class)] = &[
("tiff", Class::Layer(0)),
("sysmem", Class::Layer(0)),
("fetch", Class::Layer(0)),
+ ("denoise", Class::Layer(1)),
("raw", Class::Layer(1)),
("codecs", Class::Layer(1)),
("meta", Class::Layer(1)),