LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
LR-EDIT-DETAIL-DENOISE: non-destructive AI RAW denoise with pure-Rust CPU/GPU
Daniel Voshart7935999d9449
diff --git a/Cargo.lock b/Cargo.lockindex 86d4bca64..6de52a46f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -209,7 +209,7 @@ version = "0.6.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b"dependencies = [- "equator",+ "equator 0.4.2",][[package]]@@ -967,6 +967,12 @@ dependencies = ["unicode-width",]+[[package]]+name = "coe-rs"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698"+[[package]]name = "color"version = "0.3.3"@@ -1256,6 +1262,16 @@ version = "0.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"+[[package]]+name = "dyn-stack"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"+dependencies = [+ "bytemuck",+ "reborrow",+]+[[package]]name = "dyn-stack"version = "0.13.2"@@ -1537,13 +1553,33 @@ version = "0.36.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13"+[[package]]+name = "equator"+version = "0.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a3b0a88aa91d0ad2b9684e4479aed31a17d3f9051bdbbc634bd2c01bc5a5eee8"+dependencies = [+ "equator-macro 0.1.9",+]+[[package]]name = "equator"version = "0.4.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc"dependencies = [- "equator-macro",+ "equator-macro 0.4.2",+]++[[package]]+name = "equator-macro"+version = "0.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "60d08acb9849f7fb4401564f251be5a526829183a3645a90197dea8e786cf3ae"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",][[package]]@@ -1608,6 +1644,42 @@ dependencies = ["pin-project-lite",]+[[package]]+name = "faer-core"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32da0f6262445261106407bdfb548df4f8f1c89bcc3cff1bb6d08e7f525b575a"+dependencies = [+ "bytemuck",+ "coe-rs",+ "dyn-stack 0.10.0",+ "equator 0.1.10",+ "faer-entity",+ "gemm 0.17.1",+ "matrixcompare-core",+ "num-complex",+ "num-traits",+ "paste",+ "pulp 0.18.22",+ "reborrow",+ "seq-macro",+]++[[package]]+name = "faer-entity"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93fb12f36da3162b3fefc0c7343d905d5bd2c2bf444a7f63ceeaa2bc003311d3"+dependencies = [+ "bytemuck",+ "coe-rs",+ "libm",+ "num-complex",+ "num-traits",+ "pulp 0.18.22",+ "reborrow",+]+[[package]]name = "fastrand"version = "2.5.0"@@ -1783,13 +1855,33 @@ dependencies = ["slab",]+[[package]]+name = "gemm"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-c32 0.17.1",+ "gemm-c64 0.17.1",+ "gemm-common 0.17.1",+ "gemm-f16 0.17.1",+ "gemm-f32 0.17.1",+ "gemm-f64 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "seq-macro",+]+[[package]]name = "gemm"version = "0.18.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-c32 0.18.2","gemm-c64 0.18.2","gemm-common 0.18.2",@@ -1799,7 +1891,7 @@ dependencies = ["num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0","seq-macro",]@@ -1809,7 +1901,7 @@ version = "0.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-c32 0.19.0","gemm-c64 0.19.0","gemm-common 0.19.0",@@ -1819,7 +1911,22 @@ dependencies = ["num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "gemm-c32"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0","seq-macro",]@@ -1829,12 +1936,12 @@ version = "0.18.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.18.2","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0","seq-macro",]@@ -1844,12 +1951,27 @@ version = "0.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.19.0","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "gemm-c64"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0","seq-macro",]@@ -1859,12 +1981,12 @@ version = "0.18.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.18.2","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0","seq-macro",]@@ -1874,13 +1996,32 @@ version = "0.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.19.0","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "gemm-common"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8"+dependencies = [+ "bytemuck",+ "dyn-stack 0.10.0",+ "half",+ "num-complex",+ "num-traits",+ "once_cell",+ "paste",+ "pulp 0.18.22",+ "raw-cpuid 10.7.0","seq-macro",+ "sysctl 0.5.5",][[package]]@@ -1890,7 +2031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3"dependencies = ["bytemuck",- "dyn-stack",+ "dyn-stack 0.13.2","half","libm","num-complex",@@ -1898,10 +2039,10 @@ dependencies = ["once_cell","paste","pulp 0.21.5",- "raw-cpuid",+ "raw-cpuid 11.6.0","rayon","seq-macro",- "sysctl",+ "sysctl 0.6.0",][[package]]@@ -1911,7 +2052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e"dependencies = ["bytemuck",- "dyn-stack",+ "dyn-stack 0.13.2","half","libm","num-complex",@@ -1919,10 +2060,27 @@ dependencies = ["once_cell","paste","pulp 0.22.3",- "raw-cpuid",+ "raw-cpuid 11.6.0","rayon","seq-macro",- "sysctl",+ "sysctl 0.6.0",+]++[[package]]+name = "gemm-f16"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "gemm-f32 0.17.1",+ "half",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "seq-macro",][[package]]@@ -1931,14 +2089,14 @@ version = "0.18.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.18.2","gemm-f32 0.18.2","half","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0","rayon","seq-macro",]@@ -1949,30 +2107,45 @@ version = "0.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.19.0","gemm-f32 0.19.0","half","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0","rayon","seq-macro",]+[[package]]+name = "gemm-f32"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0",+ "seq-macro",+]+[[package]]name = "gemm-f32"version = "0.18.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.18.2","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0","seq-macro",]@@ -1982,12 +2155,27 @@ version = "0.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.19.0","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0",+ "seq-macro",+]++[[package]]+name = "gemm-f64"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0"+dependencies = [+ "dyn-stack 0.10.0",+ "gemm-common 0.17.1",+ "num-complex",+ "num-traits",+ "paste",+ "raw-cpuid 10.7.0","seq-macro",]@@ -1997,12 +2185,12 @@ version = "0.18.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.18.2","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0","seq-macro",]@@ -2012,12 +2200,12 @@ version = "0.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a"dependencies = [- "dyn-stack",+ "dyn-stack 0.13.2","gemm-common 0.19.0","num-complex","num-traits","paste",- "raw-cpuid",+ "raw-cpuid 11.6.0","seq-macro",]@@ -2859,6 +3047,22 @@ dependencies = ["serde",]+[[package]]+name = "lightcraft-denoise"+version = "0.4.0"+dependencies = [+ "faer-core",+ "lightcraft-raster",+ "miniz_oxide 0.8.9",+ "proptest",+ "rayon",+ "serde",+ "serde_json",+ "sha2",+ "thiserror 2.0.21",+ "web-time",+]+[[package]]name = "lightcraft-develop"version = "0.4.0"@@ -2881,7 +3085,9 @@ dependencies = ["lightcraft-catalog","lightcraft-codecs","lightcraft-color",+ "lightcraft-denoise","lightcraft-develop",+ "lightcraft-fetch","lightcraft-geom","lightcraft-gpu","lightcraft-merge",@@ -2928,6 +3134,7 @@ version = "0.4.0"dependencies = ["bytemuck","lightcraft-color",+ "lightcraft-denoise","lightcraft-develop","lightcraft-geom","lightcraft-pipeline",@@ -3082,6 +3289,7 @@ dependencies = ["lightcraft-catalog","lightcraft-codecs","lightcraft-color",+ "lightcraft-denoise","lightcraft-develop","lightcraft-engine","lightcraft-geom",@@ -3174,6 +3382,12 @@ dependencies = ["libc",]+[[package]]+name = "matrixcompare-core"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73"+[[package]]name = "maybe-rayon"version = "0.1.1"@@ -4291,6 +4505,18 @@ dependencies = ["unarray",]+[[package]]+name = "pulp"+version = "0.18.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6"+dependencies = [+ "bytemuck",+ "libm",+ "num-complex",+ "reborrow",+]+[[package]]name = "pulp"version = "0.21.5"@@ -4317,7 +4543,7 @@ dependencies = ["num-complex","paste","pulp-wasm-simd-flag",- "raw-cpuid",+ "raw-cpuid 11.6.0","reborrow","version_check",]@@ -4518,6 +4744,15 @@ dependencies = ["rgb",]+[[package]]+name = "raw-cpuid"+version = "10.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"+dependencies = [+ "bitflags 1.3.2",+]+[[package]]name = "raw-cpuid"version = "11.6.0"@@ -5245,6 +5480,20 @@ dependencies = ["syn 3.0.6",]+[[package]]+name = "sysctl"+version = "0.5.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"+dependencies = [+ "bitflags 2.13.2",+ "byteorder",+ "enum-as-inner",+ "libc",+ "thiserror 1.0.69",+ "walkdir",+]+[[package]]name = "sysctl"version = "0.6.0"diff --git a/Cargo.toml b/Cargo.tomlindex 77b206669..b23fae21c 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -11,6 +11,7 @@ repository = "https://github.com/storytold/lightcraft"[workspace.dependencies]# internal+lightcraft-denoise = { path = "crates/denoise" }lightcraft-geom = { path = "crates/geom" }lightcraft-sysmem = { path = "crates/sysmem" }lightcraft-color = { path = "crates/color" }@@ -34,6 +35,7 @@ lightcraft-segment = { path = "crates/segment" }# foundationthiserror = "2"+sha2 = "0.10"serde = { version = "1", features = ["derive", "rc"] }serde_json = { version = "1", features = ["float_roundtrip"] }log = "0.4"diff --git a/ROADMAP.md b/ROADMAP.mdindex 5e793e8be..18d4dae10 100644--- a/ROADMAP.md+++ b/ROADMAP.md@@ -25,7 +25,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).| **Feature checklist** | 79.7% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P0: full-resolution zoom detail; P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim || **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) || **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW, NEF, RW2, RAF, CR3, CR2 and PEF have guarded camera-JPEG colour estimates, with bundled ILCE-7M4, X-H2S and X-T4 profiles (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |-| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics | Real segmentation masks (subject, sky, people, objects, landscape, depth), AI denoise, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |+| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics; non-destructive Bayer RAW denoise with opt-in weights on pure-Rust CPU/GPU | Real segmentation masks (subject, sky, people, objects, landscape, depth), denoise model policy / linear RGB / quality verification, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) || **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering || **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic) | **Map view, Book, Print, Slideshow module, Web, publish services**: ~40 tracker rows ⬜ || **HDR & video** | 0% | | HDR edit/display/export; video play/trim/edit/export |@@ -40,7 +40,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).| Fujifilm raw shooter | ~65% | Uncompressed and lossless/lossy compressed RAF decoded; camera colour calibration remains missing. Verified on 13 bodies, including 14/16-bit GFX; see `docs/raf-compression.md` || Canon CR3 / Olympus compressed-raw shooter | ~45% | CR3 lossless Bayer and version 0x100/0x200 C-RAW develop from sensor data (M50, R100, R8 verified pixel for pixel); other CRX variants and compressed ORF open as embedded JPEG previews. Camera colour and broader model verification remain || Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering |-| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise |+| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); Bayer AI denoise needs separately installed weights; X-Trans/linear RGB, model policy and quality parity remain open |## Where we're going@@ -59,9 +59,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par(references stay in the local `plan/`), then tune against the numbers.5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1).6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for- segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Object / Describe masks now+ segmentation masks and denoise. Bayer denoise inference is implemented in pure Rust with an opt-in GPL model; release policy and independently licensed weights remain open. Unblocks M12 and Enhance. Object / Describe masks nowrun Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still- to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open.+ to be set up, docs/ai-masks.md); Subject / Sky / People, Raw Details and Super Resolution are still open.7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisationand accessibility.@@ -86,7 +86,7 @@ yet guarantee original-pixel detail; visible-region rendering remains a gap (LR-| M4 | Crop, geometry, optics | crop tool + overlays, straighten, Upright (auto/level/vertical/full/guided), manual transforms, CA, defringe, manual lens corrections | 6–10 | ✅ || M5 | Performance | source pyramids, wgpu compute pipeline (CPU oracle), draft/full renders, prefetch, budgets (16 ms slider updates on 24 MP) | 10–15 | 🚧 (stage cache, source pyramid, wgpu pipeline, prefetch, memory budget ✅; colour NR at half resolution, GPU histogram ⬜) || M6 | Masking | brush, linear/radial gradients, colour/luminance/depth range, add/subtract/intersect/invert, all local adjustments, masks panel | 8–12 | 🚧 (brush/linear/radial/colour/luminance range, add/subtract/intersect, masks panel ✅; depth range, AI masks ⬜) |-| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise, Raw Details, Super Resolution ⬜) |+| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise 🟡 (Bayer, optional weights; policy/fidelity open); Raw Details, Super Resolution ⬜) || M8 | Heal / Remove | content-aware remove (PatchMatch), heal, clone, brush spots, visualize spots, red/pet eye | 6–10 | 🚧 (heal, clone, auto source, visualize spots, red/pet eye ✅; PatchMatch remove ⬜) || M9 | Presets, profiles, versions, sync | preset browser + amount, create/import presets, profile browser, versions, history, copy/paste/sync settings | 5–8 | ✅ || M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs ✅; JXL encode, HDR export ⬜) |diff --git a/apps/lightcraft-cli/Cargo.toml b/apps/lightcraft-cli/Cargo.tomlindex 3f60fe32a..72d468586 100644--- a/apps/lightcraft-cli/Cargo.toml+++ b/apps/lightcraft-cli/Cargo.toml@@ -25,6 +25,8 @@ serde_json = { workspace = true }dhat = { workspace = true, optional = true }[features]+default = ["denoise"]+denoise = ["lightcraft-engine/denoise"]# Heap profiling: `cargo run --release -p lightcraft-cli --features dhat-heap -- snapshot …` counts# every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json# (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work.diff --git a/apps/lightcraft-cli/src/main.rs b/apps/lightcraft-cli/src/main.rsindex 6c5e6e442..c6dfdc2ac 100644--- a/apps/lightcraft-cli/src/main.rs+++ b/apps/lightcraft-cli/src/main.rs@@ -391,7 +391,7 @@ fn merge(args: &[String]) -> Result<(), String> {}i += 1;}- let mut s = Session::new().with_fs();+ let mut s = Session::new().with_fs().with_default_denoise_models();let paths = expand_paths(&files);let r = s.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;let mut ids: Vec<u64> = r["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default();@@ -652,7 +652,7 @@ fn render(args: &[String]) -> Result<(), String> {}let input = input.ok_or("render: missing input file")?;let output = output.ok_or("render: missing -o OUTPUT")?;- let mut s = Session::new().with_fs();+ let mut s = Session::new().with_fs().with_default_denoise_models();let abs = expand_paths(std::slice::from_ref(&input));let r = s.execute("library.import", &json!({"paths": abs})).map_err(|e| e.to_string())?;let id = r["imported"][0].as_u64().ok_or_else(|| format!("{input}: not a readable photo"))?;@@ -734,12 +734,12 @@ fn snapshot(args: &[String]) -> Result<(), String> {let t0 = Instant::now();let mut session = match &library {Some(dir) => {- let mut s = Session::new().with_fs();+ let mut s = Session::new().with_fs().with_default_denoise_models();s.open_library(dir, false).map_err(|e| library_error(dir, e))?;s}- None if files.is_empty() => Session::with_demo().with_fs(),- None => Session::new().with_fs(),+ None if files.is_empty() => Session::with_demo().with_fs().with_default_denoise_models(),+ None => Session::new().with_fs().with_default_denoise_models(),};if !files.is_empty() {let ti = Instant::now();diff --git a/apps/lightcraft/Cargo.toml b/apps/lightcraft/Cargo.tomlindex a29299da0..3f3f40ba0 100644--- a/apps/lightcraft/Cargo.toml+++ b/apps/lightcraft/Cargo.toml@@ -9,6 +9,10 @@ default-run = "lightcraft"[lints]workspace = true+[features]+default = ["denoise"]+denoise = ["lightcraft-engine/denoise"]+[dependencies]lightcraft-engine = { workspace = true, features = ["sam"] }lightcraft-sysmem = { workspace = true }diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rsindex 4c7bb4242..44f0548dd 100644--- a/apps/lightcraft/src/main.rs+++ b/apps/lightcraft/src/main.rs@@ -453,15 +453,15 @@ fn services(ctx: egui::Context, log_file: Option<&std::path::Path>) -> Services/// and offers Try Again / Choose Another Library… / Continue Without Saving / Quit (issue #100).fn open_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: bool) -> (Session, Option<LibraryProblem>) {if in_memory {- return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock(), None);+ return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_default_denoise_models().with_system_clock(), None);}- let unopened = || Session::new().with_fs().with_system_clock();+ let unopened = || Session::new().with_fs().with_default_denoise_models().with_system_clock();let Some(dir) = dir else {log::warn!("no library location (set --library or LIGHTCRAFT_LIBRARY)");return (unopened(), Some(LibraryProblem::new("", "There is no home folder to keep the library in. Choose a folder for it.")));};let t0 = std::time::Instant::now();- let mut s = Session::new().with_fs().with_system_clock();+ let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock();match s.open_library(&dir, seed_demo) {Ok(r) => {let (replayed, torn) = (r.replayed, r.torn_bytes);diff --git a/crates/denoise/Cargo.toml b/crates/denoise/Cargo.tomlnew file mode 100644index 000000000..55c44a144--- /dev/null+++ b/crates/denoise/Cargo.toml@@ -0,0 +1,28 @@+[package]+name = "lightcraft-denoise"+version.workspace = true+edition.workspace = true+license.workspace = true+description = "LightCraft AI denoise (L1): denoise-model manifests, Bayer packing, tiling and blending, the cache file, and pure-Rust CPU inference"++[lints]+workspace = true++[features]+default = []++[dependencies]+# Safe views over Rust-intrinsic gemm kernels. Pin the core-only release: newer backends+# generate assembly. No BLAS, cc/build scripts, nightly/AMX, or inner thread pool.+faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"] }+lightcraft-raster = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+sha2 = { workspace = true }+thiserror = { workspace = true }+miniz_oxide = { workspace = true }+rayon = { workspace = true }+web-time = { workspace = true }++[dev-dependencies]+proptest = { workspace = true }diff --git a/crates/denoise/README.md b/crates/denoise/README.mdnew file mode 100644index 000000000..cb69e3822--- /dev/null+++ b/crates/denoise/README.md@@ -0,0 +1,14 @@+# lightcraft-denoise++L1 pure-Rust AI RAW denoise primitives and CPU inference. No weights are bundled.+See [user workflow and limitations](../../docs/denoise.md).++`manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset+into `net`; `cpu` executes it and `reference` supplies scalar checks. `bayer`, `tiles`+and `run` pack sensor data and blend overlapping tiles while preserving clipped+highlights. `product` reads/writes the disposable `.lcdn` cache. `runtime` supplies+the installation self-test. Synthetic networks are generated in Rust for tests.++Run `cargo test -p lightcraft-denoise`. Real-model tests are ignored by default and+require an explicitly supplied local model. The GPU twin is `lightcraft-gpu::nn`;+model management, background jobs and export integration live in `lightcraft-engine`.diff --git a/crates/denoise/src/archive.rs b/crates/denoise/src/archive.rsnew file mode 100644index 000000000..ddaf7084d--- /dev/null+++ b/crates/denoise/src/archive.rs@@ -0,0 +1,271 @@+//! Taking one file out of a `.zip` (a `.dtmodel` is one), the little of the format a model download needs: stored and+//! deflated entries, found by name, checked against their CRC-32 and a size limit, written next to where they are+//! going and moved into place. It is a stranger's file, so every length and offset in it is checked, a name is only+//! ever compared (never used as a path), and zip64, encrypted and multi-disk archives are refused.++use std::io::{Read, Seek, SeekFrom, Write};+use std::path::Path;++/// How far from the end the end-of-central-directory record can start (a 22-byte record and a 64 KiB comment).+const EOCD_SEARCH: u64 = 22 + 65535;+/// Largest central directory we read.+const MAX_DIRECTORY: u64 = 1 << 20;++#[derive(Debug, PartialEq, thiserror::Error)]+pub enum ArchiveError {+ #[error("the archive could not be read: {0}")]+ Io(String),+ #[error("not a usable archive: {0}")]+ Format(String),+ #[error("the archive has no file called `{0}`")]+ Missing(String),+}++impl From<std::io::Error> for ArchiveError {+ fn from(e: std::io::Error) -> Self {+ ArchiveError::Io(e.to_string())+ }+}++fn format<T>(why: &str) -> Result<T, ArchiveError> {+ Err(ArchiveError::Format(why.into()))+}++fn u16_at(b: &[u8], at: usize) -> Option<u16> {+ Some(u16::from_le_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?))+}++fn u32_at(b: &[u8], at: usize) -> Option<u32> {+ Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))+}++/// CRC-32 (IEEE) of `bytes`.+fn crc32(bytes: &[u8]) -> u32 {+ let mut table = [0u32; 256];+ for (i, t) in table.iter_mut().enumerate() {+ let mut c = i as u32;+ for _ in 0..8 {+ c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };+ }+ *t = c;+ }+ !bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))+}++struct Entry {+ method: u16,+ crc: u32,+ compressed: u64,+ size: u64,+ local_header: u64,+}++/// Find the entry called `name` (exactly, or as the last part of a longer name: `folder/name`) in the central directory.+fn find(f: &mut (impl Read + Seek), name: &str) -> Result<Entry, ArchiveError> {+ let len = f.seek(SeekFrom::End(0))?;+ let tail_len = len.min(EOCD_SEARCH);+ f.seek(SeekFrom::Start(len - tail_len))?;+ let mut tail = vec![0u8; tail_len as usize];+ f.read_exact(&mut tail)?;+ let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| u32_at(&tail, i) == Some(0x0605_4b50));+ let Some(e) = eocd else { return format("this is not a zip file") };+ let (entries, size, offset) = (u16_at(&tail, e + 10), u32_at(&tail, e + 12), u32_at(&tail, e + 16));+ let (Some(entries), Some(size), Some(offset)) = (entries, size, offset) else { return format("the end of the archive is cut short") };+ if entries == 0xffff || size == u32::MAX || offset == u32::MAX {+ return format("zip64 archives are not supported");+ }+ if u64::from(size) > MAX_DIRECTORY || u64::from(offset) + u64::from(size) > len {+ return format("the list of files is not where it says");+ }+ f.seek(SeekFrom::Start(u64::from(offset)))?;+ let mut dir = vec![0u8; size as usize];+ f.read_exact(&mut dir)?;+ let mut at = 0usize;+ for _ in 0..entries {+ if u32_at(&dir, at) != Some(0x0201_4b50) {+ return format("the list of files is damaged");+ }+ let get = |o: usize| at.checked_add(o);+ let (flags, method, crc) = (u16_at(&dir, at + 8), u16_at(&dir, at + 10), u32_at(&dir, at + 16));+ let (compressed, uncompressed) = (u32_at(&dir, at + 20), u32_at(&dir, at + 24));+ let (n, x, c) = (u16_at(&dir, at + 28), u16_at(&dir, at + 30), u16_at(&dir, at + 32));+ let local = u32_at(&dir, at + 42);+ let (Some(flags), Some(method), Some(crc), Some(compressed), Some(uncompressed), Some(n), Some(x), Some(c), Some(local), Some(start)) =+ (flags, method, crc, compressed, uncompressed, n, x, c, local, get(46))+ else {+ return format("the list of files is cut short");+ };+ let Some(entry_name) = dir.get(start..start + usize::from(n)) else { return format("a file name is cut short") };+ let matches = entry_name == name.as_bytes() || entry_name.strip_suffix(name.as_bytes()).is_some_and(|p| p.ends_with(b"/"));+ if matches {+ if flags & 1 != 0 {+ return format("encrypted files are not supported");+ }+ if compressed == u32::MAX || uncompressed == u32::MAX {+ return format("zip64 files are not supported");+ }+ return Ok(Entry { method, crc, compressed: u64::from(compressed), size: u64::from(uncompressed), local_header: u64::from(local) });+ }+ at = start + usize::from(n) + usize::from(x) + usize::from(c);+ }+ Err(ArchiveError::Missing(name.into()))+}++/// Take the file called `name` out of the archive at `archive` and write it to `dest` (through a temporary file next+/// to it); `max_bytes` bounds its size. Returns the number of bytes written.+pub fn extract(archive: &Path, name: &str, dest: &Path, max_bytes: u64) -> Result<u64, ArchiveError> {+ let mut f = std::io::BufReader::new(std::fs::File::open(archive)?);+ let e = find(&mut f, name)?;+ if e.size > max_bytes || e.compressed > max_bytes {+ return format("the file in the archive is larger than allowed");+ }+ f.seek(SeekFrom::Start(e.local_header))?;+ let mut head = [0u8; 30];+ f.read_exact(&mut head)?;+ if u32_at(&head, 0) != Some(0x0403_4b50) {+ return format("the file's header is not where it says");+ }+ let skip = u64::from(u16_at(&head, 26).unwrap_or(0)) + u64::from(u16_at(&head, 28).unwrap_or(0));+ f.seek_relative(i64::try_from(skip).map_err(|_| ArchiveError::Format("a header is too long".into()))?)?;+ let mut packed = vec![0u8; e.compressed as usize];+ f.read_exact(&mut packed)?;+ let data = match e.method {+ 0 => packed,+ 8 => {+ miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes as usize).map_err(|err| ArchiveError::Format(format!("{err:?}")))?+ }+ m => return format(&format!("compression method {m} is not supported")),+ };+ if data.len() as u64 != e.size || crc32(&data) != e.crc {+ return format("the file in the archive is damaged");+ }+ let tmp = dest.with_extension("part");+ let written = (|| -> std::io::Result<()> {+ let mut out = std::fs::File::create(&tmp)?;+ out.write_all(&data)?;+ out.flush()?;+ drop(out);+ std::fs::rename(&tmp, dest)+ })();+ if let Err(err) = written {+ let _ = std::fs::remove_file(&tmp);+ return Err(err.into());+ }+ Ok(data.len() as u64)+}++#[cfg(test)]+mod tests {+ use super::*;++ /// A zip with the given (name, bytes, deflate) entries, written by hand.+ pub(crate) fn make_zip(files: &[(&str, &[u8], bool)]) -> Vec<u8> {+ let mut out = Vec::new();+ let mut dir = Vec::new();+ for (name, data, deflate) in files {+ let packed = if *deflate { miniz_oxide::deflate::compress_to_vec(data, 6) } else { data.to_vec() };+ let offset = out.len() as u32;+ let method: u16 = if *deflate { 8 } else { 0 };+ out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());+ out.extend_from_slice(&[20, 0, 0, 0]);+ out.extend_from_slice(&method.to_le_bytes());+ out.extend_from_slice(&[0, 0, 0, 0]);+ out.extend_from_slice(&crc32(data).to_le_bytes());+ out.extend_from_slice(&(packed.len() as u32).to_le_bytes());+ out.extend_from_slice(&(data.len() as u32).to_le_bytes());+ out.extend_from_slice(&(name.len() as u16).to_le_bytes());+ out.extend_from_slice(&[0, 0]);+ out.extend_from_slice(name.as_bytes());+ out.extend_from_slice(&packed);+ dir.extend_from_slice(&0x0201_4b50u32.to_le_bytes());+ dir.extend_from_slice(&[20, 0, 20, 0, 0, 0]);+ dir.extend_from_slice(&method.to_le_bytes());+ dir.extend_from_slice(&[0, 0, 0, 0]);+ dir.extend_from_slice(&crc32(data).to_le_bytes());+ dir.extend_from_slice(&(packed.len() as u32).to_le_bytes());+ dir.extend_from_slice(&(data.len() as u32).to_le_bytes());+ dir.extend_from_slice(&(name.len() as u16).to_le_bytes());+ dir.extend_from_slice(&[0; 12]);+ dir.extend_from_slice(&offset.to_le_bytes());+ dir.extend_from_slice(name.as_bytes());+ }+ let cd_offset = out.len() as u32;+ out.extend_from_slice(&dir);+ out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());+ out.extend_from_slice(&[0, 0, 0, 0]);+ out.extend_from_slice(&(files.len() as u16).to_le_bytes());+ out.extend_from_slice(&(files.len() as u16).to_le_bytes());+ out.extend_from_slice(&(dir.len() as u32).to_le_bytes());+ out.extend_from_slice(&cd_offset.to_le_bytes());+ out.extend_from_slice(&[0, 0]);+ out+ }++ fn temp(name: &str) -> std::path::PathBuf {+ std::env::temp_dir().join(format!("lc-denoise-zip-{name}-{}", std::process::id()))+ }++ #[test]+ fn a_stored_and_a_deflated_file_come_out_whole() {+ let big: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();+ let zip = make_zip(&[("m/config.json", b"{}", false), ("m/model.onnx", &big, true), ("m/other.onnx", b"zz", true)]);+ let (a, out) = (temp("a.zip"), temp("a.onnx"));+ std::fs::write(&a, &zip).unwrap();+ assert_eq!(extract(&a, "model.onnx", &out, 1 << 20).unwrap(), big.len() as u64);+ assert_eq!(std::fs::read(&out).unwrap(), big);+ assert_eq!(extract(&a, "m/config.json", &out, 1 << 20).unwrap(), 2);+ assert_eq!(std::fs::read(&out).unwrap(), b"{}");+ // a name must match whole path parts: `del.onnx` is not `model.onnx`+ assert!(matches!(extract(&a, "del.onnx", &out, 1 << 20), Err(ArchiveError::Missing(_))));+ assert!(matches!(extract(&a, "nothing", &out, 1 << 20), Err(ArchiveError::Missing(_))));+ // the size limit applies+ assert!(matches!(extract(&a, "model.onnx", &out, 1000), Err(ArchiveError::Format(_))));+ for p in [a, out] {+ let _ = std::fs::remove_file(p);+ }+ }++ /// Opt-in: `LC_DENOISE_DTMODEL=<rawdenoise-nind.dtmodel> cargo test -p lightcraft-denoise -- --ignored archive`+ #[test]+ #[ignore = "needs the real download: set LC_DENOISE_DTMODEL"]+ fn the_real_download_gives_the_model_it_should() {+ let Some(path) = std::env::var_os("LC_DENOISE_DTMODEL") else { return };+ let known = crate::known::find("rawnind-bayer").unwrap();+ let (d, entry) = (known.download.unwrap(), known.entry.unwrap());+ assert_eq!(crate::hash::sha256_file(Path::new(&path)).unwrap(), d.sha256);+ let out = temp("real.onnx");+ let n = extract(Path::new(&path), entry, &out, crate::manifest::MAX_MODEL_BYTES).unwrap();+ assert_eq!(Some(n), known.manifest.size_bytes);+ assert_eq!(Some(crate::hash::sha256_file(&out).unwrap()), known.manifest.sha256);+ let _ = std::fs::remove_file(out);+ }++ #[test]+ fn damaged_and_hostile_archives_are_errors_not_panics() {+ let zip = make_zip(&[("m/model.onnx", &vec![9u8; 5000], true)]);+ let (a, out) = (temp("b.zip"), temp("b.onnx"));+ // cut anywhere+ for n in (0..zip.len()).step_by(3) {+ std::fs::write(&a, &zip[..n]).unwrap();+ assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err(), "cut at {n}");+ }+ // one byte changed anywhere: an error, or (when it falls in an unused field) the right file, never a panic+ for at in 0..zip.len() {+ let mut bad = zip.clone();+ bad[at] ^= 0x5a;+ std::fs::write(&a, &bad).unwrap();+ if extract(&a, "model.onnx", &out, 1 << 20).is_ok() {+ assert_eq!(std::fs::read(&out).unwrap(), vec![9u8; 5000], "a changed byte at {at} gave another file");+ }+ }+ // not a zip at all, and an empty file+ for junk in [Vec::new(), b"PK".to_vec(), vec![0u8; 100_000]] {+ std::fs::write(&a, junk).unwrap();+ assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err());+ }+ assert!(extract(&temp("missing.zip"), "model.onnx", &out, 1 << 20).is_err());+ for p in [a, out] {+ let _ = std::fs::remove_file(p);+ }+ }+}diff --git a/crates/denoise/src/bayer.rs b/crates/denoise/src/bayer.rsnew file mode 100644index 000000000..a4a00d947--- /dev/null+++ b/crates/denoise/src/bayer.rs@@ -0,0 +1,191 @@+//! Bayer geometry: which way a mosaic is laid out, and how a patch of it is packed into the four planes+//! (red, green on the red row, green on the blue row, blue) a denoise model takes.+//!+//! Every Bayer layout is the same 2 × 2 cell shifted by one pixel at most, so a layout is just where its+//! first red sample is. The cell grid starts one pixel before the picture when that red sample is in+//! column or row 1, and the picture is read with its edge mirrored (without repeating the edge sample, so+//! the colour of a mirrored sample is the colour that belongs at that place).++/// Where the first red sample of a Bayer mosaic sits, `(dx, dy)` in `{0, 1}²`: RGGB is `(0, 0)`, GRBG `(1, 0)`,+/// GBRG `(0, 1)`, BGGR `(1, 1)`.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct Layout {+ pub dx: usize,+ pub dy: usize,+}++impl Layout {+ /// The layout of a 2 × 2 colour pattern, row-major, colours 0 = red, 1 = green, 2 = blue; `None` when+ /// it is not a Bayer cell (one red, one blue, the greens on the other diagonal).+ pub fn from_cell(cell: [u8; 4]) -> Option<Layout> {+ let at = |c: u8| cell.iter().position(|&v| v == c);+ let red = at(0)?;+ let blue = at(2)?;+ let greens = cell.iter().filter(|&&v| v == 1).count();+ // red and blue sit on one diagonal: indices 0+3 or 1+2+ if greens != 2 || red + blue != 3 {+ return None;+ }+ Some(Layout { dx: red % 2, dy: red / 2 })+ }++ /// The cell grid's origin: `-1` along an axis whose first red sample is at 1, so that cell 0 starts on a red column or row.+ fn origin(self) -> (isize, isize) {+ (isize::try_from(self.dx).unwrap_or(isize::MAX).saturating_neg(), isize::try_from(self.dy).unwrap_or(isize::MAX).saturating_neg())+ }++ /// Cells across and down covering a `width × height` mosaic.+ pub fn cells(self, width: usize, height: usize) -> (usize, usize) {+ let cells = |n: usize, phase: usize| n / 2 + (n % 2 + phase.min(1)).div_ceil(2);+ (cells(width, self.dx), cells(height, self.dy))+ }++ /// The mosaic position of cell `(cx, cy)`'s top-left sample (may lie outside the picture).+ pub fn position(self, cx: isize, cy: isize) -> (isize, isize) {+ let (ox, oy) = self.origin();+ (ox.saturating_add(cx.saturating_mul(2)), oy.saturating_add(cy.saturating_mul(2)))+ }+}++/// `i` mirrored into `0..n` (ping-pong, edge not repeated), so `-1` is `1` and `n` is `n - 2`. `n == 0` gives 0.+pub fn reflect(i: isize, n: usize) -> usize {+ if n <= 1 {+ return 0;+ }+ if let Some(period) = isize::try_from(n).ok().and_then(|n| (n - 1).checked_mul(2)) {+ let m = i.rem_euclid(period);+ return (if m < n as isize { m } else { period - m }) as usize;+ }+ let period = 2 * (n as i128 - 1);+ let m = (i as i128).rem_euclid(period);+ // The reflected value is in 0..n, so this conversion always fits usize.+ (if m < n as i128 { m } else { period - m }) as usize+}++/// Pack the `tile × tile` cells whose top-left cell is `(cx0, cy0)` into `out` as four planes `[R, G1, G2, B]`+/// (`4 · tile²` values, NCHW). G1 is the green beside the red sample, G2 the green under it. Samples outside+/// the picture are mirrored. Returns `false` (and writes nothing) when the sizes do not fit.+pub fn pack_tile(mosaic: &[f32], width: usize, height: usize, layout: Layout, cx0: isize, cy0: isize, tile: usize, out: &mut [f32]) -> bool {+ let Some(plane) = tile.checked_mul(tile) else { return false };+ let Some(values) = plane.checked_mul(4) else { return false };+ if width == 0+ || height == 0+ || tile == 0+ || layout.dx > 1+ || layout.dy > 1+ || width.checked_mul(height) != Some(mosaic.len())+ || out.len() != values+ {+ return false;+ }+ let Ok(last) = isize::try_from(tile - 1) else { return false };+ let (ox, oy) = layout.origin();+ for (start, origin) in [(cx0, ox), (cy0, oy)] {+ if start.checked_mul(2).and_then(|v| v.checked_add(origin)).is_none()+ || start.checked_add(last).and_then(|v| v.checked_mul(2)).and_then(|v| v.checked_add(origin)).and_then(|v| v.checked_add(1)).is_none()+ {+ return false;+ }+ }+ // the four samples of a cell, as (dx, dy) from its top-left sample+ const SUB: [(isize, isize); 4] = [(0, 0), (1, 0), (0, 1), (1, 1)];+ for (p, &(sx, sy)) in SUB.iter().enumerate() {+ let Some(dst) = out.get_mut(p * plane..(p + 1) * plane) else { return false };+ for (j, row) in dst.chunks_exact_mut(tile).enumerate() {+ let (_, y) = layout.position(cx0, cy0 + j as isize);+ let ry = reflect(y + sy, height);+ let Some(src) = mosaic.get(ry * width..(ry + 1) * width) else { return false };+ for (i, v) in row.iter_mut().enumerate() {+ let (x, _) = layout.position(cx0 + i as isize, cy0);+ *v = src.get(reflect(x + sx, width)).copied().unwrap_or(0.0);+ }+ }+ }+ true+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn hostile_sizes_and_positions_never_panic_or_write_output() {+ let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();+ let mut out = [7.0; 4];+ for (w, h, tile, x) in [(usize::MAX, 2, 1, 0), (1, 1, usize::MAX, 0), (1, 1, 0, 0), (1, 1, 1, isize::MAX), (1, 1, 1, isize::MIN)] {+ assert!(!pack_tile(&[0.5], w, h, layout, x, 0, tile, &mut out));+ assert_eq!(out, [7.0; 4]);+ }+ for n in [1, 2, isize::MAX as usize, usize::MAX] {+ for i in [isize::MIN, -1, 0, isize::MAX] {+ assert!(reflect(i, n) < n);+ }+ }+ assert_eq!(layout.cells(usize::MAX, usize::MAX), (usize::MAX / 2 + 1, usize::MAX / 2 + 1));+ }++ #[test]+ fn layouts_from_cells() {+ assert_eq!(Layout::from_cell([0, 1, 1, 2]), Some(Layout { dx: 0, dy: 0 }));+ assert_eq!(Layout::from_cell([1, 0, 2, 1]), Some(Layout { dx: 1, dy: 0 }));+ assert_eq!(Layout::from_cell([1, 2, 0, 1]), Some(Layout { dx: 0, dy: 1 }));+ assert_eq!(Layout::from_cell([2, 1, 1, 0]), Some(Layout { dx: 1, dy: 1 }));+ // not Bayer: two reds, a missing blue, greens on the wrong diagonal+ assert_eq!(Layout::from_cell([0, 0, 1, 2]), None);+ assert_eq!(Layout::from_cell([0, 1, 2, 1]), None);+ assert_eq!(Layout::from_cell([1, 1, 1, 1]), None);+ }++ #[test]+ fn reflection_keeps_the_colour_of_a_site() {+ assert_eq!(reflect(-1, 10), 1);+ assert_eq!(reflect(-2, 10), 2);+ assert_eq!(reflect(10, 10), 8);+ assert_eq!(reflect(11, 10), 7);+ assert_eq!(reflect(25, 10), 7);+ assert_eq!(reflect(5, 0), 0);+ assert_eq!(reflect(5, 1), 0);+ for i in -40..60 {+ assert_eq!(reflect(i, 11) % 2, (i.rem_euclid(2)) as usize, "parity at {i}");+ }+ }++ #[test]+ fn packing_puts_each_colour_in_its_plane_for_every_layout() {+ // a mosaic whose sample value says which colour it is: R = 10, G = 20, B = 30+ for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {+ let layout = Layout::from_cell(cell).unwrap();+ let (w, h) = (10usize, 8usize);+ let mosaic: Vec<f32> = (0..w * h).map(|i| [10.0, 20.0, 30.0][cell[(i / w % 2) * 2 + i % w % 2] as usize]).collect();+ let (nx, ny) = layout.cells(w, h);+ let tile = nx.max(ny);+ let mut out = vec![0.0; 4 * tile * tile];+ assert!(pack_tile(&mosaic, w, h, layout, 0, 0, tile, &mut out));+ let plane = tile * tile;+ for (p, want) in [10.0, 20.0, 20.0, 30.0].into_iter().enumerate() {+ assert!(out[p * plane..(p + 1) * plane].iter().all(|&v| v == want), "layout {cell:?} plane {p}");+ }+ }+ }++ #[test]+ fn g1_is_beside_the_red_sample_and_g2_under_it() {+ // RGGB with a distinct value at every site+ let (w, h) = (4, 4);+ let mosaic: Vec<f32> = (0..16).map(|i| i as f32).collect();+ let mut out = vec![0.0; 4 * 4];+ assert!(pack_tile(&mosaic, w, h, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));+ // cell (0,0): R = site (0,0)=0, G1 = site (1,0)=1, G2 = site (0,1)=4, B = site (1,1)=5+ assert_eq!([out[0], out[4], out[8], out[12]], [0.0, 1.0, 4.0, 5.0]);+ // cell (1,0): sites (2,0)=2, (3,0)=3, (2,1)=6, (3,1)=7+ assert_eq!([out[1], out[5], out[9], out[13]], [2.0, 3.0, 6.0, 7.0]);+ }++ #[test]+ fn packing_rejects_wrong_sizes() {+ let mut out = vec![0.0; 16];+ assert!(!pack_tile(&[0.0; 15], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));+ assert!(!pack_tile(&[0.0; 16], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 3, &mut out));+ assert!(!pack_tile(&[], 0, 0, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));+ }+}diff --git a/crates/denoise/src/cpu.rs b/crates/denoise/src/cpu.rsnew file mode 100644index 000000000..a856fc256--- /dev/null+++ b/crates/denoise/src/cpu.rs@@ -0,0 +1,386 @@+//! Single-thread CPU execution of the checked denoise network.+//!+//! Convolutions lower at most 256 spatial positions at a time, then use faer's safe views over pure-Rust GEMM.+//! No inner thread pool: `run::denoise_bayer` already runs tiles on the application's Rayon pool. Tensor lifetimes+//! are planned once; buffers and im2col storage are recycled between calls, independently for concurrent callers.++use crate::net::{Net, NetError, Op};+use crate::run::{Error, TileRunner};+use faer::{Parallelism, mat};+use std::sync::{Arc, Mutex};++const BLOCK: usize = 256;+/// Cap all tensor buffers and convolution scratch for one tile together (512 MiB).+const MAX_VALUES: usize = 128 * 1024 * 1024;+const MAX_TILE: usize = 2048;+const MAX_IDLE: usize = 16;++type Result<T> = std::result::Result<T, NetError>;+fn bad(why: &str) -> NetError {+ NetError::Invalid(why.into())+}+fn count(c: usize, side: usize) -> Result<usize> {+ c.checked_mul(side)+ .and_then(|n| n.checked_mul(side))+ .filter(|&n| n > 0 && n <= MAX_VALUES)+ .ok_or_else(|| bad("a tensor has no size or is too large"))+}+fn zeros(n: usize) -> Result<Vec<f32>> {+ let mut v = Vec::new();+ v.try_reserve_exact(n).map_err(|_| bad("not enough memory for a denoise tile"))?;+ v.resize(n, 0.0);+ Ok(v)+}+fn sources(op: &Op) -> (usize, Option<usize>) {+ match op {+ Op::Conv(c) => (c.src, c.src2),+ Op::ConvT2(c) => (c.src, None),+ Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => (*src, None),+ }+}+#[derive(Clone, Copy)]+struct Tensor {+ slot: usize,+ side: usize,+ channels: usize,+}+struct Plan {+ net: Net,+ tensors: Vec<Tensor>,+ sizes: Vec<usize>,+ lower: usize,+ product: usize,+ up_weights: Vec<Option<Vec<f32>>>,+}+struct Workspace {+ buffers: Vec<Vec<f32>>,+ lower: Vec<f32>,+ product: Vec<f32>,+}++/// A checked Net on square CHW tiles. Clones share immutable weights and the idle workspace cache.+#[derive(Clone)]+pub struct NetRunner {+ plan: Arc<Plan>,+ idle: Arc<Mutex<Vec<Workspace>>>,+}++impl NetRunner {+ pub fn new(net: &Net, tile: usize) -> Result<Self> {+ if tile == 0+ || tile > MAX_TILE+ || !net.fits_tile(tile)+ || net.in_channels != 4+ || net.out_channels() != 3+ || net.shape(0).is_none_or(|s| s.channels != net.in_channels)+ {+ return Err(bad("the tile or Bayer-to-RGB contract does not fit the network"));+ }+ let mut last = vec![0; net.ops().len() + 1];+ for (i, op) in net.ops().iter().enumerate() {+ let (a, b) = sources(op);+ for id in std::iter::once(a).chain(b) {+ *last.get_mut(id).ok_or_else(|| bad("missing tensor"))? = i + 1;+ }+ }+ let mut tensors = vec![Tensor { slot: 0, side: tile, channels: net.in_channels }];+ let mut sizes = vec![count(net.in_channels, tile)?];+ let mut busy = vec![true];+ let (mut lower, mut product) = (0, 0);+ let mut up_weights = Vec::new();+ for (i, op) in net.ops().iter().enumerate() {+ let (src, src2) = sources(op);+ let a = *tensors.get(src).ok_or_else(|| bad("missing tensor"))?;+ let (channels, side) = match op {+ Op::Conv(c) => {+ if let Some(s) = src2+ && tensors.get(s).is_none_or(|b| b.side != a.side)+ {+ return Err(bad("a join has different sizes"));+ }+ (c.cout, a.side)+ }+ Op::ConvT2(c) => (c.cout, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),+ Op::MaxPool2 { .. } => {+ if a.side < 2 || !a.side.is_multiple_of(2) {+ return Err(bad("pooling does not divide the tile"));+ }+ (a.channels, a.side / 2)+ }+ Op::DepthToSpace2 { .. } => (a.channels / 4, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),+ };+ let n = count(channels, side)?;+ let slot = busy+ .iter()+ .zip(&sizes)+ .enumerate()+ .filter(|(_, (used, _))| !**used)+ .min_by_key(|(_, (_, size))| (n.saturating_sub(**size), **size))+ .map_or(busy.len(), |(i, _)| i);+ if slot == sizes.len() {+ sizes.push(n);+ busy.push(true);+ } else {+ let sz = sizes.get_mut(slot).ok_or_else(|| bad("missing buffer"))?;+ *sz = (*sz).max(n);+ *busy.get_mut(slot).ok_or_else(|| bad("missing buffer"))? = true;+ }+ tensors.push(Tensor { slot, side, channels });+ // Release sources only after the output has its own buffer (also handles reading the same tensor twice).+ for id in std::iter::once(src).chain(src2) {+ if last.get(id) == Some(&(i + 1)) {+ let t = tensors.get(id).ok_or_else(|| bad("missing tensor"))?;+ *busy.get_mut(t.slot).ok_or_else(|| bad("missing buffer"))? = false;+ }+ }+ let (k, m, positions) = match op {+ Op::Conv(c) => (c.cin * c.k * c.k, c.cout, a.side * a.side),+ Op::ConvT2(c) => (c.cin, c.cout * 4, a.side * a.side),+ _ => (0, 0, 0),+ };+ lower = lower.max(k.saturating_mul(BLOCK.min(positions)));+ product = product.max(m.saturating_mul(BLOCK.min(positions)));+ let packed = if let Op::ConvT2(c) = op {+ let mut w = zeros(c.weight.len())?;+ // [cin][cout][dy][dx] -> [cout*4][cin], so an up-convolution is one GEMM and a shuffle.+ for (ci, row) in c.weight.chunks_exact(c.cout * 4).enumerate() {+ for (co, &v) in row.iter().enumerate() {+ *w.get_mut(co * c.cin + ci).ok_or_else(|| bad("invalid transposed weights"))? = v;+ }+ }+ Some(w)+ } else {+ None+ };+ up_weights.push(packed);+ }+ let total = sizes+ .iter()+ .try_fold(lower.saturating_add(product), |n, &s| n.checked_add(s))+ .filter(|&n| n <= MAX_VALUES)+ .ok_or_else(|| bad("the tile needs too much working memory"))?;+ let _ = total;+ Ok(Self { plan: Arc::new(Plan { net: net.clone(), tensors, sizes, lower, product, up_weights }), idle: Arc::new(Mutex::new(Vec::new())) })+ }+ /// Immutable data for the GPU runner, without re-reading the model.+ pub fn net(&self) -> &Net {+ &self.plan.net+ }+ pub fn bytes_per_tile(&self) -> usize {+ (self.plan.sizes.iter().sum::<usize>() + self.plan.lower + self.plan.product) * 4+ }+ fn workspace(&self) -> Result<Workspace> {+ if let Some(w) = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop() {+ return Ok(w);+ }+ Ok(Workspace {+ buffers: self.plan.sizes.iter().map(|&n| zeros(n)).collect::<Result<_>>()?,+ lower: zeros(self.plan.lower)?,+ product: zeros(self.plan.product)?,+ })+ }+ fn execute(&self, input: &[f32], w: &mut Workspace) -> Result<Vec<f32>> {+ let first = self.plan.tensors.first().ok_or_else(|| bad("no input"))?;+ let want = count(first.channels, first.side)?;+ w.buffers.first_mut().and_then(|b| b.get_mut(..want)).ok_or_else(|| bad("missing input buffer"))?.copy_from_slice(input);+ let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some();+ for (i, op) in self.plan.net.ops().iter().enumerate() {+ let started = web_time::Instant::now();+ let out = *self.plan.tensors.get(i + 1).ok_or_else(|| bad("missing output tensor"))?;+ let (before, rest) = w.buffers.split_at_mut_checked(out.slot).ok_or_else(|| bad("missing output buffer"))?;+ let (dest, after) = rest.split_first_mut().ok_or_else(|| bad("missing output buffer"))?;+ let get = |id: usize| -> Result<(&[f32], Tensor)> {+ let t = *self.plan.tensors.get(id).ok_or_else(|| bad("missing source tensor"))?;+ let data = if t.slot < out.slot {+ before.get(t.slot)+ } else {+ after.get(t.slot.checked_sub(out.slot + 1).ok_or_else(|| bad("aliased buffers"))?)+ }+ .and_then(|b| b.get(..count(t.channels, t.side).ok()?))+ .ok_or_else(|| bad("missing source buffer"))?;+ Ok((data, t))+ };+ let (src, src2) = sources(op);+ let (a, shape) = get(src)?;+ let b = src2.map(get).transpose()?.map(|v| v.0);+ let dest = dest.get_mut(..count(out.channels, out.side)?).ok_or_else(|| bad("short output buffer"))?;+ match op {+ Op::Conv(c) => conv(c, a, b, shape.side, dest, &mut w.lower, &mut w.product)?,+ Op::ConvT2(c) => up(+ c,+ a,+ shape.side,+ dest,+ self.plan.up_weights.get(i).and_then(Option::as_deref).ok_or_else(|| bad("missing up weights"))?,+ &mut w.lower,+ &mut w.product,+ )?,+ Op::MaxPool2 { .. } => pool(a, shape.side, dest)?,+ Op::DepthToSpace2 { .. } => shuffle(a, shape.side, dest)?,+ }+ if dest.iter().any(|v| !v.is_finite()) {+ return Err(bad("the model produced non-finite values"));+ }+ if profile {+ eprintln!("[denoise-cpu] layer {i}: {}x{}x{} {:.2} ms", out.channels, out.side, out.side, started.elapsed().as_secs_f64() * 1e3);+ }+ }+ let t = self.plan.tensors.last().ok_or_else(|| bad("no output"))?;+ let data = w.buffers.get(t.slot).and_then(|b| b.get(..count(t.channels, t.side).ok()?)).ok_or_else(|| bad("missing result"))?;+ let mut out = zeros(data.len())?;+ out.copy_from_slice(data);+ Ok(out)+ }+}+impl TileRunner for NetRunner {+ fn run(&self, input: &[f32]) -> std::result::Result<Vec<f32>, Error> {+ let first = self.plan.tensors.first().ok_or_else(|| Error::Runtime("missing input shape".into()))?;+ let want = count(first.channels, first.side).map_err(|e| Error::Runtime(e.to_string()))?;+ if input.len() != want || input.iter().any(|v| !v.is_finite()) {+ return Err(Error::Input("the tile has the wrong size or non-finite values".into()));+ }+ let mut w = self.workspace().map_err(|e| Error::Runtime(e.to_string()))?;+ let result = self.execute(input, &mut w).map_err(|e| Error::Runtime(e.to_string()));+ let mut idle = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner);+ if idle.len() < MAX_IDLE {+ idle.push(w);+ }+ result+ }+}+fn activate(x: f32, bias: f32, alpha: Option<f32>) -> f32 {+ let x = x + bias;+ if x < 0.0 { x * alpha.unwrap_or(1.0) } else { x }+}++/// All views are over exact, checked slices. The matrix library's assertions therefore never see file-derived sizes.+fn multiply(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) -> Result<()> {+ if m.checked_mul(k) != Some(w.len()) || k.checked_mul(n) != Some(x.len()) || m.checked_mul(n) != Some(y.len()) || m == 0 || k == 0 || n == 0 {+ return Err(bad("invalid matrix dimensions"));+ }+ faer::mul::matmul(+ mat::from_row_major_slice_mut::<f32>(y, m, n),+ mat::from_row_major_slice::<f32>(w, m, k),+ mat::from_row_major_slice::<f32>(x, k, n),+ None,+ 1.0,+ Parallelism::None,+ );+ Ok(())+}+fn lower(a: &[f32], b: Option<&[f32]>, side: usize, k: usize, start: usize, n: usize, dest: &mut [f32]) -> Result<()> {+ let pad = k / 2;+ let pixels = side * side;+ // Whole row segments are copied, so bounds checks and edge branches stay out of the multiply loop.+ for (ci, plane) in a.chunks_exact(pixels).chain(b.into_iter().flat_map(|b| b.chunks_exact(pixels))).enumerate() {+ for ky in 0..k {+ for kx in 0..k {+ let row = (ci * k * k + ky * k + kx) * n;+ let out = dest.get_mut(row..row + n).ok_or_else(|| bad("short im2col buffer"))?;+ let mut at = 0;+ while at < n {+ let p = start + at;+ let (y, x) = (p / side, p % side);+ let len = (side - x).min(n - at);+ if let Some(yy) = y.checked_add(ky).and_then(|v| v.checked_sub(pad)).filter(|&v| v < side) {+ let lo = x.max(pad.saturating_sub(kx));+ let hi = (x + len).min((side + pad).saturating_sub(kx));+ if lo < hi {+ let from = yy * side + lo + kx - pad;+ let to = at + lo - x;+ out.get_mut(at..to).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);+ out.get_mut(to..to + hi - lo)+ .ok_or_else(|| bad("short im2col row"))?+ .copy_from_slice(plane.get(from..from + hi - lo).ok_or_else(|| bad("short source row"))?);+ out.get_mut(to + hi - lo..at + len).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);+ } else {+ out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);+ }+ } else {+ out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);+ }+ at += len;+ }+ }+ }+ }+ Ok(())+}+fn conv(c: &crate::net::Conv, a: &[f32], b: Option<&[f32]>, side: usize, dest: &mut [f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {+ let pixels = side * side;+ let k = c.cin * c.k * c.k;+ for start in (0..pixels).step_by(BLOCK) {+ let n = BLOCK.min(pixels - start);+ let x = scratch.get_mut(..k * n).ok_or_else(|| bad("short im2col storage"))?;+ lower(a, b, side, c.k, start, n, x)?;+ let y = product.get_mut(..c.cout * n).ok_or_else(|| bad("short product storage"))?;+ multiply(&c.weight, x, y, c.cout, k, n)?;+ for ((row, &bias), out) in y.chunks_exact(n).zip(&c.bias).zip(dest.chunks_exact_mut(pixels)) {+ let to = out.get_mut(start..start + n).ok_or_else(|| bad("short convolution output"))?;+ for (d, &v) in to.iter_mut().zip(row) {+ *d = activate(v, bias, c.leaky);+ }+ }+ }+ Ok(())+}+fn up(c: &crate::net::ConvT2, a: &[f32], side: usize, dest: &mut [f32], weights: &[f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {+ let pixels = side * side;+ let out_side = 2 * side;+ for start in (0..pixels).step_by(BLOCK) {+ let n = BLOCK.min(pixels - start);+ let x = scratch.get_mut(..c.cin * n).ok_or_else(|| bad("short up input storage"))?;+ for (to, from) in x.chunks_exact_mut(n).zip(a.chunks_exact(pixels)) {+ to.copy_from_slice(from.get(start..start + n).ok_or_else(|| bad("short up input"))?);+ }+ let y = product.get_mut(..c.cout * 4 * n).ok_or_else(|| bad("short up product storage"))?;+ multiply(weights, x, y, c.cout * 4, c.cin, n)?;+ for (co, (&bias, out)) in c.bias.iter().zip(dest.chunks_exact_mut(4 * pixels)).enumerate() {+ for d in 0..4 {+ let row = y.get((4 * co + d) * n..(4 * co + d + 1) * n).ok_or_else(|| bad("short up product"))?;+ for (j, &v) in row.iter().enumerate() {+ let p = start + j;+ let at = (2 * (p / side) + d / 2) * out_side + 2 * (p % side) + d % 2;+ *out.get_mut(at).ok_or_else(|| bad("short up output"))? = activate(v, bias, c.leaky);+ }+ }+ }+ }+ Ok(())+}+fn pool(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {+ let s = side / 2;+ for (input, out) in a.chunks_exact(side * side).zip(dest.chunks_exact_mut(s * s)) {+ for (y, row) in out.chunks_exact_mut(s).enumerate() {+ let lo = input.get(2 * y * side..(2 * y + 1) * side).ok_or_else(|| bad("short pool input"))?;+ let hi = input.get((2 * y + 1) * side..(2 * y + 2) * side).ok_or_else(|| bad("short pool input"))?;+ for ((d, lo), hi) in row.iter_mut().zip(lo.as_chunks::<2>().0).zip(hi.as_chunks::<2>().0) {+ let [a, b] = *lo;+ let [c, e] = *hi;+ *d = a.max(b).max(c).max(e);+ }+ }+ }+ Ok(())+}+fn shuffle(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {+ let pixels = side * side;+ for (input, out) in a.chunks_exact(4 * pixels).zip(dest.chunks_exact_mut(4 * pixels)) {+ for d in 0..4 {+ let plane = input.get(d * pixels..(d + 1) * pixels).ok_or_else(|| bad("short shuffle input"))?;+ for (y, row) in plane.chunks_exact(side).enumerate() {+ let at = (2 * y + d / 2) * 2 * side;+ let to = out.get_mut(at..at + 2 * side).ok_or_else(|| bad("short shuffle output"))?;+ for (to, &v) in to.as_chunks_mut::<2>().0.iter_mut().zip(row) {+ if let Some(t) = to.get_mut(d % 2) {+ *t = v;+ }+ }+ }+ }+ }+ Ok(())+}++#[cfg(test)]+mod tests;diff --git a/crates/denoise/src/cpu/tests.rs b/crates/denoise/src/cpu/tests.rsnew file mode 100644index 000000000..16a496b53--- /dev/null+++ b/crates/denoise/src/cpu/tests.rs@@ -0,0 +1,151 @@+use super::*;+use crate::{+ net::{Conv, ConvT2},+ reference, synthetic,+};++fn random(n: usize, seed: u32) -> Vec<f32> {+ let mut s = seed | 1;+ (0..n)+ .map(|_| {+ s ^= s << 13;+ s ^= s >> 17;+ s ^= s << 5;+ (s >> 8) as f32 / (1u32 << 23) as f32 - 1.0+ })+ .collect()+}+fn net(tile: usize, ch: u64, depth: usize, seed: u64) -> Net {+ let p = std::env::temp_dir().join(format!("lc-purecpu-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id()));+ std::fs::write(&p, synthetic::unet_onnx(tile as u64, ch, depth, seed)).unwrap();+ let n = crate::onnx::read(&p).unwrap();+ std::fs::remove_file(p).unwrap();+ n+}+fn agree(a: &[f32], b: &[f32]) {+ assert_eq!(a.len(), b.len());+ assert!(a.iter().chain(b).all(|v| v.is_finite()));+ let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs()));+ let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;+ assert!(error <= 1e-3, "relative maximum error {error:e}");+}+#[test]+fn all_operators_match_scalar_on_random_nets_and_odd_and_tiny_shapes() {+ for (t, ch, d) in [(1, 3, 0), (3, 5, 0), (5, 7, 0), (6, 3, 1), (20, 5, 2), (36, 9, 2), (64, 8, 3)] {+ for seed in [1, 73] {+ let n = net(t, ch, d, seed);+ let r = NetRunner::new(&n, t).unwrap();+ let x = random(4 * t * t, seed as u32);+ let want = reference::run(&n, t, &x).unwrap();+ agree(&r.run(&x).unwrap(), &want);+ assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());+ }+ }+}++#[test]+fn malformed_values_fail_before_execution_and_workspace_sizes_are_bounded() {+ let mut n = net(4, 3, 1, 11);+ for t in [0, 1, 3, usize::MAX, 4096] {+ assert!(NetRunner::new(&n, t).is_err());+ }+ for ch in [0, 3, usize::MAX] {+ n.in_channels = ch;+ assert!(NetRunner::new(&n, 4).is_err());+ }+ let n = Net::new(+ 4,+ vec![+ Op::Conv(Conv { src: 0, src2: None, cin: 4, cout: 4096, k: 1, weight: vec![0.1; 4 * 4096], bias: vec![0.0; 4096], leaky: None }),+ Op::Conv(Conv { src: 1, src2: None, cin: 4096, cout: 12, k: 1, weight: vec![0.1; 4096 * 12], bias: vec![0.0; 12], leaky: None }),+ Op::DepthToSpace2 { src: 2 },+ ],+ )+ .unwrap();+ assert!(NetRunner::new(&n, 2048).is_err());+ let ops = (0..100).map(|i| Op::MaxPool2 { src: i }).collect();+ let n = Net::new(4, ops).unwrap();+ assert!(NetRunner::new(&n, 16).is_err());+ let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![0.25; 48], bias: vec![0.0; 12], leaky: None };+ for mode in 0..6 {+ let mut bad = c.clone();+ match mode {+ 0 => bad.src = usize::MAX,+ 1 => bad.weight.pop().map(|_| ()).unwrap(),+ 2 => bad.bias[0] = f32::NAN,+ 3 => bad.leaky = Some(f32::INFINITY),+ 4 => bad.cout = usize::MAX,+ _ => bad.k = usize::MAX,+ }+ assert!(Net::new(4, vec![Op::Conv(bad), Op::DepthToSpace2 { src: 1 }]).is_err());+ }+}+#[test]+fn bad_tiles_and_arithmetic_overflow_are_errors_and_do_not_poison_reuse() {+ let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![f32::MAX; 48], bias: vec![0.0; 12], leaky: None };+ let n = Net::new(4, vec![Op::Conv(c), Op::DepthToSpace2 { src: 1 }]).unwrap();+ let r = NetRunner::new(&n, 1).unwrap();+ assert!(r.run(&[]).is_err());+ assert!(r.run(&[f32::NAN; 4]).is_err());+ assert!(r.run(&[f32::INFINITY; 4]).is_err());+ assert_eq!(r.idle.lock().unwrap().len(), 0, "invalid input never allocates a workspace");+ assert!(r.run(&[2.0; 4]).is_err());+ assert_eq!(r.run(&[0.0; 4]).unwrap(), vec![0.0; 12]);+ // Matrix shape checks are the boundary in front of the dependency's asserting constructors.+ assert!(multiply(&[], &[], &mut [], usize::MAX, 2, 2).is_err());+}+#[test]+fn shared_runner_serves_concurrent_tiles_without_cross_contamination() {+ let n = net(12, 5, 1, 97);+ let r = NetRunner::new(&n, 12).unwrap();+ let inputs: Vec<_> = (0..8).map(|s| random(4 * 12 * 12, s)).collect();+ let answers: Vec<_> = inputs.iter().map(|x| reference::run(&n, 12, x).unwrap()).collect();+ std::thread::scope(|scope| {+ let mut handles = Vec::new();+ for (x, want) in inputs.iter().zip(&answers) {+ let r = &r;+ handles.push(scope.spawn(move || {+ for _ in 0..3 {+ agree(&r.run(x).unwrap(), want);+ }+ }));+ }+ for h in handles {+ h.join().unwrap();+ }+ });+}+#[test]+fn join_may_read_the_same_tensor_twice_and_leaky_up_convolution_is_supported() {+ let c = ConvT2 { src: 0, cin: 4, cout: 3, weight: random(48, 2), bias: vec![0.1, -0.1, 0.0], leaky: Some(0.13) };+ let n = Net::new(+ 4,+ vec![+ Op::ConvT2(c),+ Op::Conv(Conv { src: 1, src2: Some(1), cin: 6, cout: 3, k: 3, weight: random(162, 3), bias: vec![0.0; 3], leaky: Some(0.0) }),+ ],+ )+ .unwrap();+ let x = random(4 * 3 * 3, 7);+ agree(&NetRunner::new(&n, 3).unwrap().run(&x).unwrap(), &reference::run(&n, 3, &x).unwrap());+}++#[test]+fn reused_im2col_clears_all_edge_padding_between_images() {+ for t in [1, 3, 5, 36] {+ let n = net(t, 5, 0, 121);+ let r = NetRunner::new(&n, t).unwrap();+ for x in [vec![1.0; 4 * t * t], vec![0.0; 4 * t * t], random(4 * t * t, 9)] {+ agree(&r.run(&x).unwrap(), &reference::run(&n, t, &x).unwrap());+ }+ }+}+#[test]+#[ignore = "needs LC_DENOISE_MODEL; one tile with LIGHTCRAFT_PROFILE=1"]+fn profile_one_real_tile() {+ let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL");+ let m = crate::known::find("rawnind-bayer").unwrap().manifest;+ let r = crate::runtime::CpuRunner::load(std::path::Path::new(&model), &m).unwrap();+ println!("workspace {:.1} MB", r.bytes_per_tile() as f64 / 1e6);+ r.run(&crate::runtime::check_tile(512)).unwrap();+}diff --git a/crates/denoise/src/hash.rs b/crates/denoise/src/hash.rsnew file mode 100644index 000000000..50f83be39--- /dev/null+++ b/crates/denoise/src/hash.rs@@ -0,0 +1,46 @@+//! SHA-256 of a model file, streamed (a model can be hundreds of megabytes).++use std::io::{self, Read};+use std::path::Path;++use sha2::{Digest, Sha256};++/// Lowercase hex SHA-256 of everything `r` yields.+pub fn sha256_reader<R: Read>(mut r: R) -> io::Result<String> {+ let mut h = Sha256::new();+ let mut buf = vec![0u8; 1 << 20];+ loop {+ let n = r.read(&mut buf)?;+ if n == 0 {+ break;+ }+ h.update(buf.get(..n).unwrap_or(&[]));+ }+ Ok(hex(&h.finalize()))+}++/// Lowercase hex SHA-256 of a file.+pub fn sha256_file(path: &Path) -> io::Result<String> {+ sha256_reader(io::BufReader::new(std::fs::File::open(path)?))+}++/// Lowercase hex SHA-256 of `bytes`.+pub fn sha256_hex(bytes: &[u8]) -> String {+ hex(&Sha256::digest(bytes))+}++fn hex(bytes: &[u8]) -> String {+ bytes.iter().map(|b| format!("{b:02x}")).collect()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn known_digests() {+ assert_eq!(sha256_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");+ assert_eq!(sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");+ assert_eq!(sha256_reader(&b"abc"[..]).unwrap(), sha256_hex(b"abc"));+ }+}diff --git a/crates/denoise/src/known.rs b/crates/denoise/src/known.rsnew file mode 100644index 000000000..9e0e7ea1e--- /dev/null+++ b/crates/denoise/src/known.rs@@ -0,0 +1,112 @@+//! The denoise models LightCraft knows about: what each one is, who may use it, and where to fetch it.+//!+//! None is part of LightCraft. A model with an address here can be downloaded when the user asks and has accepted+//! its terms; anything else is brought by the user as an `.onnx` file with a `denoise-model.json` beside it.++use crate::licence::{Commercial, Licence};++use crate::manifest::{DenoiserManifest, Domain, Gain};++/// A known model, and how to get it.+pub struct Known {+ pub manifest: DenoiserManifest,+ /// What to fetch (an archive or the `.onnx` itself); `None` when the user must bring the file.+ pub download: Option<Download>,+ /// The `.onnx` inside the downloaded archive (`None`: the download is the `.onnx`).+ pub entry: Option<&'static str>,+}++/// RawNIND's UtNet2 for Bayer sensors, as published for darktable (`rawdenoise-nind.dtmodel`, a zip with a Bayer and a+/// linear model). Its weights are GPL-3.0; the training pictures are CC BY 4.0 and CC0.+fn rawnind_bayer() -> Known {+ let manifest = DenoiserManifest {+ id: "rawnind-bayer".into(),+ name: "RawNIND UtNet2 (Bayer)".into(),+ version: "1.0".into(),+ licence: Licence {+ name: "GPL-3.0".into(),+ commercial: Commercial::Yes,+ url: Some("https://www.gnu.org/licenses/gpl-3.0.html".into()),+ notice: "The weights are published under the GNU General Public License v3. LightCraft does not include them: they \+ are downloaded to your computer at your request and used by it only. If you pass the model file on, the \+ GPL applies to that copy."+ .into(),+ },+ source: Some("https://github.com/darktable-org/darktable-ai/tree/master/models/rawdenoise-nind".into()),+ sha256: Some("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901".into()),+ size_bytes: Some(31_056_425),+ provenance: "Trained on RawNIND, real photographs of the same scenes taken noisy and clean (images CC BY 4.0 and CC0, \+ published on Wikimedia Commons and by the University of Louvain). Paper: arXiv 2501.08924."+ .into(),+ domain: Domain::BayerToRgb,+ tile: 512,+ overlap: 64,+ gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },+ };+ let download = Download {+ id: manifest.id.clone(),+ url: "https://github.com/darktable-org/darktable-ai/releases/download/release-5.6.0/rawdenoise-nind.dtmodel".into(),+ file_name: "rawdenoise-nind.dtmodel".into(),+ size_bytes: 57_700_134,+ sha256: "d71b5f1e727c85a359e6f74dca9e2016c9d8fc3e2f7ac3e9b347d80ceca969af".into(),+ };+ Known { manifest, download: Some(download), entry: Some("model_bayer.onnx") }+}++/// Every model LightCraft knows.+pub fn all() -> Vec<Known> {+ vec![rawnind_bayer()]+}++/// The known model with this id.+pub fn find(id: &str) -> Option<Known> {+ all().into_iter().find(|k| k.manifest.id == id)+}++/// The known model whose `.onnx` has this SHA-256.+pub fn by_sha256(sha256: &str) -> Option<DenoiserManifest> {+ all().into_iter().map(|k| k.manifest).find(|m| m.sha256.as_deref() == Some(sha256))+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::manifest::validate;++ #[test]+ fn known_models_are_valid_unique_and_downloaded_over_https() {+ let all = all();+ assert!(!all.is_empty());+ for (i, k) in all.iter().enumerate() {+ validate(&k.manifest).unwrap();+ assert!(all.iter().skip(i + 1).all(|o| o.manifest.id != k.manifest.id), "unique ids");+ assert!(!k.manifest.licence.name.is_empty() && !k.manifest.licence.notice.is_empty(), "terms are shown before use");+ if let Some(d) = &k.download {+ assert_eq!(d.id, k.manifest.id);+ assert!(d.url.starts_with("https://") && d.sha256.len() == 64 && d.size_bytes > 0);+ // a name that cannot leave the staging folder+ assert!(!d.file_name.contains(['/', '\\']) && !d.file_name.starts_with('.'));+ }+ // a model that may not be used commercially is never offered for download+ if k.manifest.licence.commercial == Commercial::No {+ assert!(k.download.is_none(), "{}", k.manifest.id);+ }+ }+ assert!(find("rawnind-bayer").is_some() && find("nothing").is_none());+ assert_eq!(by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id), Some("rawnind-bayer".into()));+ }+}++#[derive(Clone, Debug, PartialEq)]+pub struct Download {+ pub id: String,+ pub url: String,+ /// The name the file is kept under while it waits for the user to accept its terms.+ pub file_name: String,+ pub size_bytes: u64,+ pub sha256: String,+}++pub fn host(url: &str) -> &str {+ url.strip_prefix("https://").and_then(|r| r.split('/').next()).unwrap_or("")+}diff --git a/crates/denoise/src/lib.rs b/crates/denoise/src/lib.rsnew file mode 100644index 000000000..c7c76b585--- /dev/null+++ b/crates/denoise/src/lib.rs@@ -0,0 +1,42 @@+//! AI denoise for LightCraft, as a step between the sensor and the demosaic that the rest of the develop+//! pipeline never sees: the raw file stays untouched, the denoised picture is cached data that can always be+//! made again, and the amount is an ordinary develop setting.+//!+//! - [`manifest`]: what a denoise model takes and gives, and who may use it (untrusted JSON, validated).+//! - [`known`] and [`archive`]: the models LightCraft can fetch, and taking the model file out of the `.zip` it comes in.+//! - [`bayer`]: Bayer layouts and the packing of a mosaic into the four planes a model takes.+//! - [`tiles`] and [`run`]: cutting a picture into overlapping tiles for a fixed-size model, bringing each tile's+//! output scale back to the input's, keeping clipped highlights as they were, and blending the tiles together.+//! The model itself is a [`run::TileRunner`], so all of this is tested without one.+//! - [`product`]: the cache file the denoised picture is kept in (half floats, compressed in strips) and the+//! blend with the plain demosaic that the Amount slider controls.+//! - [`cpu`] and [`runtime`]: a single-thread pure-Rust CPU runner and its installation self-test.+//! - [`net`] and [`onnx`]: checked plain network data and bounded ONNX decoding, shared by CPU and GPU.+//!+//! Everything here treats model files, manifests and cache files as hostile input: sizes are capped, numbers must+//! be finite, and a malformed file is an error, never a panic.+#![forbid(unsafe_code)]+#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]++pub mod archive;+pub mod bayer;+pub mod cpu;+pub mod known;+pub mod manifest;+pub mod net;+pub mod onnx;+mod onnx_proto;+pub mod product;+pub mod reference;+pub mod run;+pub mod runtime;+#[doc(hidden)]+pub mod synthetic;+pub mod tiles;++pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError};+pub use run::{Control, Error, Params, TileRunner, denoise_bayer};++pub mod hash;+pub mod licence;+pub mod synthetic_proto;diff --git a/crates/denoise/src/licence.rs b/crates/denoise/src/licence.rsnew file mode 100644index 000000000..1d89146b1--- /dev/null+++ b/crates/denoise/src/licence.rs@@ -0,0 +1,34 @@+use serde::{Deserialize, Serialize};++#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub enum Commercial {+ Yes,+ /// Research or personal use only. Never bundled or redistributed by LightCraft.+ No,+ /// The licence or the training data is unclear: the user is told so before installing.+ Unknown,+}++#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(default, rename_all = "camelCase")]+pub struct Licence {+ /// SPDX id or a short name ("MIT", "Apache-2.0", "InsightFace non-commercial research").+ pub name: String,+ pub commercial: Commercial,+ pub url: Option<String>,+ /// Plain-language terms shown before the model is enabled.+ pub notice: String,+}++impl Default for Licence {+ fn default() -> Self {+ Self { name: String::new(), commercial: Commercial::Unknown, url: None, notice: String::new() }+ }+}++pub fn valid_id(id: &str) -> bool {+ let mut bytes = id.bytes();+ let first_ok = bytes.next().is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit());+ first_ok && id.len() <= 64 && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-'))+}diff --git a/crates/denoise/src/manifest.rs b/crates/denoise/src/manifest.rsnew file mode 100644index 000000000..9fbc9f49f--- /dev/null+++ b/crates/denoise/src/manifest.rs@@ -0,0 +1,217 @@+//! A denoise model's manifest: what the model takes and gives, how big its tiles are and who may use it, as+//! plain data.+//!+//! It comes from a `denoise-model.json` next to the `.onnx` file or from the user's `catalog.json`. Both are+//! untrusted: [`validate`] must pass before a manifest is used, and [`parse`] applies it after reading JSON of+//! bounded size. The licence, id and address rules are the face models' (`crate::licence`).++use crate::licence::{Licence, valid_id};+use serde::{Deserialize, Serialize};++/// Largest `denoise-model.json` we read.+pub const MAX_MANIFEST_BYTES: usize = 64 * 1024;+/// Largest model file we accept (bytes).+pub const MAX_MODEL_BYTES: u64 = 2 * 1024 * 1024 * 1024;++/// What the model takes and what it gives.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub enum Domain {+ /// The sensor's Bayer mosaic (black subtracted, white at 1, not white balanced) packed as four planes+ /// `[R, G1, G2, B]` of its 2 × 2 cells, forced to RGGB; gives camera RGB at the mosaic's resolution, so+ /// the picture is denoised and demosaiced in one step and must not be demosaiced again.+ BayerToRgb,+}++/// How the model's output scale is brought back to the input's.+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]+#[serde(tag = "kind", rename_all = "camelCase")]+pub enum Gain {+ /// The output already has the input's scale.+ None,+ /// The output has a scale of its own, `nominal` times the input's (a million, for one model) and a little+ /// different from tile to tile: each tile is scaled so that its channel means equal the input's. A tile too+ /// dark to measure, or whose measured scale is further than `maxDeviation` (a fraction, 0.05 = 5 %) from+ /// `nominal`, is scaled as if it were within that limit.+ MatchMean { nominal: f32, max_deviation: f32 },+}++#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct DenoiserManifest {+ /// Lowercase letters, digits, `.` `_` `-`.+ pub id: String,+ pub name: String,+ pub version: String,+ #[serde(default)]+ pub licence: Licence,+ /// Where the weights come from (an `https://` page or file), for the user's information.+ #[serde(default)]+ pub source: Option<String>,+ /// Lowercase hex SHA-256 of the `.onnx` file, when known.+ #[serde(default)]+ pub sha256: Option<String>,+ #[serde(default)]+ pub size_bytes: Option<u64>,+ /// What the model was trained on, in plain words; says "undisclosed" when it is.+ #[serde(default)]+ pub provenance: String,+ pub domain: Domain,+ /// Side of the square the model runs on, in packed cells (the file's fixed input size).+ pub tile: u32,+ /// Cells neighbouring tiles share; the picture is blended across them.+ pub overlap: u32,+ pub gain: Gain,+}++#[derive(Debug, PartialEq, thiserror::Error)]+pub enum ManifestError {+ #[error("`{0}` is not valid: {1}")]+ Field(&'static str, String),+ #[error("not a denoise-model manifest: {0}")]+ Parse(String),+ #[error("a manifest may be at most {MAX_MANIFEST_BYTES} bytes")]+ TooLarge,+}++fn bad<T>(field: &'static str, why: impl Into<String>) -> Result<T, ManifestError> {+ Err(ManifestError::Field(field, why.into()))+}++fn text(field: &'static str, s: &str, min: usize, max: usize) -> Result<(), ManifestError> {+ let n = s.chars().count();+ if n < min || n > max {+ return bad(field, format!("must be {min} to {max} characters"));+ }+ if s.chars().any(|c| c.is_control() && c != '\n') {+ return bad(field, "must not contain control characters");+ }+ Ok(())+}++fn url(field: &'static str, u: &Option<String>) -> Result<(), ManifestError> {+ match u {+ None => Ok(()),+ Some(u) if u.len() <= 500 && u.starts_with("https://") && !u.chars().any(|c| c.is_control() || c.is_whitespace()) => Ok(()),+ Some(_) => bad(field, "must be an https:// address of at most 500 characters"),+ }+}++/// Check a manifest from outside before trusting it.+pub fn validate(m: &DenoiserManifest) -> Result<(), ManifestError> {+ if !valid_id(&m.id) {+ return bad("id", "1 to 64 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit");+ }+ text("name", &m.name, 1, 80)?;+ text("version", &m.version, 1, 40)?;+ text("provenance", &m.provenance, 0, 1000)?;+ text("licence.name", &m.licence.name, 0, 80)?;+ text("licence.notice", &m.licence.notice, 0, 2000)?;+ url("licence.url", &m.licence.url)?;+ url("source", &m.source)?;+ if let Some(h) = &m.sha256+ && !(h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))+ {+ return bad("sha256", "must be 64 lowercase hex digits");+ }+ if let Some(n) = m.size_bytes+ && (n == 0 || n > MAX_MODEL_BYTES)+ {+ return bad("sizeBytes", format!("must be between 1 and {MAX_MODEL_BYTES}"));+ }+ // a U-Net with four poolings wants a multiple of 16; nothing smaller than 64 or larger than 2048 is a tile+ if !(64..=2048).contains(&m.tile) || !m.tile.is_multiple_of(16) {+ return bad("tile", "must be a multiple of 16 from 64 to 2048");+ }+ // tiles must overlap by less than half, or one tile's blend ramps would run into its other neighbour's+ if !m.overlap.is_multiple_of(2) || m.overlap < 16 || m.overlap.saturating_mul(2) >= m.tile {+ return bad("overlap", "must be even, at least 16 and less than half the tile");+ }+ if let Gain::MatchMean { nominal, max_deviation } = m.gain {+ if !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal)) {+ return bad("gain.nominal", "must be between 0.000001 and 1000000000000");+ }+ if !(max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) {+ return bad("gain.maxDeviation", "must be between 0 and 0.5");+ }+ }+ Ok(())+}++/// Read and validate a `denoise-model.json`.+pub fn parse(bytes: &[u8]) -> Result<DenoiserManifest, ManifestError> {+ if bytes.len() > MAX_MANIFEST_BYTES {+ return Err(ManifestError::TooLarge);+ }+ let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes);+ let m: DenoiserManifest = serde_json::from_slice(bytes).map_err(|e| ManifestError::Parse(e.to_string()))?;+ validate(&m)?;+ Ok(m)+}++#[cfg(test)]+mod tests {+ use super::*;++ pub(crate) fn sample() -> DenoiserManifest {+ DenoiserManifest {+ id: "nind-bayer".into(),+ name: "Bayer denoiser".into(),+ version: "1".into(),+ licence: Licence::default(),+ source: Some("https://example.org/model.onnx".into()),+ sha256: None,+ size_bytes: Some(31_000_000),+ provenance: "trained on real raw noise pairs".into(),+ domain: Domain::BayerToRgb,+ tile: 512,+ overlap: 64,+ gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },+ }+ }++ #[test]+ fn a_good_manifest_round_trips() {+ let m = sample();+ let json = serde_json::to_vec(&m).unwrap();+ assert_eq!(parse(&json).unwrap(), m);+ }++ #[test]+ fn bad_fields_are_named() {+ let cases: Vec<(&str, Box<dyn Fn(&mut DenoiserManifest)>)> = vec![+ ("id", Box::new(|m| m.id = "../evil".into())),+ ("id", Box::new(|m| m.id = String::new())),+ ("name", Box::new(|m| m.name = "x\u{7}".into())),+ ("source", Box::new(|m| m.source = Some("http://insecure".into()))),+ ("sha256", Box::new(|m| m.sha256 = Some("XYZ".into()))),+ ("sizeBytes", Box::new(|m| m.size_bytes = Some(0))),+ ("tile", Box::new(|m| m.tile = 500)),+ ("tile", Box::new(|m| m.tile = 4096)),+ ("overlap", Box::new(|m| m.overlap = 300)),+ ("overlap", Box::new(|m| m.overlap = 63)),+ ("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN })),+ ("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: 2.0 })),+ ("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: 0.0, max_deviation: 0.1 })),+ ("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 })),+ ];+ for (field, edit) in cases {+ let mut m = sample();+ edit(&mut m);+ match validate(&m) {+ Err(ManifestError::Field(f, _)) => assert_eq!(f, field),+ other => panic!("{field}: {other:?}"),+ }+ }+ }++ #[test]+ fn parse_is_bounded_and_tolerant() {+ assert_eq!(parse(&vec![b' '; MAX_MANIFEST_BYTES + 1]), Err(ManifestError::TooLarge));+ assert!(matches!(parse(b"{ nope"), Err(ManifestError::Parse(_))));+ assert!(matches!(parse(b""), Err(ManifestError::Parse(_))));+ let mut with_bom = vec![0xEF, 0xBB, 0xBF];+ with_bom.extend(serde_json::to_vec(&sample()).unwrap());+ assert!(parse(&with_bom).is_ok());+ }+}diff --git a/crates/denoise/src/net.rs b/crates/denoise/src/net.rsnew file mode 100644index 000000000..422418336--- /dev/null+++ b/crates/denoise/src/net.rs@@ -0,0 +1,345 @@+//! A denoise network as plain data: the handful of layer kinds a U-Net denoiser is made of, with their weights.+//!+//! [`onnx`](crate::onnx) reads the file into this description, shared by the pure-Rust CPU and GPU runners.+//! Unsupported operators, groups, dilation or activations are reported as errors when loading the model.+//!+//! The description is checked ([`Net::new`]) because it comes from a stranger's file: channel counts must chain,+//! weights must have the sizes the shapes need and be finite, and the depth must stay small.++/// Most layers a network may have.+pub const MAX_OPS: usize = 512;+/// Most channels a layer may have.+pub const MAX_CHANNELS: usize = 4096;+/// Most weights (numbers) a network may have in all.+pub const MAX_WEIGHTS: usize = 128 * 1024 * 1024;++#[derive(Debug, PartialEq, thiserror::Error)]+pub enum NetError {+ /// The model uses something this denoise network description cannot express.+ #[error("not supported by the denoise runners: {0}")]+ Unsupported(String),+ /// The model is not a well-formed network.+ #[error("not a valid network: {0}")]+ Invalid(String),+}++fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {+ Err(NetError::Invalid(why.into()))+}++/// A tensor is named by its number: 0 is the network's input, and layer `i` makes tensor `i + 1`.+pub type TensorId = usize;++/// A convolution with stride 1 and "same" padding (`k` is 1 or 3), over one tensor or two joined along the channels+/// (`src` first), then a bias and optionally a leaky ReLU.+#[derive(Clone, Debug, PartialEq)]+pub struct Conv {+ pub src: TensorId,+ /// A second source, whose channels follow `src`'s (a skip connection's concatenation).+ pub src2: Option<TensorId>,+ pub cin: usize,+ pub cout: usize,+ pub k: usize,+ /// `[cout][cin][k][k]`, as ONNX has it.+ pub weight: Vec<f32>,+ pub bias: Vec<f32>,+ /// `Some(alpha)`: a leaky ReLU with this slope follows.+ pub leaky: Option<f32>,+}++/// A transposed convolution with a 2 × 2 kernel and stride 2: every input pixel becomes a 2 × 2 block.+#[derive(Clone, Debug, PartialEq)]+pub struct ConvT2 {+ pub src: TensorId,+ pub cin: usize,+ pub cout: usize,+ /// `[cin][cout][2][2]`, as ONNX has it.+ pub weight: Vec<f32>,+ pub bias: Vec<f32>,+ pub leaky: Option<f32>,+}++#[derive(Clone, Debug, PartialEq)]+pub enum Op {+ Conv(Conv),+ ConvT2(ConvT2),+ /// 2 × 2 maximum, stride 2.+ MaxPool2 {+ src: TensorId,+ },+ /// Depth-to-space with block size 2 in "CRD" order: output channel `c` at `(2y + dy, 2x + dx)` is input channel+ /// `4c + 2dy + dx` at `(y, x)`.+ DepthToSpace2 {+ src: TensorId,+ },+}++/// What a tensor looks like: its channels, and its resolution as a power of two below the input's (a negative+/// level is above it).+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct Shape {+ pub channels: usize,+ pub level: i32,+}++#[derive(Clone, Debug, PartialEq)]+pub struct Net {+ pub in_channels: usize,+ ops: Vec<Op>,+ shapes: Vec<Shape>,+}++impl Net {+ /// Check `ops` (layer `i` makes tensor `i + 1`) over an input of `in_channels` and describe every tensor.+ pub fn new(in_channels: usize, ops: Vec<Op>) -> Result<Net, NetError> {+ if in_channels == 0 || in_channels > MAX_CHANNELS {+ return invalid("the input has no channels or too many");+ }+ if ops.is_empty() || ops.len() > MAX_OPS {+ return invalid("the network has no layers or too many");+ }+ let mut shapes = vec![Shape { channels: in_channels, level: 0 }];+ let mut weights = 0usize;+ for (i, op) in ops.iter().enumerate() {+ let shape_of = |t: TensorId| {+ shapes+ .get(t)+ .copied()+ .filter(|_| t <= i)+ .ok_or_else(|| NetError::Invalid(format!("layer {i} reads tensor {t}, which does not exist yet")))+ };+ let out = match op {+ Op::Conv(c) => {+ let a = shape_of(c.src)?;+ let (cin, level) = match c.src2 {+ Some(s2) => {+ let b = shape_of(s2)?;+ if b.level != a.level {+ return invalid(format!("layer {i} joins tensors of different resolutions"));+ }+ (a.channels + b.channels, a.level)+ }+ None => (a.channels, a.level),+ };+ if c.cin != cin || c.cout == 0 || c.cout > MAX_CHANNELS || !(c.k == 1 || c.k == 3) {+ return invalid(format!(+ "layer {i}: a convolution of {} to {} channels with a {}-wide kernel does not fit",+ c.cin, c.cout, c.k+ ));+ }+ check_weights(+ i,+ &c.weight,+ c.cout.checked_mul(cin).and_then(|n| n.checked_mul(c.k * c.k)),+ &c.bias,+ c.cout,+ c.leaky,+ &mut weights,+ )?;+ Shape { channels: c.cout, level }+ }+ Op::ConvT2(c) => {+ let a = shape_of(c.src)?;+ if c.cin != a.channels || c.cout == 0 || c.cout > MAX_CHANNELS {+ return invalid(format!("layer {i}: a transposed convolution of {} to {} channels does not fit", c.cin, c.cout));+ }+ check_weights(i, &c.weight, c.cin.checked_mul(c.cout).and_then(|n| n.checked_mul(4)), &c.bias, c.cout, c.leaky, &mut weights)?;+ Shape { channels: c.cout, level: a.level - 1 }+ }+ Op::MaxPool2 { src } => {+ let a = shape_of(*src)?;+ Shape { channels: a.channels, level: a.level + 1 }+ }+ Op::DepthToSpace2 { src } => {+ let a = shape_of(*src)?;+ if !a.channels.is_multiple_of(4) {+ return invalid(format!("layer {i}: depth-to-space needs a multiple of 4 channels"));+ }+ Shape { channels: a.channels / 4, level: a.level - 1 }+ }+ };+ shapes.push(out);+ }+ Ok(Net { in_channels, ops, shapes })+ }++ /// The layers: layer `i` makes tensor `i + 1`.+ pub fn ops(&self) -> &[Op] {+ &self.ops+ }++ /// The shape of tensor `t` (0 is the input).+ pub fn shape(&self, t: TensorId) -> Option<Shape> {+ self.shapes.get(t).copied()+ }++ /// The tensor the network gives (the last layer's).+ pub fn output(&self) -> TensorId {+ self.ops.len()+ }++ pub fn out_channels(&self) -> usize {+ self.shapes.last().map_or(0, |s| s.channels)+ }++ /// The deepest resolution level: a tile must be a multiple of `2^this` cells wide.+ pub fn depth(&self) -> u32 {+ self.shapes.iter().map(|s| s.level.max(0) as u32).max().unwrap_or(0)+ }++ /// Can the network run on a `tile × tile` input and give `[out_channels, 2·tile, 2·tile]`? (Pooling must divide+ /// the tile, and the output must be one level above the input.)+ pub fn fits_tile(&self, tile: usize) -> bool {+ let Some(multiple) = 1usize.checked_shl(self.depth()) else { return false };+ tile > 0 && tile.is_multiple_of(multiple) && self.shapes.last().is_some_and(|s| s.level == -1)+ }++ /// Multiply-adds for one tile, for estimating time.+ pub fn macs(&self, tile: usize) -> u64 {+ let pixels = |level: i32| -> u64 {+ let side = if level >= 0 {+ tile.checked_shr(level as u32).unwrap_or(0)+ } else {+ tile.saturating_mul(1usize.checked_shl(level.unsigned_abs()).unwrap_or(usize::MAX))+ };+ (side as u64).saturating_mul(side as u64)+ };+ self.ops+ .iter()+ .enumerate()+ .map(|(i, op)| {+ let out = self.shapes.get(i + 1).map_or(0, |s| pixels(s.level));+ match op {+ Op::Conv(c) => out.saturating_mul((c.cin * c.cout * c.k * c.k) as u64),+ // every input pixel makes a 2 × 2 block: as many multiply-adds as the 2 × 2-wide kernel has weights+ Op::ConvT2(c) => pixels(self.shapes.get(c.src).map_or(0, |s| s.level)).saturating_mul((c.cin * c.cout * 4) as u64),+ _ => 0,+ }+ })+ .fold(0u64, u64::saturating_add)+ }+}++fn check_weights(+ layer: usize,+ weight: &[f32],+ want: Option<usize>,+ bias: &[f32],+ cout: usize,+ leaky: Option<f32>,+ total: &mut usize,+) -> Result<(), NetError> {+ if want != Some(weight.len()) || bias.len() != cout {+ return invalid(format!("layer {layer}: the weights are not the size its shape needs"));+ }+ *total = total.saturating_add(weight.len()).saturating_add(bias.len());+ if *total > MAX_WEIGHTS {+ return invalid("the network has too many weights");+ }+ if weight.iter().chain(bias).any(|v| !v.is_finite()) || leaky.is_some_and(|a| !a.is_finite()) {+ return invalid(format!("layer {layer}: a weight is not a finite number"));+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ fn conv(src: TensorId, src2: Option<TensorId>, cin: usize, cout: usize, k: usize) -> Op {+ Op::Conv(Conv { src, src2, cin, cout, k, weight: vec![0.1; cout * cin * k * k], bias: vec![0.0; cout], leaky: Some(0.2) })+ }++ fn convt(src: TensorId, cin: usize, cout: usize) -> Op {+ Op::ConvT2(ConvT2 { src, cin, cout, weight: vec![0.1; cin * cout * 4], bias: vec![0.0; cout], leaky: None })+ }++ /// A two-level network of the shape the denoisers have: 4 → 8 channels, pool, 16 channels, up, join, 12, and the+ /// 1 × 1 convolution to the colours.+ fn unet() -> Vec<Op> {+ vec![+ conv(0, None, 4, 8, 3), // 1: 8 ch, level 0+ Op::MaxPool2 { src: 1 }, // 2: level 1+ conv(2, None, 8, 16, 3), // 3: 16 ch, level 1+ convt(3, 16, 8), // 4: 8 ch, level 0+ conv(4, Some(1), 16, 8, 3), // 5: join [up, skip]+ conv(5, None, 8, 12, 1), // 6: 12 ch+ Op::DepthToSpace2 { src: 6 }, // 7: 3 ch, level -1+ ]+ }++ #[test]+ fn deep_networks_and_huge_cost_queries_never_overflow_or_truncate_depth() {+ let n = Net::new(4, unet()).unwrap();+ assert_eq!(n.macs(usize::MAX), u64::MAX);+ assert_eq!(n.macs(0), 0);+ let mut ops = Vec::new();+ for i in 0..21 {+ ops.push(Op::MaxPool2 { src: i });+ }+ for i in 21..42 {+ ops.push(convt(i, 4, 4));+ }+ ops.push(conv(42, None, 4, 12, 1));+ ops.push(Op::DepthToSpace2 { src: 43 });+ let n = Net::new(4, ops).unwrap();+ assert!(!n.fits_tile(1 << 20));+ assert!(n.fits_tile(1 << 21));+ let _ = n.macs(16);+ }++ #[test]+ fn a_u_net_is_described_with_its_shapes_and_costs() {+ let net = Net::new(4, unet()).unwrap();+ assert_eq!(net.out_channels(), 3);+ assert_eq!(net.shape(3), Some(Shape { channels: 16, level: 1 }));+ assert_eq!(net.shape(7), Some(Shape { channels: 3, level: -1 }));+ assert_eq!(net.depth(), 1);+ assert!(net.fits_tile(64) && !net.fits_tile(63) && !net.fits_tile(0));+ // 64 × 64 cells: layer 1 = 4096·4·8·9, and so on; just check it is the sum of what the layers cost+ let t = 64u64;+ let want = t * t * 4 * 8 * 9 + (t / 2) * (t / 2) * 8 * 16 * 9 + (t / 2) * (t / 2) * 16 * 8 * 4 + t * t * 16 * 8 * 9 + t * t * 8 * 12;+ assert_eq!(net.macs(64), want);+ }++ #[test]+ fn a_network_that_does_not_chain_is_refused() {+ // wrong input channel count for the first convolution+ assert!(Net::new(3, unet()).is_err());+ // a layer reading a tensor that does not exist yet+ let mut ops = unet();+ ops[2] = conv(9, None, 8, 16, 3);+ assert!(Net::new(4, ops).is_err());+ // joining tensors of different resolutions+ let mut ops = unet();+ ops[4] = conv(3, Some(1), 24, 8, 3);+ assert!(Net::new(4, ops).is_err());+ // weights of the wrong size, and not finite+ let mut ops = unet();+ if let Op::Conv(c) = &mut ops[0] {+ c.weight.pop();+ }+ assert!(Net::new(4, ops).is_err());+ let mut ops = unet();+ if let Op::Conv(c) = &mut ops[0] {+ c.bias[0] = f32::NAN;+ }+ assert!(Net::new(4, ops).is_err());+ // a kernel that is neither 1 nor 3 wide, no layers, depth-to-space on a channel count it cannot split+ let mut ops = unet();+ if let Op::Conv(c) = &mut ops[0] {+ c.k = 5;+ }+ assert!(Net::new(4, ops).is_err());+ assert!(Net::new(4, vec![]).is_err());+ assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }]).is_ok());+ assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }, Op::DepthToSpace2 { src: 1 }]).is_err());+ assert!(Net::new(2, vec![Op::DepthToSpace2 { src: 0 }]).is_err());+ }++ #[test]+ fn a_network_that_does_not_end_one_level_up_does_not_fit_a_tile() {+ let net = Net::new(4, unet()[..6].to_vec()).unwrap();+ assert!(!net.fits_tile(64), "no depth-to-space: the output is not at twice the resolution");+ }+}diff --git a/crates/denoise/src/onnx.rs b/crates/denoise/src/onnx.rsnew file mode 100644index 000000000..fc84a36cf--- /dev/null+++ b/crates/denoise/src/onnx.rs@@ -0,0 +1,369 @@+//! Reading an `.onnx` file into a [`Net`] with bounded, pure-Rust protobuf decoding.+//!+//! Only what the denoise U-Nets are made of is understood: `Conv` (3 × 3 "same" or 1 × 1, stride 1, no groups, no+//! dilation) optionally followed by `LeakyRelu`, `ConvTranspose` (2 × 2, stride 2) likewise, `MaxPool` (2 × 2, stride 2),+//! a two-input `Concat` on the channels that feeds a convolution, and a final `DepthToSpace` (2, `CRD`). Any other+//! operator or attribute is [`NetError::Unsupported`], which is reported to the user by both runners.++use std::collections::HashMap;+use std::path::Path;++use crate::onnx_proto::{self, AttributeProto, NodeProto, TensorProto, ValueInfoProto};+use std::io::Read;++use crate::net::{Conv, ConvT2, MAX_WEIGHTS, Net, NetError, Op, TensorId};++fn unsupported<T>(why: impl Into<String>) -> Result<T, NetError> {+ Err(NetError::Unsupported(why.into()))+}++fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {+ Err(NetError::Invalid(why.into()))+}++/// Read the model at `path`.+pub fn read(path: &Path) -> Result<Net, NetError> {+ let file = std::fs::File::open(path).map_err(|e| NetError::Invalid(e.to_string()))?;+ let size = usize::try_from(file.metadata().map_err(|e| NetError::Invalid(e.to_string()))?.len())+ .ok()+ .filter(|&n| n <= onnx_proto::MAX_BYTES)+ .ok_or_else(|| NetError::Invalid("model file too large".into()))?;+ let mut bytes = Vec::new();+ bytes.try_reserve_exact(size).map_err(|_| NetError::Invalid("not enough memory to load the model".into()))?;+ file.take(onnx_proto::MAX_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|e| NetError::Invalid(e.to_string()))?;+ read_bytes(&bytes)+}++fn read_bytes(bytes: &[u8]) -> Result<Net, NetError> {+ let graph = onnx_proto::decode(bytes)?;++ let initialisers: HashMap<&str, &TensorProto> = graph.initializer.iter().map(|t| (t.name.as_str(), t)).collect();+ if initialisers.len() != graph.initializer.len() {+ return invalid("duplicate initializer names");+ }+ let inputs: Vec<_> = graph.input.iter().filter(|i| !initialisers.contains_key(i.name.as_str())).collect();+ let ([input], [output]) = (inputs.as_slice(), graph.output.as_slice()) else { return unsupported("not one input and one output") };+ let in_channels = match channel_dim(input) {+ Some(c) => c,+ None => return unsupported("the input's channel count is not written in the file"),+ };++ // how many nodes read each name (a graph output counts as a reader)+ let mut readers: HashMap<&str, usize> = HashMap::new();+ for n in &graph.node {+ for i in &n.input {+ *readers.entry(i.as_str()).or_default() += 1;+ }+ }+ *readers.entry(output.name.as_str()).or_default() += 1;++ let mut ids: HashMap<String, TensorId> = HashMap::new();+ ids.insert(input.name.clone(), 0);+ // a Concat is not a layer of its own: the convolution that reads it takes both sources+ let mut joins: HashMap<String, (TensorId, TensorId)> = HashMap::new();+ let mut ops: Vec<Op> = Vec::new();+ let mut weights_total = 0usize;++ let nodes = &graph.node;+ let mut i = 0;+ while i < nodes.len() {+ let Some(node) = nodes.get(i) else { break };+ i += 1;+ if node.output.len() != 1 || node.output.first().is_none_or(|n| n.is_empty() || ids.contains_key(n) || joins.contains_key(n)) {+ return invalid("a layer has missing, repeated or multiple outputs");+ }+ let allowed: &[&str] = match node.op_type.as_str() {+ "Conv" => &["kernel_shape", "pads", "strides", "dilations", "group", "auto_pad"],+ "ConvTranspose" => &["kernel_shape", "pads", "strides", "dilations", "group", "output_padding", "auto_pad"],+ "MaxPool" => &["kernel_shape", "pads", "strides", "dilations", "ceil_mode", "storage_order", "auto_pad"],+ "Concat" => &["axis"],+ "DepthToSpace" => &["blocksize", "mode"],+ _ => &[],+ };+ if node.attribute.iter().any(|a| !allowed.contains(&a.name.as_str())) {+ return unsupported("an unknown operator attribute");+ }+ let this = ops.len() + 1;+ let lookup = |name: &str| -> Result<TensorId, NetError> {+ ids.get(name).copied().ok_or_else(|| NetError::Invalid(format!("`{}` reads `{name}`, which nothing makes", node.op_type)))+ };+ // a LeakyRelu straight after this layer, which nothing else reads, is part of it+ let mut fuse_leaky = |out: &str| -> Result<(Option<f32>, String), NetError> {+ if let Some(next) = nodes.get(i)+ && next.op_type == "LeakyRelu"+ && next.input.first().map(String::as_str) == Some(out)+ && readers.get(out).copied() == Some(1)+ {+ if next.input.len() != 1 || next.output.len() != 1 || next.attribute.iter().any(|a| a.name != "alpha") {+ return invalid("malformed LeakyRelu");+ }+ let alpha = attr(next, "alpha").map_or(0.01, |a| a.f);+ i += 1;+ let name = next.output.first().cloned().ok_or_else(|| NetError::Invalid("LeakyRelu without an output".into()))?;+ return Ok((Some(alpha), name));+ }+ Ok((None, out.to_string()))+ };+ match node.op_type.as_str() {+ "Concat" => {+ if attr(node, "axis").map(|a| a.i) != Some(1) {+ return unsupported("Concat on another axis than the channels");+ }+ let [a, b] = node.input.as_slice() else { return unsupported("Concat of other than two tensors") };+ let out = node.output.first().ok_or_else(|| NetError::Invalid("Concat without an output".into()))?;+ if readers.get(out.as_str()).copied() != Some(1) {+ return unsupported("a Concat that is read by more than one layer");+ }+ joins.insert(out.clone(), (lookup(a)?, lookup(b)?));+ }+ "Conv" => {+ let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;+ let (cout, cin, kh, kw) = four(&w.0)?;+ let b = if b.is_empty() { vec![0.0; cout] } else { b };+ if kh != kw || !(kh == 1 || kh == 3) {+ return unsupported(format!("a {kh} × {kw} convolution"));+ }+ if ints_or(node, "strides", &[1, 1]) != [1, 1]+ || ints_or(node, "dilations", &[1, 1]) != [1, 1]+ || attr(node, "group").is_some_and(|a| a.i != 1)+ {+ return unsupported("a convolution with strides, dilation or groups");+ }+ if attr(node, "kernel_shape").is_some_and(|a| a.ints != [kh as i64, kw as i64]) {+ return invalid("kernel shape does not match the weights");+ }+ let pad = (kh as i64 - 1) / 2;+ if ints_or(node, "pads", &[0; 4]) != [pad; 4] || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {+ return unsupported("a convolution that is not padded to keep its size");+ }+ let src_name = node.input.first().ok_or_else(|| NetError::Invalid("Conv without an input".into()))?;+ let (src, src2) = match joins.remove(src_name) {+ Some((a, b)) => (a, Some(b)),+ None => (lookup(src_name)?, None),+ };+ let out = node.output.first().ok_or_else(|| NetError::Invalid("Conv without an output".into()))?;+ let (leaky, out_name) = fuse_leaky(out)?;+ ids.insert(out_name, this);+ ops.push(Op::Conv(Conv { src, src2, cin, cout, k: kh, weight: w.1, bias: b, leaky }));+ }+ "ConvTranspose" => {+ let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;+ let (cin, cout, kh, kw) = four(&w.0)?;+ let b = if b.is_empty() { vec![0.0; cout] } else { b };+ if (kh, kw) != (2, 2)+ || ints_or(node, "strides", &[1, 1]) != [2, 2]+ || ints_or(node, "pads", &[0; 4]) != [0; 4]+ || ints_or(node, "dilations", &[1, 1]) != [1, 1]+ || attr(node, "group").is_some_and(|a| a.i != 1)+ || attr(node, "output_padding").is_some_and(|a| a.ints.iter().any(|&v| v != 0))+ {+ return unsupported("a transposed convolution other than 2 × 2 with stride 2");+ }+ if attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {+ return unsupported("transposed convolution auto padding");+ }+ if attr(node, "kernel_shape").is_some_and(|a| a.ints != [2, 2]) {+ return invalid("transposed kernel shape does not match the weights");+ }+ let src = lookup(node.input.first().map_or("", String::as_str))?;+ let out = node.output.first().ok_or_else(|| NetError::Invalid("ConvTranspose without an output".into()))?;+ let (leaky, out_name) = fuse_leaky(out)?;+ ids.insert(out_name, this);+ ops.push(Op::ConvT2(ConvT2 { src, cin, cout, weight: w.1, bias: b, leaky }));+ }+ "MaxPool" => {+ if ints_or(node, "kernel_shape", &[]) != [2, 2]+ || ints_or(node, "strides", &[1, 1]) != [2, 2]+ || ints_or(node, "pads", &[0; 4]) != [0; 4]+ || ints_or(node, "dilations", &[1, 1]) != [1, 1]+ || attr(node, "ceil_mode").is_some_and(|a| a.i != 0)+ || attr(node, "storage_order").is_some_and(|a| a.i != 0)+ || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET")+ {+ return unsupported("a max-pool other than 2 × 2 with stride 2");+ }+ let src = lookup(node.input.first().map_or("", String::as_str))?;+ let out = node.output.first().ok_or_else(|| NetError::Invalid("MaxPool without an output".into()))?;+ ids.insert(out.clone(), this);+ ops.push(Op::MaxPool2 { src });+ }+ "DepthToSpace" => {+ if attr(node, "blocksize").map(|a| a.i) != Some(2) || attr(node, "mode").map(|a| a.s.as_slice()) != Some(&b"CRD"[..]) {+ return unsupported("depth-to-space other than block size 2 in CRD order");+ }+ let src = lookup(node.input.first().map_or("", String::as_str))?;+ let out = node.output.first().ok_or_else(|| NetError::Invalid("DepthToSpace without an output".into()))?;+ ids.insert(out.clone(), this);+ ops.push(Op::DepthToSpace2 { src });+ }+ other => return unsupported(format!("the `{other}` operator")),+ }+ }+ if !joins.is_empty() {+ return unsupported("a Concat whose result no convolution reads");+ }+ if ids.get(output.name.as_str()) != Some(&ops.len()) {+ return unsupported("the output is not the last layer");+ }+ Net::new(in_channels, ops)+}++fn attr<'a>(n: &'a NodeProto, name: &str) -> Option<&'a AttributeProto> {+ n.attribute.iter().find(|a| a.name == name)+}++/// The integer list attribute `name`, or `default` when the node does not have it.+fn ints_or(n: &NodeProto, name: &str, default: &[i64]) -> Vec<i64> {+ attr(n, name).map_or_else(|| default.to_vec(), |a| a.ints.clone())+}++fn four(dims: &[i64]) -> Result<(usize, usize, usize, usize), NetError> {+ match dims {+ [a, b, c, d] if dims.iter().all(|&v| v > 0 && v <= 1 << 20) => Ok((*a as usize, *b as usize, *c as usize, *d as usize)),+ _ => invalid("a weight tensor that is not four-dimensional"),+ }+}++type Weights = ((Vec<i64>, Vec<f32>), Vec<f32>);++/// The weight and bias initialisers of a convolution node.+fn weights_of(node: &NodeProto, initialisers: &HashMap<&str, &TensorProto>, total: &mut usize) -> Result<Weights, NetError> {+ let mut get = |idx: usize| -> Result<(Vec<i64>, Vec<f32>), NetError> {+ let name = node.input.get(idx).ok_or_else(|| NetError::Invalid(format!("`{}` without its weights", node.op_type)))?;+ let t = initialisers.get(name.as_str()).ok_or_else(|| NetError::Unsupported(format!("weights that are computed (`{name}`)")))?;+ let values = floats(t)?;+ *total = total.saturating_add(values.len());+ if *total > MAX_WEIGHTS {+ return invalid("the network has too many weights");+ }+ Ok((t.dims.clone(), values))+ };+ let w = get(1)?;+ let b = if node.input.len() > 2 {+ let (dims, values) = get(2)?;+ if dims.len() != 1 || values.is_empty() {+ return invalid("a bias tensor must be a nonempty vector");+ }+ values+ } else {+ Vec::new()+ };+ Ok((w, b))+}++fn floats(t: &TensorProto) -> Result<Vec<f32>, NetError> {+ if t.data_type != 1 {+ return unsupported("weights that are not 32-bit floats");+ }+ let want: usize = t+ .dims+ .iter()+ .try_fold(1usize, |n, &d| usize::try_from(d).ok().and_then(|d| n.checked_mul(d)))+ .ok_or_else(|| NetError::Invalid("a tensor with a negative or huge size".into()))?;+ if want > MAX_WEIGHTS {+ return invalid("a tensor with too many numbers");+ }+ if !t.raw_data.is_empty() && (!t.float_data.is_empty() || !t.raw_data.len().is_multiple_of(4)) {+ return invalid("ambiguous or truncated tensor data");+ }+ let actual = if t.raw_data.is_empty() { t.float_data.len() } else { t.raw_data.len() / 4 };+ if actual != want {+ return invalid("tensor data does not match its dimensions");+ }+ let mut values = Vec::new();+ values.try_reserve_exact(want).map_err(|_| NetError::Invalid("not enough memory for weights".into()))?;+ if !t.raw_data.is_empty() {+ values.extend(t.raw_data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)));+ } else {+ values.extend_from_slice(&t.float_data);+ }+ if values.len() != want {+ return invalid(format!("tensor `{}` has {} numbers, not the {want} its shape says", t.name, values.len()));+ }+ Ok(values)+}++/// The channel count in an input's declared shape `[batch, channels, …]`.+fn channel_dim(v: &ValueInfoProto) -> Option<usize> {+ v.channels+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::synthetic::unet_onnx;++ fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {+ let p = std::env::temp_dir().join(format!("lc-denoise-onnx-{name}-{}.onnx", std::process::id()));+ std::fs::write(&p, bytes).unwrap();+ p+ }++ #[test]+ fn a_u_net_is_read_with_its_layers_fused() {+ let (tile, depth) = (32usize, 2usize);+ let p = temp_model("unet", &unet_onnx(tile as u64, 8, depth, 7));+ let net = read(&p).unwrap();+ // per level two convolutions; a pool per level but the last; going up a transposed convolution and two convolutions+ // per level (the join is part of the first); then the 1 × 1 convolution and depth-to-space+ assert_eq!(net.ops().len(), 2 * (depth + 1) + depth + depth * 3 + 2);+ assert!(matches!(net.ops().last(), Some(crate::net::Op::DepthToSpace2 { .. })));+ assert!(net.ops().iter().any(|o| matches!(o, crate::net::Op::Conv(c) if c.src2.is_some())), "the joins became two-source convolutions");+ assert!(+ net.ops().iter().all(|o| !matches!(o, crate::net::Op::Conv(c) if c.leaky.is_none() && c.k == 3)),+ "every 3 × 3 convolution has its activation"+ );+ assert_eq!((net.in_channels, net.out_channels(), net.depth()), (4, 3, 2));+ assert!(net.fits_tile(tile) && !net.fits_tile(30));++ let _ = std::fs::remove_file(p);+ }++ #[test]+ fn an_explicit_empty_or_nonvector_bias_is_not_treated_as_missing() {+ let w = TensorProto { name: "w".into(), dims: vec![12, 4, 1, 1], data_type: 1, float_data: vec![0.1; 48], ..Default::default() };+ let n = NodeProto { input: vec!["data".into(), "w".into(), "b".into()], ..Default::default() };+ for dims in [vec![0], vec![3, 4]] {+ let b = TensorProto {+ name: "b".into(),+ float_data: vec![0.0; dims.iter().product::<i64>() as usize],+ dims,+ data_type: 1,+ ..Default::default()+ };+ let inits = HashMap::from([("w", &w), ("b", &b)]);+ assert!(weights_of(&n, &inits, &mut 0).is_err());+ }+ }++ #[test]+ fn truncated_and_mutated_protobuf_is_bounded_and_never_panics() {+ let good = unet_onnx(4, 3, 1, 7);+ for n in (0..good.len()).step_by(31) {+ let _ = read_bytes(&good[..n]);+ }+ for at in (0..good.len()).step_by(37) {+ let mut bad = good.clone();+ bad[at] ^= 0xff;+ let _ = read_bytes(&bad);+ }+ assert!(read_bytes(&[0x3a, 0xff, 0xff, 0xff, 0xff, 0x7f]).is_err());+ assert!(read_bytes(&[0x3a, 0, 0x3a, 0]).is_err());+ }+ proptest::proptest! {+ #[test]+ fn arbitrary_model_bytes_never_panic(bytes in proptest::collection::vec(proptest::prelude::any::<u8>(), 0..4096)) {+ let _ = read_bytes(&bytes);+ }+ }++ #[test]+ fn what_the_description_cannot_say_is_unsupported_and_garbage_is_an_error() {+ // not a model+ let p = temp_model("junk", b"this is not an onnx file");+ assert!(matches!(read(&p), Err(NetError::Invalid(_)) | Err(NetError::Unsupported(_))));+ let _ = std::fs::remove_file(p);+ // a missing file+ assert!(read(Path::new("/does/not/exist.onnx")).is_err());+ }+}diff --git a/crates/denoise/src/onnx_proto.rs b/crates/denoise/src/onnx_proto.rsnew file mode 100644index 000000000..d8bbba411--- /dev/null+++ b/crates/denoise/src/onnx_proto.rs@@ -0,0 +1,295 @@+//! Bounded, borrowed decoding of the ONNX fields used by denoise, from the public Apache-2.0 schema:+//! https://github.com/onnx/onnx/blob/main/onnx/onnx.proto3+//! No generated code, protoc, recursive graph decoding, or external tensor files.++use crate::net::{MAX_OPS, MAX_WEIGHTS, NetError};++const MAX_ITEMS: usize = MAX_OPS * 4;+const MAX_STRING: usize = 4096;+pub(super) const MAX_BYTES: usize = MAX_WEIGHTS * 4 + 16 * 1024 * 1024;++type Result<T> = std::result::Result<T, NetError>;+fn bad() -> NetError {+ NetError::Invalid("malformed or oversized ONNX protobuf".into())+}++struct Fields<'a> {+ rest: &'a [u8],+ count: usize,+}+enum Wire<'a> {+ Int(u64),+ Bytes(&'a [u8]),+ F32(f32),+ Other,+}++fn varint(rest: &mut &[u8]) -> Result<u64> {+ let mut value = 0u64;+ for shift in (0..70).step_by(7) {+ let (b, tail) = rest.split_first().ok_or_else(bad)?;+ *rest = tail;+ if shift == 63 && *b > 1 {+ return Err(bad());+ }+ value |= u64::from(*b & 127) << shift;+ if b & 128 == 0 {+ return Ok(value);+ }+ }+ Err(bad())+}++impl<'a> Fields<'a> {+ fn new(rest: &'a [u8]) -> Self {+ Self { rest, count: 0 }+ }+ fn take(&mut self, n: usize) -> Result<&'a [u8]> {+ let out = self.rest.get(..n).ok_or_else(bad)?;+ self.rest = self.rest.get(n..).ok_or_else(bad)?;+ Ok(out)+ }+ fn next(&mut self) -> Result<Option<(u32, Wire<'a>)>> {+ if self.rest.is_empty() {+ return Ok(None);+ }+ self.count += 1;+ if self.count > MAX_WEIGHTS {+ return Err(bad());+ }+ let tag = varint(&mut self.rest)?;+ let number = u32::try_from(tag >> 3).map_err(|_| bad())?;+ if number == 0 || number >= 1 << 29 {+ return Err(bad());+ }+ let wire = match tag & 7 {+ 0 => Wire::Int(varint(&mut self.rest)?),+ 1 => {+ self.take(8)?;+ Wire::Other+ }+ 2 => {+ let n = usize::try_from(varint(&mut self.rest)?).map_err(|_| bad())?;+ Wire::Bytes(self.take(n)?)+ }+ 5 => {+ let b: [u8; 4] = self.take(4)?.try_into().map_err(|_| bad())?;+ Wire::F32(f32::from_le_bytes(b))+ }+ _ => return Err(bad()),+ };+ Ok(Some((number, wire)))+ }+}++fn string(b: &[u8]) -> Result<String> {+ if b.len() > MAX_STRING {+ return Err(bad());+ }+ std::str::from_utf8(b).map(str::to_owned).map_err(|_| bad())+}+fn push<T>(v: &mut Vec<T>, value: T, cap: usize) -> Result<()> {+ if v.len() >= cap {+ return Err(bad());+ }+ v.try_reserve(1).map_err(|_| bad())?;+ v.push(value);+ Ok(())+}+fn ints(w: Wire<'_>, out: &mut Vec<i64>, cap: usize) -> Result<()> {+ match w {+ Wire::Int(v) => push(out, v as i64, cap),+ Wire::Bytes(mut b) => {+ while !b.is_empty() {+ let v = varint(&mut b)?;+ push(out, v as i64, cap)?;+ }+ Ok(())+ }+ _ => Err(bad()),+ }+}++#[derive(Default)]+pub(super) struct AttributeProto {+ pub name: String,+ pub f: f32,+ pub i: i64,+ pub s: Vec<u8>,+ pub ints: Vec<i64>,+}+#[derive(Default)]+pub(super) struct NodeProto {+ pub input: Vec<String>,+ pub output: Vec<String>,+ pub op_type: String,+ pub attribute: Vec<AttributeProto>,+}+#[derive(Default)]+pub(super) struct TensorProto<'a> {+ pub name: String,+ pub dims: Vec<i64>,+ pub data_type: i32,+ pub raw_data: &'a [u8],+ pub float_data: Vec<f32>,+}+pub(super) struct ValueInfoProto {+ pub name: String,+ pub channels: Option<usize>,+}+#[derive(Default)]+pub(super) struct Graph<'a> {+ pub node: Vec<NodeProto>,+ pub initializer: Vec<TensorProto<'a>>,+ pub input: Vec<ValueInfoProto>,+ pub output: Vec<ValueInfoProto>,+}++fn attribute(b: &[u8]) -> Result<AttributeProto> {+ let mut a = AttributeProto::default();+ let mut fields = Fields::new(b);+ while let Some((n, w)) = fields.next()? {+ match (n, w) {+ (1, Wire::Bytes(b)) => a.name = string(b)?,+ (2, Wire::F32(v)) => a.f = v,+ (3, Wire::Int(v)) => a.i = v as i64,+ (4, Wire::Bytes(b)) => {+ if b.len() > MAX_STRING {+ return Err(bad());+ }+ a.s = b.to_vec();+ }+ (8, w) => ints(w, &mut a.ints, 16)?,+ (20, Wire::Int(1 | 2 | 3 | 7)) => {}+ _ => return Err(NetError::Unsupported("unknown attribute value".into())),+ }+ }+ Ok(a)+}+fn node(b: &[u8]) -> Result<NodeProto> {+ let mut a = NodeProto::default();+ let mut fields = Fields::new(b);+ while let Some((n, w)) = fields.next()? {+ match (n, w) {+ (1, Wire::Bytes(b)) => push(&mut a.input, string(b)?, 4)?,+ (2, Wire::Bytes(b)) => push(&mut a.output, string(b)?, 2)?,+ (4, Wire::Bytes(b)) => a.op_type = string(b)?,+ (5, Wire::Bytes(b)) => push(&mut a.attribute, attribute(b)?, 16)?,+ (7, Wire::Bytes(b)) if !b.is_empty() => return Err(NetError::Unsupported("a custom operator domain".into())),+ _ => {}+ }+ }+ Ok(a)+}+fn tensor(b: &[u8]) -> Result<TensorProto<'_>> {+ let mut a = TensorProto::default();+ let mut fields = Fields::new(b);+ while let Some((n, w)) = fields.next()? {+ match (n, w) {+ (1, w) => ints(w, &mut a.dims, 8)?,+ (2, Wire::Int(v)) => a.data_type = i32::try_from(v).map_err(|_| bad())?,+ (4, Wire::F32(v)) => push(&mut a.float_data, v, MAX_WEIGHTS)?,+ (4, Wire::Bytes(b)) => {+ if !b.len().is_multiple_of(4) {+ return Err(bad());+ }+ for chunk in b.as_chunks::<4>().0 {+ push(&mut a.float_data, f32::from_le_bytes(*chunk), MAX_WEIGHTS)?;+ }+ }+ (8, Wire::Bytes(b)) => a.name = string(b)?,+ (9, Wire::Bytes(b)) => {+ if b.len() > MAX_WEIGHTS * 4 {+ return Err(bad());+ }+ a.raw_data = b;+ }+ (13, _) | (14, Wire::Int(1..)) => return Err(NetError::Unsupported("external tensor data".into())),+ _ => {}+ }+ }+ Ok(a)+}+fn channel_shape(b: &[u8]) -> Result<Option<usize>> {+ let mut fields = Fields::new(b);+ let mut dim = 0;+ let mut channels = None;+ while let Some((n, w)) = fields.next()? {+ if let (1, Wire::Bytes(b)) = (n, w) {+ if dim >= 8 {+ return Err(bad());+ }+ let mut df = Fields::new(b);+ while let Some((n, w)) = df.next()? {+ if let (1, Wire::Int(v)) = (n, w)+ && dim == 1+ {+ channels = usize::try_from(v).ok().filter(|&v| v > 0);+ }+ }+ dim += 1;+ }+ }+ Ok(channels)+}+fn channel_type(b: &[u8]) -> Result<Option<usize>> {+ let mut fields = Fields::new(b);+ while let Some((n, w)) = fields.next()? {+ if let (1, Wire::Bytes(b)) = (n, w) {+ let mut tf = Fields::new(b);+ let (mut elem, mut channels) = (0, None);+ while let Some((n, w)) = tf.next()? {+ match (n, w) {+ (1, Wire::Int(v)) => elem = v,+ (2, Wire::Bytes(b)) => channels = channel_shape(b)?,+ _ => {}+ }+ }+ return Ok(if elem == 1 { channels } else { None });+ }+ }+ Ok(None)+}+fn value_info(b: &[u8]) -> Result<ValueInfoProto> {+ let mut out = ValueInfoProto { name: String::new(), channels: None };+ let mut fields = Fields::new(b);+ while let Some((n, w)) = fields.next()? {+ match (n, w) {+ (1, Wire::Bytes(b)) => out.name = string(b)?,+ (2, Wire::Bytes(b)) => out.channels = channel_type(b)?,+ _ => {}+ }+ }+ Ok(out)+}+fn graph(b: &[u8]) -> Result<Graph<'_>> {+ let mut out = Graph::default();+ let mut fields = Fields::new(b);+ while let Some((n, w)) = fields.next()? {+ match (n, w) {+ (1, Wire::Bytes(b)) => push(&mut out.node, node(b)?, MAX_ITEMS)?,+ (5, Wire::Bytes(b)) => push(&mut out.initializer, tensor(b)?, MAX_ITEMS)?,+ (11, Wire::Bytes(b)) => push(&mut out.input, value_info(b)?, MAX_ITEMS)?,+ (12, Wire::Bytes(b)) => push(&mut out.output, value_info(b)?, 1)?,+ (15, _) => return Err(NetError::Unsupported("sparse initializers".into())),+ _ => {}+ }+ }+ Ok(out)+}+pub(super) fn decode(b: &[u8]) -> Result<Graph<'_>> {+ if b.len() > MAX_BYTES {+ return Err(bad());+ }+ let mut fields = Fields::new(b);+ let mut out = None;+ while let Some((n, w)) = fields.next()? {+ if let (7, Wire::Bytes(b)) = (n, w) {+ if out.is_some() {+ return Err(bad());+ }+ out = Some(graph(b)?);+ }+ }+ out.ok_or_else(bad)+}diff --git a/crates/denoise/src/product.rs b/crates/denoise/src/product.rsnew file mode 100644index 000000000..1e3938855--- /dev/null+++ b/crates/denoise/src/product.rs@@ -0,0 +1,595 @@+//! The denoised picture as cached data, and the blend the Amount slider controls.+//!+//! A product is camera RGB at the sensor's resolution (what the model gave, before the raw's own opcodes and+//! crop), kept as half floats in strips of rows, each strip byte-shuffled and deflated with its own checksum.+//! It is derived data: any problem reading it (missing, truncated, corrupt, made for another photo or model) is+//! just a cache miss, never a panic, and the picture can be made again.+//!+//! File: `LCDN`, version, header length, header JSON (`width`, `height`, `strip`, `key`), then for each strip its+//! compressed length and CRC-32, then the strips' bytes. `key` says what the product was made from (the photo's+//! content, the model, this code's version, the settings it depends on); a file with another key is stale.++use std::io::{Read, Write};+use std::path::Path;++use lightcraft_raster::Rgb32f;+use rayon::prelude::*;+use serde::{Deserialize, Serialize};++const MAGIC: &[u8; 4] = b"LCDN";+const VERSION: u8 = 1;+const MAX_HEADER: usize = 4096;+const MAX_SIDE: usize = 1 << 17;+/// Most pixels a product may have (the denoiser's own limit, [`crate::run::MAX_PIXELS`]).+const MAX_PIXELS: usize = crate::run::MAX_PIXELS;+/// Rows per strip: about a megapixel of three planes at 24 MP widths.+const STRIP_ROWS: usize = 128;+/// Bumped when the meaning of a product changes (included in keys by the caller).+pub const ALGORITHM: u32 = 1;++#[derive(Debug, thiserror::Error)]+pub enum ProductError {+ #[error("the denoised picture could not be read or written: {0}")]+ Io(String),+ #[error("not a usable denoised picture: {0}")]+ Format(String),+ /// A product made from something else (another photo state, model or version).+ #[error("the denoised picture is out of date")]+ Stale,+}++impl From<std::io::Error> for ProductError {+ fn from(e: std::io::Error) -> Self {+ ProductError::Io(e.to_string())+ }+}++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct Header {+ pub width: usize,+ pub height: usize,+ pub strip: usize,+ pub key: String,+}++/// `f32` as IEEE half-float bits, rounded to nearest even; beyond the range it saturates, NaN becomes 0.+pub fn to_f16(v: f32) -> u16 {+ if !v.is_finite() {+ return if v.is_nan() {+ 0+ } else if v > 0.0 {+ 0x7bff+ } else {+ 0xfbff+ };+ }+ let b = v.to_bits();+ let sign = ((b >> 16) & 0x8000) as u16;+ let exp = ((b >> 23) & 0xff) as i32 - 127 + 15;+ let man = b & 0x7f_ffff;+ if exp >= 31 {+ return sign | 0x7bff;+ }+ if exp <= 0 {+ if exp < -10 {+ return sign;+ }+ let man = man | 0x80_0000;+ let shift = (14 - exp) as u32;+ let (half, rem, halfway) = (man >> shift, man & ((1u32 << shift) - 1), 1u32 << (shift - 1));+ let mut h = half as u16;+ if rem > halfway || (rem == halfway && h & 1 == 1) {+ h += 1;+ }+ return sign | h;+ }+ let mut h = (((exp as u32) << 10) | (man >> 13)) as u16;+ let rem = man & 0x1fff;+ if rem > 0x1000 || (rem == 0x1000 && h & 1 == 1) {+ h += 1;+ }+ if h >= 0x7c00 {+ h = 0x7bff;+ }+ sign | h+}++/// Half-float bits as `f32`.+pub fn from_f16(h: u16) -> f32 {+ let sign = if h & 0x8000 != 0 { -1.0f32 } else { 1.0 };+ let exp = i32::from((h >> 10) & 0x1f);+ let man = f32::from(h & 0x3ff);+ match exp {+ 0 => sign * man * 2f32.powi(-24),+ 31 => sign * 65504.0,+ _ => sign * (1.0 + man / 1024.0) * 2f32.powi(exp - 15),+ }+}++fn crc(bytes: &[u8]) -> u32 {+ // CRC-32 (IEEE), bitwise: strips are small and this runs once per strip+ let mut table = [0u32; 256];+ for (i, t) in table.iter_mut().enumerate() {+ let mut c = i as u32;+ for _ in 0..8 {+ c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };+ }+ *t = c;+ }+ !bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))+}++/// One strip as bytes: for each plane (R, G, B) the low bytes of its half floats, then the high bytes.+fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Vec<u8> {+ let n = (y1 - y0) * img.width;+ let mut raw = vec![0u8; 6 * n];+ let rows = img.data.get(y0 * img.width..y1 * img.width).unwrap_or(&[]);+ for (i, px) in rows.iter().enumerate() {+ for (c, &v) in px.iter().enumerate() {+ let h = to_f16(v);+ if let Some(lo) = raw.get_mut(2 * c * n + i) {+ *lo = h as u8;+ }+ if let Some(hi) = raw.get_mut((2 * c + 1) * n + i) {+ *hi = (h >> 8) as u8;+ }+ }+ }+ miniz_oxide::deflate::compress_to_vec(&raw, 3)+}++/// Write `img` to `path` (through a temporary file next to it, so a reader never sees half a product).+pub fn write(path: &Path, img: &Rgb32f, key: &str) -> Result<(), ProductError> {+ if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || img.data.len() != img.width * img.height {+ return Err(ProductError::Format("the picture has no usable size".into()));+ }+ let strips: Vec<(usize, usize)> = (0..img.height).step_by(STRIP_ROWS).map(|y| (y, (y + STRIP_ROWS).min(img.height))).collect();+ let packed: Vec<Vec<u8>> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect();+ let header = serde_json::to_vec(&Header { width: img.width, height: img.height, strip: STRIP_ROWS, key: key.to_string() })+ .map_err(|e| ProductError::Format(e.to_string()))?;+ let tmp = path.with_extension("tmp");+ let result = (|| -> Result<(), ProductError> {+ if let Some(dir) = path.parent() {+ std::fs::create_dir_all(dir)?;+ }+ let mut f = std::io::BufWriter::new(std::fs::File::create(&tmp)?);+ f.write_all(MAGIC)?;+ f.write_all(&[VERSION, 0, 0, 0])?;+ f.write_all(&(header.len() as u32).to_le_bytes())?;+ f.write_all(&header)?;+ for p in &packed {+ f.write_all(&(p.len() as u32).to_le_bytes())?;+ f.write_all(&crc(p).to_le_bytes())?;+ }+ for p in &packed {+ f.write_all(p)?;+ }+ f.flush()?;+ drop(f);+ std::fs::rename(&tmp, path)?;+ Ok(())+ })();+ if result.is_err() {+ let _ = std::fs::remove_file(&tmp);+ }+ result+}++/// The parsed start of a product file: header, each strip's (compressed length, CRC-32), and the file length the+/// start says the whole file has.+struct Start {+ header: Header,+ table: Vec<(usize, u32)>,+ expected_len: u64,+}++fn read_start(r: &mut impl Read) -> Result<Start, ProductError> {+ let mut fixed = [0u8; 12];+ r.read_exact(&mut fixed)?;+ if fixed.get(0..4) != Some(&MAGIC[..]) || fixed.get(4) != Some(&VERSION) {+ return Err(ProductError::Format("not a denoised picture of this version".into()));+ }+ let len = fixed.get(8..12).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_le_bytes).unwrap_or(u32::MAX) as usize;+ if len > MAX_HEADER {+ return Err(ProductError::Format("the header is too large".into()));+ }+ let mut hb = vec![0u8; len];+ r.read_exact(&mut hb)?;+ let h: Header = serde_json::from_slice(&hb).map_err(|e| ProductError::Format(e.to_string()))?;+ let pixels = h.width.saturating_mul(h.height);+ if h.width == 0 || h.height == 0 || h.width > MAX_SIDE || h.height > MAX_SIDE || pixels > MAX_PIXELS || h.strip != STRIP_ROWS {+ return Err(ProductError::Format("the size is not usable".into()));+ }+ let n = h.height.div_ceil(h.strip);+ let mut table = Vec::with_capacity(n);+ let mut total = 12 + len as u64 + 8 * n as u64;+ for _ in 0..n {+ let mut e = [0u8; 8];+ r.read_exact(&mut e)?;+ let l = u32::from_le_bytes([e[0], e[1], e[2], e[3]]) as usize;+ let c = u32::from_le_bytes([e[4], e[5], e[6], e[7]]);+ // a strip never compresses to much more than its raw size+ if l > 12 * h.strip * h.width + 4096 {+ return Err(ProductError::Format("a strip is larger than it can be".into()));+ }+ total += l as u64;+ table.push((l, c));+ }+ Ok(Start { header: h, table, expected_len: total })+}++/// The header of the product at `path`, when it is one.+pub fn read_header_at(path: &Path) -> Result<Header, ProductError> {+ let mut f = std::io::BufReader::new(std::fs::File::open(path)?);+ Ok(read_start(&mut f)?.header)+}++/// Whether `path` holds a complete-looking product made from `key` (the header and the file's length are checked;+/// strips are checked when read).+pub fn is_current(path: &Path, key: &str) -> bool {+ let check = || -> Result<bool, ProductError> {+ let mut f = std::io::BufReader::new(std::fs::File::open(path)?);+ let s = read_start(&mut f)?;+ Ok(s.header.key == key && std::fs::metadata(path)?.len() == s.expected_len)+ };+ check().unwrap_or(false)+}++/// A rectangle of a product, in the product's pixels.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct Window {+ pub x: usize,+ pub y: usize,+ pub width: usize,+ pub height: usize,+}++/// Half floats as `f32`, for all 65536 bit patterns (a table is much faster than computing each).+fn f16_table() -> &'static [f32] {+ static TABLE: std::sync::OnceLock<Vec<f32>> = std::sync::OnceLock::new();+ TABLE.get_or_init(|| (0..=u16::MAX).map(from_f16).collect())+}++/// Strips decoded together (they are inflated in parallel, then folded in order).+const GROUP: usize = 8;++/// One strip, checked and inflated: for each plane the low bytes of its half floats, then the high bytes.+fn decode_strip(index: usize, packed: &[u8], expect_crc: u32, n: usize) -> Result<Vec<u8>, ProductError> {+ if crc(packed) != expect_crc {+ return Err(ProductError::Format(format!("strip {index} is damaged")));+ }+ let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(packed, 6 * n).map_err(|e| ProductError::Format(format!("strip {index}: {e:?}")))?;+ if raw.len() != 6 * n {+ return Err(ProductError::Format(format!("strip {index} has the wrong size")));+ }+ Ok(raw)+}++/// Read the product at `path`, made from `key`, with every `factor × factor` block of pixels averaged into one+/// (`factor` 1 = full size). The result is camera RGB.+///+/// With a `window`, only that rectangle is read and the blocks start at its corner; only whole blocks are made, so+/// the result is `window.width / factor × window.height / factor` (the same blocks a binned development of the same+/// crop makes). Without one the whole product is read and blocks at the right and bottom edges may be smaller.+///+/// With `clip` (white is 1), a block in which any sample of a colour is at or above it takes that colour's+/// maximum instead of the mean, so a clipped highlight stays clipped for highlight reconstruction.+pub fn read_window(path: &Path, key: &str, window: Option<Window>, factor: usize, clip: Option<f32>) -> Result<Rgb32f, ProductError> {+ let factor = factor.clamp(1, 1 << 12);+ let mut f = std::io::BufReader::new(std::fs::File::open(path)?);+ let Start { header: h, table, expected_len } = read_start(&mut f)?;+ if h.key != key {+ return Err(ProductError::Stale);+ }+ if std::fs::metadata(path)?.len() != expected_len {+ return Err(ProductError::Format("the file is not the length its header says".into()));+ }+ let w = window.unwrap_or(Window { x: 0, y: 0, width: h.width, height: h.height });+ let fits = |a: usize, b: usize, limit: usize| a.checked_add(b).is_some_and(|e| e <= limit);+ if w.width == 0 || w.height == 0 || !fits(w.x, w.width, h.width) || !fits(w.y, w.height, h.height) {+ return Err(ProductError::Format("the window does not lie inside the picture".into()));+ }+ // whole blocks for a window; partial ones at the edges when the whole picture is read+ let whole = window.is_none();+ let (ow, oh) = if whole { (w.width.div_ceil(factor), w.height.div_ceil(factor)) } else { (w.width / factor, w.height / factor) };+ if ow == 0 || oh == 0 {+ return Err(ProductError::Format("the window is smaller than one block".into()));+ }+ // the picture rows and columns the blocks cover+ let (x_end, y_end) = if whole { (w.x + w.width, w.y + w.height) } else { (w.x + ow * factor, w.y + oh * factor) };+ let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] };+ let lut = f16_table();+ let value = |raw: &[u8], c: usize, n: usize, i: usize| -> f32 {+ let (lo, hi) = (raw.get(2 * c * n + i).copied().unwrap_or(0), raw.get((2 * c + 1) * n + i).copied().unwrap_or(0));+ lut.get(usize::from(lo) | usize::from(hi) << 8).copied().unwrap_or(0.0)+ };+ // which strips are needed, and where each starts in the file (after the header and the strip table)+ let wanted = |s: usize| s * h.strip < y_end && (s + 1) * h.strip > w.y;+ let (mut sum, mut peak) = (vec![[0f32; 3]; ow], vec![[f32::MIN; 3]; ow]);+ let (mut rows_in, mut oy) = (0usize, 0usize);+ let strips: Vec<usize> = (0..table.len()).filter(|&s| wanted(s)).collect();+ // skip the strips before the first one wanted+ let skipped: usize = table.iter().take(strips.first().copied().unwrap_or(0)).map(|e| e.0).sum();+ f.seek_relative(i64::try_from(skipped).map_err(|_| ProductError::Format("the file is too large".into()))?)?;+ for group in strips.chunks(GROUP) {+ let mut packed = Vec::with_capacity(group.len());+ for &s in group {+ let Some(&(len, _)) = table.get(s) else { return Err(ProductError::Format("a strip is missing".into())) };+ let mut p = vec![0u8; len];+ f.read_exact(&mut p)?;+ packed.push(p);+ }+ let decoded: Vec<Result<Vec<u8>, ProductError>> = group+ .par_iter()+ .zip(packed.par_iter())+ .map(|(&s, p)| {+ let rows = h.height.saturating_sub(s * h.strip).min(h.strip);+ decode_strip(s, p, table.get(s).map_or(0, |e| e.1), rows * h.width)+ })+ .collect();+ for (&s, raw) in group.iter().zip(decoded) {+ let raw = raw?;+ let y0 = s * h.strip;+ let rows = h.height.saturating_sub(y0).min(h.strip);+ let n = rows * h.width;+ for r in 0..rows {+ let y = y0 + r;+ if y < w.y || y >= y_end {+ continue;+ }+ let base = r * h.width;+ for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {+ let x0 = w.x + ox * factor;+ let x1 = (x0 + factor).min(x_end);+ for x in x0..x1 {+ for c in 0..3 {+ let v = value(&raw, c, n, base + x);+ if let (Some(s_), Some(p_)) = (sm.get_mut(c), pk.get_mut(c)) {+ *s_ += v;+ *p_ = p_.max(v);+ }+ }+ }+ }+ rows_in += 1;+ if rows_in == factor || y + 1 == y_end {+ for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {+ // the block's size at the right and bottom edges may be smaller+ let bw = (x_end - (w.x + ox * factor)).min(factor);+ let k = 1.0 / (rows_in * bw).max(1) as f32;+ if let Some(o) = out.data.get_mut(oy * ow + ox) {+ for c in 0..3 {+ let (s_, p_) = (sm.get(c).copied().unwrap_or(0.0), pk.get(c).copied().unwrap_or(0.0));+ if let Some(slot) = o.get_mut(c) {+ *slot = if factor > 1 && clip.is_some_and(|t| p_ >= t) { p_ } else { s_ * k };+ }+ }+ }+ *sm = [0.0; 3];+ *pk = [f32::MIN; 3];+ }+ rows_in = 0;+ oy += 1;+ }+ }+ }+ }+ Ok(out)+}++/// [`read_window`] of the whole product: every `factor × factor` block averaged into one.+pub fn read(path: &Path, key: &str, factor: usize) -> Result<Rgb32f, ProductError> {+ read_window(path, key, None, factor, None)+}++/// `base + (denoised − base) · amount` for every pixel (`amount` 0 to 1), in place in `base`. The two pictures must+/// be the same size; if they are not, `base` is left as it was and `false` is returned.+pub fn blend(base: &mut Rgb32f, denoised: &Rgb32f, amount: f32) -> bool {+ if base.width != denoised.width || base.height != denoised.height {+ return false;+ }+ let a = if amount.is_finite() { amount.clamp(0.0, 1.0) } else { 0.0 };+ base.data.par_iter_mut().zip(denoised.data.par_iter()).for_each(|(b, d)| {+ for c in 0..3 {+ b[c] += (d[c] - b[c]) * a;+ }+ });+ true+}++#[cfg(test)]+mod tests {+ use super::*;++ fn temp(name: &str) -> std::path::PathBuf {+ std::env::temp_dir().join(format!("lc-denoise-{name}-{}.lcdn", std::process::id()))+ }++ fn picture(w: usize, h: usize) -> Rgb32f {+ Rgb32f {+ width: w,+ height: h,+ data: (0..w * h).map(|i| [(i % w) as f32 / w as f32, (i / w) as f32 / h as f32, 0.001 + 0.5 * ((i % 7) as f32 / 7.0)]).collect(),+ }+ }++ #[test]+ fn half_floats_round_trip_closely() {+ for v in [0.0f32, 1.0, -1.0, 0.5, 0.333_333, 1e-4, 6.1e-5, 3e-6, 123.456, 65000.0, -0.002] {+ let back = from_f16(to_f16(v));+ assert!((back - v).abs() <= v.abs() * 0.001 + 1e-7, "{v} -> {back}");+ }+ assert_eq!(from_f16(to_f16(1e10)), 65504.0);+ assert_eq!(to_f16(f32::NAN), 0);+ assert_eq!(from_f16(to_f16(f32::NEG_INFINITY)), -65504.0);+ }++ #[test]+ fn a_product_survives_a_round_trip_at_full_and_reduced_size() {+ let (w, h) = (301, 517);+ let img = picture(w, h);+ let p = temp("round");+ write(&p, &img, "k1").unwrap();+ assert!(is_current(&p, "k1") && !is_current(&p, "k2"));+ assert_eq!(read_header_at(&p).unwrap().width, w);+ let full = read(&p, "k1", 1).unwrap();+ assert_eq!((full.width, full.height), (w, h));+ for (a, b) in full.data.iter().zip(&img.data) {+ for c in 0..3 {+ assert!((a[c] - b[c]).abs() < 0.002, "{a:?} vs {b:?}");+ }+ }+ // averaged into blocks of 4: the sizes round up, and the mean of the picture is kept+ let small = read(&p, "k1", 4).unwrap();+ assert_eq!((small.width, small.height), (w.div_ceil(4), h.div_ceil(4)));+ let mean = |i: &Rgb32f| i.data.iter().map(|p| f64::from(p[1])).sum::<f64>() / i.data.len() as f64;+ assert!((mean(&small) - mean(&img)).abs() < 0.01);+ // a factor that does not divide the strip height is the same average+ let odd = read(&p, "k1", 5).unwrap();+ assert_eq!((odd.width, odd.height), (w.div_ceil(5), h.div_ceil(5)));+ assert!((mean(&odd) - mean(&img)).abs() < 0.01);+ assert!(matches!(read(&p, "other", 1), Err(ProductError::Stale)));+ std::fs::remove_file(&p).unwrap();+ }++ /// What binning a window of `img` by `k` means, written the slow obvious way.+ fn blocks(img: &Rgb32f, win: Window, k: usize, clip: Option<f32>) -> Rgb32f {+ let (ow, oh) = (win.width / k, win.height / k);+ let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] };+ for by in 0..oh {+ for bx in 0..ow {+ for c in 0..3 {+ let vals: Vec<f32> = (0..k * k).map(|i| img.data[(win.y + by * k + i / k) * img.width + win.x + bx * k + i % k][c]).collect();+ let max = vals.iter().copied().fold(f32::MIN, f32::max);+ out.data[by * ow + bx][c] =+ if k > 1 && clip.is_some_and(|t| max >= t) { max } else { vals.iter().sum::<f32>() / vals.len() as f32 };+ }+ }+ }+ out+ }++ #[test]+ fn a_window_is_read_as_the_blocks_of_that_crop() {+ let (w, h) = (301, 517);+ let mut img = picture(w, h);+ // a clipped sample in the middle of a block+ img.data[(130 + 2) * w + 61 + 1] = [1.0, 0.4, 0.4];+ let p = temp("window");+ write(&p, &img, "k1").unwrap();+ let near = |a: &Rgb32f, b: &Rgb32f| {+ assert_eq!((a.width, a.height), (b.width, b.height));+ for (x, y) in a.data.iter().zip(&b.data) {+ for c in 0..3 {+ assert!((x[c] - y[c]).abs() < 0.003, "{x:?} vs {y:?}");+ }+ }+ };+ // windows at odd places, across strip boundaries (128 rows), with factors that do and do not divide them+ for (win, k) in [+ (Window { x: 3, y: 5, width: 200, height: 300 }, 1),+ (Window { x: 3, y: 5, width: 200, height: 300 }, 4),+ (Window { x: 61, y: 130, width: 121, height: 250 }, 5),+ (Window { x: 0, y: 0, width: w, height: h }, 2),+ (Window { x: 1, y: 380, width: 90, height: 137 }, 3),+ ] {+ let got = read_window(&p, "k1", Some(win), k, None).unwrap();+ near(&got, &blocks(&img, win, k, None));+ }+ // with a clip level the block with the clipped sample keeps its highest value, the others stay means+ let win = Window { x: 61, y: 130, width: 40, height: 20 };+ let clipped = read_window(&p, "k1", Some(win), 4, Some(0.99)).unwrap();+ near(&clipped, &blocks(&img, win, 4, Some(0.99)));+ assert!(clipped.data[0][0] > 0.99, "the clipped block stays clipped: {:?}", clipped.data[0]);+ let plain = read_window(&p, "k1", Some(win), 4, None).unwrap();+ assert!(plain.data[0][0] < 0.99, "an average falls below the clip level");+ // windows that do not fit, or are smaller than a block, are errors+ for bad in [+ Window { x: 0, y: 0, width: 0, height: 5 },+ Window { x: 290, y: 0, width: 20, height: 5 },+ Window { x: 0, y: 510, width: 5, height: 10 },+ Window { x: usize::MAX, y: 0, width: 2, height: 2 },+ Window { x: 0, y: 0, width: 3, height: 3 },+ ] {+ assert!(read_window(&p, "k1", Some(bad), 4, None).is_err(), "{bad:?}");+ }+ std::fs::remove_file(&p).unwrap();+ }++ #[test]+ fn a_smooth_picture_compresses_well() {+ let img = Rgb32f {+ width: 512,+ height: 512,+ data: (0..512 * 512)+ .map(|i| {+ let v = 0.2 + 0.0001 * (i % 512) as f32;+ [v, v * 0.8, v * 0.6]+ })+ .collect(),+ };+ let p = temp("size");+ write(&p, &img, "k").unwrap();+ let bytes = std::fs::metadata(&p).unwrap().len() as usize;+ assert!(bytes < 512 * 512 * 6 / 2, "{bytes} bytes for a smooth picture");+ std::fs::remove_file(&p).unwrap();+ }++ #[test]+ fn damaged_files_are_errors_not_panics() {+ let img = picture(100, 300);+ let p = temp("damage");+ write(&p, &img, "k").unwrap();+ let good = std::fs::read(&p).unwrap();+ // truncated at every length+ for cut in (0..good.len()).step_by(37) {+ std::fs::write(&p, &good[..cut]).unwrap();+ assert!(read(&p, "k", 1).is_err(), "cut at {cut}");+ assert!(!is_current(&p, "k"));+ }+ // a flipped byte anywhere is caught by a checksum, the header parse, or the key+ for at in (0..good.len()).step_by(53) {+ let mut bad = good.clone();+ bad[at] ^= 0xff;+ std::fs::write(&p, &bad).unwrap();+ let _ = read(&p, "k", 1); // must not panic; most are errors+ }+ // a header that claims an absurd size+ let mut huge = b"LCDN\x01\0\0\0".to_vec();+ let hdr = br#"{"width":4000000000,"height":4000000000,"strip":128,"key":"k"}"#;+ huge.extend((hdr.len() as u32).to_le_bytes());+ huge.extend(hdr);+ std::fs::write(&p, &huge).unwrap();+ assert!(matches!(read(&p, "k", 1), Err(ProductError::Format(_))));+ std::fs::write(&p, b"").unwrap();+ assert!(read(&p, "k", 1).is_err());+ assert!(read(&temp("missing"), "k", 1).is_err());+ std::fs::remove_file(&p).unwrap();+ }++ #[test]+ fn writing_rejects_unusable_pictures() {+ let p = temp("bad");+ assert!(write(&p, &Rgb32f { width: 0, height: 0, data: vec![] }, "k").is_err());+ assert!(write(&p, &Rgb32f { width: 3, height: 3, data: vec![[0.0; 3]; 8] }, "k").is_err());+ assert!(!p.exists());+ }++ #[test]+ fn blend_mixes_by_amount_and_rejects_other_sizes() {+ let mut base = Rgb32f { width: 2, height: 1, data: vec![[0.0; 3], [1.0; 3]] };+ let dn = Rgb32f { width: 2, height: 1, data: vec![[1.0; 3], [0.0; 3]] };+ assert!(blend(&mut base, &dn, 0.25));+ assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);+ assert!(blend(&mut base, &dn, f32::NAN));+ assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);+ assert!(!blend(&mut base, &Rgb32f { width: 1, height: 1, data: vec![[0.0; 3]] }, 1.0));+ assert!(blend(&mut base, &dn, 1.0));+ assert_eq!(base.data, dn.data);+ }++ #[test]+ fn crc_matches_the_standard_check_value() {+ assert_eq!(crc(b"123456789"), 0xcbf4_3926);+ }+}diff --git a/crates/denoise/src/reference.rs b/crates/denoise/src/reference.rsnew file mode 100644index 000000000..ef58b5b9c--- /dev/null+++ b/crates/denoise/src/reference.rs@@ -0,0 +1,133 @@+//! A plain-loop interpreter of a [`Net`], slow and obviously right: the reference the parser and the GPU runner are+//! tested against (tract's answer on the real model is the other). It is not used on photos.++use crate::net::{Net, NetError, Op, TensorId};+use crate::run::{Error, TileRunner};++/// A tensor in `[channels][side][side]` order.+struct T {+ channels: usize,+ side: usize,+ data: Vec<f32>,+}++/// Run `net` on `input` (`in_channels` planes of `tile × tile`).+pub fn run(net: &Net, tile: usize, input: &[f32]) -> Result<Vec<f32>, NetError> {+ if !net.fits_tile(tile) || input.len() != net.in_channels * tile * tile {+ return Err(NetError::Invalid("the tile does not fit the network".into()));+ }+ let mut tensors: Vec<T> = vec![T { channels: net.in_channels, side: tile, data: input.to_vec() }];+ let get = |tensors: &[T], id: TensorId| -> Result<usize, NetError> {+ if id < tensors.len() { Ok(id) } else { Err(NetError::Invalid("a layer reads a tensor that does not exist yet".into())) }+ };+ for op in net.ops() {+ let out = match op {+ Op::Conv(c) => {+ let a = get(&tensors, c.src)?;+ let b = c.src2.map(|s| get(&tensors, s)).transpose()?;+ let side = tensors[a].side;+ let pad = (c.k - 1) / 2;+ let mut data = vec![0f32; c.cout * side * side];+ // the input channels in order: the first source's, then the second's+ let planes: Vec<&[f32]> =+ std::iter::once(&tensors[a]).chain(b.map(|b| &tensors[b])).flat_map(|t| t.data.chunks(side * side)).collect();+ for co in 0..c.cout {+ for y in 0..side {+ for x in 0..side {+ let mut sum = c.bias[co];+ for (ci, plane) in planes.iter().enumerate() {+ for ky in 0..c.k {+ for kx in 0..c.k {+ let (yy, xx) = ((y + ky) as isize - pad as isize, (x + kx) as isize - pad as isize);+ if yy < 0 || xx < 0 || yy >= side as isize || xx >= side as isize {+ continue;+ }+ sum += plane[yy as usize * side + xx as usize] * c.weight[((co * c.cin + ci) * c.k + ky) * c.k + kx];+ }+ }+ }+ data[(co * side + y) * side + x] = leaky(sum, c.leaky);+ }+ }+ }+ T { channels: c.cout, side, data }+ }+ Op::ConvT2(c) => {+ let a = get(&tensors, c.src)?;+ let side = tensors[a].side;+ let out_side = 2 * side;+ let mut data = vec![0f32; c.cout * out_side * out_side];+ for co in 0..c.cout {+ for y in 0..side {+ for x in 0..side {+ for dy in 0..2 {+ for dx in 0..2 {+ let mut sum = c.bias[co];+ for ci in 0..c.cin {+ sum += tensors[a].data[(ci * side + y) * side + x] * c.weight[((ci * c.cout + co) * 2 + dy) * 2 + dx];+ }+ data[(co * out_side + 2 * y + dy) * out_side + 2 * x + dx] = leaky(sum, c.leaky);+ }+ }+ }+ }+ }+ T { channels: c.cout, side: out_side, data }+ }+ Op::MaxPool2 { src } => {+ let a = get(&tensors, *src)?;+ let (channels, side) = (tensors[a].channels, tensors[a].side / 2);+ let mut data = vec![0f32; channels * side * side];+ for ch in 0..channels {+ for y in 0..side {+ for x in 0..side {+ let at = |dy: usize, dx: usize| tensors[a].data[(ch * 2 * side + 2 * y + dy) * 2 * side + 2 * x + dx];+ data[(ch * side + y) * side + x] = at(0, 0).max(at(0, 1)).max(at(1, 0)).max(at(1, 1));+ }+ }+ }+ T { channels, side, data }+ }+ Op::DepthToSpace2 { src } => {+ let a = get(&tensors, *src)?;+ let (channels, side) = (tensors[a].channels / 4, tensors[a].side);+ let out_side = 2 * side;+ let mut data = vec![0f32; channels * out_side * out_side];+ for ch in 0..channels {+ for y in 0..side {+ for x in 0..side {+ for dy in 0..2 {+ for dx in 0..2 {+ data[(ch * out_side + 2 * y + dy) * out_side + 2 * x + dx] =+ tensors[a].data[((4 * ch + 2 * dy + dx) * side + y) * side + x];+ }+ }+ }+ }+ }+ T { channels, side: out_side, data }+ }+ };+ tensors.push(out);+ }+ tensors.pop().map(|t| t.data).ok_or_else(|| NetError::Invalid("no layers".into()))+}++fn leaky(v: f32, alpha: Option<f32>) -> f32 {+ match alpha {+ Some(a) if v < 0.0 => v * a,+ _ => v,+ }+}++/// The reference as a [`TileRunner`] (for tests of the code around the model).+pub struct ReferenceRunner {+ pub net: Net,+ pub tile: usize,+}++impl TileRunner for ReferenceRunner {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {+ run(&self.net, self.tile, input).map_err(|e| Error::Runtime(e.to_string()))+ }+}diff --git a/crates/denoise/src/run.rs b/crates/denoise/src/run.rsnew file mode 100644index 000000000..5bc763ce5--- /dev/null+++ b/crates/denoise/src/run.rs@@ -0,0 +1,582 @@+//! Denoising a whole Bayer mosaic with a tile model: cut it into overlapping tiles, run the model on each, bring+//! each tile's scale back to the input's, keep clipped highlights as they were, and blend the tiles together.+//!+//! The model itself is a [`TileRunner`], so everything around it (packing, tiling, scale matching, highlight+//! protection, blending, cancelling, progress) is tested without one.++use std::sync::atomic::{AtomicBool, Ordering};++use lightcraft_raster::Rgb32f;+use rayon::prelude::*;++use crate::bayer::{Layout, pack_tile};+use crate::manifest::Gain;+use crate::tiles::{starts, weight};++/// Most pixels one picture may have (guards the allocations that follow from its size).+pub const MAX_PIXELS: usize = 400_000_000;+/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from.+const MIN_SIGNAL: f32 = 0.003;++#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("{0}")]+ Input(String),+ #[error("the model gave {0}")]+ Model(String),+ #[error("the model could not run: {0}")]+ Runtime(String),+ #[error("the model could not be loaded: {0}")]+ Load(String),+ #[error("cancelled")]+ Cancelled,+}++/// Something that runs the model on one tile.+pub trait TileRunner: Sync {+ /// `input` is the tile as four planes `[R, G1, G2, B]` of `tile × tile` cells (NCHW, batch 1); the answer is+ /// camera RGB, three planes of `2·tile × 2·tile` pixels (CHW), in the model's own scale.+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error>;+}++pub struct Params {+ /// Side of the model's square, in cells.+ pub tile: usize,+ /// Cells neighbouring tiles share.+ pub overlap: usize,+ pub gain: Gain,+ /// White is 1; samples at or above this (minus a little) are clipped and keep their value.+ pub clip: Option<f32>,+ /// Tiles run at once (each model call may itself be single-threaded).+ pub parallel: usize,+}++#[derive(Default)]+pub struct Control<'a> {+ pub cancel: Option<&'a AtomicBool>,+ /// `(tiles done, tiles in all)`.+ pub progress: Option<&'a (dyn Fn(usize, usize) + Sync)>,+}++/// The tile grid of a picture: `(starts across, starts down)`.+pub fn grid(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> (Vec<usize>, Vec<usize>) {+ let (nx, ny) = layout.cells(width, height);+ (starts(nx, tile, overlap), starts(ny, tile, overlap))+}++/// How many model calls [`denoise_bayer`] makes for a picture.+pub fn tile_count(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> usize {+ let (sx, sy) = grid(width, height, layout, tile, overlap);+ sx.len().saturating_mul(sy.len())+}++/// Denoise (and demosaic) the normalised mosaic `mosaic` of `width × height` samples, laid out as `layout`.+/// Gives camera RGB at the mosaic's resolution.+pub fn denoise_bayer(+ mosaic: &[f32],+ width: usize,+ height: usize,+ layout: Layout,+ runner: &dyn TileRunner,+ p: &Params,+ ctl: &Control<'_>,+) -> Result<Rgb32f, Error> {+ let pixels = width+ .checked_mul(height)+ .filter(|&n| n > 0 && n <= MAX_PIXELS)+ .ok_or_else(|| Error::Input("the picture has no size, or is too large".into()))?;+ if mosaic.len() != pixels {+ return Err(Error::Input("the mosaic does not have width × height samples".into()));+ }+ if !(16..=2048).contains(&p.tile) || p.overlap >= p.tile.div_ceil(2) || layout.dx > 1 || layout.dy > 1 {+ return Err(Error::Input("the tile size and overlap do not fit".into()));+ }+ if let Gain::MatchMean { nominal, max_deviation } = p.gain+ && !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal) && max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation))+ {+ return Err(Error::Input("the model gain is not finite or within its supported range".into()));+ }+ if p.clip.is_some_and(|v| !v.is_finite() || v <= 0.0) {+ return Err(Error::Input("the highlight clip must be finite and positive".into()));+ }+ let (nx, ny) = layout.cells(width, height);+ let (sx, sy) = (starts(nx, p.tile, p.overlap), starts(ny, p.tile, p.overlap));+ let (cols, rows) = (sx.len(), sy.len());+ if cols == 0 || rows == 0 {+ return Err(Error::Input("the picture needs too many tiles".into()));+ }+ let tiles: Vec<(usize, usize, bool, bool, bool, bool)> = sy+ .iter()+ .enumerate()+ .flat_map(|(j, &y0)| sx.iter().enumerate().map(move |(i, &x0)| (x0, y0, i > 0, i + 1 < cols, j > 0, j + 1 < rows)))+ .collect();+ let total = tiles.len();++ let mut acc = vec![[0f32; 3]; pixels];+ let mut wsum = vec![0f32; nx * ny];+ let group = p.parallel.clamp(1, 64);+ let mut done = 0;+ // where the time goes, printed under `LIGHTCRAFT_PROFILE`+ let (mut t_wait, mut t_total) = (std::time::Duration::ZERO, std::time::Duration::ZERO);+ let began = std::time::Instant::now();++ // One group of tiles at a time: pack, run the model, check and bring each to the input's scale, in parallel.+ // Blending (sequential: the tiles add into one picture, in tile order so the result does not depend on timing) of+ // the previous group runs while the next group is on the model, so neither the card nor the cores wait for it.+ type Tile = (usize, usize, bool, bool, bool, bool);+ let compute = |chunk: &[Tile]| -> Vec<Result<Vec<f32>, Error>> {+ chunk+ .par_iter()+ .map(|&(x0, y0, bx, ax, by, ay)| {+ let mut input = vec![0f32; 4 * p.tile * p.tile];+ if !pack_tile(mosaic, width, height, layout, x0 as isize, y0 as isize, p.tile, &mut input) {+ return Err(Error::Input("a tile could not be packed".into()));+ }+ let mut out = runner.run(&input)?;+ let side = 2 * p.tile;+ if out.len() != 3 * side * side {+ return Err(Error::Model(format!("{} numbers for a tile, not the {} expected", out.len(), 3 * side * side)));+ }+ if out.iter().any(|v| !v.is_finite()) {+ return Err(Error::Model("numbers that are not finite".into()));+ }+ finish_tile(&input, &mut out, p, (bx, ax, by, ay));+ Ok(out)+ })+ .collect()+ };+ let mut blend = |chunk: &[Tile], outs: Vec<Result<Vec<f32>, Error>>| -> Result<(), Error> {+ for (&(x0, y0, bx, ax, by, ay), r) in chunk.iter().zip(outs) {+ blend_tile(&r?, &mut acc, &mut wsum, (width, height), (nx, ny), layout, (x0, y0), p, (bx, ax, by, ay));+ done += 1;+ if let Some(f) = ctl.progress {+ f(done, total);+ }+ }+ Ok(())+ };+ let mut previous: Option<(&[Tile], Vec<Result<Vec<f32>, Error>>)> = None;+ for chunk in tiles.chunks(group) {+ if ctl.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {+ return Err(Error::Cancelled);+ }+ let (outs, blended) = rayon::join(+ || compute(chunk),+ || match previous.take() {+ Some((c, o)) => {+ let started = std::time::Instant::now();+ let r = blend(c, o);+ (r, started.elapsed())+ }+ None => (Ok(()), std::time::Duration::ZERO),+ },+ );+ blended.0?;+ t_wait += blended.1;+ previous = Some((chunk, outs));+ }+ if let Some((c, o)) = previous.take() {+ let started = std::time::Instant::now();+ blend(c, o)?;+ t_wait += started.elapsed();+ }+ t_total += began.elapsed();+ let started = std::time::Instant::now();+ // normalise by the weights+ let (ox, oy) = (layout.position(0, 0).0, layout.position(0, 0).1);+ acc.par_chunks_mut(width).enumerate().for_each(|(y, row)| {+ let cy = ((y as isize - oy) / 2) as usize;+ for (x, px) in row.iter_mut().enumerate() {+ let cx = ((x as isize - ox) / 2) as usize;+ let w = wsum.get(cy * nx + cx).copied().unwrap_or(0.0);+ if w > 0.0 {+ *px = px.map(|v| v / w);+ }+ }+ });+ if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {+ eprintln!(+ "[denoise] {total} tiles, {} at once: tiles {:.0} ms (blending, {:.0} ms of it, overlaps the model), normalise {:.0} ms",+ group,+ t_total.as_secs_f64() * 1e3,+ t_wait.as_secs_f64() * 1e3,+ started.elapsed().as_secs_f64() * 1e3+ );+ }+ Ok(Rgb32f { width, height, data: acc })+}++/// Bring a tile's output to the input's scale and keep clipped highlights as they were (in place).+fn finish_tile(input: &[f32], out: &mut [f32], p: &Params, edges: (bool, bool, bool, bool)) {+ let t = p.tile;+ let side = 2 * t;+ if let Gain::MatchMean { nominal, max_deviation } = p.gain {+ // the part of the tile whose scale is measured: away from the edges that blend into a neighbour+ let m = p.overlap / 2;+ let (i0, i1) = (if edges.0 { m } else { 0 }, t - if edges.1 { m } else { 0 });+ let (j0, j1) = (if edges.2 { m } else { 0 }, t - if edges.3 { m } else { 0 });+ let cells = ((i1 - i0) * (j1 - j0)) as f64;+ let plane_sum = |pl: usize| -> f64 {+ (j0..j1)+ .map(|j| input.get(pl * t * t + j * t + i0..pl * t * t + j * t + i1).map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>()))+ .sum()+ };+ let in_mean = [plane_sum(0) / cells, (plane_sum(1) + plane_sum(2)) / (2.0 * cells), plane_sum(3) / cells];+ for (c, &mean_in) in in_mean.iter().enumerate() {+ let sum_out: f64 = (2 * j0..2 * j1)+ .map(|y| {+ out.get(c * side * side + y * side + 2 * i0..c * side * side + y * side + 2 * i1)+ .map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>())+ })+ .sum();+ let mean_out = sum_out / (4.0 * cells);+ let measured = if mean_in as f32 >= MIN_SIGNAL && mean_out > 0.0 { (mean_out / mean_in) as f32 } else { nominal };+ let g = measured.clamp(nominal * (1.0 - max_deviation), nominal * (1.0 + max_deviation));+ if g.is_finite() && g > 0.0 {+ let inv = 1.0 / g;+ if let Some(plane) = out.get_mut(c * side * side..(c + 1) * side * side) {+ plane.iter_mut().for_each(|v| *v *= inv);+ }+ }+ }+ }+ if let Some(clip) = p.clip {+ protect_clipped(input, out, t, clip);+ }+}++/// Where the mosaic is at (or within a cell of) white, the output becomes the plain cell values again, so a clipped+/// area stays exactly as clipped as the camera recorded it (highlight recovery later depends on that).+fn protect_clipped(input: &[f32], out: &mut [f32], t: usize, clip: f32) {+ let side = 2 * t;+ let plane = t * t;+ let (Some(r), Some(g1), Some(g2), Some(b)) =+ (input.get(0..plane), input.get(plane..2 * plane), input.get(2 * plane..3 * plane), input.get(3 * plane..4 * plane))+ else {+ return;+ };+ let mut m: Vec<f32> = (0..plane).map(|i| r[i].max(g1[i]).max(g2[i]).max(b[i])).collect();+ if !m.iter().any(|&v| v >= clip) {+ return;+ }+ // 3×3 maximum over cells (separable)+ let dilate = |src: &[f32], horizontal: bool| -> Vec<f32> {+ (0..plane)+ .map(|i| {+ let (x, y) = (i % t, i / t);+ let mut best = src[i];+ for d in [-1isize, 1] {+ let (nx, ny) = if horizontal { (x as isize + d, y as isize) } else { (x as isize, y as isize + d) };+ if nx >= 0 && ny >= 0 && (nx as usize) < t && (ny as usize) < t {+ best = best.max(src[ny as usize * t + nx as usize]);+ }+ }+ best+ })+ .collect()+ };+ m = dilate(&dilate(&m, true), false);+ let lo = clip - 0.03;+ for j in 0..t {+ for i in 0..t {+ let k = j * t + i;+ let s = ((m[k] - lo) / (clip - lo)).clamp(0.0, 1.0);+ if s <= 0.0 {+ continue;+ }+ let s = s * s * (3.0 - 2.0 * s);+ let plain = [r[k], 0.5 * (g1[k] + g2[k]), b[k]];+ for (c, &pv) in plain.iter().enumerate() {+ for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {+ if let Some(v) = out.get_mut(c * side * side + (2 * j + sy) * side + 2 * i + sx) {+ *v += (pv - *v) * s;+ }+ }+ }+ }+ }+}++/// Add a finished tile to the picture with its blend weights.+#[allow(clippy::too_many_arguments)]+fn blend_tile(+ out: &[f32],+ acc: &mut [[f32; 3]],+ wsum: &mut [f32],+ size: (usize, usize),+ cells: (usize, usize),+ layout: Layout,+ at: (usize, usize),+ p: &Params,+ edges: (bool, bool, bool, bool),+) {+ let (width, height) = size;+ let (nx, ny) = cells;+ let (t, side) = (p.tile, 2 * p.tile);+ let fade = p.overlap;+ let wx: Vec<f32> = (0..t).map(|i| weight(i, t, fade, edges.0, edges.1)).collect();+ let wy: Vec<f32> = (0..t).map(|j| weight(j, t, fade, edges.2, edges.3)).collect();+ for j in 0..t {+ let cy = at.1 + j;+ for i in 0..t {+ let cx = at.0 + i;+ // cells past the picture's edge (a picture smaller than one tile is padded) belong to no pixel+ if cx >= nx || cy >= ny {+ continue;+ }+ let (Some(&a), Some(&b)) = (wx.get(i), wy.get(j)) else { continue };+ let w = a * b;+ if let Some(s) = wsum.get_mut(cy * nx + cx) {+ *s += w;+ }+ let (px0, py0) = layout.position(cx as isize, cy as isize);+ for (sy, sx) in [(0isize, 0isize), (0, 1), (1, 0), (1, 1)] {+ let (x, y) = (px0 + sx, py0 + sy);+ if x < 0 || y < 0 || x as usize >= width || y as usize >= height {+ continue;+ }+ let o = (2 * j + sy as usize) * side + 2 * i + sx as usize;+ let (Some(&r), Some(&g), Some(&bl)) = (out.get(o), out.get(side * side + o), out.get(2 * side * side + o)) else { continue };+ if let Some(px) = acc.get_mut(y as usize * width + x as usize) {+ px[0] += w * r;+ px[1] += w * g;+ px[2] += w * bl;+ }+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ /// A stand-in model: the cell's samples as an RGB picture at twice the resolution (a plain 2 × 2 "demosaic"),+ /// times `scale`, so the output has a scale of its own like the real model's.+ struct Mock {+ tile: usize,+ scale: f32,+ }+ impl TileRunner for Mock {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {+ let (t, side) = (self.tile, 2 * self.tile);+ let plane = t * t;+ let mut out = vec![0.0; 3 * side * side];+ for j in 0..t {+ for i in 0..t {+ let k = j * t + i;+ let rgb = [input[k], 0.5 * (input[plane + k] + input[2 * plane + k]), input[3 * plane + k]];+ for (c, v) in rgb.iter().enumerate() {+ for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {+ out[c * side * side + (2 * j + sy) * side + 2 * i + sx] = v * self.scale;+ }+ }+ }+ }+ Ok(out)+ }+ }++ fn params(tile: usize, overlap: usize, gain: Gain) -> Params {+ Params { tile, overlap, gain, clip: None, parallel: 2 }+ }++ #[test]+ fn overflowing_tiles_and_nan_gain_are_errors_before_the_model_runs() {+ let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();+ let model = Mock { tile: 16, scale: 1.0 };+ for (tile, overlap, gain) in [+ (usize::MAX, 0, Gain::None),+ (16, usize::MAX, Gain::None),+ (16, 0, Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN }),+ (16, 0, Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 }),+ ] {+ assert!(matches!(denoise_bayer(&[0.5], 1, 1, layout, &model, ¶ms(tile, overlap, gain), &Control::default()), Err(Error::Input(_))));+ }+ }++ /// A smooth picture whose colour at a sample is `base[colour] · (1 + shading)`.+ fn mosaic(w: usize, h: usize, cell: [u8; 4]) -> Vec<f32> {+ (0..w * h)+ .map(|i| {+ let (x, y) = (i % w, i / w);+ let c = cell[(y % 2) * 2 + x % 2] as usize;+ [0.5, 0.3, 0.2][c] * (0.6 + 0.4 * ((x as f32 / 23.0).sin() * (y as f32 / 31.0).cos()))+ })+ .collect()+ }++ fn reference(m: &[f32], w: usize, h: usize, layout: Layout) -> Rgb32f {+ // what the mock gives for the whole picture in one tile+ let (nx, ny) = layout.cells(w, h);+ let t = nx.max(ny).div_ceil(16) * 16;+ denoise_bayer(m, w, h, layout, &Mock { tile: t, scale: 1.0 }, ¶ms(t, 16, Gain::None), &Control::default()).unwrap()+ }++ #[test]+ fn tiles_blend_to_the_same_picture_as_one_tile_for_every_layout() {+ for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {+ let layout = Layout::from_cell(cell).unwrap();+ let (w, h) = (150usize, 110usize);+ let m = mosaic(w, h, cell);+ let want = reference(&m, w, h, layout);+ let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &Control::default()).unwrap();+ assert_eq!((got.width, got.height), (w, h));+ for (a, b) in got.data.iter().zip(&want.data) {+ for c in 0..3 {+ assert!((a[c] - b[c]).abs() < 1e-5, "layout {cell:?}: {a:?} vs {b:?}");+ }+ }+ }+ }++ #[test]+ fn the_picture_does_not_depend_on_how_many_tiles_run_at_once_or_how_long_each_takes() {+ // tiles that finish in a different order every time: blending still adds them in tile order+ struct Jittery(Mock);+ impl TileRunner for Jittery {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {+ let wait = (input.iter().take(64).sum::<f32>() * 1000.0) as u64 % 7;+ std::thread::sleep(std::time::Duration::from_millis(wait));+ self.0.run(input)+ }+ }+ let layout = Layout { dx: 0, dy: 1 };+ let (w, h) = (150usize, 110usize);+ let m = mosaic(w, h, [1, 0, 2, 1]);+ let run = |parallel: usize| {+ let mut p = params(32, 8, Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 });+ p.parallel = parallel;+ denoise_bayer(&m, w, h, layout, &Jittery(Mock { tile: 32, scale: 1.01e6 }), &p, &Control::default()).unwrap()+ };+ let one = run(1);+ for parallel in [2, 3, 8] {+ assert_eq!(run(parallel).data, one.data, "{parallel} at once");+ }+ }++ #[test]+ fn a_failing_tile_or_a_cancel_ends_the_picture_without_a_panic() {+ struct Fails(Mock, std::sync::atomic::AtomicUsize);+ impl TileRunner for Fails {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {+ if self.1.fetch_add(1, Ordering::Relaxed) == 4 {+ return Err(Error::Runtime("boom".into()));+ }+ self.0.run(input)+ }+ }+ let layout = Layout { dx: 0, dy: 0 };+ let (w, h) = (150usize, 110usize);+ let m = mosaic(w, h, [0, 1, 1, 2]);+ let r = denoise_bayer(+ &m,+ w,+ h,+ layout,+ &Fails(Mock { tile: 32, scale: 1.0 }, Default::default()),+ ¶ms(32, 8, Gain::None),+ &Control::default(),+ );+ assert!(matches!(r, Err(Error::Runtime(_))), "{r:?}");+ let stop = AtomicBool::new(true);+ let ctl = Control { cancel: Some(&stop), progress: None };+ let r = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &ctl);+ assert!(matches!(r, Err(Error::Cancelled)));+ }++ #[test]+ fn a_scale_of_its_own_is_taken_out_tile_by_tile() {+ let layout = Layout { dx: 0, dy: 0 };+ let (w, h) = (150usize, 110usize);+ let m = mosaic(w, h, [0, 1, 1, 2]);+ let want = reference(&m, w, h, layout);+ let gain = Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 };+ // the scale differs a little from nominal; each tile's own scale is measured and removed+ let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.02e6 }, ¶ms(32, 8, gain), &Control::default()).unwrap();+ let mean = |img: &Rgb32f, c: usize| img.data.iter().map(|p| f64::from(p[c])).sum::<f64>() / img.data.len() as f64;+ for c in 0..3 {+ let (a, b) = (mean(&got, c), mean(&want, c));+ assert!((a - b).abs() / b < 0.01, "channel {c}: {a} vs {b}");+ }+ // a scale far from nominal is clamped rather than trusted+ let wild = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 3.0e6 }, ¶ms(32, 8, gain), &Control::default()).unwrap();+ assert!(mean(&wild, 1) > 2.0 * mean(&want, 1));+ }++ #[test]+ fn a_picture_smaller_than_a_tile_is_padded_by_mirroring() {+ let layout = Layout { dx: 1, dy: 1 };+ let (w, h) = (20usize, 14usize);+ let m = mosaic(w, h, [2, 1, 1, 0]);+ let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 64, scale: 1.0 }, ¶ms(64, 16, Gain::None), &Control::default()).unwrap();+ assert_eq!((got.width, got.height), (w, h));+ assert!(got.data.iter().all(|p| p.iter().all(|v| v.is_finite() && *v > 0.0)));+ }++ #[test]+ fn clipped_highlights_stay_clipped() {+ // a model that darkens everything by 10 % would pull a clipped plateau below white+ struct Dim(Mock);+ impl TileRunner for Dim {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {+ Ok(self.0.run(input)?.into_iter().map(|v| v * 0.9).collect())+ }+ }+ let layout = Layout { dx: 0, dy: 0 };+ let (w, h) = (96usize, 96usize);+ let m: Vec<f32> = (0..w * h).map(|i| if i % w > 40 && i % w < 70 { 1.0 } else { 0.4 }).collect();+ let mut p = params(32, 8, Gain::None);+ p.clip = Some(0.99);+ let got = denoise_bayer(&m, w, h, layout, &Dim(Mock { tile: 32, scale: 1.0 }), &p, &Control::default()).unwrap();+ let mid = got.data[48 * w + 55];+ assert!(mid.iter().all(|&v| v > 0.995), "plateau {mid:?}");+ let dark = got.data[48 * w + 10];+ assert!(dark.iter().all(|&v| (v - 0.36).abs() < 0.01), "unclipped area still goes through the model: {dark:?}");+ }++ #[test]+ fn cancelling_and_progress_work() {+ let layout = Layout { dx: 0, dy: 0 };+ let (w, h) = (150usize, 110usize);+ let m = mosaic(w, h, [0, 1, 1, 2]);+ let cancel = AtomicBool::new(true);+ let r = denoise_bayer(+ &m,+ w,+ h,+ layout,+ &Mock { tile: 32, scale: 1.0 },+ ¶ms(32, 8, Gain::None),+ &Control { cancel: Some(&cancel), progress: None },+ );+ assert!(matches!(r, Err(Error::Cancelled)));+ let seen = std::sync::Mutex::new(Vec::new());+ let f = |d: usize, n: usize| seen.lock().unwrap().push((d, n));+ denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &Control { cancel: None, progress: Some(&f) })+ .unwrap();+ let seen = seen.into_inner().unwrap();+ let total = tile_count(w, h, layout, 32, 8);+ assert!(total > 1);+ assert_eq!(seen.last(), Some(&(total, total)));+ assert!(seen.windows(2).all(|p| p[1].0 == p[0].0 + 1));+ }++ #[test]+ fn hostile_inputs_are_errors() {+ let layout = Layout { dx: 0, dy: 0 };+ let mock = Mock { tile: 32, scale: 1.0 };+ let p = params(32, 8, Gain::None);+ assert!(matches!(denoise_bayer(&[], 0, 0, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));+ assert!(matches!(denoise_bayer(&[0.0; 10], 4, 4, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));+ assert!(matches!(denoise_bayer(&[0.0; 16], 4, 4, layout, &mock, ¶ms(32, 16, Gain::None), &Control::default()), Err(Error::Input(_))));+ struct Bad(Vec<f32>);+ impl TileRunner for Bad {+ fn run(&self, _: &[f32]) -> Result<Vec<f32>, Error> {+ Ok(self.0.clone())+ }+ }+ assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_))));+ assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_))));+ }+}diff --git a/crates/denoise/src/runtime.rs b/crates/denoise/src/runtime.rsnew file mode 100644index 000000000..4314537a5--- /dev/null+++ b/crates/denoise/src/runtime.rs@@ -0,0 +1,423 @@+//! Model loading, a shared installation self-test, and deterministic test tiles.+//! `CpuRunner` is the pure-Rust executor; scalar operator tests live in `reference`.++use std::path::Path;++use web_time::Instant;++use serde::Serialize;++use crate::manifest::{DenoiserManifest, Gain};+use crate::run::{Error, TileRunner};++/// What `self_test` found.+#[derive(Clone, Debug, PartialEq, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct SelfTest {+ pub ok: bool,+ /// Milliseconds to load and optimise the model.+ pub load_ms: f64,+ /// Median milliseconds to run one tile.+ pub tile_ms: f64,+ /// Megapixels of picture one tile covers, for estimating a whole photo's time.+ pub tile_megapixels: f64,+ /// The model's output scale relative to its input on the test picture (what `Gain::MatchMean` takes out).+ pub scale: f64,+ /// Each check: what was checked and whether it held.+ pub checks: Vec<(String, bool)>,+}++/// The default pure-Rust CPU model, with the same manifest and installation self-test as the reference.+#[derive(Clone)]+pub struct CpuRunner {+ runner: crate::cpu::NetRunner,+ manifest: DenoiserManifest,+ load_ms: f64,+}+impl CpuRunner {+ pub fn load(path: &Path, manifest: &DenoiserManifest) -> Result<Self, Error> {+ crate::manifest::validate(manifest).map_err(|e| Error::Load(e.to_string()))?;+ let started = Instant::now();+ let net = crate::onnx::read(path).map_err(|e| Error::Load(e.to_string()))?;+ let runner = crate::cpu::NetRunner::new(&net, manifest.tile as usize).map_err(|e| Error::Load(e.to_string()))?;+ Ok(Self { runner, manifest: manifest.clone(), load_ms: started.elapsed().as_secs_f64() * 1e3 })+ }+ pub fn manifest(&self) -> &DenoiserManifest {+ &self.manifest+ }+ pub fn net(&self) -> &crate::net::Net {+ self.runner.net()+ }+ pub fn bytes_per_tile(&self) -> usize {+ self.runner.bytes_per_tile()+ }+ pub fn self_test(&self) -> SelfTest {+ self_test(self, &self.manifest, self.load_ms)+ }+}+impl TileRunner for CpuRunner {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {+ self.runner.run(input)+ }+}+fn self_test(runner: &dyn TileRunner, manifest: &DenoiserManifest, load_ms: f64) -> SelfTest {+ let t = manifest.tile as usize;+ let mut checks: Vec<(String, bool)> = Vec::new();+ let (input, base) = test_tile(t);+ let (mut times, mut scale) = (Vec::new(), 0.0f64);+ let mut run = || {+ let started = Instant::now();+ let r = runner.run(&input);+ times.push(started.elapsed().as_secs_f64() * 1000.0);+ r+ };+ let (a, b) = (run(), run());+ match (&a, &b) {+ (Ok(a), Ok(b)) => {+ checks.push((format!("gives {} finite numbers per tile", a.len()), true));+ checks.push((+ "gives the same answer for the same tile".into(),+ a.iter().zip(b).all(|(x, y)| (x - y).abs() <= 1e-4 * x.abs().max(y.abs()).max(1e-9)),+ ));+ let side = 2 * t;+ let mean_in = base.iter().map(|&v| f64::from(v)).sum::<f64>() / base.len().max(1) as f64;+ let mean_out = a.iter().map(|&v| f64::from(v)).sum::<f64>() / a.len().max(1) as f64;+ scale = if mean_in > 0.0 { mean_out / mean_in } else { 0.0 };+ if let Gain::MatchMean { nominal, max_deviation } = manifest.gain {+ let ratio = scale / f64::from(nominal);+ let tolerance = f64::from(max_deviation).max(0.05) * 4.0;+ checks.push((format!("its scale is the {nominal:.3e} the description says (found {scale:.3e})"), (ratio - 1.0).abs() <= tolerance));+ }+ // noise left after bringing the answer back to the input's scale, against the noise that went in+ let g = if scale > 0.0 { 1.0 / scale } else { 1.0 };+ let (mut noise_in, mut noise_out, mut n_in, mut n_out) = (0.0f64, 0.0f64, 0usize, 0usize);+ for (i, (&v, &s)) in input.iter().zip(base.iter().cycle()).enumerate() {+ // centre of each plane only: edges are the model's guess+ let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t);+ if x >= t / 4 && x < 3 * t / 4 && y >= t / 4 && y < 3 * t / 4 {+ noise_in += f64::from(v - s).powi(2);+ n_in += 1;+ }+ }+ for c in 0..3 {+ for y in side / 4..3 * side / 4 {+ for x in side / 4..3 * side / 4 {+ if let (Some(&o), Some(&s)) = (a.get(c * side * side + y * side + x), base.get((y / 2) * t + x / 2)) {+ noise_out += (f64::from(o) * g - f64::from(s)).powi(2);+ n_out += 1;+ }+ }+ }+ }+ let (rms_in, rms_out) = ((noise_in / n_in.max(1) as f64).sqrt(), (noise_out / n_out.max(1) as f64).sqrt());+ checks.push((format!("takes noise down (from {rms_in:.4} to {rms_out:.4})"), rms_out < 0.8 * rms_in));+ }+ (Err(e), _) | (_, Err(e)) => checks.push((format!("runs: {e}"), false)),+ }+ times.sort_by(|x, y| x.total_cmp(y));+ SelfTest {+ ok: !checks.is_empty() && checks.iter().all(|(_, ok)| *ok),+ load_ms,+ tile_ms: times.get(times.len() / 2).copied().unwrap_or(0.0),+ tile_megapixels: (2 * t * 2 * t) as f64 / 1e6,+ scale,+ checks,+ }+}++/// A deterministic test tile: four planes of a smooth gradient with noise on top, and the gradient alone for one+/// plane (`tile × tile`).+pub fn test_tile(t: usize) -> (Vec<f32>, Vec<f32>) {+ if t == 0 || t > 2048 {+ return (Vec::new(), Vec::new());+ }+ let mut seed = 0x2545_f491u32;+ let mut next = move || {+ seed ^= seed << 13;+ seed ^= seed >> 17;+ seed ^= seed << 5;+ (seed >> 8) as f32 / (1u32 << 24) as f32+ };+ let base: Vec<f32> = (0..t * t).map(|i| 0.12 + 0.3 * (((i % t) + (i / t)) as f32 / (2 * t) as f32)).collect();+ let input = (0..4 * t * t)+ .map(|i| {+ // sum of four uniforms: close to Gaussian, deviation 0.0577 × 2 before scaling+ let n = next() + next() + next() + next() - 2.0;+ base.get(i % (t * t)).copied().unwrap_or(0.0) + 0.05 * n+ })+ .collect();+ (input, base)+}++/// A tile for checking that two runners of one model agree: the whole range a raw photo has rather than the test tile's+/// mid-greys — a noisy ramp from black to white across the tile, a flat black corner and a clipped white one (four+/// planes of `tile × tile`).+pub fn check_tile(t: usize) -> Vec<f32> {+ if t == 0 || t > 2048 {+ return Vec::new();+ }+ let mut seed = 0x9e37_79b9u32;+ let mut next = move || {+ seed ^= seed << 13;+ seed ^= seed >> 17;+ seed ^= seed << 5;+ (seed >> 8) as f32 / (1u32 << 24) as f32+ };+ let (edge, span) = ((t / 8).max(1), (2 * t).saturating_sub(2).max(1) as f32);+ (0..4 * t * t)+ .map(|i| {+ let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t);+ let n = next() + next() + next() + next() - 2.0;+ if x < edge && y < edge {+ 0.0+ } else if x + edge >= t && y + edge >= t {+ 1.0+ } else {+ ((x + y) as f32 / span + 0.05 * n).clamp(0.0, 1.0)+ }+ })+ .collect()+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::licence::Licence;+ use crate::manifest::Domain;+ use crate::synthetic_proto::{len_field, value_info_bytes};++ fn varint(mut v: u64, out: &mut Vec<u8>) {+ loop {+ let b = (v & 0x7f) as u8;+ v >>= 7;+ if v == 0 {+ out.push(b);+ return;+ }+ out.push(b | 0x80);+ }+ }++ fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {+ varint(n << 3, out);+ varint(v, out);+ }++ fn node(op: &str, inputs: &[&str], output: &str, ints: &[(&str, i64)]) -> Vec<u8> {+ let mut n = Vec::new();+ for i in inputs {+ len_field(1, i.as_bytes(), &mut n);+ }+ len_field(2, output.as_bytes(), &mut n);+ len_field(4, op.as_bytes(), &mut n);+ for (name, value) in ints {+ let mut a = Vec::new();+ len_field(1, name.as_bytes(), &mut a);+ varint_field(3, *value as u64, &mut a);+ varint_field(20, 2, &mut a);+ len_field(5, &a, &mut n);+ }+ n+ }++ fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec<u8> {+ let mut t = Vec::new();+ for d in dims {+ varint_field(1, *d, &mut t);+ }+ varint_field(2, 1, &mut t);+ len_field(8, name.as_bytes(), &mut t);+ let raw: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();+ len_field(9, &raw, &mut t);+ t+ }++ /// A tiny working denoiser-shaped network: `[1, 4, t, t]` → a 1 × 1 convolution to 12 channels (each output+ /// channel is `scale` times one input plane, so the scale is known) → depth-to-space → `[1, 3, 2t, 2t]`.+ fn tiny_model(t: u64, scale: f32) -> Vec<u8> {+ tiny_model_with(t, scale, 12)+ }++ /// The same with another number of convolution channels; only 12 can be arranged into the three colours.+ fn tiny_model_with(t: u64, scale: f32, channels: u64) -> Vec<u8> {+ let mut graph = Vec::new();+ len_field(1, &node("Conv", &["data", "w"], if channels == 12 { "c" } else { "rgb" }, &[("group", 1)]), &mut graph);+ if channels == 12 {+ let mut depth_to_space = node("DepthToSpace", &["c"], "rgb", &[("blocksize", 2)]);+ let mut mode = Vec::new();+ len_field(1, b"mode", &mut mode);+ len_field(4, b"CRD", &mut mode);+ varint_field(20, 3, &mut mode);+ len_field(5, &mode, &mut depth_to_space);+ len_field(1, &depth_to_space, &mut graph);+ }+ // CRD output channel colour·4 + dy·2 + dx reads input plane [R, G1, B] of the tile.+ let mut w = vec![0f32; channels as usize * 4];+ for oc in 0..channels as usize {+ let colour = (oc / 4).min(2);+ let plane = [0, 1, 3][colour];+ w[oc * 4 + plane] = scale;+ }+ len_field(5, &tensor("w", &[channels, 4, 1, 1], &w), &mut graph);+ len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(t), Ok(t)]), &mut graph);+ len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * t), Ok(2 * t)]), &mut graph);+ let mut ops = Vec::new();+ len_field(1, b"", &mut ops);+ varint_field(2, 13, &mut ops);+ let mut model = Vec::new();+ varint_field(1, 8, &mut model);+ len_field(7, &graph, &mut model);+ len_field(8, &ops, &mut model);+ model+ }++ fn manifest(tile: u32, nominal: f32) -> DenoiserManifest {+ DenoiserManifest {+ id: "tiny".into(),+ name: "Tiny".into(),+ version: "1".into(),+ licence: Licence::default(),+ source: None,+ sha256: None,+ size_bytes: None,+ provenance: String::new(),+ domain: Domain::BayerToRgb,+ tile,+ overlap: 16,+ gain: Gain::MatchMean { nominal, max_deviation: 0.05 },+ }+ }++ fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {+ let p = std::env::temp_dir().join(format!("lc-denoise-runtime-{name}-{}.onnx", std::process::id()));+ std::fs::write(&p, bytes).unwrap();+ p+ }++ #[test]+ fn a_working_model_loads_runs_and_has_the_scale_it_says() {+ let p = temp_model("ok", &tiny_model(64, 1000.0));+ let r = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap();+ let input: Vec<f32> = (0..4 * 64 * 64).map(|i| (i % 97) as f32 / 97.0).collect();+ let out = r.run(&input).unwrap();+ assert_eq!(out.len(), 3 * 128 * 128);+ // output (colour c, pixel (2y+dy, 2x+dx)) = scale × input plane [R, G1, B][c] at (y, x)+ for (c, plane) in [0usize, 1, 3].into_iter().enumerate() {+ let (y, x) = (5usize, 9usize);+ assert!((out[c * 128 * 128 + (2 * y + 1) * 128 + 2 * x] - 1000.0 * input[plane * 64 * 64 + y * 64 + x]).abs() < 1e-2);+ }+ let _ = std::fs::remove_file(p);+ }++ #[test]+ fn the_self_test_checks_scale_and_noise() {+ // a model that only multiplies cannot take noise down: it must fail that check but pass the others+ let p = temp_model("selftest", &tiny_model(64, 1000.0));+ let t = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap().self_test();+ assert!(!t.ok, "{t:?}");+ assert!(t.checks.iter().any(|(c, ok)| c.contains("same answer") && *ok), "{t:?}");+ assert!(t.checks.iter().any(|(c, ok)| c.contains("scale") && *ok), "{t:?}");+ assert!(t.checks.iter().any(|(c, ok)| c.contains("noise") && !*ok), "{t:?}");+ // and one that claims the wrong scale fails that check+ let q = CpuRunner::load(&p, &manifest(64, 5.0)).unwrap().self_test();+ assert!(q.checks.iter().any(|(c, ok)| c.contains("scale") && !*ok), "{q:?}");+ let _ = std::fs::remove_file(p);+ }++ #[test]+ fn wrong_sizes_and_broken_files_are_errors() {+ let p = temp_model("sizes", &tiny_model(64, 1.0));+ let r = CpuRunner::load(&p, &manifest(64, 1.0)).unwrap();+ assert!(matches!(r.run(&[0.0; 10]), Err(Error::Input(_))));+ // a fully convolutional model works at whatever tile size the manifest asks for+ let wide = CpuRunner::load(&p, &manifest(128, 1.0)).unwrap();+ assert_eq!(wide.run(&vec![0.5; 4 * 128 * 128]).unwrap().len(), 3 * 256 * 256);+ // a model that gives another number of values than the manifest promises is an error+ let q = temp_model("channels", &tiny_model_with(64, 1.0, 8));+ assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "shape is refused before execution");+ let _ = std::fs::remove_file(q);+ for (name, bytes) in [("junk", b"nonsense".to_vec()), ("empty", Vec::new())] {+ let q = temp_model(name, &bytes);+ assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "{name}");+ let _ = std::fs::remove_file(q);+ }+ assert!(CpuRunner::load(&std::env::temp_dir().join("does-not-exist.onnx"), &manifest(64, 1.0)).is_err());+ let _ = std::fs::remove_file(p);+ }++ #[test]+ fn the_check_tile_covers_black_to_white() {+ for t in [1, 2, 7, 64] {+ let c = check_tile(t);+ assert_eq!(c.len(), 4 * t * t);+ assert!(c.iter().all(|v| (0.0..=1.0).contains(v)), "{t}");+ }+ let c = check_tile(64);+ let near = |lo: f32, hi: f32| c.iter().filter(|v| (lo..=hi).contains(*v)).count();+ assert!(near(0.0, 0.0) >= 4 * 8 * 8 && near(1.0, 1.0) >= 4 * 8 * 8, "a flat black and a clipped white corner");+ for band in 0..10 {+ let lo = band as f32 / 10.0;+ assert!(near(lo, lo + 0.1) > 100, "values around {lo}");+ }+ assert_eq!(check_tile(64), c, "the same tile every time");+ }++ #[test]+ fn truncated_and_corrupted_models_never_panic() {+ let good = tiny_model(64, 1.0);+ for n in (0..good.len()).step_by(5) {+ let q = temp_model("trunc", &good[..n]);+ let _ = CpuRunner::load(&q, &manifest(64, 1.0));+ let _ = std::fs::remove_file(q);+ }+ for at in (0..good.len()).step_by(7) {+ let mut bad = good.clone();+ if let Some(b) = bad.get_mut(at) {+ *b = b.wrapping_add(0x55);+ }+ let q = temp_model("flip", &bad);+ if let Ok(r) = CpuRunner::load(&q, &manifest(64, 1.0)) {+ let _ = r.run(&vec![0.5; 4 * 64 * 64]);+ }+ let _ = std::fs::remove_file(q);+ }+ }++ /// Opt-in: `LC_DENOISE_MODEL=<model_bayer.onnx> cargo test -p lightcraft-denoise --release+ /// -- --ignored --nocapture` self-tests a real model (RawNIND's Bayer model: tile 512, scale about 1e6).+ #[test]+ #[ignore = "needs a real model file: set LC_DENOISE_MODEL"]+ fn a_real_model_passes_the_self_test() {+ let Some(path) = std::env::var_os("LC_DENOISE_MODEL") else { return };+ let m = manifest(512, 1.0e6);+ let r = CpuRunner::load(Path::new(&path), &m).unwrap();+ let t = r.self_test();+ println!("load {:.0} ms | {:.0} ms per tile | scale {:.4e} | {:?}", t.load_ms, t.tile_ms, t.scale, t.checks);+ assert!(t.ok);+ }+}++#[cfg(test)]+mod cpu_tests {+ use super::*;+ #[test]+ fn manifest_and_test_tile_sizes_are_checked_without_tract() {+ for t in [0, 2049, usize::MAX] {+ assert!(test_tile(t).0.is_empty());+ assert!(check_tile(t).is_empty());+ }+ let path = std::env::temp_dir().join(format!("lc-cpu-load-{}.onnx", std::process::id()));+ std::fs::write(&path, crate::synthetic::unet_onnx(64, 5, 1, 7)).unwrap();+ let mut m = crate::known::find("rawnind-bayer").unwrap().manifest;+ m.tile = 64;+ m.overlap = 16;+ m.gain = Gain::None;+ let r = CpuRunner::load(&path, &m).unwrap();+ let x = check_tile(64);+ assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());+ assert!(r.self_test().checks.iter().any(|(text, ok)| text.contains("same answer") && *ok));+ m.tile = u32::MAX;+ assert!(CpuRunner::load(&path, &m).is_err());+ std::fs::remove_file(path).unwrap();+ }+}diff --git a/crates/denoise/src/synthetic.rs b/crates/denoise/src/synthetic.rsnew file mode 100644index 000000000..8ceafca77--- /dev/null+++ b/crates/denoise/src/synthetic.rs@@ -0,0 +1,234 @@+//! A small ONNX U-Net built in memory, for tests of the layers that read and run denoise networks (here, in the GPU+//! runner and in the engine): the same layer kinds and wiring as the real denoisers, with deterministic random weights.+//! Nothing here is a model anyone would use.++use crate::synthetic_proto::{len_field, value_info_bytes};++fn varint(mut v: u64, out: &mut Vec<u8>) {+ loop {+ let b = (v & 0x7f) as u8;+ v >>= 7;+ if v == 0 {+ out.push(b);+ return;+ }+ out.push(b | 0x80);+ }+}++fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {+ varint(n << 3, out);+ varint(v, out);+}++enum A<'a> {+ Int(i64),+ Ints(&'a [i64]),+ Float(f32),+ Str(&'a str),+}++fn node(op: &str, inputs: &[&str], output: &str, attrs: &[(&str, A)]) -> Vec<u8> {+ let mut n = Vec::new();+ for i in inputs {+ len_field(1, i.as_bytes(), &mut n);+ }+ len_field(2, output.as_bytes(), &mut n);+ len_field(4, op.as_bytes(), &mut n);+ for (name, value) in attrs {+ let mut a = Vec::new();+ len_field(1, name.as_bytes(), &mut a);+ match value {+ A::Int(v) => {+ varint_field(3, *v as u64, &mut a);+ varint_field(20, 2, &mut a);+ }+ A::Ints(vs) => {+ for v in *vs {+ varint_field(8, *v as u64, &mut a);+ }+ varint_field(20, 7, &mut a);+ }+ A::Float(v) => {+ varint((2 << 3) | 5, &mut a);+ a.extend_from_slice(&v.to_le_bytes());+ varint_field(20, 1, &mut a);+ }+ A::Str(s) => {+ len_field(4, s.as_bytes(), &mut a);+ varint_field(20, 3, &mut a);+ }+ }+ len_field(5, &a, &mut n);+ }+ n+}++fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec<u8> {+ let mut t = Vec::new();+ for d in dims {+ varint_field(1, *d, &mut t);+ }+ varint_field(2, 1, &mut t);+ len_field(8, name.as_bytes(), &mut t);+ let raw: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();+ len_field(9, &raw, &mut t);+ t+}++struct Rng(u64);++impl Rng {+ fn next(&mut self) -> f32 {+ self.0 ^= self.0 >> 12;+ self.0 ^= self.0 << 25;+ self.0 ^= self.0 >> 27;+ ((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0+ }+}++struct Builder {+ nodes: Vec<Vec<u8>>,+ inits: Vec<Vec<u8>>,+ rng: Rng,+ n: usize,+}++impl Builder {+ fn name(&mut self, kind: &str) -> String {+ self.n += 1;+ format!("{kind}{}", self.n)+ }++ /// Random weights scaled so activations keep a sane size through the layers.+ fn weights(&mut self, name: &str, dims: &[u64], fan_in: usize) -> String {+ let count: u64 = dims.iter().product();+ let scale = (3.0 / fan_in as f32).sqrt();+ let values: Vec<f32> = (0..count).map(|_| self.rng.next() * scale).collect();+ self.inits.push(tensor(name, dims, &values));+ name.to_string()+ }++ fn bias(&mut self, name: &str, n: u64) -> String {+ let values: Vec<f32> = (0..n).map(|_| self.rng.next() * 0.1).collect();+ self.inits.push(tensor(name, &[n], &values));+ name.to_string()+ }++ /// `Conv` (+ `LeakyRelu` when `leaky`); gives the name of the tensor that results.+ fn conv(&mut self, src: &str, cin: u64, cout: u64, k: u64, leaky: bool) -> String {+ let (w, b, out) = (self.name("w"), self.name("b"), self.name("conv"));+ let w = self.weights(&w, &[cout, cin, k, k], (cin * k * k) as usize);+ let b = self.bias(&b, cout);+ let pad = (k as i64 - 1) / 2;+ self.nodes.push(node(+ "Conv",+ &[src, &w, &b],+ &out,+ &[("kernel_shape", A::Ints(&[k as i64, k as i64])), ("pads", A::Ints(&[pad; 4])), ("strides", A::Ints(&[1, 1])), ("group", A::Int(1))],+ ));+ if !leaky {+ return out;+ }+ let act = self.name("act");+ self.nodes.push(node("LeakyRelu", &[&out], &act, &[("alpha", A::Float(0.2))]));+ act+ }++ fn pool(&mut self, src: &str) -> String {+ let out = self.name("pool");+ self.nodes.push(node(+ "MaxPool",+ &[src],+ &out,+ &[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))],+ ));+ out+ }++ fn up(&mut self, src: &str, cin: u64, cout: u64) -> String {+ let (w, b, out) = (self.name("uw"), self.name("ub"), self.name("up"));+ let w = self.weights(&w, &[cin, cout, 2, 2], (cin * 4) as usize);+ let b = self.bias(&b, cout);+ self.nodes.push(node(+ "ConvTranspose",+ &[src, &w, &b],+ &out,+ &[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))],+ ));+ out+ }++ fn concat(&mut self, a: &str, b: &str) -> String {+ let out = self.name("cat");+ self.nodes.push(node("Concat", &[a, b], &out, &[("axis", A::Int(1))]));+ out+ }+}++/// A denoiser-shaped U-Net as ONNX bytes: `[1, 4, tile, tile]` → `[1, 3, 2·tile, 2·tile]`, with `depth` poolings+/// (so `tile` must be a multiple of `2^depth`) and `ch` channels on the first level, doubling on each (every count a+/// multiple of 4). Each level is two 3 × 3 convolutions with a leaky ReLU, as in the real networks; the output is a+/// 1 × 1 convolution to 12 channels and depth-to-space.+pub fn unet_onnx(tile: u64, ch: u64, depth: usize, seed: u64) -> Vec<u8> {+ let mut b = Builder { nodes: Vec::new(), inits: Vec::new(), rng: Rng(seed | 1), n: 0 };+ let mut skips: Vec<(String, u64)> = Vec::new();+ let (mut cur, mut c_in, mut c) = ("data".to_string(), 4u64, ch);+ for level in 0..=depth {+ let x = b.conv(&cur, c_in, c, 3, true);+ let x = b.conv(&x, c, c, 3, true);+ if level < depth {+ skips.push((x.clone(), c));+ cur = b.pool(&x);+ c_in = c;+ c *= 2;+ } else {+ cur = x;+ }+ }+ for (skip, sc) in skips.into_iter().rev() {+ let up = b.up(&cur, c, sc);+ let joined = b.concat(&up, &skip);+ let x = b.conv(&joined, 2 * sc, sc, 3, true);+ cur = b.conv(&x, sc, sc, 3, true);+ c = sc;+ }+ let rgb = b.conv(&cur, c, 12, 1, false);+ b.nodes.push(node("DepthToSpace", &[&rgb], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]));++ let mut graph = Vec::new();+ for n in &b.nodes {+ len_field(1, n, &mut graph);+ }+ for t in &b.inits {+ len_field(5, t, &mut graph);+ }+ len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(tile), Ok(tile)]), &mut graph);+ len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * tile), Ok(2 * tile)]), &mut graph);+ let mut ops = Vec::new();+ len_field(1, b"", &mut ops);+ varint_field(2, 13, &mut ops);+ let mut model = Vec::new();+ varint_field(1, 8, &mut model);+ len_field(7, &graph, &mut model);+ len_field(8, &ops, &mut model);+ model+}++/// A tiny smoothing network for installing a real CPU model in setup-flow tests.+/// It averages a 3 × 3 neighborhood of the four Bayer planes and returns three identical channels.+pub fn smoothing_onnx() -> Vec<u8> {+ let mut graph = Vec::new();+ len_field(1, &node("Conv", &["data", "weights"], "samples", &[("kernel_shape", A::Ints(&[3, 3])), ("pads", A::Ints(&[1, 1, 1, 1]))]), &mut graph);+ len_field(1, &node("DepthToSpace", &["samples"], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]), &mut graph);+ len_field(5, &tensor("weights", &[12, 4, 3, 3], &[1.0 / 36.0; 12 * 4 * 3 * 3]), &mut graph);+ len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(64), Ok(64)]), &mut graph);+ len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(128), Ok(128)]), &mut graph);+ let mut model = Vec::new();+ varint_field(1, 8, &mut model);+ len_field(7, &graph, &mut model);+ let mut ops = Vec::new();+ varint_field(2, 13, &mut ops);+ len_field(8, &ops, &mut model);+ model+}diff --git a/crates/denoise/src/synthetic_proto.rs b/crates/denoise/src/synthetic_proto.rsnew file mode 100644index 000000000..a78d586bd--- /dev/null+++ b/crates/denoise/src/synthetic_proto.rs@@ -0,0 +1,44 @@+pub(crate) fn varint(mut v: u64, out: &mut Vec<u8>) {+ loop {+ let b = (v & 0x7f) as u8;+ v >>= 7;+ if v == 0 {+ out.push(b);+ return;+ }+ out.push(b | 0x80);+ }+}++pub fn len_field(n: u64, body: &[u8], out: &mut Vec<u8>) {+ varint(n << 3 | 2, out);+ varint(body.len() as u64, out);+ out.extend_from_slice(body);+}++pub(crate) fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {+ varint(n << 3, out);+ varint(v, out);+}++/// A graph input or output: `Ok(n)` is a fixed dimension, `Err(name)` a named (dynamic) one.+pub fn value_info_bytes(name: &str, elem: u64, dims: &[std::result::Result<u64, &str>]) -> Vec<u8> {+ let mut shape = Vec::new();+ for d in dims {+ let mut dim = Vec::new();+ match d {+ Ok(v) => varint_field(1, *v, &mut dim),+ Err(p) => len_field(2, p.as_bytes(), &mut dim),+ }+ len_field(1, &dim, &mut shape);+ }+ let mut tensor = Vec::new();+ varint_field(1, elem, &mut tensor);+ len_field(2, &shape, &mut tensor);+ let mut ty = Vec::new();+ len_field(1, &tensor, &mut ty);+ let mut vi = Vec::new();+ len_field(1, name.as_bytes(), &mut vi);+ len_field(2, &ty, &mut vi);+ vi+}diff --git a/crates/denoise/src/tiles.rs b/crates/denoise/src/tiles.rsnew file mode 100644index 000000000..59898c5a4--- /dev/null+++ b/crates/denoise/src/tiles.rs@@ -0,0 +1,100 @@+//! Cutting a picture into overlapping tiles and putting the results back together.+//!+//! A model runs on a fixed square, so a picture is covered by tiles spaced evenly along each axis, each sharing+//! at least `overlap` cells with its neighbours. Each tile's influence fades towards its own edges (except at the+//! picture's edge), and the result is the sum of tile × weight divided by the sum of weights, so any set of+//! overlapping tiles blends consistently and a seam between two tiles of slightly different scale is a smooth ramp.++/// Where the tiles start along one axis of `n` cells: `[0]` when the axis fits one tile (the tile is then padded by+/// mirroring), else evenly spaced starts, the first at 0 and the last at `n − tile`, each tile sharing at least+/// `overlap` cells with the next.+pub fn starts(n: usize, tile: usize, overlap: usize) -> Vec<usize> {+ if n <= tile || tile == 0 {+ return vec![0];+ }+ let step = tile.saturating_sub(overlap).max(1);+ // tiles needed so that start k·spacing with spacing ≤ step reaches n − tile+ let Some(k) = (n - tile).div_ceil(step).checked_add(1).filter(|&k| k <= 1_000_000) else { return Vec::new() };+ let last = n - tile;+ (0..k).map(|i| if k == 1 { 0 } else { ((i as u128 * last as u128 + (k - 1) as u128 / 2) / (k - 1) as u128) as usize }).collect()+}++/// Weight of the cell at offset `p` (0-based, inside a tile of `tile` cells) along one axis: 1 in the middle, easing+/// down to a sliver over `fade` cells at an edge that has a neighbour, 1 right up to an edge that is the picture's.+pub fn weight(p: usize, tile: usize, fade: usize, has_before: bool, has_after: bool) -> f32 {+ let fade = fade.max(1) as f32;+ let ramp = |d: usize| {+ let t = ((d as f32 + 0.5) / fade).min(1.0);+ // smoothstep, never exactly 0 so that a cell is always covered+ (t * t * (3.0 - 2.0 * t)).max(1e-3)+ };+ let before = if has_before { ramp(p) } else { 1.0 };+ let after = if has_after { ramp(tile.saturating_sub(1).saturating_sub(p)) } else { 1.0 };+ before.min(after)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn hostile_axes_have_bounded_tile_counts_and_checked_spacing() {+ assert!(starts(usize::MAX, 1, 0).is_empty());+ assert!(starts(usize::MAX, 2, usize::MAX).is_empty());+ let s = starts(usize::MAX, usize::MAX / 2, 0);+ assert_eq!(s.first(), Some(&0));+ assert_eq!(s.last(), Some(&(usize::MAX - usize::MAX / 2)));+ assert!(s.windows(2).all(|s| s[0] < s[1]));+ }++ #[test]+ fn one_tile_when_it_fits() {+ assert_eq!(starts(100, 512, 64), vec![0]);+ assert_eq!(starts(512, 512, 64), vec![0]);+ assert_eq!(starts(0, 512, 64), vec![0]);+ assert_eq!(starts(10, 0, 0), vec![0]);+ }++ #[test]+ fn tiles_cover_the_axis_and_overlap_enough() {+ for n in [513usize, 600, 900, 1000, 1024, 1500, 3024, 3600, 5000, 12345] {+ let s = starts(n, 512, 64);+ assert_eq!(s[0], 0, "n={n}");+ assert_eq!(*s.last().unwrap(), n - 512, "n={n}");+ for w in s.windows(2) {+ assert!(w[1] > w[0], "starts increase, n={n}");+ assert!(w[0] + 512 >= w[1] + 64, "at least 64 shared cells between {} and {}, n={n}", w[0], w[1]);+ }+ }+ }++ #[test]+ fn weights_fade_only_towards_neighbours() {+ let (t, f) = (512, 64);+ // a lone tile: full weight everywhere+ assert!((0..t).all(|p| weight(p, t, f, false, false) == 1.0));+ // the first tile fades on its far side only+ assert_eq!(weight(0, t, f, false, true), 1.0);+ assert!(weight(t - 1, t, f, false, true) < 0.01);+ // the last tile fades on its near side only+ assert!(weight(0, t, f, true, false) < 0.01);+ assert_eq!(weight(t - 1, t, f, true, false), 1.0);+ // a middle tile is 1 in the middle and everywhere positive+ assert_eq!(weight(t / 2, t, f, true, true), 1.0);+ assert!((0..t).all(|p| weight(p, t, f, true, true) > 0.0));+ }++ #[test]+ fn two_overlapping_tiles_blend_to_a_partition_of_unity() {+ // at every cell of the shared region the normalised weights are in [0, 1] and sum to 1+ let (t, f) = (512usize, 64usize);+ let s = starts(900, t, 64);+ assert_eq!(s.len(), 2);+ for cell in s[1]..s[0] + t {+ let a = weight(cell - s[0], t, f, false, true);+ let b = weight(cell - s[1], t, f, true, false);+ let (na, nb) = (a / (a + b), b / (a + b));+ assert!((na + nb - 1.0).abs() < 1e-6 && na >= 0.0 && nb >= 0.0);+ }+ }+}diff --git a/crates/denoise/tests/real_model.rs b/crates/denoise/tests/real_model.rsnew file mode 100644index 000000000..bea7e712d--- /dev/null+++ b/crates/denoise/tests/real_model.rs@@ -0,0 +1,248 @@+//! Opt-in end-to-end check of the real RawNIND Bayer model on real mosaics with noise added:+//!+//! ```text+//! LC_DENOISE_MODEL=<model_bayer.onnx> LC_DENOISE_MOSAICS=<folder> LC_DENOISE_OUT=<folder> \+//! cargo test --release -p lightcraft-denoise --test real_model -- --ignored --nocapture+//! ```+//!+//! `LC_DENOISE_MOSAICS` holds `<name>.mosaic.f32` (little-endian f32, normalised: black 0, white 1, not white+//! balanced), `<name>.rgb.f32` (a clean demosaic of it, camera RGB, interleaved) and `<name>.txt` (`width`,+//! `height`, `cfa`, `wb` lines). Noise is Poisson-Gaussian, made here. Optional: `LC_DENOISE_PARALLEL` (tiles at+//! once, default 8), `LC_DENOISE_ONLY` (one name), `LC_DENOISE_ISO` (noise strength, default 1.0; 0 adds none),+//! `LC_DENOISE_FULL` (the whole frame instead of a 2048 × 1536 crop).++use std::path::{Path, PathBuf};+use std::time::Instant;++use lightcraft_denoise::bayer::Layout;+use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};+use lightcraft_denoise::run::{Control, Params, denoise_bayer};+use lightcraft_denoise::runtime::CpuRunner;++struct Rng(u64);+impl Rng {+ fn next(&mut self) -> f64 {+ self.0 ^= self.0 >> 12;+ self.0 ^= self.0 << 25;+ self.0 ^= self.0 >> 27;+ ((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 11) as f64 + 0.5) / (1u64 << 53) as f64+ }+ fn gauss(&mut self) -> f64 {+ (-2.0 * self.next().ln()).sqrt() * (std::f64::consts::TAU * self.next()).cos()+ }+}++fn read_f32(path: &Path) -> Vec<f32> {+ std::fs::read(path).unwrap().as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()+}++struct Dump {+ width: usize,+ height: usize,+ layout: Layout,+ wb: [f32; 3],+ mosaic: Vec<f32>,+ rgb: Vec<f32>,+}++fn load(dir: &Path, name: &str) -> Dump {+ let text = std::fs::read_to_string(dir.join(format!("{name}.txt"))).unwrap();+ let field = |k: &str| text.lines().find_map(|l| l.strip_prefix(k).map(|v| v.trim().to_string())).unwrap();+ let cell = match field("cfa").as_str() {+ "RGGB" => [0, 1, 1, 2],+ "BGGR" => [2, 1, 1, 0],+ "GRBG" => [1, 0, 2, 1],+ "GBRG" => [1, 2, 0, 1],+ other => panic!("cfa {other}"),+ };+ let wb: Vec<f32> = field("wb").split_whitespace().map(|v| v.parse().unwrap()).collect();+ Dump {+ width: field("width").parse().unwrap(),+ height: field("height").parse().unwrap(),+ layout: Layout::from_cell(cell).unwrap(),+ wb: [wb[0], wb[1], wb[2]],+ mosaic: read_f32(&dir.join(format!("{name}.mosaic.f32"))),+ rgb: read_f32(&dir.join(format!("{name}.rgb.f32"))),+ }+}++/// Crop with an even origin so the Bayer phase is kept.+fn crop(d: &Dump, x0: usize, y0: usize, w: usize, h: usize) -> (Vec<f32>, Vec<f32>) {+ let mut m = Vec::with_capacity(w * h);+ let mut c = Vec::with_capacity(w * h * 3);+ for y in y0..y0 + h {+ m.extend_from_slice(&d.mosaic[y * d.width + x0..y * d.width + x0 + w]);+ c.extend_from_slice(&d.rgb[(y * d.width + x0) * 3..(y * d.width + x0 + w) * 3]);+ }+ (m, c)+}++/// 0 = red, 1 = green, 2 = blue at a mosaic position.+fn site(layout: Layout, x: usize, y: usize) -> usize {+ match ((x + layout.dx) % 2, (y + layout.dy) % 2) {+ (0, 0) => 0,+ (1, 1) => 2,+ _ => 1,+ }+}++/// The simplest demosaic (each missing colour is the mean of the same-coloured samples in the 3 × 3 around),+/// as the baseline a user would get from the noisy picture without AI.+fn bilinear(m: &[f32], w: usize, h: usize, layout: Layout) -> Vec<f32> {+ let mut out = vec![0f32; w * h * 3];+ for y in 0..h {+ for x in 0..w {+ let mut sum = [0f32; 3];+ let mut n = [0f32; 3];+ for yy in y.saturating_sub(1)..(y + 2).min(h) {+ for xx in x.saturating_sub(1)..(x + 2).min(w) {+ let c = site(layout, xx, yy);+ sum[c] += m[yy * w + xx];+ n[c] += 1.0;+ }+ }+ for c in 0..3 {+ out[(y * w + x) * 3 + c] = if n[c] > 0.0 { sum[c] / n[c] } else { 0.0 };+ }+ }+ }+ out+}++/// Signal-to-error ratio in dB of `got` against `want` (both interleaved RGB), ignoring a border.+fn snr(got: &[f32], want: &[f32], w: usize, h: usize, border: usize) -> f64 {+ let (mut sig, mut err) = (0f64, 0f64);+ for y in border..h - border {+ for x in border..w - border {+ for c in 0..3 {+ let (g, t) = (f64::from(got[(y * w + x) * 3 + c]), f64::from(want[(y * w + x) * 3 + c]));+ sig += t * t;+ err += (g - t).powi(2);+ }+ }+ }+ 10.0 * (sig / err.max(1e-30)).log10()+}++fn crc32(data: &[u8]) -> u32 {+ let mut c = !0u32;+ for &b in data {+ c ^= u32::from(b);+ for _ in 0..8 {+ c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };+ }+ }+ !c+}++/// A viewable PNG of interleaved camera-RGB panels side by side: white balanced, one exposure for all, gamma.+fn write_png(path: &Path, panels: &[&[f32]], w: usize, h: usize, wb: [f32; 3], exposure: f32) {+ let total_w = w * panels.len();+ let mut raw = Vec::with_capacity((total_w * 3 + 1) * h);+ for y in 0..h {+ raw.push(0u8);+ for p in panels {+ for x in 0..w {+ for c in 0..3 {+ let v = (p[(y * w + x) * 3 + c] * wb[c] * exposure).clamp(0.0, 1.0);+ raw.push((v.powf(1.0 / 2.2) * 255.0 + 0.5) as u8);+ }+ }+ }+ }+ let z = miniz_oxide::deflate::compress_to_vec_zlib(&raw, 6);+ let mut png = b"\x89PNG\r\n\x1a\n".to_vec();+ let mut chunk = |kind: &[u8; 4], body: &[u8]| {+ png.extend_from_slice(&(body.len() as u32).to_be_bytes());+ let mut k = kind.to_vec();+ k.extend_from_slice(body);+ png.extend_from_slice(&k);+ png.extend_from_slice(&crc32(&k).to_be_bytes());+ };+ let mut ihdr = Vec::new();+ ihdr.extend_from_slice(&(total_w as u32).to_be_bytes());+ ihdr.extend_from_slice(&(h as u32).to_be_bytes());+ ihdr.extend_from_slice(&[8, 2, 0, 0, 0]);+ chunk(b"IHDR", &ihdr);+ chunk(b"IDAT", &z);+ chunk(b"IEND", &[]);+ std::fs::write(path, png).unwrap();+}++#[test]+#[ignore = "needs a real model and mosaics: see the module docs"]+fn the_real_model_denoises_real_mosaics() {+ let (Some(model), Some(dir)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_MOSAICS")) else { return };+ let dir = PathBuf::from(dir);+ let out_dir = std::env::var_os("LC_DENOISE_OUT").map(PathBuf::from);+ let parallel: usize = std::env::var("LC_DENOISE_PARALLEL").ok().and_then(|v| v.parse().ok()).unwrap_or(8);+ let iso: f64 = std::env::var("LC_DENOISE_ISO").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0);+ let names: Vec<String> = match std::env::var("LC_DENOISE_ONLY") {+ Ok(n) => vec![n],+ Err(_) => ["arw-sony-a7m3-compressed", "cr2-canon-6d", "nef-nikon-d5100-lossless", "dng-google-pixel2xl", "pef-pentax-k3"]+ .map(String::from)+ .to_vec(),+ };+ let manifest = DenoiserManifest {+ id: "rawnind-bayer".into(),+ name: "RawNIND Bayer".into(),+ version: "1".into(),+ licence: Default::default(),+ source: None,+ sha256: None,+ size_bytes: None,+ provenance: String::new(),+ domain: Domain::BayerToRgb,+ tile: 512,+ overlap: 64,+ gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },+ };+ let runner = CpuRunner::load(Path::new(&model), &manifest).unwrap();+ let params = Params { tile: 512, overlap: 64, gain: manifest.gain, clip: Some(1.0), parallel };+ for name in names {+ let d = load(&dir, &name);+ // a crop of about 2 × 1.5 tiles, away from the edges, even origin+ let full = std::env::var_os("LC_DENOISE_FULL").is_some();+ let (w, h) = if full { (d.width & !1, d.height & !1) } else { (2048usize, 1536usize) };+ let (x0, y0) = (((d.width - w) / 3) & !1, ((d.height - h) / 2) & !1);+ let (clean, reference) = crop(&d, x0, y0, w, h);+ // Poisson-Gaussian noise: variance a·signal + b, as a sensor at high ISO+ let (a, b) = (1.5e-3 * iso, (4.0e-3 * iso.sqrt()).powi(2));+ let mut rng = Rng(0x9e37_79b9_7f4a_7c15 ^ name.len() as u64);+ let noisy: Vec<f32> = clean+ .iter()+ .map(|&v| {+ let v = f64::from(v);+ let sigma = (a * v.max(0.0) + b).sqrt();+ ((v + sigma * rng.gauss()) as f32).min(1.0)+ })+ .collect();+ let started = Instant::now();+ let done = std::sync::atomic::AtomicUsize::new(0);+ let progress = |n: usize, total: usize| done.store(n * 1000 / total, std::sync::atomic::Ordering::Relaxed);+ let denoised = denoise_bayer(&noisy, w, h, d.layout, &runner, ¶ms, &Control { cancel: None, progress: Some(&progress) }).unwrap();+ let secs = started.elapsed().as_secs_f64();+ let baseline = bilinear(&noisy, w, h, d.layout);+ let clean_bilinear = bilinear(&clean, w, h, d.layout);+ let den: Vec<f32> = denoised.data.iter().flat_map(|p| p.iter().copied()).collect();+ let (s_noisy, s_den) = (snr(&baseline, &reference, w, h, 64), snr(&den, &reference, w, h, 64));+ let s_floor = snr(&clean_bilinear, &reference, w, h, 64);+ println!(+ "{name}: {w}×{h} crop, {} tiles, parallel {parallel}, {secs:.1} s | SNR vs clean demosaic: noisy bilinear {s_noisy:.2} dB, denoised {s_den:.2} dB (clean bilinear {s_floor:.2} dB)",+ lightcraft_denoise::run::tile_count(w, h, d.layout, 512, 64)+ );+ if let Some(out) = &out_dir {+ std::fs::create_dir_all(out).unwrap();+ let mean = reference.iter().map(|&v| f64::from(v)).sum::<f64>() / reference.len() as f64;+ let exposure = (0.18 / mean.max(1e-6)) as f32;+ // three panels, a 512 × 384 window each, at 1:1+ let (pw, ph, px, py) = (512usize, 384usize, 700usize, 500usize);+ let window =+ |buf: &[f32]| -> Vec<f32> { (0..ph).flat_map(|y| buf[((py + y) * w + px) * 3..((py + y) * w + px + pw) * 3].to_vec()).collect() };+ write_png(&out.join(format!("{name}.png")), &[&window(&baseline), &window(&den), &window(&reference)], pw, ph, d.wb, exposure);+ }+ if iso >= 4.0 {+ assert!(s_den > s_noisy + 2.0, "{name}: denoised {s_den:.2} dB is not clearly better than noisy {s_noisy:.2} dB");+ }+ }+}diff --git a/crates/develop/src/controls.rs b/crates/develop/src/controls.rsindex 5696d6706..c8a001b79 100644--- a/crates/develop/src/controls.rs+++ b/crates/develop/src/controls.rs@@ -207,6 +207,7 @@ controls! {"grain.amount" => grain.amount, "Grain", Grain, 0, 100, 0, 1, 0, Plain;"grain.size" => grain.size, "Size", Grain, 0, 100, 25, 1, 0, Plain;"grain.roughness" => grain.roughness, "Roughness", Grain, 0, 100, 50, 1, 0, Plain;+ "enhance.denoise" => enhance.denoise, "Amount", Detail, 0, 100, 0, 1, 0, Plain;"detail.sharpenAmount" => detail.sharpen_amount, "Sharpening", Detail, 0, 150, 0, 1, 0, Plain;"detail.sharpenRadius" => detail.sharpen_radius, "Radius", Detail, 0.5, 3, 1, 0.1, 1, Plain;"detail.sharpenDetail" => detail.sharpen_detail, "Detail", Detail, 0, 100, 25, 1, 0, Plain;diff --git a/crates/develop/src/lib.rs b/crates/develop/src/lib.rsindex bfce1893e..b7c952511 100644--- a/crates/develop/src/lib.rs+++ b/crates/develop/src/lib.rs@@ -94,6 +94,13 @@ impl DevelopSettings {std::borrow::Cow::Owned(d)}+ /// How much of the AI-denoised picture to mix in, 0 (none) to 1 (all of it): the Denoise amount, unless the Detail+ /// section is switched off or the value is not a number.+ pub fn denoise_amount(&self) -> f32 {+ let a = self.enhance.denoise;+ if a.is_finite() && self.enhance.denoise_enabled() && self.section_enabled("detail") { (a / 100.0).clamp(0.0, 1.0) as f32 } else { 0.0 }+ }+pub fn set_section_enabled(&mut self, section: &str, on: bool) {self.disabled_sections.retain(|s| s != section);if !on {@@ -112,6 +119,7 @@ impl DevelopSettings {self.curve = d;}Section::Color => self.wb.mode = WbMode::AsShot,+ Section::Detail => self.enhance.denoise_on = None,_ => {}}}@@ -193,4 +201,40 @@ mod tests {s.set_section_enabled("effects", true);assert!(s.section_enabled("effects"));}++ #[test]+ fn the_denoise_amount_is_a_slider_and_follows_the_detail_section() {+ let mut s = DevelopSettings::default();+ assert_eq!(s.denoise_amount(), 0.0);+ assert!(controls::set(&mut s, "enhance.denoise", 60.0));+ assert_eq!(controls::get(&s, "enhance.denoise"), Some(60.0));+ assert!((s.denoise_amount() - 0.6).abs() < 1e-6);+ // the slider's range is 0 to 100; values outside it are brought in+ controls::set(&mut s, "enhance.denoise", 500.0);+ assert_eq!(s.denoise_amount(), 1.0);+ controls::set(&mut s, "enhance.denoise", -5.0);+ assert_eq!(s.denoise_amount(), 0.0);+ // switching the Detail section off switches the denoise off with it; a settings file with a bad number does nothing+ controls::set(&mut s, "enhance.denoise", 80.0);+ s.set_section_enabled("detail", false);+ assert_eq!(s.denoise_amount(), 0.0);+ s.set_section_enabled("detail", true);+ s.enhance.denoise = f64::NAN;+ assert_eq!(s.denoise_amount(), 0.0);+ // it is part of Detail: a reset of the section clears it+ controls::set(&mut s, "enhance.denoise", 40.0);+ s.reset_section(Section::Detail);+ assert_eq!(s.enhance.denoise, 0.0);+ // The later independent switch preserves Amount when off, and can be on at zero.+ controls::set(&mut s, "enhance.denoise", 75.0);+ s.enhance.denoise_on = Some(false);+ assert_eq!(s.denoise_amount(), 0.0);+ assert_eq!(s.enhance.denoise, 75.0);+ s.enhance.denoise_on = Some(true);+ assert_eq!(s.denoise_amount(), 0.75);+ controls::set(&mut s, "enhance.denoise", 0.0);+ assert!(s.enhance.denoise_enabled());+ s.reset_section(Section::Detail);+ assert_eq!(s.enhance.denoise_on, None);+ }}diff --git a/crates/develop/src/settings.rs b/crates/develop/src/settings.rsindex 910cd07dd..5f0db8ab7 100644--- a/crates/develop/src/settings.rs+++ b/crates/develop/src/settings.rs@@ -915,6 +915,14 @@ pub struct LensBlur {#[serde(default)]pub struct Enhance {pub denoise: f64,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub denoise_on: Option<bool>,pub raw_details: bool,pub super_resolution: bool,}++impl Enhance {+ pub fn denoise_enabled(&self) -> bool {+ self.denoise_on.unwrap_or(self.denoise > 0.0)+ }+}diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.tomlindex a916d1806..536df7160 100644--- a/crates/engine/Cargo.toml+++ b/crates/engine/Cargo.toml@@ -21,6 +21,7 @@ web-time = { workspace = true }lightcraft-preview = { workspace = true }lightcraft-codecs = { workspace = true }lightcraft-raw = { workspace = true }+lightcraft-denoise = { workspace = true }lightcraft-meta = { workspace = true }miniz_oxide = { workspace = true }lightcraft-merge = { workspace = true }@@ -38,6 +39,10 @@ lightcraft-segment = { workspace = true, optional = true }[features]# AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds.sam = ["dep:lightcraft-segment"]+denoise = []++[target.'cfg(not(target_arch = "wasm32"))'.dependencies]+lightcraft-fetch = { workspace = true }[dev-dependencies]libc = { workspace = true }diff --git a/crates/engine/src/cmd/denoise.rs b/crates/engine/src/cmd/denoise.rsnew file mode 100644index 000000000..c51037917--- /dev/null+++ b/crates/engine/src/cmd/denoise.rs@@ -0,0 +1,543 @@+//! Denoise commands (`denoise.*`): the models (list, install, download, test, remove, choose), the settings, and the+//! work (queue, cancel, status, pump, clear). The machinery they drive is [`crate::denoise`].+//!+//! Models live in the folder the host provides (`Session::denoise.models_dir`): one subfolder per model with+//! `model.onnx`, `denoise-model.json` (its manifest) and `installed.json` (what the user accepted and the file's hash),+//! plus `settings.json` beside them. Model files, archives and manifests are hostile input: sizes are capped, ids are+//! validated before they become folder names, and nothing outside a model's own folder is ever touched.++use std::path::{Path, PathBuf};+use std::sync::PoisonError;++use lightcraft_catalog::PhotoId;+use lightcraft_denoise::archive;+use lightcraft_denoise::hash::sha256_file;+use lightcraft_denoise::known::{self, Known};+use lightcraft_denoise::manifest::{self, DenoiserManifest, MAX_MANIFEST_BYTES, MAX_MODEL_BYTES};+use serde_json::{Value, json};++use super::{CommandSpec, always, bad, bool_or, cmd, str_param};+use crate::denoise::{Pace, RunOn, Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings};+use crate::model_download::{self, State};+use crate::{EngineError, Result, Session};++fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {+ EngineError::Other(format!("{what}: {e}"))+}++fn models_dir(s: &Session, cmd: &str) -> Result<PathBuf> {+ s.denoise.models_dir.clone().ok_or_else(|| bad(cmd, "this build has nowhere to keep denoise models (the desktop app does)"))+}++/// What a command says about a model.+fn row(m: &DenoiserManifest, download_host: Option<String>, installed: bool, selected: bool, accepted: &Value) -> Value {+ json!({+ "id": m.id,+ "name": m.name,+ "version": m.version,+ "licence": m.licence,+ "provenance": m.provenance,+ "source": m.source,+ "sizeBytes": m.size_bytes,+ "sha256": m.sha256,+ "tile": m.tile,+ "known": known::find(&m.id).is_some(),+ // a pinned address LightCraft can fetch it from (the user presses Download), and which site that is+ "downloadHost": download_host,+ "installed": installed,+ "selected": selected,+ "accepted": accepted,+ })+}++fn host_of(k: &Known) -> Option<String> {+ k.download.as_ref().map(|d| lightcraft_denoise::known::host(&d.url).to_string())+}++fn list(s: &mut Session, _: &Value) -> Result<Value> {+ s.denoise_refresh_active(true);+ let dir = s.denoise.models_dir.clone();+ let st = dir.as_deref().map(read_settings).unwrap_or_default();+ let installed = dir.as_deref().map(installed_models).unwrap_or_default();+ let on_disk = |id: &str| installed.iter().find(|i| i.manifest.id == id);+ let chosen = |id: &str| st.model.as_deref() == Some(id);+ let mut models: Vec<Value> = Vec::new();+ for k in known::all() {+ let found = on_disk(&k.manifest.id);+ models.push(row(&k.manifest, host_of(&k), found.is_some(), chosen(&k.manifest.id), found.map_or(&Value::Null, |f| &f.accepted)));+ }+ for i in installed.iter().filter(|i| known::find(&i.manifest.id).is_none()) {+ models.push(row(&i.manifest, None, true, chosen(&i.manifest.id), &i.accepted));+ }+ Ok(json!({+ "dir": dir.as_ref().map(|d| d.display().to_string()),+ "productsDir": s.denoise_products_dir().map(|d| d.display().to_string()),+ "model": st.model,+ // whether this build can run denoise models (the `denoise` feature: our pure-Rust CPU/GPU runners)+ "runtime": cfg!(feature = "denoise"),+ "auto": st.auto(),+ "cacheGb": st.cache_bytes() >> 30,+ "threads": st.threads,+ "runOn": st.run_on().name(),+ "models": models,+ }))+}++/// What a file the user pointed at is, as far as installing it goes.+struct Source {+ manifest: DenoiserManifest,+ /// The `.onnx` inside the archive; `None` when the file is the `.onnx`.+ entry: Option<&'static str>,+ file_name: String,+}++fn identify(path: &Path, c: &str, sha_known: Option<&str>) -> Result<Source> {+ let meta = std::fs::metadata(path).map_err(|e| bad(c, format!("cannot read `{}`: {e}", path.display())))?;+ if !meta.is_file() {+ return Err(bad(c, format!("`{}` is not a file", path.display())));+ }+ if meta.len() == 0 || meta.len() > MAX_MODEL_BYTES {+ return Err(bad(c, "a model file must be between 1 byte and 2 GiB"));+ }+ let sha256 = match sha_known {+ Some(h) => h.to_string(),+ None => sha256_file(path).map_err(|e| fail(&format!("could not read {}", path.display()), e))?,+ };+ let file_name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();+ // an archive LightCraft knows (the file darktable publishes the model in)+ if let Some(k) = known::all().into_iter().find(|k| k.download.as_ref().is_some_and(|d| d.sha256 == sha256)) {+ return Ok(Source { manifest: k.manifest, entry: k.entry, file_name });+ }+ // the `.onnx` of a known model+ if let Some(m) = known::by_sha256(&sha256) {+ return Ok(Source { manifest: m, entry: None, file_name });+ }+ // an `.onnx` with its manifest beside it+ let sibling = path.with_file_name("denoise-model.json");+ if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx"))+ && let Some(bytes) = read_capped(&sibling, MAX_MANIFEST_BYTES)+ {+ let m = manifest::parse(&bytes).map_err(|e| bad(c, format!("{}: {e}", sibling.display())))?;+ return Ok(Source { manifest: m, entry: None, file_name });+ }+ Err(bad(+ c,+ "LightCraft does not know this file. A denoise model is an .onnx file with a denoise-model.json beside it that says how to use it \+ (see docs/denoise.md), or an archive LightCraft can fetch itself (`denoise.models.download`)",+ ))+}++/// Install the model in `src`. A model that passes its self-test becomes the one in use (`activate`, the default): installing+/// a model is how someone says they want it. `verified` is for a download whose hash was already checked.+fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activate: bool, verified: bool) -> Result<Value> {+ let dir = models_dir(s, c)?;+ let ident = identify(src, c, None)?;+ let m = ident.manifest.clone();+ manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?;+ if !acknowledged {+ return Err(bad(+ c,+ "the licence has not been accepted: show the user its terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree",+ ));+ }+ let home = dir.join(&m.id);+ std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?;+ let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx"));+ let _ = std::fs::remove_file(&part);+ let placed: Result<()> = match ident.entry {+ Some(entry) => {+ archive::extract(src, entry, &part, MAX_MODEL_BYTES).map(|_| ()).map_err(|e| fail("could not take the model out of the archive", e))+ }+ None if verified && std::fs::rename(src, &part).is_ok() => Ok(()),+ None => std::fs::copy(src, &part).map(|_| ()).map_err(|e| fail("could not copy the model (is the disk full?)", e)),+ };+ let sha256 = placed.and_then(|_| sha256_file(&part).map_err(|e| fail("could not check the model", e))).and_then(|h| match &m.sha256 {+ Some(want) if *want != h => Err(EngineError::Other("the model file is not the one its description names (its SHA-256 differs)".into())),+ _ => Ok(h),+ });+ let sha256 = match sha256 {+ Ok(h) => h,+ Err(e) => {+ let _ = std::fs::remove_file(&part);+ return Err(e);+ }+ };+ let _ = std::fs::remove_file(&final_path);+ if let Err(e) = std::fs::rename(&part, &final_path) {+ let _ = std::fs::remove_file(&part);+ return Err(fail("could not finish the copy", e));+ }+ // a model that does not work is never left installed+ let run_on = s.denoise.settings.run_on();+ let test = (s.denoise.loader)(&final_path, &m).and_then(|model| model.self_test(run_on));+ let test = match test {+ Ok(t) => t,+ Err(e) => {+ let _ = std::fs::remove_file(&final_path);+ let _ = std::fs::remove_file(home.join("denoise-model.json"));+ let _ = std::fs::remove_dir(&home);+ return Err(bad(c, e));+ }+ };+ write_atomic(&home.join("denoise-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?).map_err(EngineError::Other)?;+ let accepted = json!({+ "acceptedAt": (s.clock)(),+ "licence": m.licence.name,+ "commercial": m.licence.commercial,+ "fileName": ident.file_name,+ "sha256": sha256,+ "selfTest": test,+ });+ write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?).map_err(EngineError::Other)?;+ // a file that was downloaded and waited for the user's acceptance has served: the installed copy is the model now+ if src.parent() == Some(dir.join(model_download::STAGING).as_path()) {+ let _ = std::fs::remove_file(src);+ s.denoise.downloads.staged_file_used(src);+ }+ let mut st = read_settings(&dir);+ if activate {+ st.model = Some(m.id.clone());+ write_settings(&dir, &st).map_err(EngineError::Other)?;+ }+ s.denoise.touch();+ s.denoise_refresh_active(true);+ let host = known::find(&m.id).and_then(|k| host_of(&k));+ Ok(json!({"installed": row(&m, host, true, st.model.as_deref() == Some(m.id.as_str()), &accepted), "model": st.model}))+}++fn install(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.models.install";+ let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;+ install_file(s, C, Path::new(path), bool_or(p, "acknowledged", false), bool_or(p, "activate", true), false)+}++/// What an error says, without the command it came from (for a line shown to the user).+fn plain(e: &EngineError) -> String {+ match e {+ EngineError::BadParams { msg, .. } => msg.clone(),+ other => other.to_string(),+ }+}++/// Install every download that has arrived: each was accepted when it was started, so it is installed and chosen without+/// another question. Called by `denoise.models.downloads` and by every frame's `denoise.pump`.+pub(crate) fn finish_downloads(s: &mut Session) {+ const C: &str = "denoise.models.download";+ for (id, path, _sha) in s.denoise.downloads.finished() {+ match install_file(s, C, &path, true, true, true) {+ Ok(_) => s.denoise.downloads.set_outcome(&id, State::Installed),+ Err(e) => {+ // a file that cannot be installed is of no use to anyone: throw it away, say why+ let _ = std::fs::remove_file(&path);+ s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e)));+ }+ }+ }+}++fn download(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.models.download";+ let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;+ let dir = models_dir(s, C)?;+ let k = known::find(id).ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;+ let spec = k.download.clone().ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;+ if !cfg!(feature = "denoise") {+ return Err(bad(C, "this build cannot run denoise models, so there is nothing to download for it"));+ }+ if !bool_or(p, "acknowledged", false) {+ return Err(bad(+ C,+ "the licence has not been accepted: show the user the model's terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree",+ ));+ }+ if installed_models(&dir).iter().any(|i| i.manifest.id == spec.id) {+ return Err(bad(C, "that model is already installed"));+ }+ let host = lightcraft_denoise::known::host(&spec.url).to_string();+ s.denoise.downloads.start(spec, &dir).map_err(|e| bad(C, e))?;+ Ok(json!({"started": id, "from": host}))+}++fn download_row(id: &str, state: &State) -> Value {+ let from = known::find(id).and_then(|k| host_of(&k));+ match state {+ State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}),+ State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}),+ State::Installed => json!({"id": id, "state": "installed", "from": from}),+ State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}),+ State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}),+ }+}++fn downloads(s: &mut Session, _: &Value) -> Result<Value> {+ finish_downloads(s);+ let all: Vec<Value> = s.denoise.downloads.snapshot().iter().map(|(id, st)| download_row(id, st)).collect();+ Ok(json!({"running": s.denoise.downloads.running(), "downloads": all}))+}++fn download_cancel(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.models.downloadCancel";+ let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;+ Ok(json!({"discarded": s.denoise.downloads.discard(id)}))+}++fn test(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.models.test";+ let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;+ let dir = models_dir(s, C)?;+ let found = installed_models(&dir).into_iter().find(|i| i.manifest.id == id).ok_or_else(|| bad(C, "that model is not installed"))?;+ let run_on = s.denoise.settings.run_on();+ let result = (s.denoise.loader)(&found.onnx, &found.manifest).and_then(|m| m.self_test(run_on));+ let (ok, detail) = match &result {+ Ok(v) => (true, v.clone()),+ Err(e) => (false, json!(e)),+ };+ // keep the latest outcome with the model's record+ let mut record = found.accepted;+ if let Some(o) = record.as_object_mut() {+ o.insert("selfTest".into(), if ok { detail.clone() } else { json!({"ok": false, "error": detail}) });+ write_atomic(&found.onnx.with_file_name("installed.json"), &serde_json::to_vec_pretty(&record).map_err(|e| fail("record", e))?)+ .map_err(EngineError::Other)?;+ }+ Ok(json!({"id": id, "ok": ok, "result": detail}))+}++fn remove(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.models.remove";+ let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;+ let dir = models_dir(s, C)?;+ if !lightcraft_denoise::licence::valid_id(id) {+ return Err(bad(C, "not a model id"));+ }+ let home = dir.join(id);+ // only a folder that is a model (it has a manifest) is ever deleted+ if !home.join("denoise-model.json").is_file() {+ return Err(bad(C, "that model is not installed"));+ }+ std::fs::remove_dir_all(&home).map_err(|e| fail("could not remove the model", e))?;+ let mut st = read_settings(&dir);+ if st.model.as_deref() == Some(id) {+ st.model = installed_models(&dir).into_iter().map(|i| i.manifest.id).find(|m| m != id);+ write_settings(&dir, &st).map_err(EngineError::Other)?;+ }+ s.denoise.touch();+ s.denoise_refresh_active(true);+ Ok(json!({"removed": id, "model": st.model}))+}++fn select(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.models.select";+ let dir = models_dir(s, C)?;+ let id = match p.get("id") {+ None | Some(Value::Null) => None,+ Some(v) => Some(v.as_str().ok_or_else(|| bad(C, "`id` must be a model id or null"))?.to_string()),+ };+ if let Some(id) = &id+ && !installed_models(&dir).iter().any(|i| &i.manifest.id == id)+ {+ return Err(bad(C, "that denoise model is not installed"));+ }+ let mut st = read_settings(&dir);+ st.model = id;+ write_settings(&dir, &st).map_err(EngineError::Other)?;+ s.denoise.touch();+ s.denoise_refresh_active(true);+ Ok(json!({"model": st.model}))+}++fn settings(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.settings";+ let dir = models_dir(s, C)?;+ let mut st: Settings = read_settings(&dir);+ if let Some(v) = p.get("auto") {+ st.auto = Some(v.as_bool().ok_or_else(|| bad(C, "`auto` must be true or false"))?);+ }+ if let Some(v) = p.get("cacheGb") {+ let n =+ v.as_u64().filter(|n| (1..=10_000).contains(n)).ok_or_else(|| bad(C, "`cacheGb` must be a whole number of gigabytes from 1 to 10000"))?;+ st.cache_gb = Some(n as u32);+ }+ if let Some(v) = p.get("threads") {+ st.threads = match v {+ Value::Null => None,+ v => Some(v.as_u64().filter(|n| (1..=64).contains(n)).ok_or_else(|| bad(C, "`threads` must be from 1 to 64, or null"))? as u32),+ };+ }+ let run_on = match p.get("runOn") {+ None => None,+ Some(v) => Some(v.as_str().and_then(RunOn::parse).ok_or_else(|| bad(C, "`runOn` must be auto, gpu or cpu"))?),+ };+ if let Some(r) = run_on {+ st.run_on = Some(r.name().to_string());+ st.gpu = None;+ }+ write_settings(&dir, &st).map_err(EngineError::Other)?;+ if run_on.is_some() {+ // a choice of where denoise runs is also a go-ahead to try the card again: a set-up that crashed or gave up+ // before is forgotten, and the loaded model sets the card up afresh for the next photo+ forget_gpu_set_ups(&dir);+ if let Some(a) = &s.denoise.active {+ *a.model.lock().unwrap_or_else(PoisonError::into_inner) = None;+ }+ }+ s.denoise.touch();+ s.denoise_refresh_active(true);+ // a smaller limit takes effect now+ s.denoise_evict(None);+ Ok(json!({"auto": st.auto(), "cacheGb": st.cache_bytes() >> 30, "threads": st.threads, "runOn": st.run_on().name()}))+}++fn ids_param(p: &Value) -> Option<Vec<PhotoId>> {+ p.get("ids").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_u64).map(PhotoId).collect())+}++fn status_json(s: &Session, photo: Option<PhotoId>) -> Value {+ let (files, bytes) = s.denoise_cache_usage();+ let st = &s.denoise.settings;+ let model = s.denoise.active.as_ref().map(|a| json!({"id": a.id, "name": a.manifest.name}));+ let mut v = json!({+ "enabled": s.denoise.active.is_some(),+ "model": model,+ "runtime": cfg!(feature = "denoise"),+ "productsDir": s.denoise_products_dir().map(|d| d.display().to_string()),+ "queued": s.denoise.queued(),+ "running": s.denoise_running_json(),+ "ready": s.media.denoise.len(),+ "made": s.denoise_made(),+ "auto": st.auto(),+ "runOn": st.run_on().name(),+ "device": s.denoise_device(),+ "cache": {"files": files, "bytes": bytes, "limitBytes": st.cache_bytes()},+ "generation": s.denoise.generation,+ });+ if let (Some(id), Some(o)) = (photo, v.as_object_mut()) {+ o.insert("photo".into(), serde_json::to_value(s.denoise_photo_state(id)).unwrap_or(Value::Null));+ }+ v+}++/// A per-photo switch, independent of the selection and Auto Sync. Turning on starts at 50%.+fn toggle(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.toggle";+ let id = match p.get("id") {+ Some(v) => PhotoId(v.as_u64().ok_or_else(|| bad(C, "`id` must be a photo id"))?),+ None => s.active().ok_or_else(|| bad(C, "select a raw photo"))?,+ };+ let photo = s.catalog.photo(id).ok_or_else(|| bad(C, "no such photo"))?;+ if !crate::denoise::eligible(photo) {+ return Err(bad(C, "AI Denoise applies to raw photos developed from their sensor data"));+ }+ let mut d = (*s.develop_of(id).ok_or_else(|| bad(C, "no such photo"))?).clone();+ let on = match p.get("enabled") {+ Some(v) => v.as_bool().ok_or_else(|| bad(C, "`enabled` must be true or false"))?,+ None => !d.enhance.denoise_enabled(),+ };+ if on {+ if !d.enhance.denoise.is_finite() {+ return Err(bad(C, "the photo has an invalid Denoise amount"));+ }+ s.denoise_refresh_active(true);+ if s.denoise.active.is_none() {+ return Err(bad(C, "AI Denoise needs a model: install one in Settings > AI Denoise"));+ }+ }+ // The switch is its own setting: the Amount stays where it is (0 included) when it is flipped, so flipping compares+ // with and without. A photo that never had an Amount starts at 50 the first time.+ let amount = if on && d.enhance.denoise_on.is_none() && d.enhance.denoise <= 0.0 { 50.0 } else { d.enhance.denoise };+ if d.enhance.denoise != amount || d.enhance.denoise_on != Some(on) {+ s.end_interaction()?;+ d.enhance.denoise = amount;+ d.enhance.denoise_on = Some(on);+ let op = s.develop_op(id, d, "AI Denoise").ok_or_else(|| bad(C, "no such photo"))?;+ s.commit("AI Denoise", op)?;+ }+ if on {+ s.denoise_enqueue(&[id], true, false);+ } else {+ s.denoise_cancel(Some(id), false);+ }+ Ok(json!({"id": id.0, "enabled": on, "amount": amount}))+}++fn queue(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "denoise.queue";+ s.denoise_refresh_active(true);+ if s.denoise.active.is_none() {+ return Err(bad(C, "no denoise model is chosen: install one first (`denoise.models.download` or `denoise.models.install`)"));+ }+ if s.denoise_products_dir().is_none() {+ return Err(bad(C, "denoise keeps its pictures in the library's folder: open a library on disk first"));+ }+ let ids = match ids_param(p) {+ Some(ids) => ids,+ None => match str_param(p, "scope").unwrap_or("selected") {+ "selected" => s.selection.active.iter().chain(s.selection.ids.iter()).copied().collect(),+ "visible" => s.visible_cloned().to_vec(),+ "withAmount" => s.catalog.photos().filter(|ph| ph.develop.denoise_amount() > 0.0).map(|ph| ph.id).collect(),+ other => return Err(bad(C, format!("unknown scope `{other}`: use selected, visible or withAmount"))),+ },+ };+ let queued = s.denoise_enqueue(&ids, false, bool_or(p, "retry", false));+ let mut v = status_json(s, None);+ if let Some(o) = v.as_object_mut() {+ o.insert("added".into(), json!(queued));+ }+ Ok(v)+}++fn cancel(s: &mut Session, p: &Value) -> Result<Value> {+ let photo = p.get("id").and_then(Value::as_u64).map(PhotoId);+ let n = s.denoise_cancel(photo, bool_or(p, "all", photo.is_none()));+ Ok(json!({"cancelled": n}))+}++fn status(s: &mut Session, p: &Value) -> Result<Value> {+ s.denoise_refresh_active(false);+ Ok(status_json(s, p.get("id").and_then(Value::as_u64).map(PhotoId)))+}++/// `denoise.pump {pace?}`: cheap to call every frame. Takes in what the background job has finished and keeps one going.+fn pump(s: &mut Session, p: &Value) -> Result<Value> {+ s.denoise_pump(Pace::parse(p.get("pace").and_then(Value::as_str)));+ Ok(json!({+ "active": s.denoise.active.is_some(),+ "running": s.denoise_running_json(),+ "queued": s.denoise.queued(),+ "ready": s.media.denoise.len(),+ "generation": s.denoise.generation,+ "downloading": s.denoise.downloads.running(),+ }))+}++fn clear(s: &mut Session, _: &Value) -> Result<Value> {+ let (files, bytes) = s.denoise_clear();+ Ok(json!({"deleted": files, "bytes": bytes}))+}++pub fn specs() -> Vec<CommandSpec> {+ vec![+ cmd!(query "denoise.models.list", "Denoise Models", [], None, "{} → {dir, productsDir, model, runtime, auto, cacheGb, threads, runOn, models: [{id, name, version, licence{name, commercial, url, notice}, provenance, source, sizeBytes, sha256, tile, known, downloadHost, installed, selected, accepted}]}", always, list),+ cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true} → {installed, model} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install),+ cmd!(query "denoise.models.download", "Download Denoise Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a model LightCraft has a pinned address for (see `downloadHost` in the list) in the background over HTTPS with lightcraft-fetch. `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, installed and chosen by itself; `denoise.models.downloads` shows how far it is", always, download),+ cmd!(query "denoise.models.downloads", "Denoise Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived", always, downloads),+ cmd!(query "denoise.models.downloadCancel", "Cancel Denoise Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel),+ cmd!(query "denoise.models.test", "Test Denoise Model", [], None, "{id} → {ok, result} — load an installed model and check it gives sensible pictures; needs the denoise runtime", always, test),+ cmd!(query "denoise.models.remove", "Remove Denoise Model", [], None, "{id} → {removed, model} — delete an installed model (its cached pictures stay until they are evicted)", always, remove),+ cmd!(query "denoise.models.select", "Choose Denoise Model", [], None, "{id: installed model | null} → {model} — null switches denoise off", always, select),+ cmd!(query "denoise.settings", "Denoise Settings", [], None, "{auto?: bool, cacheGb?: 1..10000, threads?: 1..64 | null, runOn?: auto | gpu | cpu} → the settings — `auto`: make the picture of the photos being looked at that have a Denoise amount; `cacheGb`: how much the cached pictures may take; `threads`: most tiles run at once; `runOn`: where the model runs — `auto` (the default) the graphics card where it is faster than the processor (both timed on this computer when the card is set up), `gpu` the card whenever it can run the model, `cpu` the processor. Setting `runOn` also lets the card be tried again after a set-up that failed or closed LightCraft", always, settings),+ cmd!(+ "denoise.toggle",+ "AI Denoise",+ [],+ None,+ "{id?: photoId, enabled?: bool} → {id, enabled, amount} — turn AI Denoise on (50% initially) or off for one raw photo, independent of selection and Auto Sync; needs an installed model to turn on",+ always,+ toggle+ ),+ cmd!(query "denoise.queue", "Denoise Photos", [], None, "{ids?: [photoId], scope?: selected | visible | withAmount, retry?: false} → {added, …status} — make the denoised pictures of raw photos that have none, in the background. `retry` tries photos that failed before", always, queue),+ cmd!(query "denoise.cancel", "Cancel Denoise", [], None, "{id?, all?} → {cancelled} — stop the photo being made and/or forget the ones waiting (all of them when `id` is omitted)", always, cancel),+ cmd!(query "denoise.status", "Denoise Status", [], None, "{id?} → {enabled, model, runtime, productsDir, queued, running: {photo, done, total, seconds} | null, ready, made, auto, runOn, device: {kind: gpu | cpu | pending | none, adapter?, reason?, cardMs?, cpuMs?, fellBack?}, cache: {files, bytes, limitBytes}, generation, photo?: {state: notApplicable | noModel | ready | queued | running | failed | idle, …}}", always, status),+ cmd!(query "denoise.pump", "Denoise Pump", [], None, "{pace?: pause | light | normal | full} → {active, running, queued, ready, generation, downloading} — called every frame by the app: takes in finished work and keeps one photo going", always, pump),+ cmd!(query "denoise.clear", "Clear Denoise Cache", [], None, "{} → {deleted, bytes} — delete every cached denoised picture of this library (they are made again when wanted)", always, clear),+ ]+}diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rsindex 9fc831161..9f1d255c1 100644--- a/crates/engine/src/cmd/mod.rs+++ b/crates/engine/src/cmd/mod.rs@@ -10,6 +10,7 @@ mod color;pub(crate) mod convert;mod cull;pub mod curves;+pub(crate) mod denoise;mod develop;mod edit;mod export;@@ -140,6 +141,7 @@ pub fn command_specs() -> &'static [CommandSpec] {v.extend(missing::specs());v.extend(metadata::specs());v.extend(filters::specs());+ v.extend(denoise::specs());v})}diff --git a/crates/engine/src/config.rs b/crates/engine/src/config.rsnew file mode 100644index 000000000..96632f7ac--- /dev/null+++ b/crates/engine/src/config.rs@@ -0,0 +1,23 @@+//! Shared, opt-in per-user denoise model storage for native hosts.++use crate::Session;+use std::path::PathBuf;++pub fn default_denoise_models_dir() -> Option<PathBuf> {+ std::env::var_os("LIGHTCRAFT_DENOISE_MODELS")+ .filter(|v| !v.is_empty())+ .map(PathBuf::from)+ .or_else(|| crate::camera_profiles::config_dir().map(|d| d.join("denoise-models")))+}++impl Session {+ pub fn with_default_denoise_models(mut self) -> Self {+ self.set_denoise_models_dir(default_denoise_models_dir());+ self+ }++ pub fn set_denoise_models_dir(&mut self, dir: Option<PathBuf>) {+ self.denoise.models_dir = dir;+ self.denoise.touch();+ }+}diff --git a/crates/engine/src/denoise.rs b/crates/engine/src/denoise.rsnew file mode 100644index 000000000..37d79dcd2--- /dev/null+++ b/crates/engine/src/denoise.rs@@ -0,0 +1,1629 @@+//! AI denoise, the session's side (`docs/denoise.md`): which photos have a denoised picture, making the missing ones+//! in the background, and keeping the cache within its size.+//!+//! The denoised picture of a raw photo is *cached data*, never a file in the library: it is made from the untouched+//! original by the model the user chose, kept in `<library>/denoise/<key>.lcdn`, and thrown away whenever space is+//! needed (it is made again when it is next wanted). Its name is a hash of the model, the algorithm and the photo's+//! content, so a changed file or another model simply does not find it. The Denoise *amount* is an ordinary develop+//! setting (`enhance.denoise`): a render mixes the plain and the denoised picture by it ([`crate::media`]).+//!+//! What runs where: the session thread only decides (which photos, which model) and keeps an index of what is ready+//! ([`crate::media::DenoiseIndex`], memory only). One background thread at a time decodes a photo, runs the model on+//! its mosaic tile by tile and writes the product ([`make_product`]); an export that finds its photo's product missing+//! makes it itself, on its own thread, so it never comes out without the denoise it was asked for.+//!+//! Failure is a message, never a panic: a file that is not a Bayer raw, a model that does not load, a full disk, a+//! panic inside the runtime and a cancelled job all end as an [`Outcome`] the pump records.++use std::collections::{HashMap, HashSet, VecDeque};+use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError};+use std::time::{Duration, SystemTime};++use lightcraft_catalog::{Photo, PhotoId, Source};+use lightcraft_denoise::bayer::Layout;+use lightcraft_denoise::manifest::DenoiserManifest;+use lightcraft_denoise::run::{Control, Params, TileRunner, denoise_bayer};+use lightcraft_denoise::{Error as RunError, product};+use lightcraft_develop::DevelopSettings;+use lightcraft_preview::Hasher128;+use serde::{Deserialize, Serialize};+use serde_json::{Value, json};++use crate::media::{DenoiseSpec, MakeProduct, PairLoader};+use crate::merge::ByteReader;+use crate::model_download::Downloads;+use crate::{Selection, Session};++/// Extension of a product file.+pub const EXT: &str = "lcdn";+/// Where the products of a library are kept, inside the library's folder (safe to delete, like `thumbs/`).+pub const DIR: &str = "denoise";+/// What the cache may take up unless the user says otherwise (GB). A product is about as large as the raw file's+/// decoded samples, compressed: ~100 MB for 24 megapixels.+pub const DEFAULT_CACHE_GB: u32 = 20;+/// Samples at or above this (white = 1) are clipped: the model leaves them as the camera recorded them.+const CLIP: f32 = 0.99;+/// Most photos waiting in the queue.+const MAX_QUEUE: usize = 5000;+/// Most selected photos the automatic queue looks at in one pump.+const AUTO_LOOK: usize = 64;+/// How long a look at the model settings is trusted.+const SETTINGS_TTL: Duration = Duration::from_secs(2);+/// Stack of the threads that run the model (the runtime recurses through its graph).+const STACK: usize = 16 << 20;++// ---------------------------------------------------------------------------------------------------------------------+// the model++/// Where the model runs (Settings ▸ AI Denoise ▸ Speed; `denoise.settings {runOn}`).+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+pub enum RunOn {+ /// The graphics card where it is faster than the processor at the pace the work goes at (both are timed on this+ /// computer when the card is set up), else the processor.+ #[default]+ Auto,+ /// The graphics card whenever it can run the model, even where the processor would be faster.+ Gpu,+ /// Always the processor.+ Cpu,+}++impl RunOn {+ /// `auto`, `gpu` or `cpu`.+ pub fn parse(name: &str) -> Option<RunOn> {+ match name {+ "auto" => Some(RunOn::Auto),+ "gpu" => Some(RunOn::Gpu),+ "cpu" => Some(RunOn::Cpu),+ _ => None,+ }+ }++ pub fn name(self) -> &'static str {+ match self {+ RunOn::Auto => "auto",+ RunOn::Gpu => "gpu",+ RunOn::Cpu => "cpu",+ }+ }+}++/// A loaded denoise model: what runs tiles, and a self-check.+pub(crate) trait Model: Send + Sync {+ /// What runs a photo's tiles, and how many to run at once, when `threads` may be used and the setting is `run_on`+ /// (a graphics card is kept busy by a few). The first time the card may be used it is set up and checked here.+ fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize);+ /// Check it does something sensible: the test's result, or why not. Unless `run_on` is the processor the graphics+ /// card is set up and checked too, and the result says where the work will run.+ fn self_test(&self, run_on: RunOn) -> Result<Value, String>;+ /// Where the tiles run: `{kind: "gpu", adapter}`, `{kind: "cpu", reason?}` or `{kind: "pending"}` (the card is not+ /// set up yet), with the card's and the processor's time for the check tile (`cardMs`, `cpuMs`) once both were+ /// timed. Never starts anything.+ fn device(&self, _run_on: RunOn) -> Value {+ json!({"kind": "cpu"})+ }+}++/// Loads the model at a path as its manifest describes it.+pub(crate) type Loader = Arc<dyn Fn(&Path, &DenoiserManifest) -> Result<Arc<dyn Model>, String> + Send + Sync>;++/// Tiles run at once on a graphics card: enough to keep it fed while others are packed and blended.+#[cfg(feature = "denoise")]+const GPU_PARALLEL: usize = 4;+/// How far the GPU's answer on the check tile may be from the CPU's (of the answer's largest value) before it is not used.+#[cfg(feature = "denoise")]+const GPU_AGREE: f32 = 1e-3;+/// Longest the card may take to set up (build its kernels, run the check tile, time it) before the processor does the+/// work instead.+#[cfg(feature = "denoise")]+const GPU_SETUP_LIMIT: Duration = Duration::from_secs(30);+/// Tiles the card may get wrong (an error, numbers that are not finite) before it stops being used for the model.+#[cfg(feature = "denoise")]+const GPU_MAX_FALLBACKS: usize = 3;+/// The file beside a model while the graphics card is set up for it (see [`set_up_card`]).+pub(crate) const GPU_SETUP_MARKER: &str = "gpu-setup.marker";+/// Why the card is not tried when a set-up marker is found.+#[cfg(feature = "denoise")]+const GPU_CRASHED: &str = "LightCraft closed while it was setting up the graphics card for AI Denoise last time";+/// Why the card is not used when its set-up ran out of time (the start of it).+#[cfg(feature = "denoise")]+const GPU_SLOW: &str = "setting up the graphics card took longer than";+/// Why the card is not used while a set-up that ran out of time is still going.+#[cfg(feature = "denoise")]+const GPU_STUCK: &str = "the graphics card is still setting up from before (it took too long)";++/// A set-up that may work another time (a crash, the time limit), unlike one the card or the model cannot do.+#[cfg(feature = "denoise")]+fn worth_retrying(why: &str) -> bool {+ why == GPU_CRASHED || why == GPU_STUCK || why.starts_with(GPU_SLOW)+}++/// The model on our pure-Rust CPU runner, and on the graphics card when there is one that can run it: the card is set up for the+/// first photo that may use it, checked against the CPU's answer on a check tile and timed against it, and used while+/// it keeps working; a tile it gets wrong is run on the CPU, so a photo is always finished.+#[cfg(feature = "denoise")]+struct Network {+ cpu: lightcraft_denoise::runtime::CpuRunner,+ path: PathBuf,+ tile: usize,+ gpu: OnceLock<Result<GpuSide, String>>,+ /// Tiles the card got wrong (an error, numbers that are not finite) and the CPU ran.+ fallbacks: AtomicUsize,+ /// Tiles at once of the photo started last (0: none yet), to say where the work runs.+ last_threads: AtomicUsize,+}++#[cfg(feature = "denoise")]+struct GpuSide {+ runner: Box<dyn TileRunner + Send>,+ adapter: String,+ /// The check tile on the card, the fastest of a few runs (ms).+ card_ms: f64,+ /// The check tile on the CPU, on one thread (ms).+ cpu_ms: f64,+}++/// Whether the card (`card_ms` a tile) is faster than the CPU running `threads` tiles at once, from the CPU's time for+/// one tile alone (`cpu_ms`). Tiles at once share the memory and the cores' second threads, so each one more counts as+/// half a tile's speed: on a 16-core laptop one tile alone took 1.4 s and sixteen at once finished one every 0.2 s.+#[cfg(feature = "denoise")]+fn card_is_faster(card_ms: f64, cpu_ms: f64, threads: usize) -> bool {+ let at_once = 1.0 + 0.5 * threads.saturating_sub(1) as f64;+ card_ms.is_finite() && card_ms <= cpu_ms / at_once+}++/// Card set-ups of this process that have not finished, by marker (their markers are not a crash's).+#[cfg(feature = "denoise")]+static SETTING_UP: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());++/// Removes a set-up's marker when the set-up ends, however it ends — but not when the process dies inside the driver.+#[cfg(feature = "denoise")]+struct SetUpEnded(PathBuf);++#[cfg(feature = "denoise")]+impl Drop for SetUpEnded {+ fn drop(&mut self) {+ let _ = std::fs::remove_file(&self.0);+ SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner).retain(|m| m != &self.0);+ }+}++/// Set the graphics card up with `card` on a thread of its own while `meanwhile` runs here, and give up on the card when+/// it takes longer than `limit` (a driver building kernels for minutes, or stuck). The `marker` file exists while `card`+/// runs: one found before it starts was left by a LightCraft that closed during a set-up (a driver crash takes the app+/// with it), so the card is not tried again until the user chooses where denoise runs (`denoise.settings {runOn}`).+#[cfg(feature = "denoise")]+fn set_up_card<T: Send + 'static, R>(+ marker: &Path,+ limit: Duration,+ card: impl FnOnce() -> Result<T, String> + Send + 'static,+ meanwhile: impl FnOnce() -> R,+) -> Result<(T, R), String> {+ let ended = {+ let mut running = SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner);+ if running.iter().any(|m| m == marker) {+ return Err(GPU_STUCK.into());+ }+ if marker.exists() {+ return Err(GPU_CRASHED.into());+ }+ let secs = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map_or(0, |d| d.as_secs());+ // a marker that cannot be written only loses the crash guard+ let _ = std::fs::write(marker, format!("setting up the graphics card for AI Denoise (unix time {secs})\n"));+ running.push(marker.to_path_buf());+ SetUpEnded(marker.to_path_buf())+ };+ let started = std::time::Instant::now();+ let (tx, rx) = std::sync::mpsc::channel();+ std::thread::Builder::new()+ .name("denoise-gpu-setup".into())+ .stack_size(STACK)+ .spawn(move || {+ let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(card)).unwrap_or_else(|_| Err("the GPU runner gave up".into()));+ drop(ended);+ let _ = tx.send(r);+ })+ .map_err(|e| format!("could not start setting up the graphics card: {e}"))?;+ let here = meanwhile();+ match rx.recv_timeout(limit.saturating_sub(started.elapsed())) {+ Ok(r) => r.map(|t| (t, here)),+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(format!("{GPU_SLOW} {} s, so it is not used", limit.as_secs_f64())),+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err("the GPU runner gave up".into()),+ }+}++/// The card's time for one tile once it is warm: the fastest of up to three runs, stopping after about two seconds (a+/// slow card is slow enough to tell at once).+#[cfg(feature = "denoise")]+fn card_tile_ms(runner: &dyn TileRunner, input: &[f32]) -> f64 {+ if runner.run(input).is_err() {+ return f64::INFINITY;+ }+ let began = std::time::Instant::now();+ let mut best = f64::INFINITY;+ for _ in 0..3 {+ let one = std::time::Instant::now();+ if runner.run(input).is_err() {+ break;+ }+ best = best.min(one.elapsed().as_secs_f64() * 1000.0);+ if began.elapsed() > Duration::from_secs(2) {+ break;+ }+ }+ best+}++/// Set the card up for the model at `path`: kernels built, the check tile run on both and compared, both timed.+#[cfg(feature = "denoise")]+fn make_gpu(cpu: &lightcraft_denoise::runtime::CpuRunner, path: &Path, tile: usize) -> Result<GpuSide, String> {+ let input = Arc::new(lightcraft_denoise::runtime::check_tile(tile));+ let (on_card, net) = (input.clone(), cpu.net().clone());+ let card = move || -> Result<_, String> {+ let runner = lightcraft_gpu::nn::runner(&net, tile)?;+ let got = runner.run(&on_card).map_err(|e| format!("the GPU could not run the check tile: {e}"))?;+ let card_ms = card_tile_ms(&runner, &on_card);+ Ok((runner, got, card_ms))+ };+ let on_cpu = || {+ let started = std::time::Instant::now();+ (cpu.run(&input), started.elapsed().as_secs_f64() * 1000.0)+ };+ let ((runner, got, card_ms), (want, cpu_ms)) = set_up_card(&path.with_file_name(GPU_SETUP_MARKER), GPU_SETUP_LIMIT, card, on_cpu)?;+ let want = want.map_err(|e| format!("the CPU could not check the GPU: {e}"))?;+ let scale = want.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-12);+ let worst = want.iter().zip(&got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;+ if got.len() != want.len() || !got.iter().all(|v| v.is_finite()) || worst > GPU_AGREE {+ return Err(format!("its answer on the check tile is not the CPU's (off by {worst:.1e} of the largest value): not used"));+ }+ Ok(GpuSide { adapter: runner.adapter(), runner: Box::new(runner), card_ms, cpu_ms })+}++#[cfg(feature = "denoise")]+impl Network {+ /// The card when it can be used now, else why not. `start` sets it up when that has not been tried yet; without it+ /// nothing is started.+ fn card(&self, start: bool) -> Result<&GpuSide, String> {+ let side = if start {+ Some(self.gpu.get_or_init(|| {+ let made = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_gpu(&self.cpu, &self.path, self.tile)))+ .unwrap_or_else(|_| Err("the GPU runner gave up".into()));+ if let Err(why) = &made {+ log::info!("denoise: running on the CPU ({why})");+ }+ made+ }))+ } else {+ self.gpu.get()+ };+ let g = match side {+ None => return Err("the graphics card is not set up yet".into()),+ Some(Err(why)) => return Err(why.clone()),+ Some(Ok(g)) => g,+ };+ if let Some(why) = lightcraft_gpu::nn::broken_reason() {+ return Err(format!("the graphics card stopped working ({why})"));+ }+ if self.fallbacks.load(Ordering::Relaxed) >= GPU_MAX_FALLBACKS {+ return Err(format!("{} got {GPU_MAX_FALLBACKS} tiles wrong", g.adapter));+ }+ Ok(g)+ }++ /// The card, when a photo made `threads` tiles at a time under `run_on` runs on it; else why the CPU does.+ fn choose(&self, run_on: RunOn, threads: usize, start: bool) -> Result<&GpuSide, String> {+ if run_on == RunOn::Cpu {+ return Err("the processor is chosen in Settings".into());+ }+ let g = self.card(start)?;+ if run_on == RunOn::Auto && !card_is_faster(g.card_ms, g.cpu_ms, threads) {+ return Err(format!("it is faster than {} on this computer", g.adapter));+ }+ Ok(g)+ }++ fn fell_back(&self, why: &str) {+ let n = self.fallbacks.fetch_add(1, Ordering::Relaxed) + 1;+ if n == 1 {+ log::warn!("denoise: a tile came back wrong from the GPU ({why}); the CPU runs it");+ } else if n == GPU_MAX_FALLBACKS {+ log::warn!("denoise: the GPU got {n} tiles wrong; the CPU does the rest");+ }+ }+}++/// The card, with the CPU running any tile it gets wrong. Handed out only for photos that run on the card.+#[cfg(feature = "denoise")]+impl TileRunner for Network {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, RunError> {+ if let Ok(g) = self.card(false) {+ match g.runner.run(input) {+ Ok(out) if out.iter().all(|v| v.is_finite()) => return Ok(out),+ Ok(_) => self.fell_back("numbers that are not finite"),+ Err(e) => self.fell_back(&e.to_string()),+ }+ }+ self.cpu.run(input)+ }+}++#[cfg(feature = "denoise")]+impl Model for Network {+ fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize) {+ let threads = threads.max(1);+ self.last_threads.store(threads, Ordering::Relaxed);+ match self.choose(run_on, threads, true) {+ Ok(_) => (self, threads.min(GPU_PARALLEL)),+ Err(_) => (&self.cpu, threads),+ }+ }++ fn self_test(&self, run_on: RunOn) -> Result<Value, String> {+ let t = self.cpu.self_test();+ if !t.ok {+ let failed: Vec<&str> = t.checks.iter().filter(|(_, ok)| !*ok).map(|(c, _)| c.as_str()).collect();+ return Err(format!("the model failed its self-test: it does not {}", failed.join(", and does not ")));+ }+ let mut v = serde_json::to_value(&t).map_err(|e| e.to_string())?;+ // the card is set up and checked here, so the test says where the work will run+ if run_on != RunOn::Cpu {+ let _ = self.card(true);+ }+ if let Some(o) = v.as_object_mut() {+ o.insert("device".into(), self.device(run_on));+ }+ Ok(v)+ }++ fn device(&self, run_on: RunOn) -> Value {+ if run_on != RunOn::Cpu && self.gpu.get().is_none() {+ return json!({"kind": "pending"});+ }+ let threads = match self.last_threads.load(Ordering::Relaxed) {+ 0 => Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None),+ n => n,+ };+ let mut v = match self.choose(run_on, threads, false) {+ Ok(g) => json!({"kind": "gpu", "adapter": g.adapter}),+ // a set-up that crashed or ran out of time can be tried again (`denoise.settings {runOn}`)+ Err(why) => json!({"kind": "cpu", "retry": worth_retrying(&why), "reason": why}),+ };+ if let (Some(Ok(g)), Some(o)) = (self.gpu.get(), v.as_object_mut()) {+ // measured on this computer, on the check tile: the card's best run, and one tile alone on the CPU+ o.insert("cardMs".into(), json!(g.card_ms));+ o.insert("cpuMs".into(), json!(g.cpu_ms));+ o.insert("fellBack".into(), json!(self.fallbacks.load(Ordering::Relaxed)));+ }+ v+ }+}++/// The loader of this build: our pure-Rust executor with the `denoise` feature, else one that says why nothing can run.+pub(crate) fn default_loader() -> Loader {+ #[cfg(feature = "denoise")]+ {+ Arc::new(|path, manifest| {+ let cpu = lightcraft_denoise::runtime::CpuRunner::load(path, manifest).map_err(|e| e.to_string())?;+ Ok(Arc::new(Network {+ cpu,+ path: path.to_path_buf(),+ tile: manifest.tile as usize,+ gpu: OnceLock::new(),+ fallbacks: AtomicUsize::new(0),+ last_threads: AtomicUsize::new(0),+ }) as Arc<dyn Model>)+ })+ }+ #[cfg(not(feature = "denoise"))]+ {+ Arc::new(|_, _| Err("this build cannot run denoise models".to_string()))+ }+}++/// A model that is loaded on first use and kept, shared by the background job and the exports.+pub(crate) type ModelSlot = Arc<Mutex<Option<Arc<dyn Model>>>>;++// ---------------------------------------------------------------------------------------------------------------------+// settings and installed models++/// `settings.json` in the models folder.+#[derive(Clone, Default, Serialize, Deserialize)]+#[serde(default, rename_all = "camelCase")]+pub(crate) struct Settings {+ /// The installed model in use; `None`: denoise is off.+ pub model: Option<String>,+ /// Photos with a Denoise amount above 0 that are being looked at (active, selected) get their picture made without+ /// being asked (`None`: yes).+ pub auto: Option<bool>,+ /// Cache size limit (GB; `None`: [`DEFAULT_CACHE_GB`]).+ pub cache_gb: Option<u32>,+ /// Most tiles run at once (`None`: by how hard the pump says to work).+ pub threads: Option<u32>,+ /// Where the model runs: `auto`, `gpu` or `cpu` (`None` or anything else: `auto`; see [`RunOn`]).+ pub run_on: Option<String>,+ /// What came before `runOn`: `false` was the processor only.+ pub gpu: Option<bool>,+}++impl Settings {+ pub fn auto(&self) -> bool {+ self.auto.unwrap_or(true)+ }++ pub fn run_on(&self) -> RunOn {+ match self.run_on.as_deref().and_then(RunOn::parse) {+ Some(r) => r,+ None if self.gpu == Some(false) => RunOn::Cpu,+ None => RunOn::Auto,+ }+ }++ pub fn cache_bytes(&self) -> u64 {+ u64::from(self.cache_gb.unwrap_or(DEFAULT_CACHE_GB).clamp(1, 10_000)) << 30+ }+}++pub(crate) fn read_capped(path: &Path, max: usize) -> Option<Vec<u8>> {+ use std::io::Read;+ let mut buf = Vec::new();+ std::fs::File::open(path).ok()?.take(max as u64 + 1).read_to_end(&mut buf).ok()?;+ (buf.len() <= max).then_some(buf)+}++pub(crate) fn read_settings(dir: &Path) -> Settings {+ read_capped(&dir.join("settings.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()+}++/// Write a file next to its final name and rename it into place.+pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> {+ let part = path.with_extension("part");+ std::fs::write(&part, bytes).map_err(|e| format!("could not write {}: {e}", path.display()))?;+ std::fs::rename(&part, path).map_err(|e| {+ let _ = std::fs::remove_file(&part);+ format!("could not write {}: {e}", path.display())+ })+}++pub(crate) fn write_settings(dir: &Path, st: &Settings) -> Result<(), String> {+ std::fs::create_dir_all(dir).map_err(|e| format!("could not create the denoise models folder: {e}"))?;+ write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| e.to_string())?)+}++/// An installed model folder: `<id>/model.onnx`, `denoise-model.json`, `installed.json`.+pub(crate) struct Installed {+ pub manifest: DenoiserManifest,+ pub onnx: PathBuf,+ /// What was recorded when it was installed (terms accepted, hash of the file, self-test).+ pub accepted: Value,+}++impl Installed {+ /// Names the model file the cache is made with: a hash of the file as installed, so another file with the same id+ /// never finds this one's products.+ pub fn fingerprint(&self) -> String {+ let sha = self.accepted.get("sha256").and_then(Value::as_str).or(self.manifest.sha256.as_deref()).unwrap_or("");+ let len = std::fs::metadata(&self.onnx).map(|m| m.len()).unwrap_or(0);+ format!("{}@{}:{len}", self.manifest.id, sha)+ }+}++/// The models in the folder. A folder that is not a valid model (wrong name, no model file, a manifest that does not+/// validate or does not match its folder) is ignored, never trusted.+pub(crate) fn installed_models(dir: &Path) -> Vec<Installed> {+ let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };+ let mut out = Vec::new();+ for e in entries.flatten() {+ let p = e.path();+ let Some(name) = p.file_name().and_then(|n| n.to_str()).map(str::to_string) else { continue };+ let onnx = p.join("model.onnx");+ if !lightcraft_denoise::licence::valid_id(&name) || !p.is_dir() || !onnx.is_file() {+ continue;+ }+ let Some(m) = read_capped(&p.join("denoise-model.json"), lightcraft_denoise::manifest::MAX_MANIFEST_BYTES)+ .and_then(|b| lightcraft_denoise::manifest::parse(&b).ok())+ else {+ continue;+ };+ if m.id != name {+ continue;+ }+ let accepted = read_capped(&p.join("installed.json"), 8192).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null);+ out.push(Installed { manifest: m, onnx, accepted });+ }+ out.sort_by(|a, b| a.manifest.id.cmp(&b.manifest.id));+ out+}++/// Forget that setting the graphics card up for a model closed LightCraft before (the markers [`set_up_card`] leaves):+/// the user chose where denoise runs, so the card may be tried again.+pub(crate) fn forget_gpu_set_ups(dir: &Path) {+ for i in installed_models(dir) {+ let _ = std::fs::remove_file(i.onnx.with_file_name(GPU_SETUP_MARKER));+ }+}++/// The model in use.+pub(crate) struct Active {+ pub id: String,+ pub manifest: DenoiserManifest,+ pub onnx: PathBuf,+ pub fingerprint: String,+ pub model: ModelSlot,+}++// ---------------------------------------------------------------------------------------------------------------------+// products++/// The photos whose files can be denoised at all: raw files developed from their sensor data and read from disk.+pub(crate) fn eligible(p: &Photo) -> bool {+ p.develops_raw() && matches!(p.source, Source::File { .. }) && !p.deleted+}++/// A product's name: what it was made from and with.+pub(crate) fn product_key(fingerprint: &str, p: &Photo) -> String {+ let mut h = Hasher128::new();+ h.str("lcdn").u64(u64::from(product::ALGORITHM)).str(fingerprint).str(&crate::media::content_key(p));+ h.finish().to_string()+}++pub(crate) fn product_path(dir: &Path, key: &str) -> PathBuf {+ dir.join(format!("{key}.{EXT}"))+}++/// The `.lcdn` files in `dir`: path, size, last change.+fn products_in(dir: &Path) -> Vec<(PathBuf, u64, SystemTime)> {+ let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };+ entries+ .flatten()+ .filter(|e| e.path().extension().is_some_and(|x| x == EXT))+ .filter_map(|e| {+ let m = e.metadata().ok()?;+ m.is_file().then(|| (e.path(), m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH)))+ })+ .collect()+}++/// Make sure only one thread writes a product at a time: a second one waits, then finds it made.+fn making() -> &'static (Mutex<HashSet<PathBuf>>, Condvar) {+ static MAKING: OnceLock<(Mutex<HashSet<PathBuf>>, Condvar)> = OnceLock::new();+ MAKING.get_or_init(|| (Mutex::new(HashSet::new()), Condvar::new()))+}++struct Slot(PathBuf);++impl Slot {+ fn enter(path: &Path) -> Slot {+ let (set, cv) = making();+ let mut g = set.lock().unwrap_or_else(PoisonError::into_inner);+ while g.contains(path) {+ g = cv.wait(g).unwrap_or_else(PoisonError::into_inner);+ }+ g.insert(path.to_path_buf());+ Slot(path.to_path_buf())+ }+}++impl Drop for Slot {+ fn drop(&mut self) {+ let (set, cv) = making();+ set.lock().unwrap_or_else(PoisonError::into_inner).remove(&self.0);+ cv.notify_all();+ }+}++// ---------------------------------------------------------------------------------------------------------------------+// making a product++/// Everything a thread needs to make one product, detached from the session.+#[derive(Clone)]+pub(crate) struct JobSpec {+ /// The original file.+ pub path: String,+ pub read: Option<ByteReader>,+ pub product: PathBuf,+ pub key: String,+ pub manifest: DenoiserManifest,+ pub onnx: PathBuf,+ pub model: ModelSlot,+ pub loader: Loader,+ /// Tiles run at once.+ pub parallel: usize,+ /// Where the model runs (the setting).+ pub run_on: RunOn,+}++/// Why a product was not made.+#[derive(Debug, PartialEq)]+pub(crate) enum MakeError {+ /// Denoise does not apply to this file (not a Bayer raw): nothing is wrong.+ Unsupported(String),+ Failed(String),+ Cancelled,+}++impl From<RunError> for MakeError {+ fn from(e: RunError) -> Self {+ match e {+ RunError::Cancelled => MakeError::Cancelled,+ other => MakeError::Failed(other.to_string()),+ }+ }+}++/// Progress and cancel of a running job.+#[derive(Default)]+pub(crate) struct Progress {+ done: AtomicUsize,+ total: AtomicUsize,+ cancel: AtomicBool,+}++impl Progress {+ /// Ask the job to stop at the next tile.+ pub(crate) fn cancel_now(&self) {+ self.cancel.store(true, Ordering::Relaxed);+ }+}++/// The loaded model, loading it first when it is not (kept for the next photo).+fn loaded(spec: &JobSpec) -> Result<Arc<dyn Model>, MakeError> {+ if let Some(m) = spec.model.lock().unwrap_or_else(PoisonError::into_inner).clone() {+ return Ok(m);+ }+ let m = (spec.loader)(&spec.onnx, &spec.manifest).map_err(|e| MakeError::Failed(format!("the denoise model could not be loaded: {e}")))?;+ *spec.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(m.clone());+ Ok(m)+}++/// The cell `[(0,0), (1,0), (0,1), (1,1)]` colours of a Bayer mosaic and so its layout.+fn layout_of(cfa: &lightcraft_raw::Cfa) -> Option<Layout> {+ Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)])+}++/// Make the product of `spec` unless it is there: decode the photo, run the model over its mosaic, write the result.+/// `Ok(true)` when it was made now, `Ok(false)` when it already existed.+pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Result<bool, MakeError> {+ let _slot = Slot::enter(&spec.product);+ if product::is_current(&spec.product, &spec.key) {+ return Ok(false);+ }+ let began = std::time::Instant::now();+ let cancelled = || progress.is_some_and(|p| p.cancel.load(Ordering::Relaxed));+ if cancelled() {+ return Err(MakeError::Cancelled);+ }+ let bytes = match &spec.read {+ Some(r) => r(&spec.path).map_err(MakeError::Failed)?,+ None => std::fs::read(&spec.path).map_err(|e| MakeError::Failed(format!("{}: {e}", spec.path)))?,+ };+ // the decode and the normalised mosaic wait their turn at the process-wide memory gate, as every decode does+ let (mosaic, layout) = {+ let _held = crate::memory::work_gate().acquire(bytes.len().saturating_mul(4));+ if lightcraft_raw::probe(&bytes).is_none() {+ return Err(MakeError::Unsupported("not a raw file".into()));+ }+ let raw = match lightcraft_raw::decode(&bytes) {+ Ok(r) => r,+ Err(lightcraft_raw::RawError::Unsupported(why)) => return Err(MakeError::Unsupported(why)),+ Err(e) => return Err(MakeError::Failed(e.to_string())),+ };+ drop(bytes);+ if raw.cpp != 1 || !raw.cfa.as_ref().is_some_and(lightcraft_raw::Cfa::is_bayer) {+ return Err(MakeError::Unsupported(+ "only Bayer raw files can be denoised (this one has another sensor layout or is already demosaiced)".into(),+ ));+ }+ let n = raw.normalized().map_err(|e| MakeError::Failed(e.to_string()))?;+ drop(raw);+ let layout =+ n.cfa.as_ref().and_then(layout_of).ok_or_else(|| MakeError::Unsupported("the colour filter layout is not a Bayer pattern".into()))?;+ (n, layout)+ };+ if cancelled() {+ return Err(MakeError::Cancelled);+ }+ let decoded = began.elapsed();+ let model = loaded(spec)?;+ let (runner, parallel) = model.runner(spec.run_on, spec.parallel.max(1));+ let params = Params {+ tile: spec.manifest.tile as usize,+ overlap: spec.manifest.overlap as usize,+ gain: spec.manifest.gain,+ clip: Some(CLIP),+ parallel: parallel.max(1),+ };+ let report = |done: usize, total: usize| {+ if let Some(p) = progress {+ p.total.store(total, Ordering::Relaxed);+ p.done.store(done, Ordering::Relaxed);+ }+ };+ let ctl = Control { cancel: progress.map(|p| &p.cancel), progress: Some(&report) };+ // the tiles run on threads of their own, so a busy window's renders keep theirs+ let pool = rayon::ThreadPoolBuilder::new()+ .num_threads(params.parallel)+ .stack_size(STACK)+ .thread_name(|i| format!("denoise-{i}"))+ .build()+ .map_err(|e| MakeError::Failed(format!("could not start the worker threads: {e}")))?;+ let rgb = pool.install(|| denoise_bayer(&mosaic.data, mosaic.width, mosaic.height, layout, runner, ¶ms, &ctl))?;+ drop(mosaic);+ if cancelled() {+ return Err(MakeError::Cancelled);+ }+ let ran = began.elapsed();+ product::write(&spec.product, &rgb, &spec.key).map_err(|e| MakeError::Failed(format!("could not save the denoised picture: {e}")))?;+ if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {+ eprintln!(+ "[denoise] read + decode {:.0} ms, model (set-up, tiles, blend) {:.0} ms, write {:.0} ms",+ decoded.as_secs_f64() * 1e3,+ (ran - decoded).as_secs_f64() * 1e3,+ (began.elapsed() - ran).as_secs_f64() * 1e3+ );+ }+ Ok(true)+}++// ---------------------------------------------------------------------------------------------------------------------+// the session's state++/// How hard the background work may go (the app picks it from what the user is doing, like the face scan).+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum Pace {+ /// Start nothing new.+ Pause,+ /// Two tiles at once.+ Light,+ /// Half the machine.+ Normal,+ /// Most of the machine: the user is watching the progress.+ Full,+}++impl Pace {+ /// `pause`, `light`, `normal` or `full`; anything else is `normal`.+ pub fn parse(name: Option<&str>) -> Pace {+ match name {+ Some("pause") => Pace::Pause,+ Some("light") => Pace::Light,+ Some("full") => Pace::Full,+ _ => Pace::Normal,+ }+ }++ /// Tiles to run at once on a machine with `cores` threads (`cap`: the user's own limit).+ pub fn parallel(self, cores: usize, cap: Option<u32>) -> usize {+ let n = match self {+ Pace::Pause | Pace::Light => 2,+ Pace::Normal => cores / 2,+ Pace::Full => cores * 4 / 5,+ }+ .clamp(1, 16);+ cap.map_or(n, |c| n.min(c.max(1) as usize))+ }+}++/// How a photo stands, for the interface.+#[derive(Clone, Debug, PartialEq, Serialize)]+#[serde(tag = "state", rename_all = "camelCase")]+pub enum PhotoState {+ /// Not a photo denoise applies to (not a raw, offline, a demo).+ NotApplicable,+ /// No model is chosen: nothing can be made.+ NoModel,+ /// Its picture is made: the Amount slider works.+ Ready,+ Queued {+ /// Photos ahead of it (0 = next).+ ahead: usize,+ },+ Running {+ done: usize,+ total: usize,+ },+ /// Making it failed (`why`), or denoise does not apply to this file (`unsupported`).+ Failed {+ why: String,+ unsupported: bool,+ },+ /// Nothing has asked for it yet.+ Idle,+}++#[derive(Clone)]+struct Failure {+ key: String,+ why: String,+ unsupported: bool,+}++enum Outcome {+ Done,+ Failed(String, bool),+ Cancelled,+}++struct Running {+ photo: PhotoId,+ key: String,+ product: PathBuf,+ progress: Arc<Progress>,+ outcome: Arc<Mutex<Option<Outcome>>>,+ started: web_time::Instant,+}++pub(crate) struct State {+ /// Where the models are kept (`None` where there is no file system).+ pub models_dir: Option<PathBuf>,+ /// Models being downloaded at the user's request.+ pub downloads: Downloads,+ /// How a model is loaded (tests put another in).+ pub loader: Loader,+ pub(crate) active: Option<Active>,+ active_seen: Option<web_time::Instant>,+ /// `settings.json` as last read (with the model choice).+ pub(crate) settings: Settings,+ /// The index must be made again (the library, the model or the cache changed).+ dirty: bool,+ /// The catalog revision and time of the last index, to look again when photos changed.+ reindexed: Option<(u64, web_time::Instant)>,+ queue: VecDeque<PhotoId>,+ running: Option<Running>,+ failed: HashMap<PhotoId, Failure>,+ /// Counts every change of what is ready, so the interface knows to draw again.+ pub generation: u64,+ /// Photos made since the session started.+ made: u64,+}++impl Default for State {+ fn default() -> Self {+ State {+ models_dir: None,+ downloads: Downloads::default(),+ loader: default_loader(),+ active: None,+ active_seen: None,+ settings: Settings::default(),+ dirty: true,+ reindexed: None,+ queue: VecDeque::new(),+ running: None,+ failed: HashMap::new(),+ generation: 0,+ made: 0,+ }+ }+}++impl Drop for State {+ /// Quitting stops the job instead of leaving it to finish a photo nobody will see.+ fn drop(&mut self) {+ if let Some(r) = &self.running {+ r.progress.cancel_now();+ }+ }+}++impl State {+ /// The library was replaced: nothing carries over (photo ids are per library).+ pub fn library_changed(&mut self) {+ if let Some(r) = self.running.take() {+ r.progress.cancel_now();+ }+ self.queue.clear();+ self.failed.clear();+ self.dirty = true;+ self.generation += 1;+ }++ /// The model choice or the cache changed on disk: look again at the next pump.+ pub fn touch(&mut self) {+ self.active_seen = None;+ self.dirty = true;+ }++ pub fn queued(&self) -> usize {+ self.queue.len()+ }+}++impl Session {+ /// Where the active model's tiles run (see [`Model::device`]); `{kind: "none"}` while no model is loaded.+ pub(crate) fn denoise_device(&self) -> Value {+ let loaded = self.denoise.active.as_ref().and_then(|a| a.model.lock().unwrap_or_else(PoisonError::into_inner).clone());+ match loaded {+ Some(m) => m.device(self.denoise.settings.run_on()),+ None => json!({"kind": "none"}),+ }+ }++ /// The folder the products of the open library are kept in; `None` without a library on disk.+ pub(crate) fn denoise_products_dir(&self) -> Option<PathBuf> {+ self.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join(DIR))+ }++ /// Read which model is in use (at most every couple of seconds): the manifest and file of the model the settings+ /// name, if it is installed.+ pub(crate) fn denoise_refresh_active(&mut self, force: bool) {+ let fresh = self.denoise.active_seen.is_some_and(|t| t.elapsed() < SETTINGS_TTL);+ if fresh && !force {+ return;+ }+ self.denoise.active_seen = Some(web_time::Instant::now());+ self.denoise.settings = self.denoise.models_dir.as_deref().map(read_settings).unwrap_or_default();+ let chosen = self.denoise.models_dir.as_deref().and_then(|dir| {+ let id = self.denoise.settings.model.clone()?;+ installed_models(dir).into_iter().find(|i| i.manifest.id == id)+ });+ let found = chosen.map(|i| (i.fingerprint(), i));+ let same = match (&self.denoise.active, &found) {+ (None, None) => true,+ (Some(a), Some((fp, _))) => a.fingerprint == *fp,+ _ => false,+ };+ if same {+ return;+ }+ // another model: what was made with the old one is not this one's, and what failed may work now+ self.denoise.active = found.map(|(fingerprint, i)| Active {+ id: i.manifest.id.clone(),+ manifest: i.manifest,+ onnx: i.onnx,+ fingerprint,+ model: Arc::new(Mutex::new(None)),+ });+ if let Some(r) = self.denoise.running.take() {+ r.progress.cancel_now();+ }+ self.denoise.queue.clear();+ self.denoise.failed.clear();+ self.denoise.dirty = true;+ }++ fn denoise_spec(&self, p: &Photo) -> Option<(DenoiseSpec, String)> {+ let (dir, active) = (self.denoise_products_dir()?, self.denoise.active.as_ref()?);+ let key = product_key(&active.fingerprint, p);+ Some((DenoiseSpec { product: product_path(&dir, &key), key: key.clone(), make: None }, key))+ }++ /// Make the index of ready photos from the products on disk (a directory listing and a hash per photo: no file is+ /// opened). Pictures that appeared or vanished drop the decoded sources they affect.+ pub(crate) fn denoise_reindex(&mut self) {+ self.denoise.dirty = false;+ self.denoise.reindexed = Some((self.catalog.revision, web_time::Instant::now()));+ let mut now: HashMap<PhotoId, DenoiseSpec> = HashMap::new();+ if let (Some(dir), Some(active)) = (self.denoise_products_dir(), self.denoise.active.as_ref()) {+ let names: HashSet<String> =+ products_in(&dir).into_iter().filter_map(|(p, _, _)| p.file_stem().and_then(|s| s.to_str()).map(str::to_string)).collect();+ if !names.is_empty() {+ for p in self.catalog.photos().filter(|p| eligible(p)) {+ let key = product_key(&active.fingerprint, p);+ if names.contains(&key) {+ now.insert(p.id, DenoiseSpec { product: product_path(&dir, &key), key, make: None });+ }+ }+ }+ }+ let mut changed = false;+ for id in self.media.denoise.ids() {+ if !now.contains_key(&id) {+ self.media.denoise.remove(id);+ self.media.forget(id);+ changed = true;+ }+ }+ for (id, spec) in now {+ if self.media.denoise.spec(id).is_none_or(|old| old.key != spec.key) {+ self.media.denoise.set(id, spec);+ self.media.forget(id);+ changed = true;+ }+ }+ if changed {+ self.denoise.generation += 1;+ }+ }++ /// A photo whose file changed (its content key is not the one its index entry was made for) has no picture.+ fn denoise_check_photo(&mut self, id: PhotoId) {+ let Some(p) = self.catalog.photo(id).cloned() else { return };+ let Some((spec, _)) = self.denoise_spec(&p) else { return };+ match self.media.denoise.spec(id) {+ Some(have) if have.key != spec.key => {+ self.media.denoise.remove(id);+ self.media.forget(id);+ self.denoise.generation += 1;+ }+ None if spec.product.is_file() && product::is_current(&spec.product, &spec.key) => {+ // made by an export, or by another session+ self.media.denoise.set(id, spec);+ self.media.forget(id);+ self.denoise.generation += 1;+ }+ _ => {}+ }+ }++ /// How photo `id` stands.+ pub fn denoise_photo_state(&self, id: PhotoId) -> PhotoState {+ let Some(p) = self.catalog.photo(id) else { return PhotoState::NotApplicable };+ if !eligible(p) {+ return PhotoState::NotApplicable;+ }+ if self.denoise.active.is_none() {+ // the model folder has not been looked at yet (just started): not "no model" until it has been+ return if self.denoise.active_seen.is_none() { PhotoState::Idle } else { PhotoState::NoModel };+ }+ if self.denoise_products_dir().is_none() {+ return PhotoState::Failed {+ why: "AI Denoise keeps its pictures in the library folder, and this session has none.".into(),+ unsupported: true,+ };+ }+ if self.media.denoise.spec(id).is_some() {+ return PhotoState::Ready;+ }+ if let Some(r) = self.denoise.running.as_ref().filter(|r| r.photo == id) {+ return PhotoState::Running { done: r.progress.done.load(Ordering::Relaxed), total: r.progress.total.load(Ordering::Relaxed) };+ }+ if let Some(ahead) = self.denoise.queue.iter().position(|q| *q == id) {+ return PhotoState::Queued { ahead: ahead + usize::from(self.denoise.running.is_some()) };+ }+ let key = self.denoise_spec(p).map(|s| s.1);+ match self.denoise.failed.get(&id) {+ Some(f) if Some(&f.key) == key.as_ref() => PhotoState::Failed { why: f.why.clone(), unsupported: f.unsupported },+ _ => PhotoState::Idle,+ }+ }++ /// Ask for photos' pictures to be made (the ones that are not made, queued or known to be impossible). With `retry`+ /// a photo that failed before is tried again. A photo that is not raw is skipped. Returns how many were queued.+ pub(crate) fn denoise_enqueue(&mut self, ids: &[PhotoId], front: bool, retry: bool) -> usize {+ self.denoise_refresh_active(false);+ if self.denoise.active.is_none() || self.denoise_products_dir().is_none() {+ return 0;+ }+ let mut added = 0;+ for id in ids {+ let Some(p) = self.catalog.photo(*id).cloned() else { continue };+ if !eligible(&p) || self.media.denoise.spec(*id).is_some() || self.denoise.queue.contains(id) {+ continue;+ }+ if self.denoise.running.as_ref().is_some_and(|r| r.photo == *id) || self.denoise.queue.len() >= MAX_QUEUE {+ continue;+ }+ let Some((_, key)) = self.denoise_spec(&p) else { continue };+ if retry {+ self.denoise.failed.remove(id);+ } else if self.denoise.failed.get(id).is_some_and(|f| f.key == key) {+ continue;+ }+ if front {+ self.denoise.queue.push_front(*id);+ } else {+ self.denoise.queue.push_back(*id);+ }+ added += 1;+ }+ added+ }++ /// Stop the job that is running, and with `all` forget everything that waits.+ pub(crate) fn denoise_cancel(&mut self, photo: Option<PhotoId>, all: bool) -> usize {+ let mut n = 0;+ if let Some(r) = &self.denoise.running+ && photo.is_none_or(|p| p == r.photo)+ {+ r.progress.cancel_now();+ n += 1;+ }+ let before = self.denoise.queue.len();+ match photo {+ Some(p) => self.denoise.queue.retain(|q| *q != p),+ None if all => self.denoise.queue.clear(),+ None => {}+ }+ n + before - self.denoise.queue.len()+ }++ /// The photos the automatic queue looks at: the one open and the ones selected, those with a Denoise amount.+ fn denoise_wanted(&self) -> Vec<PhotoId> {+ let Selection { active, ids, .. } = &self.selection;+ active+ .iter()+ .chain(ids.iter().take(AUTO_LOOK))+ .copied()+ .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0))+ .collect()+ }++ fn denoise_job_spec(&self, p: &Photo, parallel: usize) -> Option<JobSpec> {+ let Source::File { path } = &p.source else { return None };+ let (spec, _) = self.denoise_spec(p)?;+ let active = self.denoise.active.as_ref()?;+ Some(JobSpec {+ path: path.clone(),+ read: self.media.file_bytes.clone(),+ product: spec.product,+ key: spec.key,+ manifest: active.manifest.clone(),+ onnx: active.onnx.clone(),+ model: active.model.clone(),+ loader: self.denoise.loader.clone(),+ parallel,+ run_on: self.denoise.settings.run_on(),+ })+ }++ /// Start the next queued photo on its own thread.+ fn denoise_start_next(&mut self, parallel: usize) {+ while let Some(id) = self.denoise.queue.pop_front() {+ let Some(p) = self.catalog.photo(id).cloned() else { continue };+ if !eligible(&p) || self.media.denoise.spec(id).is_some() {+ continue;+ }+ let Some(spec) = self.denoise_job_spec(&p, parallel) else { continue };+ // another session or an export may have made it since it was queued+ if product::is_current(&spec.product, &spec.key) {+ self.denoise_check_photo(id);+ continue;+ }+ let progress = Arc::new(Progress::default());+ let outcome: Arc<Mutex<Option<Outcome>>> = Arc::new(Mutex::new(None));+ let (key, product) = (spec.key.clone(), spec.product.clone());+ let (worker_progress, worker_outcome) = (progress.clone(), outcome.clone());+ let started = std::thread::Builder::new().name("denoise".into()).stack_size(STACK).spawn(move || {+ let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_product(&spec, Some(&worker_progress))));+ let o = match result {+ Ok(Ok(_)) => Outcome::Done,+ Ok(Err(MakeError::Cancelled)) => Outcome::Cancelled,+ Ok(Err(MakeError::Unsupported(why))) => Outcome::Failed(why, true),+ Ok(Err(MakeError::Failed(why))) => Outcome::Failed(why, false),+ Err(_) => Outcome::Failed("the denoiser stopped unexpectedly".into(), false),+ };+ *worker_outcome.lock().unwrap_or_else(PoisonError::into_inner) = Some(o);+ });+ match started {+ Ok(_) => {+ self.denoise.running = Some(Running { photo: id, key, product, progress, outcome, started: web_time::Instant::now() });+ }+ Err(e) => {+ self.denoise.failed.insert(id, Failure { key, why: format!("could not start the denoiser: {e}"), unsupported: false });+ continue;+ }+ }+ return;+ }+ }++ /// Take in the running job's result.+ fn denoise_poll(&mut self) {+ let Some(r) = &self.denoise.running else { return };+ let Some(outcome) = r.outcome.lock().unwrap_or_else(PoisonError::into_inner).take() else { return };+ let Some(r) = self.denoise.running.take() else { return };+ match outcome {+ Outcome::Done => {+ self.denoise.made += 1;+ self.denoise_check_photo(r.photo);+ self.denoise_evict(Some(&r.product));+ }+ Outcome::Failed(why, unsupported) => {+ self.denoise.failed.insert(r.photo, Failure { key: r.key, why, unsupported });+ }+ Outcome::Cancelled => {}+ }+ // the picture's state changed: the interface draws it again+ self.denoise.generation += 1;+ }++ /// Keep the cache within its limit: the pictures nobody asks for go first, the oldest first; `keep` (the one just+ /// made) never goes. The index forgets what was deleted.+ pub(crate) fn denoise_evict(&mut self, keep: Option<&Path>) {+ let Some(models) = self.denoise.models_dir.clone() else { return };+ self.denoise_evict_to(read_settings(&models).cache_bytes(), keep);+ }++ /// [`Self::denoise_evict`] for a limit of `limit` bytes.+ pub(crate) fn denoise_evict_to(&mut self, limit: u64, keep: Option<&Path>) {+ let Some(dir) = self.denoise_products_dir() else { return };+ let mut files = products_in(&dir);+ let mut total: u64 = files.iter().map(|f| f.1).sum();+ if total <= limit {+ return;+ }+ // the pictures photos use now (a Denoise amount above 0) are the last to go+ let wanted: HashSet<PathBuf> = self+ .media+ .denoise+ .ids()+ .into_iter()+ .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0))+ .filter_map(|id| self.media.denoise.spec(id).map(|s| s.product.clone()))+ .collect();+ files.sort_by_key(|(path, _, modified)| (wanted.contains(path), *modified));+ let mut gone: HashSet<PathBuf> = HashSet::new();+ for (path, size, _) in files {+ if total <= limit {+ break;+ }+ if keep == Some(path.as_path()) {+ continue;+ }+ if std::fs::remove_file(&path).is_ok() {+ total = total.saturating_sub(size);+ gone.insert(path);+ }+ }+ if gone.is_empty() {+ return;+ }+ for id in self.media.denoise.ids() {+ if self.media.denoise.spec(id).is_some_and(|s| gone.contains(&s.product)) {+ self.media.denoise.remove(id);+ self.media.forget(id);+ }+ }+ self.denoise.generation += 1;+ }++ /// Delete every product of this library (they are made again when wanted).+ pub(crate) fn denoise_clear(&mut self) -> (usize, u64) {+ self.denoise_cancel(None, true);+ let (mut files, mut bytes) = (0, 0);+ if let Some(dir) = self.denoise_products_dir() {+ for (path, size, _) in products_in(&dir) {+ if std::fs::remove_file(&path).is_ok() {+ files += 1;+ bytes += size;+ }+ }+ }+ self.denoise.failed.clear();+ self.denoise.touch();+ (files, bytes)+ }++ /// Per frame: take in what finished, look at what the user is looking at, and keep one job going. Cheap when+ /// nothing is happening (no file is opened unless a photo with a Denoise amount lacks its picture).+ pub(crate) fn denoise_pump(&mut self, pace: Pace) {+ crate::cmd::denoise::finish_downloads(self);+ self.denoise_refresh_active(false);+ // photos that changed (a replaced file has another key) must not keep the picture of what they were+ let stale = !self.media.denoise.is_empty()+ && self.denoise.reindexed.is_none_or(|(rev, at)| rev != self.catalog.revision && at.elapsed() > SETTINGS_TTL);+ if self.denoise.dirty || stale {+ self.denoise_reindex();+ }+ self.denoise_poll();+ if self.denoise.active.is_none() || self.denoise_products_dir().is_none() {+ return;+ }+ let (cap, auto) = (self.denoise.settings.threads, self.denoise.settings.auto());+ let wanted = self.denoise_wanted();+ for id in &wanted {+ self.denoise_check_photo(*id);+ }+ if auto {+ // the photo being edited goes first+ let (open, others): (Vec<PhotoId>, Vec<PhotoId>) = wanted.into_iter().partition(|id| self.selection.active == Some(*id));+ self.denoise_enqueue(&others, false, false);+ self.denoise_enqueue(&open, true, false);+ }+ if self.denoise.running.is_none() && pace != Pace::Pause {+ let cores = std::thread::available_parallelism().map_or(1, |n| n.get());+ self.denoise_start_next(pace.parallel(cores, cap));+ }+ }++ /// The statistics `denoise.status` reports besides the queue (reads the cache folder).+ pub(crate) fn denoise_cache_usage(&self) -> (usize, u64) {+ match self.denoise_products_dir() {+ Some(dir) => {+ let files = products_in(&dir);+ (files.len(), files.iter().map(|f| f.1).sum())+ }+ None => (0, 0),+ }+ }++ /// Are pictures made without being asked for photos being looked at (Settings ▸ AI Denoise)?+ pub fn denoise_auto(&self) -> bool {+ self.denoise.settings.auto()+ }++ /// Is a picture being made, or waiting for its turn with a model to make it? (Headless runs wait for this.)+ pub fn denoise_busy(&self) -> bool {+ self.denoise.running.is_some() || (!self.denoise.queue.is_empty() && self.denoise.active.is_some())+ }++ /// Seconds the running job has been going, and its progress.+ pub(crate) fn denoise_running_json(&self) -> Value {+ match &self.denoise.running {+ Some(r) => json!({+ "photo": r.photo.0,+ "done": r.progress.done.load(Ordering::Relaxed),+ "total": r.progress.total.load(Ordering::Relaxed),+ "seconds": r.started.elapsed().as_secs_f64(),+ }),+ None => Value::Null,+ }+ }++ pub(crate) fn denoise_made(&self) -> u64 {+ self.denoise.made+ }++ /// What an export of photo `id` with `settings` needs for its Denoise amount: its denoised picture, made now when the+ /// queue has not got to it. `Ok(None)` when denoise does not apply (amount 0, not a Bayer raw), `Err` when it should+ /// have and cannot (no model chosen, nowhere to keep the picture, the model failing).+ pub(crate) fn denoise_for_export(&mut self, id: PhotoId, settings: &DevelopSettings) -> Result<Option<(DenoiseSpec, PairLoader)>, String> {+ if settings.denoise_amount() <= 0.0 {+ return Ok(None);+ }+ let Some(p) = self.catalog.photo(id).cloned() else { return Ok(None) };+ let Some(loader) = self.media.denoise.loader.clone() else { return Ok(None) };+ if !eligible(&p) {+ return Ok(None);+ }+ self.denoise_refresh_active(false);+ let name = &p.file_name;+ if self.denoise.active.is_none() {+ return Err(format!("{name}: Denoise is on for this photo but no denoise model is chosen; choose one in Settings, or set Denoise to 0"));+ }+ if self.denoise_products_dir().is_none() {+ return Err(format!("{name}: Denoise needs a library on disk to keep its pictures in"));+ }+ self.denoise_check_photo(id);+ let Some(job) = self.denoise_job_spec(&p, Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None)) else {+ return Ok(None);+ };+ let label = name.clone();+ let spec = job.clone();+ let make: MakeProduct = Arc::new(move || match make_product(&spec, None) {+ Ok(_) => Ok(true),+ Err(MakeError::Unsupported(_)) => Ok(false),+ Err(MakeError::Failed(why)) => Err(format!("{label}: could not denoise: {why}")),+ Err(MakeError::Cancelled) => Err(format!("{label}: denoise was cancelled")),+ });+ Ok(Some((DenoiseSpec { product: job.product, key: job.key, make: Some(make) }, loader)))+ }+}++#[cfg(all(test, feature = "denoise"))]+mod gpu_tests {+ use super::*;+ use lightcraft_denoise::manifest::{Domain, Gain};++ /// A folder of the test's own: the card's set-up leaves its marker beside the model.+ fn folder(name: &str) -> PathBuf {+ let dir = std::env::temp_dir().join(format!("lc-engine-gpu-{}-{name}", std::process::id()));+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ dir+ }++ /// The model file is read again when the card is set up (for the first photo), so it stays until the test ends.+ fn model(name: &str, tile: u64) -> (Arc<dyn Model>, lightcraft_denoise::runtime::CpuRunner, PathBuf) {+ let path = folder(name).join("model.onnx");+ std::fs::write(&path, lightcraft_denoise::synthetic::unet_onnx(tile, 8, 2, 7)).unwrap();+ let manifest = DenoiserManifest {+ id: "synthetic".into(),+ name: "Synthetic".into(),+ version: "1".into(),+ licence: Default::default(),+ source: None,+ sha256: None,+ size_bytes: None,+ provenance: String::new(),+ domain: Domain::BayerToRgb,+ tile: tile as u32,+ overlap: 16,+ gain: Gain::MatchMean { nominal: 1.0, max_deviation: 0.05 },+ };+ let loaded = default_loader()(&path, &manifest).unwrap();+ let cpu = lightcraft_denoise::runtime::CpuRunner::load(&path, &manifest).unwrap();+ (loaded, cpu, path)+ }++ fn worst(want: &[f32], got: &[f32]) -> f32 {+ assert_eq!(want.len(), got.len());+ let scale = want.iter().fold(1e-9f32, |m, v| m.max(v.abs()));+ want.iter().zip(got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale+ }++ #[test]+ fn a_tile_on_the_card_is_the_cpus_tile() {+ let (model, cpu, path) = model("same", 64);+ let (input, _) = lightcraft_denoise::runtime::test_tile(64);+ let want = cpu.run(&input).unwrap();+ let (runner, at_once) = model.runner(RunOn::Gpu, 16);+ assert!(worst(&want, &runner.run(&input).unwrap()) < 1e-3);+ let device = model.device(RunOn::Gpu);+ eprintln!("device: {device}");+ match device["kind"].as_str() {+ Some("gpu") => {+ assert!(device["adapter"].as_str().is_some_and(|a| !a.is_empty()));+ assert!(at_once <= GPU_PARALLEL, "a card is fed by a few tiles at once");+ assert!(device["cardMs"].as_f64().is_some_and(|ms| ms > 0.0) && device["cpuMs"].as_f64().is_some_and(|ms| ms > 0.0), "{device}");+ }+ Some("cpu") => {+ assert!(device["reason"].as_str().is_some_and(|r| !r.is_empty()), "the CPU is only used for a reason: {device}");+ assert_eq!(at_once, 16);+ }+ other => panic!("the first photo decides where the work runs, not {other:?}"),+ }+ assert!(!path.with_file_name(GPU_SETUP_MARKER).exists(), "a set-up that ended leaves no marker");+ let _ = std::fs::remove_dir_all(path.parent().unwrap());+ }++ #[test]+ fn with_the_processor_chosen_everything_runs_on_the_cpu() {+ let (model, cpu, path) = model("cpu", 64);+ let (input, _) = lightcraft_denoise::runtime::test_tile(64);+ let (runner, at_once) = model.runner(RunOn::Cpu, 16);+ assert_eq!(runner.run(&input).unwrap(), cpu.run(&input).unwrap(), "the CPU path is the CPU runner, bit for bit");+ assert_eq!(at_once, 16, "the threads asked for are the threads used");+ assert_eq!(model.device(RunOn::Cpu)["kind"], "cpu");+ assert_eq!(model.device(RunOn::Auto)["kind"], "pending", "the card was not even set up");+ let _ = std::fs::remove_dir_all(path.parent().unwrap());+ }++ #[test]+ fn the_card_is_used_where_it_is_faster_than_the_processor() {+ // measured: a tile alone on one CPU thread 1.4 s; an RTX 4090 Laptop 13 ms; a Radeon 610M 490 ms+ for threads in [1, 2, 8, 16, 64] {+ assert!(card_is_faster(13.0, 1400.0, threads), "a big card wins at {threads}");+ }+ assert!(card_is_faster(490.0, 1400.0, 2), "a small card beats the two threads background work gets");+ assert!(!card_is_faster(490.0, 1400.0, 16), "and loses to sixteen");+ assert!(!card_is_faster(f64::INFINITY, 1400.0, 1) && !card_is_faster(f64::NAN, 1400.0, 1), "a card that never ran is never faster");+ assert!(!card_is_faster(10.0, 0.0, 1));+ }++ /// A card that gives numbers that are not finite.+ struct Garbage(AtomicUsize);++ impl TileRunner for Garbage {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, RunError> {+ self.0.fetch_add(1, Ordering::Relaxed);+ let n = input.len() / 4 * 12;+ Ok(vec![f32::NAN; n])+ }+ }++ #[test]+ fn a_tile_the_card_gets_wrong_is_run_on_the_processor_and_three_stop_the_card() {+ let (_, cpu, path) = model("garbage", 64);+ let card = GpuSide { runner: Box::new(Garbage(AtomicUsize::new(0))), adapter: "Test card".into(), card_ms: 1.0, cpu_ms: 1000.0 };+ let model = Network {+ cpu: cpu.clone(),+ path: path.clone(),+ tile: 32,+ gpu: OnceLock::from(Ok(card)),+ fallbacks: AtomicUsize::new(0),+ last_threads: AtomicUsize::new(0),+ };+ let (input, _) = lightcraft_denoise::runtime::test_tile(64);+ let want = cpu.run(&input).unwrap();+ let (runner, at_once) = model.runner(RunOn::Auto, 8);+ assert_eq!(at_once, GPU_PARALLEL, "the (fast) card is chosen");+ for _ in 0..GPU_MAX_FALLBACKS {+ assert_eq!(runner.run(&input).unwrap(), want, "a wrong tile is the CPU's tile");+ }+ let device = model.device(RunOn::Auto);+ assert_eq!(device["kind"], "cpu", "{device}");+ assert!(device["reason"].as_str().unwrap().contains("tiles wrong"), "{device}");+ assert_eq!(device["fellBack"], GPU_MAX_FALLBACKS);+ let (_, at_once) = model.runner(RunOn::Gpu, 8);+ assert_eq!(at_once, 8, "the next photo runs on the CPU, even with the card chosen");+ let _ = std::fs::remove_dir_all(path.parent().unwrap());+ }++ #[test]+ fn a_slow_card_is_left_out_unless_it_is_chosen() {+ let (_, cpu, path) = model("slow", 64);+ let slow = GpuSide { runner: Box::new(cpu.clone()), adapter: "Slow card".into(), card_ms: 490.0, cpu_ms: 1400.0 };+ let model = Network {+ cpu,+ path: path.clone(),+ tile: 32,+ gpu: OnceLock::from(Ok(slow)),+ fallbacks: AtomicUsize::new(0),+ last_threads: AtomicUsize::new(0),+ };+ assert_eq!(model.runner(RunOn::Auto, 16).1, 16, "sixteen threads beat it");+ let device = model.device(RunOn::Auto);+ assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some("it is faster than Slow card on this computer")));+ assert_eq!(device["cardMs"], 490.0, "the times measured are reported");+ assert_eq!(device["retry"], false, "trying again would not make it faster");+ assert_eq!(model.runner(RunOn::Auto, 2).1, GPU_PARALLEL.min(2), "two threads do not");+ assert_eq!(model.device(RunOn::Auto)["kind"], "gpu");+ assert_eq!(model.runner(RunOn::Gpu, 16).1, GPU_PARALLEL, "chosen, it is used anyway");+ let _ = std::fs::remove_dir_all(path.parent().unwrap());+ }++ #[test]+ fn a_set_up_marker_left_behind_keeps_the_card_off() {+ let (model, _, path) = model("crashed", 64);+ let marker = path.with_file_name(GPU_SETUP_MARKER);+ std::fs::write(&marker, "setting up the graphics card for AI Denoise (unix time 0)").unwrap();+ let (_, at_once) = model.runner(RunOn::Gpu, 4);+ assert_eq!(at_once, 4, "the CPU does the work");+ let device = model.device(RunOn::Gpu);+ assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some(GPU_CRASHED)), "{device}");+ assert_eq!(device["retry"], true, "the interface offers to try again");+ assert!(marker.exists(), "it stays until the user chooses where denoise runs");+ let _ = std::fs::remove_dir_all(path.parent().unwrap());+ }++ #[test]+ fn a_set_up_runs_beside_the_cpu_and_removes_its_marker() {+ let dir = folder("setup");+ let marker = dir.join(GPU_SETUP_MARKER);+ let seen = Arc::new(AtomicBool::new(false));+ let (m, s) = (marker.clone(), seen.clone());+ let r = set_up_card(+ &marker,+ Duration::from_secs(10),+ move || {+ s.store(m.exists(), Ordering::Relaxed);+ Ok(7)+ },+ || "cpu",+ );+ assert_eq!(r, Ok((7, "cpu")));+ assert!(seen.load(Ordering::Relaxed), "the marker is there while the card is set up");+ assert!(!marker.exists(), "and gone after");+ // a set-up that fails or panics removes it too+ assert_eq!(set_up_card(&marker, Duration::from_secs(10), || Err::<(), _>("no card".into()), || ()), Err("no card".into()));+ assert!(!marker.exists());+ let r = set_up_card(&marker, Duration::from_secs(10), || -> Result<(), String> { panic!("driver") }, || ());+ assert!(r.is_err() && !marker.exists(), "{r:?}");+ // a marker found before the set-up starts means the last one took LightCraft down: the card is not touched+ std::fs::write(&marker, "left behind").unwrap();+ let touched = Arc::new(AtomicBool::new(false));+ let t = touched.clone();+ let r = set_up_card(+ &marker,+ Duration::from_secs(10),+ move || {+ t.store(true, Ordering::Relaxed);+ Ok(())+ },+ || (),+ );+ assert_eq!(r, Err(GPU_CRASHED.to_string()));+ assert!(!touched.load(Ordering::Relaxed) && marker.exists());+ let _ = std::fs::remove_dir_all(&dir);+ }++ #[test]+ fn a_set_up_that_takes_too_long_is_given_up_on() {+ let dir = folder("slowsetup");+ let marker = dir.join(GPU_SETUP_MARKER);+ let started = std::time::Instant::now();+ let slow = || -> Result<(), String> {+ std::thread::sleep(Duration::from_millis(600));+ Ok(())+ };+ let r = set_up_card(&marker, Duration::from_millis(50), slow, || ());+ assert!(r.as_ref().is_err_and(|e| e.contains("took longer") && worth_retrying(e)), "{r:?}");+ assert!(started.elapsed() < Duration::from_millis(500), "the caller does not wait for the card");+ // while the first is still stuck, another set-up of the same model does not start a second one+ let again = set_up_card(&marker, Duration::from_secs(5), || Ok(()), || ());+ assert!(again.is_err_and(|e| e.contains("still setting up") && worth_retrying(&e)));+ // once it ends its marker goes, and the card can be set up again+ while marker.exists() {+ assert!(started.elapsed() < Duration::from_secs(10), "the stuck set-up ended");+ std::thread::sleep(Duration::from_millis(20));+ }+ while SETTING_UP.lock().unwrap().iter().any(|m| m == &marker) {+ std::thread::sleep(Duration::from_millis(5));+ }+ assert_eq!(set_up_card(&marker, Duration::from_secs(5), || Ok(1), || 2), Ok((1, 2)));+ let _ = std::fs::remove_dir_all(&dir);+ }+}diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rsindex c620be928..5ef823dbf 100644--- a/crates/engine/src/files.rs+++ b/crates/engine/src/files.rs@@ -11,7 +11,7 @@ use lightcraft_pipeline::SourceInfo;use lightcraft_raster::Rgb32f;use lightcraft_raster::resample::{Filter, fit};-use crate::media::{FileLoader, FileProbe, PreviewLoader, ProbeInfo};+use crate::media::{DenoiseSpec, FileLoader, FileProbe, PairLoader, PreviewLoader, ProbeInfo};fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option<String>) {let shutter = m.exposure_time.map(|t| if t >= 1.0 { format!("{t:.0}") } else { format!("1/{:.0}", 1.0 / t) }).unwrap_or_default();@@ -220,22 +220,56 @@ pub fn bin_factor(raw: &lightcraft_raw::RawImage, max_edge: usize) -> Option<usi/// instead of each one waking the pool from outside and waiting for it (which costs more than the/// loops themselves when the machine is busy).pub fn load_bytes(bytes: &[u8], max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {- rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge))+ rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge, None).map(|(img, _, info)| (img, info)))}/// [`load_bytes`] taking the file's bytes: a raw file's bytes are freed as soon as it is decoded/// (less memory held while it is developed).pub fn load_vec(bytes: Vec<u8>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {- rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge))+ rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, None).map(|(img, _, info)| (img, info)))}-fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {+/// [`load_vec`] that also develops the photo's cached denoised picture (`spec`) from the same decode: the plain+/// picture, the denoised one (`None` when it cannot be read or does not fit this raw: the plain one is then all+/// there is) and the source facts. Both pictures are the same size.+pub fn load_vec_pair(bytes: Vec<u8>, max_edge: usize, spec: &DenoiseSpec) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> {+ rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, Some(spec)))+}++/// The denoised camera RGB of `raw` from its cached product, as the plain development would have it: the default+/// crop, and `factor` × `factor` blocks averaged into one pixel when the plain one was binned. `None` when the product+/// is missing, damaged, made from something else or not the size of this raw's active area.+fn denoised_camera_rgb(raw: &lightcraft_raw::RawImage, spec: &DenoiseSpec, factor: usize) -> Option<Rgb32f> {+ use lightcraft_denoise::product::{self, Window};+ let a = raw.active_area;+ let header = product::read_header_at(&spec.product).ok()?;+ if header.key != spec.key || (header.width, header.height) != (a.width, a.height) {+ return None;+ }+ if raw.opcodes.list3.is_empty() {+ let c = raw.develop_crop(a.width, a.height);+ let window = Window { x: c.x, y: c.y, width: c.width, height: c.height };+ product::read_window(&spec.product, &spec.key, Some(window), factor, Some(HIGHLIGHT_CLIP)).ok()+ } else {+ // (the plain development is not binned either when the file has opcodes of this kind: `factor` is 1)+ let whole = product::read(&spec.product, &spec.key, 1).ok()?;+ Some(raw.finish_demosaiced(whole))+ }+}++fn load_bytes_now(+ bytes: std::borrow::Cow<'_, [u8]>,+ max_edge: usize,+ denoise: Option<&DenoiseSpec>,+) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> {if lightcraft_raw::probe(&bytes).is_some() {let mut raw = match lightcraft_raw::decode(&bytes).map_err(|e| preview_reason(&bytes, e)) {Ok(r) => r,Err(Ok(why)) => {// show the camera's embedded JPEG (rendered, not raw) until the variant is supported- return load_embedded_preview(&bytes, max_edge).ok_or(format!("unsupported raw ({why}) without an embedded preview"));+ return load_embedded_preview(&bytes, max_edge)+ .map(|(img, info)| (img, None, info))+ .ok_or(format!("unsupported raw ({why}) without an embedded preview"));}Err(Err(e)) => return Err(e),};@@ -250,71 +284,72 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<// Previews and thumbnails bin the mosaic straight to (about) the size they need; only// larger levels (exports, 1:1) demosaic the whole sensor.let t0 = web_time::Instant::now();- let binned = match bin_factor(&raw, max_edge) {+ let bin = bin_factor(&raw, max_edge);+ let binned = match bin {Some(k) => raw.develop_binned(k, HIGHLIGHT_CLIP).map_err(|e| e.to_string())?,None => None,};- let mut img = match binned {+ let factor = if binned.is_some() { bin.unwrap_or(1) } else { 1 };+ let img = match binned {Some(img) => img,None => {let method = if max_edge <= 600 { lightcraft_raw::Method::Bilinear } else { lightcraft_raw::Method::Ahd };raw.develop(method).map_err(|e| e.to_string())?}};+ // the same development from the denoised mosaic, when the photo has been denoised+ let twin = denoise.and_then(|spec| denoised_camera_rgb(&raw, spec, factor)).filter(|d| d.width == img.width && d.height == img.height);// the samples aren't needed any more (the colour model below reads only the tags)raw.data = lightcraft_raw::RawData::U16(Vec::new());- let mut stages = vec![("develop", t0.elapsed())];- stages.push(("transform", t0.elapsed()));- lightcraft_raw::highlight::reconstruct(&mut img, t.wb, HIGHLIGHT_CLIP);- stages.push(("highlights", t0.elapsed()));+ let wb = t.wb;let m = camera_look.as_ref().map(|p| p.matrix.mul(&t.matrix)).unwrap_or(t.matrix).to_f32();let hue_sat = camera_look.as_ref().and_then(|p| p.hue_sat.as_ref()).and_then(crate::camera_preview::HueSat::new);let gain = 2f32.powf(t.baseline_exposure as f32);- let wb = t.wb;// A DNG's own profile look (hue/saturation map, look table), DNG spec chapter 6.let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy));- img.map_in_place(|p| {- let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];- let rgb = [- m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],- m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],- m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],- ];- let rgb = match &tables {- Some(tables) => tables.apply(rgb, gain),- None => rgb.map(|v| v * gain),- };- // after the baseline exposure, as when it was fitted- hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))- });- stages.push(("colour", t0.elapsed()));- let img = fit(&img, max_edge, max_edge, Filter::Box);- stages.push(("fit", t0.elapsed()));- let img = img.into_oriented(raw.orientation);- stages.push(("orient", t0.elapsed()));+ // camera RGB → what the pipeline takes: clipped highlights rebuilt, white balance and the colour model applied,+ // fitted to `max_edge` and upright (the same for the plain and the denoised picture)+ let finish = |mut img: Rgb32f| -> Rgb32f {+ lightcraft_raw::highlight::reconstruct(&mut img, wb, HIGHLIGHT_CLIP);+ img.map_in_place(|p| {+ let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];+ let rgb = [+ m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],+ m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],+ m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],+ ];+ let rgb = match &tables {+ Some(tables) => tables.apply(rgb, gain),+ None => rgb.map(|v| v * gain),+ };+ hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))+ });+ let img = fit(&img, max_edge, max_edge, Filter::Box);+ img.into_oriented(raw.orientation)+ };+ let develop_ms = t0.elapsed();+ let img = finish(img);+ let twin = twin.map(finish);if lightcraft_pipeline::profiling() {- let mut prev = std::time::Duration::ZERO;- let parts: Vec<String> = stages- .iter()- .map(|(n, t)| {- let d = *t - prev;- prev = *t;- format!("{n} {:.1}", d.as_secs_f64() * 1e3)- })- .collect();- eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", "));+ let finish_ms = (t0.elapsed() - develop_ms).as_secs_f64() * 1e3;+ eprintln!(+ "[profile] raw source {}×{} (max {max_edge}): develop {:.1} ms, finish {finish_ms:.1} ms (after decode)",+ img.width,+ img.height,+ develop_ms.as_secs_f64() * 1e3+ );}let (temp, tint) = xy_to_temp_tint(xy);let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback;let camera_tone = camera_look.as_ref().map(|p| p.tone).or_else(|| raw.color.profile.tone_curve.as_ref().and_then(dng_tone_curve));let (temp, tint) = if relative { (6500.0, 0.0) } else { (temp.round(), tint.round()) };- return Ok((img, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone }));+ return Ok((img, twin, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone }));}let d = lightcraft_codecs::decode(&bytes, fit_box(max_edge)).map_err(|e| e.to_string())?;drop(bytes);let img = d.to_working();let img = if img.width.max(img.height) > max_edge { fit(&img, max_edge, max_edge, Filter::Mitchell) } else { img };- Ok((img.into_oriented(Orientation::from_exif(d.orientation)), SourceInfo::default()))+ Ok((img.into_oriented(Orientation::from_exif(d.orientation)), None, SourceInfo::default()))}/// A DNG `ProfileToneCurve` (linear in, linear out, 1.0 = white after exposure compensation) as the@@ -408,6 +443,21 @@ pub fn fs_preview_loader() -> PreviewLoader {Arc::new(|path: &str, max_edge: usize| embedded_preview_srgb(&std::fs::read(path).ok()?, max_edge))}+/// The filesystem-backed [`PairLoader`]: the file is read and decoded once, and developed as usual and from the photo's+/// denoised picture. It holds two pictures at once, so it counts for more at the memory gate than a plain load.+pub fn fs_pair_loader() -> PairLoader {+ Arc::new(|path: &str, max_edge: usize, spec: &DenoiseSpec| {+ let len = std::fs::metadata(path).map(|m| m.len() as usize).unwrap_or(0);+ let weight = len * if max_edge <= crate::media::SourceLevel::Thumb.max_edge() { 4 } else { 10 };+ let gate = crate::memory::work_gate();+ let _permit = if crate::memory::is_background() { gate.acquire(weight) } else { gate.acquire_urgent(weight) };+ let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?;+ let r = load_vec_pair(bytes, max_edge, spec);+ crate::memory::release();+ r+ })+}+/// Filesystem-backed hooks (native). On the web the host installs bytes-based hooks instead.////// Their working memory is bounded by [`crate::memory::work_gate`]: background loads (grid@@ -443,6 +493,7 @@ impl crate::Session {self.media.file_loader = Some(l);self.media.file_probe = Some(p);self.media.preview_loader = Some(fs_preview_loader());+ self.media.denoise.loader = Some(fs_pair_loader());self}}@@ -483,6 +534,37 @@ mod tests {assert!(probe_bytes("x.jpg", b"not an image").is_err());}+ #[test]+ fn cached_denoise_uses_the_same_white_balance_profile_and_orientation_as_plain_raw() {+ use lightcraft_denoise::product;+ use lightcraft_raw::{DngWriteOptions, Method, Orientation, profile::HsvTable, write_dng};+ let dir = crate::tests_xmp::temp_dir("denoise-color-finish");+ let fixture = crate::tests_xmp::synthetic_dng_with(None, Default::default());+ let mut raw = lightcraft_raw::decode(&fixture).unwrap();+ raw.orientation = Orientation::Rotate90;+ raw.color.baseline_exposure = 0.5;+ raw.color.profile.look_table =+ Some(HsvTable { hue_divisions: 1, sat_divisions: 2, val_divisions: 1, data: vec![[23.0, 0.65, 0.8]; 2], srgb_value: false });+ let bytes = write_dng(&raw, &DngWriteOptions::default()).unwrap();+ let decoded = lightcraft_raw::decode(&bytes).unwrap();+ assert!(decoded.color.profile.look_table.is_some());+ // A cache containing the ordinary demosaic must undergo the identical downstream finish.+ let camera_rgb = decoded.develop(Method::Ahd).unwrap();+ let key = "synthetic-color-finish";+ let path = dir.join("same.lcdn");+ product::write(&path, &camera_rgb, key).unwrap();+ let spec = DenoiseSpec { product: path, key: key.into(), make: None };+ let (ordinary, _) = load_bytes(&bytes, 2048).unwrap();+ let (plain, cached, _) = load_vec_pair(bytes, 2048, &spec).unwrap();+ assert_eq!(plain.data, ordinary.data, "requesting a twin must not change the ordinary path");+ let cached = cached.unwrap();+ assert_eq!((plain.width, plain.height), (24, 32));+ assert_eq!((cached.width, cached.height), (plain.width, plain.height));+ let worst = plain.data.iter().zip(&cached.data).flat_map(|(a, b)| a.iter().zip(b).map(|(a, b)| (a - b).abs())).fold(0.0f32, f32::max);+ assert!(worst < 0.002, "color finish differs beyond cache half-float rounding: {worst}");+ let _ = std::fs::remove_dir_all(dir);+ }+/// Luminance (from linear RGB) minus its 9 × 9 local mean: tone differences between a render and a camera JPEG/// mostly cancel, edges remain.fn detail(w: usize, h: usize, lum: impl Fn(usize, usize) -> f32) -> Vec<f32> {diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rsindex f26b4ca7a..5f298de4e 100644--- a/crates/engine/src/lib.rs+++ b/crates/engine/src/lib.rs@@ -14,9 +14,11 @@ pub mod availability;mod camera_preview;pub mod camera_profiles;pub mod cmd;+pub mod config;pub mod crs;pub mod crs_masks;pub mod demo;+pub mod denoise;pub mod devices;pub mod export;pub mod files;@@ -33,6 +35,7 @@ pub mod logging;pub mod media;pub mod memory;pub mod merge;+mod model_download;pub mod originals;pub mod preset_import;pub mod preset_luminar;@@ -132,8 +135,10 @@ pub struct Interaction {/// Source of [`Session::visible_shared`] generations (process-wide, so two sessions never share one).static VISIBLE_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);+static LIBRARY_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);pub struct Session {+ library_generation: u64,/// Auto Sync: edits to the active photo also change the other selected photos (the settings/// that changed, nothing else).pub auto_sync: bool,@@ -157,6 +162,8 @@ pub struct Session {/// Set by a command whose change must not rewrite the photo's XMP sidecar even with auto-write on/// (a catalog-only edit of data the sidecar writer does not emit); consumed when the command ends.pub(crate) skip_auto_write: bool,+ /// AI denoise: the model in use, the photos that have their picture and the work in progress.+ pub(crate) denoise: denoise::State,/// Copied develop settings (partial JSON) for Paste.pub clipboard: Option<Value>,/// The folder on disk the [`LibrarySource::Folder`] view browses.@@ -247,8 +254,13 @@ impl Default for Session {}impl Session {+ pub fn library_generation(&self) -> u64 {+ self.library_generation+ }+pub fn new() -> Session {Session {+ library_generation: LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed),auto_sync: false,catalog: Catalog::new(),source: LibrarySource::All,@@ -263,6 +275,7 @@ impl Session {redo: Vec::new(),interaction: None,skip_auto_write: false,+ denoise: Default::default(),clipboard: None,meta_clipboard: None,browse: None,@@ -783,6 +796,8 @@ mod tests;#[cfg(test)]mod tests_color;#[cfg(test)]+mod tests_denoise;+#[cfg(test)]mod tests_export;#[cfg(test)]mod tests_folders;diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rsindex c1b91c4cd..384d90ea3 100644--- a/crates/engine/src/library.rs+++ b/crates/engine/src/library.rs@@ -297,6 +297,9 @@ impl Session {{log::error!("library: {e}");}+ // Photo ids belong to one library; denoise work must not carry into the next.+ self.denoise.library_changed();+ self.media.denoise.clear();let (mut journal, mut catalog, report) = Journal::open(catalog)?;// A loaded catalog counts revisions from 0, like every other one. Caches (sidebar counts,// keyword tree, the grid's list…) are keyed on the revision, so give each library loaded@@ -305,6 +308,7 @@ impl Session {static LOADS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);catalog.revision = LOADS.fetch_add(1, std::sync::atomic::Ordering::Relaxed).wrapping_add(1) << 32;self.catalog = catalog;+ self.library_generation = crate::LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed);self.undo.clear();self.redo.clear();self.interaction = None;diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rsindex 1ed89e473..479831c98 100644--- a/crates/engine/src/media.rs+++ b/crates/engine/src/media.rs@@ -125,6 +125,44 @@ pub type FileLoader = Arc<dyn Fn(&str, usize) -> Result<(Rgb32f, SourceInfo), St/// `max_edge` (set by the app). `None`: no usable preview.pub type PreviewLoader = Arc<dyn Fn(&str, usize) -> Option<Rgba8> + Send + Sync>;+/// Makes a photo's denoised picture when it is not cached yet (an export does not wait for the background queue):+/// `Ok(true)` once it is there, `Ok(false)` when denoise does not apply to this photo (render it plain), `Err` when it+/// should have worked and did not.+pub type MakeProduct = Arc<dyn Fn() -> Result<bool, String> + Send + Sync>;++/// A raw photo's cached AI-denoised picture (see [`crate::denoise`]): where the file is and the key it must have been+/// made with. A file that is missing, damaged or made for something else is a cache miss, never an error.+#[derive(Clone)]+pub struct DenoiseSpec {+ pub product: std::path::PathBuf,+ pub key: String,+ /// Run before the picture is read, by a job that must not render without it (an export).+ pub make: Option<MakeProduct>,+}++/// Decodes a raw file once and develops it twice: as usual, and from the denoised picture in `spec` when that can be+/// read (`None` when it cannot). Both are at most `max_edge` long and the same size.+pub type PairLoader = Arc<dyn Fn(&str, usize, &DenoiseSpec) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> + Send + Sync>;++/// A photo's denoised picture next to its plain one, and the last mix of the two that was asked for.+pub struct Twin {+ pub image: Arc<Rgb32f>,+ /// The Amount (in thousandths) and the picture mixed for it, so dragging other sliders keeps one source picture.+ mix: std::sync::Mutex<Option<(u32, Arc<Rgb32f>)>>,+}++impl Twin {+ pub fn new(image: Arc<Rgb32f>) -> Twin {+ Twin { image, mix: std::sync::Mutex::new(None) }+ }++ /// Bytes held, the last mix included.+ fn bytes(&self) -> usize {+ let mixed = self.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref().map_or(0, |m| m.1.data.len() * 12);+ self.image.data.len() * 12 + mixed+ }+}+/// Pixels and the source interpretation learned while decoding; they must be cached together.#[derive(Clone)]pub struct DecodedSource {@@ -133,11 +171,45 @@ pub struct DecodedSource {/// A smart preview's stored camera tone curve: the one decoder fact its pixels need that the/// catalog's header facts lack (used when `info` is `None`).pub camera_tone: Option<lightcraft_pipeline::tone::CameraTone>,+ /// The same picture developed from the AI-denoised mosaic, when the photo has one cached.+ pub denoised: Option<Arc<Twin>>,}impl DecodedSource {pub fn new(image: Arc<Rgb32f>, info: Option<SourceInfo>) -> Self {- DecodedSource { image, info, camera_tone: None }+ DecodedSource { image, info, camera_tone: None, denoised: None }+ }++ /// The picture to develop for a Denoise `amount` (0 to 1, see [`DevelopSettings::denoise_amount`]): the plain one,+ /// the denoised one, or a mix of the two. A mix is made once per amount and kept, so the stages after it are+ /// reused until the amount changes.+ pub fn image_for(&self, amount: f32) -> Arc<Rgb32f> {+ let Some(twin) = &self.denoised else { return self.image.clone() };+ if amount.is_nan() || amount <= 0.0 || twin.image.width != self.image.width || twin.image.height != self.image.height {+ return self.image.clone();+ }+ if amount >= 1.0 {+ return twin.image.clone();+ }+ let thousandths = (amount * 1000.0).round().clamp(1.0, 999.0) as u32;+ let mut memo = twin.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner);+ if let Some((t, img)) = memo.as_ref()+ && *t == thousandths+ {+ return img.clone();+ }+ let mut mixed = (*self.image).clone();+ if !lightcraft_denoise::product::blend(&mut mixed, &twin.image, thousandths as f32 / 1000.0) {+ return self.image.clone();+ }+ let mixed = Arc::new(mixed);+ *memo = Some((thousandths, mixed.clone()));+ mixed+ }++ /// Bytes held: the picture, its denoised twin and the last mix.+ pub fn bytes(&self) -> usize {+ self.image.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>() + self.denoised.as_ref().map_or(0, |t| t.bytes())}/// What to render these pixels against: the decoder's facts, else `header` (the catalog's)@@ -161,6 +233,8 @@ pub enum SourceRef {/// The photo's smart preview, used when the original can't be read (offline drive). The/// UI thread never checks whether the original is there; the render worker finds out.fallback: Option<std::path::PathBuf>,+ /// The photo has a cached denoised picture: load it with the plain one (see [`PairLoader`]).+ denoise: Option<(DenoiseSpec, PairLoader)>,},/// A smart preview standing in for a missing original.Smart {@@ -177,15 +251,45 @@ impl SourceRef {let image = match self {SourceRef::Loaded(a) => return Ok((**a).clone()),SourceRef::Demo { scene, max_edge } => Arc::new(scene.render_fit(*max_edge)),- SourceRef::File { path, max_edge, loader, fallback } => {- let r = match loader {+ SourceRef::File { path, max_edge, loader, fallback, denoise } => {+ let plain = || match loader {Some(l) => l(path, *max_edge).map(|(image, info)| DecodedSource::new(Arc::new(image), Some(info))),None => Err(format!("no decoder available for {path}")),};+ // the denoised picture comes with the plain one from a single decode; when it cannot be read the+ // pair loader gives the plain picture alone (the cached file is never a reason to fail a render)+ let r = match denoise {+ Some((spec, pair)) => {+ // a job that needs the denoised picture has it made first; if that fails the job fails (no smart+ // preview stands in: an export must not quietly come out without the denoise it was asked for)+ let made = match &spec.make {+ Some(make) => make()?,+ None => true,+ };+ if made {+ pair(path, *max_edge, spec).and_then(|(image, twin, info)| {+ if spec.make.is_some() && twin.is_none() {+ return Err("the cached AI Denoise picture could not be read; clear the denoise cache and retry".into());+ }+ Ok(DecodedSource {+ image: Arc::new(image),+ info: Some(info),+ camera_tone: None,+ denoised: twin.map(|t| Arc::new(Twin::new(Arc::new(t)))),+ })+ })+ } else {+ plain()+ }+ }+ None => plain(),+ };// An offline original renders from its smart preview (no decoder facts: header ones).return match (r, fallback) {#[cfg(not(target_arch = "wasm32"))]- (Err(e), Some(sp)) if crate::smart::is_valid(sp) => crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")),+ (Err(e), Some(sp)) if denoise.as_ref().is_none_or(|(spec, _)| spec.make.is_none()) && crate::smart::is_valid(sp) => {+ crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}"))+ }(r, _) => r,};}@@ -224,6 +328,54 @@ pub struct MediaCache {scenes: Vec<lightcraft_scenes::Scene>,/// Rendered thumbnails (memory, plus disk once a library is attached).pub rendered: Arc<PreviewCache>,+ /// Which photos have a cached denoised picture, and how to read it.+ pub denoise: DenoiseIndex,+}++/// The photos that have a cached denoised picture for the denoise model in use. Only memory: nothing here touches the+/// file system, so it can be asked on the UI thread.+#[derive(Default)]+pub struct DenoiseIndex {+ ready: std::collections::HashMap<PhotoId, DenoiseSpec>,+ /// Reads the pictures (`None`: this build cannot, and nothing is ever ready).+ pub loader: Option<PairLoader>,+}++impl DenoiseIndex {+ /// What to load for `id` besides its plain picture, and how.+ pub fn get(&self, id: PhotoId) -> Option<(&DenoiseSpec, &PairLoader)> {+ self.ready.get(&id).zip(self.loader.as_ref())+ }++ pub fn spec(&self, id: PhotoId) -> Option<&DenoiseSpec> {+ self.ready.get(&id)+ }++ pub fn set(&mut self, id: PhotoId, spec: DenoiseSpec) {+ self.ready.insert(id, spec);+ }++ pub fn remove(&mut self, id: PhotoId) -> bool {+ self.ready.remove(&id).is_some()+ }++ pub fn clear(&mut self) {+ self.ready.clear();+ }++ pub fn len(&self) -> usize {+ self.ready.len()+ }++ pub fn is_empty(&self) -> bool {+ self.ready.is_empty()+ }++ pub fn ids(&self) -> Vec<PhotoId> {+ let mut v: Vec<PhotoId> = self.ready.keys().copied().collect();+ v.sort();+ v+ }}impl Default for MediaCache {@@ -244,6 +396,7 @@ impl Default for MediaCache {availability: Default::default(),scenes: Vec::new(),rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))),+ denoise: DenoiseIndex::default(),}}}@@ -253,10 +406,6 @@ fn rendered_budget(share: usize) -> usize {RENDERED_MEM_BYTES.min(share / 4)}-fn source_bytes(img: &Rgb32f) -> usize {- img.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>()-}-impl MediaCache {/// Keep rendered thumbnails on disk in `dir` as well. The cache this replaces is retired/// (its files stay): render jobs still holding it can't write through it any more, even@@ -319,8 +468,8 @@ impl MediaCache {let Some((tick, which)) = candidates.into_iter().flatten().min() else { break };let freed = match which {0 => self.thumbs.pop_oldest(),- 1 => self.previews.iter().position(|e| e.2 == tick).map(|i| source_bytes(&self.previews.remove(i).1.image)),- 2 => self.full.take().map(|e| source_bytes(&e.1.image)),+ 1 => self.previews.iter().position(|e| e.2 == tick).map(|i| self.previews.remove(i).1.bytes()),+ 2 => self.full.take().map(|e| e.1.bytes()),_ => self.rendered.evict_oldest(),};match freed {@@ -372,7 +521,7 @@ impl MediaCache {let tick = lightcraft_preview::next_tick();match level {SourceLevel::Thumb => {- let cost = source_bytes(&img.image);+ let cost = img.bytes();self.thumbs.insert(id, img, cost);}SourceLevel::Preview => {@@ -415,13 +564,42 @@ impl MediaCache {/// Decoded sources held: (thumbnail level, preview level, full size).pub fn usage(&self) -> (crate::memory::Usage, crate::memory::Usage, crate::memory::Usage) {use crate::memory::Usage;- let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| source_bytes(&e.1.image)).sum());- let full = self.full.as_ref().map(|e| Usage::new(1, source_bytes(&e.1.image))).unwrap_or_default();+ let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| e.1.bytes()).sum());+ let full = self.full.as_ref().map(|e| Usage::new(1, e.1.bytes())).unwrap_or_default();(Usage::new(self.thumbs.len(), self.thumbs.cost()), previews, full)}+ /// What a render of photo `id` with settings `s` adds to its identity for the denoised picture it will use: 0 unless+ /// it uses one (the Denoise amount is above 0 and the photo has a cached denoised picture).+ pub fn denoise_salt(&self, id: PhotoId, s: &DevelopSettings) -> u64 {+ match self.denoise.spec(id) {+ Some(spec) if s.denoise_amount() > 0.0 => Hasher128::new().str(&spec.key).finish().0 as u64 | 1,+ _ => 0,+ }+ }++ /// How to load photo `p` at `level`: from the cache when its source is there, else from its file (with its denoised+ /// picture beside it when it has one), its smart preview or the demo scene.pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {- if let Some(a) = self.get_source(p.id, level) {+ self.source_ref_with(p, level, true)+ }++ /// [`Self::source_ref`] that never takes a decoded source from memory: for a job that needs something the cached+ /// one lacks (an export, whose denoised picture is made when it runs).+ pub fn source_ref_uncached(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {+ self.source_ref_with(p, level, false)+ }++ fn source_ref_with(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {+ let mut r = self.source_ref_plain(p, level, cached);+ if let (SourceRef::File { denoise, .. }, Some((spec, loader))) = (&mut r, self.denoise.get(p.id)) {+ *denoise = Some((spec.clone(), loader.clone()));+ }+ r+ }++ fn source_ref_plain(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {+ if cached && let Some(a) = self.get_source(p.id, level) {return SourceRef::Loaded(Box::new(a));}// Procedural scenes have a nominal size: "full" is that size, not unbounded.@@ -459,10 +637,12 @@ impl MediaCache {}match self.scenes.iter().find(|s| s.id == *scene) {Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge },- None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None },+ None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None, denoise: None },}}- Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None },+ Source::File { path } => {+ SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None, denoise: None }+ }}}}@@ -573,7 +753,8 @@ impl RenderJob {let was_loaded = matches!(self.source, SourceRef::Loaded(_));match self.source.load_source() {Ok(source) => {- let src = &source.image;+ // the plain picture, or (Denoise amount above 0 and a denoised picture cached) the mix the amount asks for+ let src = &source.image_for(self.settings.denoise_amount());let info = source.info_or(self.info);// Thumbnails (many small jobs side by side) stay on the CPU; views and exports use// the GPU when there is one.@@ -757,22 +938,23 @@ impl crate::Session {let content = source_key.0 as u64;// Temporary "before" settings must not replace the photo's cached develop hash.let settings_hash = if before { settings.hash64() } else { self.media.settings_hashes.get(id, &settings) };+ let salt = self.media.denoise_salt(id, &settings);let key = settings_hash^ ((max_w as u64) << 40)^ ((max_h as u64) << 20)^ (apply_crop as u64)^ (level as u64) << 60^ id.0.wrapping_mul(0x9e37_79b9_7f4a_7c15)- ^ content.rotate_left(17);+ ^ content.rotate_left(17)+ ^ salt;let cache = thumb_bucket.map(|b| {- let k = Hasher128::new()- .str(&content_key)- .u64(settings_hash)- .u64(b as u64)- .u64(RENDER_CACHE_VERSION)- .u64(crate::camera_profiles::cache_key())- .finish();- (self.media.rendered.clone(), k)+ let mut k = Hasher128::new();+ k.str(&content_key).u64(settings_hash).u64(b as u64).u64(RENDER_CACHE_VERSION).u64(crate::camera_profiles::cache_key());+ // (only a render that uses a denoised picture has it in its key, so every other thumbnail stays valid)+ if salt != 0 {+ k.u64(salt);+ }+ (self.media.rendered.clone(), k.finish())});Some(RenderJob {request_id: 0,@@ -1052,6 +1234,15 @@ impl crate::Session {depth: lightcraft_pipeline::OutputDepth,) -> Result<RenderJob, String> {let mut job = self.render_job(id, max_w, max_h, false, true).ok_or("no such photo")?;+ // An export is made from the denoised picture its Denoise amount asks for, made when the job runs if the+ // background queue has not got to it. (A source kept in memory may lack the picture, so the job reads the file.)+ if let Some(wanted) = self.denoise_for_export(id, &job.settings)? {+ let p = self.catalog.photo(id).ok_or("no such photo")?.clone();+ job.source = self.media.source_ref_uncached(&p, job.level);+ if let SourceRef::File { denoise, .. } = &mut job.source {+ *denoise = Some(wanted);+ }+ }job.request.space = space;job.request.depth = depth;job.key ^= (space as u64 + 1).wrapping_mul(0xa076_1d64_78bd_642f) ^ (depth as u64 + 1).wrapping_mul(0xe703_7ed1_a0b4_28db);@@ -1219,6 +1410,7 @@ mod tests {Ok((image, info))})),fallback: None,+ denoise: None,};job.info = SourceInfo::default(); // Header facts cannot override decoder facts.job.settings = Arc::new(DevelopSettings::default());diff --git a/crates/engine/src/model_download.rs b/crates/engine/src/model_download.rsnew file mode 100644index 000000000..f07c373d2--- /dev/null+++ b/crates/engine/src/model_download.rs@@ -0,0 +1,363 @@+//! Downloading a denoise model explicitly requested after showing its terms.+//!+//! The transfer is [`lightcraft_fetch`] (pure-Rust HTTPS with resume, size and SHA-256 checks), on a background thread,+//! so the UI never waits for the network. Only a model listed in [`lightcraft_denoise::known::Download`] can be fetched:+//! the address and hash come from our pinned catalog or the user's model catalog, and no token is sent with it. What arrives+//! matches the model's recorded size and SHA-256 or it is thrown away; it then waits in a staging folder until the model+//! is installed (`denoise.models.install`, or by itself once the user has accepted the model's terms in the download).+//!+//! Failure is a message, never a panic: no network, a stalled or truncated transfer, a full disk, a file that is not the+//! one expected, and a panic inside the thread itself all end as [`State::Failed`].++use std::collections::HashMap;+use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{Arc, Mutex, PoisonError};+use std::time::Duration;++use lightcraft_denoise::known::Download;++/// The folder inside the models folder where a downloaded file waits to be installed. Its name starts with a dot, so it+/// can never be a model's own folder (ids start with a letter or digit).+pub const STAGING: &str = ".downloads";++/// Where a download stands.+#[derive(Clone, Debug, PartialEq)]+pub enum State {+ /// Fetching (`bytes` of `total` so far); the file is checked as the last step.+ Running {+ bytes: u64,+ total: u64,+ },+ /// Fetched and verified (its SHA-256 is `sha256`), waiting in the staging folder at `path` to be installed.+ Done {+ path: PathBuf,+ sha256: String,+ },+ /// Installed and in use: nothing more to do but tell the user.+ Installed,+ Failed(String),+ Cancelled,+}++struct Shared {+ state: Mutex<State>,+ cancel: AtomicBool,+}++impl Shared {+ fn new(total: u64) -> Self {+ Shared { state: Mutex::new(State::Running { bytes: 0, total }), cancel: AtomicBool::new(false) }+ }++ fn set(&self, s: State) {+ *self.state.lock().unwrap_or_else(PoisonError::into_inner) = s;+ }++ fn get(&self) -> State {+ self.state.lock().unwrap_or_else(PoisonError::into_inner).clone()+ }+}++/// The downloads of one session, by model id.+#[derive(Default)]+pub struct Downloads {+ jobs: HashMap<String, Arc<Shared>>,+}++impl Downloads {+ /// Start fetching `spec` into `<models dir>/.downloads/`. Different models can download side by side; the same+ /// one is started once.+ pub fn start(&mut self, spec: Download, models_dir: &Path) -> Result<(), String> {+ if self.jobs.get(&spec.id).is_some_and(|j| matches!(j.get(), State::Running { .. })) {+ return Err("that model is already downloading".into());+ }+ let staging = models_dir.join(STAGING);+ std::fs::create_dir_all(&staging).map_err(|e| format!("could not create the download folder: {e}"))?;+ let shared = Arc::new(Shared::new(spec.size_bytes));+ let worker = shared.clone();+ let id = spec.id.clone();+ std::thread::Builder::new()+ .name("denoise-model-download".into())+ .spawn(move || {+ let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&spec, &spec.url, &staging, &worker)));+ if outcome.is_err() {+ worker.set(State::Failed("the download stopped unexpectedly".into()));+ }+ })+ .map_err(|e| format!("could not start the download: {e}"))?;+ self.jobs.insert(id, shared);+ Ok(())+ }++ /// Every download this session has started and not discarded, by model id.+ pub fn snapshot(&self) -> Vec<(String, State)> {+ let mut all: Vec<_> = self.jobs.iter().map(|(id, j)| (id.clone(), j.get())).filter(|(_, s)| *s != State::Cancelled).collect();+ all.sort_by(|a, b| a.0.cmp(&b.0));+ all+ }++ pub fn running(&self) -> bool {+ self.jobs.values().any(|j| matches!(j.get(), State::Running { .. }))+ }++ /// Stop a running download, or discard a finished one (its staged file is deleted). `true` if there was one.+ pub fn discard(&mut self, id: &str) -> bool {+ let Some(job) = self.jobs.get(id) else { return false };+ job.cancel.store(true, Ordering::Relaxed);+ match job.get() {+ // the thread notices, deletes the partial file and marks itself cancelled+ State::Running { .. } => {}+ State::Done { path, .. } => {+ let _ = std::fs::remove_file(path);+ self.jobs.remove(id);+ }+ State::Installed | State::Failed(_) | State::Cancelled => {+ self.jobs.remove(id);+ }+ }+ true+ }++ /// The staged file at `path` has been installed (its copy is the model now) or is no longer wanted.+ pub fn staged_file_used(&mut self, path: &Path) {+ self.jobs.retain(|_, j| !matches!(j.get(), State::Done { path: p, .. } if p == path));+ }++ /// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256).+ pub fn finished(&self) -> Vec<(String, PathBuf, String)> {+ let mut out: Vec<_> = self+ .jobs+ .iter()+ .filter_map(|(id, j)| match j.get() {+ State::Done { path, sha256 } => Some((id.clone(), path, sha256)),+ _ => None,+ })+ .collect();+ out.sort_by(|a, b| a.0.cmp(&b.0));+ out+ }++ /// Record how installing a finished download went (`Installed` or `Failed`).+ pub fn set_outcome(&mut self, id: &str, state: State) {+ match self.jobs.get(id) {+ Some(j) => j.set(state),+ None => {+ let j = Shared::new(0);+ j.set(state);+ self.jobs.insert(id.to_string(), Arc::new(j));+ }+ }+ }+}++impl Drop for Downloads {+ /// Quitting while a download runs stops it instead of leaving it behind.+ fn drop(&mut self) {+ for job in self.jobs.values() {+ job.cancel.store(true, Ordering::Relaxed);+ }+ let started = std::time::Instant::now();+ while self.running() && started.elapsed() < Duration::from_secs(1) {+ std::thread::sleep(Duration::from_millis(10));+ }+ }+}++/// What went wrong, in words the user can act on.+#[cfg(not(target_arch = "wasm32"))]+fn explain(e: &lightcraft_fetch::DownloadError) -> String {+ use lightcraft_fetch::DownloadError;+ match e {+ DownloadError::AllMirrorsFailed { errors, .. } => {+ let detail: String = errors.join("; ").chars().take(300).collect();+ format!("could not download the model ({detail}): check the internet connection, or get the file from its page and add it yourself")+ }+ other => other.to_string(),+ }+}++/// Fetch `spec` from its exact URL into `staging`, check it and leave it there.+#[cfg(not(target_arch = "wasm32"))]+fn fetch(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) -> Result<(), lightcraft_fetch::DownloadError> {+ let file = lightcraft_fetch::FileSpec { name: &spec.file_name, size: Some(spec.size_bytes), sha256: Some(&spec.sha256), max: spec.size_bytes };+ lightcraft_fetch::download_url(&file, mirror, staging, &lightcraft_fetch::Options::default(), &shared.cancel, &mut |p| {+ shared.set(State::Running { bytes: p.done.min(spec.size_bytes), total: spec.size_bytes });+ })+}++#[cfg(target_arch = "wasm32")]+fn fetch(_: &Download, _: &str, _: &Path, _: &Shared) -> Result<(), String> {+ Err("the web build downloads no models".into())+}++/// The whole job: reuse a verified earlier copy, else fetch, check and stage. Always ends in a final [`State`].+fn run(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) {+ let target = staging.join(&spec.file_name);+ let part = staging.join(format!("{}.part", spec.file_name));+ match fetch(spec, mirror, staging, shared) {+ Ok(()) => shared.set(State::Done { path: target, sha256: spec.sha256.to_string() }),+ #[cfg(not(target_arch = "wasm32"))]+ Err(lightcraft_fetch::DownloadError::Cancelled) => {+ let _ = std::fs::remove_file(&part);+ shared.set(State::Cancelled);+ }+ #[cfg(not(target_arch = "wasm32"))]+ Err(e) => {+ shared.set(State::Failed(explain(&e)));+ }+ #[cfg(target_arch = "wasm32")]+ Err(e) => {+ let _ = std::fs::remove_file(&part);+ shared.set(State::Failed(e));+ }+ }+}++#[cfg(all(test, not(target_arch = "wasm32")))]+mod tests {+ use super::*;+ use lightcraft_denoise::hash::sha256_hex;+ use std::io::{BufRead, BufReader, Write};+ use std::net::TcpListener;++ fn scratch(name: &str) -> PathBuf {+ let d = std::env::temp_dir().join(format!("lc-face-dl-{name}-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&d);+ std::fs::create_dir_all(&d).unwrap();+ d+ }++ fn spec_for(bytes: &[u8]) -> Download {+ Download {+ id: "test-model".into(),+ url: "https://example.invalid/m/test-model.onnx".into(),+ file_name: "test-model.onnx".into(),+ size_bytes: bytes.len() as u64,+ sha256: sha256_hex(bytes),+ }+ }++ /// A local server that answers every request with `status` and `body` (the whole file, no ranges).+ fn serve(status: &'static str, body: Vec<u8>) -> String {+ let l = TcpListener::bind("127.0.0.1:0").unwrap();+ let base = format!("http://{}/m", l.local_addr().unwrap());+ std::thread::spawn(move || {+ for mut s in l.incoming().flatten() {+ let mut r = BufReader::new(s.try_clone().unwrap());+ loop {+ let mut line = String::new();+ if r.read_line(&mut line).unwrap_or(0) == 0 || line.trim().is_empty() {+ break;+ }+ }+ let _ = s.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes());+ let _ = s.write_all(&body);+ }+ });+ base+ }++ #[test]+ fn fetches_checks_and_stages_a_file() {+ let dir = scratch("ok");+ let bytes: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();+ let (staging, spec) = (dir.join(STAGING), spec_for(&bytes));+ std::fs::create_dir_all(&staging).unwrap();+ let mirror = serve("200 OK", bytes.clone());+ let shared = Shared::new(spec.size_bytes);+ run(&spec, &mirror, &staging, &shared);+ let State::Done { path, sha256 } = shared.get() else { panic!("{:?}", shared.get()) };+ assert_eq!((std::fs::read(&path).unwrap(), sha256), (bytes, spec.sha256.to_string()));+ assert!(!staging.join("test-model.onnx.part").exists());+ // asked again with the copy still staged: no fetch needed (nothing listens there)+ let again = Shared::new(spec.size_bytes);+ run(&spec, "http://127.0.0.1:1/m", &staging, &again);+ assert!(matches!(again.get(), State::Done { .. }), "{:?}", again.get());+ let _ = std::fs::remove_dir_all(&dir);+ }++ #[test]+ fn a_file_that_does_not_match_is_thrown_away() {+ let dir = scratch("bad");+ let staging = dir.join(STAGING);+ std::fs::create_dir_all(&staging).unwrap();+ // the right size and the wrong content+ let spec = spec_for(&[7u8; 5000]);+ let shared = Shared::new(spec.size_bytes);+ run(&spec, &serve("200 OK", vec![8u8; 5000]), &staging, &shared);+ assert!(matches!(shared.get(), State::Failed(_)), "{:?}", shared.get());+ // nothing is left behind: not the file, not a partial one+ assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);+ let _ = std::fs::remove_dir_all(&dir);+ }++ #[test]+ fn an_unreachable_or_refusing_server_is_a_message() {+ let dir = scratch("down");+ let staging = dir.join(STAGING);+ std::fs::create_dir_all(&staging).unwrap();+ let spec = spec_for(b"abc");+ for mirror in ["http://127.0.0.1:1/m".to_string(), serve("404 Not Found", b"not found".to_vec())] {+ let shared = Shared::new(3);+ run(&spec, &mirror, &staging, &shared);+ let State::Failed(why) = shared.get() else { panic!("{:?}", shared.get()) };+ assert!(why.contains("internet connection"), "{why}");+ }+ assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);+ let _ = std::fs::remove_dir_all(&dir);+ }++ #[test]+ fn a_cancelled_download_leaves_nothing() {+ let dir = scratch("cancel");+ let staging = dir.join(STAGING);+ std::fs::create_dir_all(&staging).unwrap();+ let spec = spec_for(b"abc");+ let shared = Shared::new(3);+ shared.cancel.store(true, Ordering::Relaxed);+ run(&spec, &serve("200 OK", b"abc".to_vec()), &staging, &shared);+ assert_eq!(shared.get(), State::Cancelled);+ assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);+ let _ = std::fs::remove_dir_all(&dir);+ }++ #[test]+ fn dropping_the_session_stops_a_running_download() {+ let shared = Arc::new(Shared::new(10));+ // a stand-in for the download thread: notices the cancel flag the way the downloader does+ let worker = shared.clone();+ let handle = std::thread::spawn(move || {+ while !worker.cancel.load(Ordering::Relaxed) {+ std::thread::sleep(Duration::from_millis(5));+ }+ worker.set(State::Cancelled);+ });+ let mut d = Downloads::default();+ d.jobs.insert("m".into(), shared.clone());+ assert!(d.running());+ let started = std::time::Instant::now();+ drop(d);+ assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish");+ assert_eq!(shared.get(), State::Cancelled);+ handle.join().unwrap();+ }++ #[test]+ fn discarding_a_finished_download_deletes_its_file() {+ let dir = scratch("discard");+ let file = dir.join("staged.onnx");+ std::fs::write(&file, b"x").unwrap();+ let mut d = Downloads::default();+ let shared = Arc::new(Shared::new(1));+ shared.set(State::Done { path: file.clone(), sha256: String::new() });+ d.jobs.insert("m".into(), shared);+ assert_eq!(d.snapshot().len(), 1);+ assert!(!d.running());+ assert!(d.discard("m"));+ assert!(!file.exists());+ assert!(d.snapshot().is_empty());+ assert!(!d.discard("m"), "nothing left to discard");+ let _ = std::fs::remove_dir_all(&dir);+ }+}diff --git a/crates/engine/src/smart.rs b/crates/engine/src/smart.rsindex 8deff7bde..a48fcd040 100644--- a/crates/engine/src/smart.rs+++ b/crates/engine/src/smart.rs@@ -190,7 +190,7 @@ pub fn is_valid(path: &Path) -> bool {/// Load the proxy at `path`.pub fn load(path: &Path) -> Result<crate::media::DecodedSource, String> {let b = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?;- decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone })+ decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone, denoised: None })}#[cfg(test)]diff --git a/crates/engine/src/tests_denoise.rs b/crates/engine/src/tests_denoise.rsnew file mode 100644index 000000000..172f90824--- /dev/null+++ b/crates/engine/src/tests_denoise.rs@@ -0,0 +1,699 @@+//! AI denoise through the engine: installing a model, making a photo's denoised picture in the background and for an+//! export, the Amount slider mixing it in, and the cache. A tiny stand-in model (it halves every sample) takes the+//! place of a real one, which is enough to see what the machinery does with its answer; how well real models denoise is+//! measured separately (`crates/denoise/tests/real_model.rs`).++use std::path::{Path, PathBuf};+use std::sync::Arc;+use std::sync::atomic::{AtomicUsize, Ordering};++use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};+use lightcraft_denoise::run::TileRunner;+use serde_json::{Value, json};++use crate::Session;+use crate::denoise::{Loader, Model, PhotoState, RunOn};+use crate::export::{ExportOptions, export_photo};+use crate::tests_xmp::{synthetic_dng_of, temp_dir};++/// A model that makes every output pixel half of its cell's colour: a picture that is plainly not the plain one.+struct Dim {+ runs: Arc<AtomicUsize>,+}++impl TileRunner for Dim {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, lightcraft_denoise::Error> {+ self.runs.fetch_add(1, Ordering::Relaxed);+ let t = ((input.len() / 4) as f64).sqrt() as usize;+ let side = 2 * t;+ let mut out = vec![0f32; 3 * side * side];+ for y in 0..side {+ for x in 0..side {+ let i = (y / 2) * t + x / 2;+ let (r, g, b) = (input[i], (input[t * t + i] + input[2 * t * t + i]) / 2.0, input[3 * t * t + i]);+ out[y * side + x] = r * 0.5;+ out[side * side + y * side + x] = g * 0.5;+ out[2 * side * side + y * side + x] = b * 0.5;+ }+ }+ Ok(out)+ }+}++impl Model for Dim {+ fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) {+ (self, threads)+ }++ fn self_test(&self, _: RunOn) -> Result<Value, String> {+ Ok(json!({"ok": true}))+ }+}++fn dim_loader(runs: &Arc<AtomicUsize>) -> Loader {+ let runs = runs.clone();+ Arc::new(move |_, _| Ok(Arc::new(Dim { runs: runs.clone() }) as Arc<dyn Model>))+}++fn manifest(id: &str) -> DenoiserManifest {+ DenoiserManifest {+ id: id.into(),+ name: "Stand-in".into(),+ version: "1".into(),+ licence: Default::default(),+ source: None,+ sha256: None,+ size_bytes: None,+ provenance: "made up for a test".into(),+ domain: Domain::BayerToRgb,+ tile: 64,+ overlap: 16,+ gain: Gain::None,+ }+}++/// The model's `.onnx` (not a real one: the stand-in loader never reads it) with its manifest beside it.+fn model_files(dir: &Path, id: &str) -> PathBuf {+ let src = dir.join(format!("src-{id}"));+ std::fs::create_dir_all(&src).unwrap();+ std::fs::write(src.join("denoise-model.json"), serde_json::to_vec(&manifest(id)).unwrap()).unwrap();+ std::fs::write(src.join("model.onnx"), format!("not an onnx file, only a stand-in for {id}")).unwrap();+ src.join("model.onnx")+}++/// A Bayer DNG that is a smooth ramp, big enough for a 64-cell tile and a second one.+fn ramp_dng(w: usize, h: usize, cfa: &str) -> Vec<u8> {+ let data: Vec<u16> = (0..w * h).map(|i| 600 + ((i % w) * 30 + (i / w) * 35) as u16).collect();+ synthetic_dng_of(w, h, cfa, data, None, Default::default())+}++struct Setup {+ dir: PathBuf,+ s: Session,+ runs: Arc<AtomicUsize>,+}++/// A session with a library on disk holding one raw photo, and the stand-in model installed and chosen.+fn setup(tag: &str, with_model: bool) -> Setup {+ let dir = temp_dir(tag);+ let originals = dir.join("originals");+ std::fs::create_dir_all(&originals).unwrap();+ std::fs::write(originals.join("ramp.dng"), ramp_dng(192, 160, "RGGB")).unwrap();+ let runs = Arc::new(AtomicUsize::new(0));+ let mut s = Session::new().with_fs();+ s.open_library(dir.join("library"), false).unwrap();+ s.execute("library.import", &json!({"paths": [originals.join("ramp.dng").to_string_lossy()]})).unwrap();+ let id = s.catalog.photos().next().unwrap().id;+ s.execute("library.select", &json!({"ids": [id.0]})).unwrap();+ s.set_denoise_models_dir(Some(dir.join("denoise-models")));+ s.denoise.loader = dim_loader(&runs);+ if with_model {+ let onnx = model_files(&dir, "stand-in");+ s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();+ }+ Setup { dir, s, runs }+}++fn photo(s: &Session) -> lightcraft_catalog::PhotoId {+ s.catalog.photos().next().unwrap().id+}++/// Pump until `n` photos are ready (the job runs on its own thread).+fn pump_until_ready(s: &mut Session, n: u64) -> Value {+ for _ in 0..600 {+ let r = s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();+ if r["ready"].as_u64() >= Some(n) {+ return r;+ }+ std::thread::sleep(std::time::Duration::from_millis(50));+ }+ panic!("not ready: {}", s.execute("denoise.status", &json!({})).unwrap());+}++fn mean(img: &lightcraft_raster::Rgba8) -> f64 {+ let sum: u64 = img.data.iter().map(|p| u64::from(p[0]) + u64::from(p[1]) + u64::from(p[2])).sum();+ sum as f64 / (img.data.len() * 3) as f64+}++fn rendered_mean(s: &mut Session, id: lightcraft_catalog::PhotoId) -> f64 {+ mean(&s.render_now(id, 160, 160).unwrap().image)+}++fn set_amount(s: &mut Session, amount: f64) {+ s.execute("develop.set", &json!({"control": "enhance.denoise", "value": amount})).unwrap();+}++#[test]+fn a_model_is_installed_chosen_and_removed_with_the_terms_accepted() {+ let mut x = setup("model", false);+ let onnx = model_files(&x.dir, "stand-in");+ // the terms come first+ let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy()})).unwrap_err().to_string();+ assert!(e.contains("licence has not been accepted"), "{e}");+ let r = x.s.execute("denoise.models.list", &json!({})).unwrap();+ assert_eq!(r["model"], Value::Null);+ assert_eq!(r["runtime"], cfg!(feature = "denoise"));+ // a file with no manifest is not guessed at+ let stray = x.dir.join("stray.onnx");+ std::fs::write(&stray, b"who knows").unwrap();+ let e = x.s.execute("denoise.models.install", &json!({"path": stray.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();+ assert!(e.contains("does not know this file"), "{e}");+ // installed, it is the model in use+ let r = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();+ assert_eq!((r["installed"]["id"].clone(), r["model"].clone()), (json!("stand-in"), json!("stand-in")), "{r}");+ assert!(r["installed"]["accepted"]["sha256"].as_str().is_some_and(|h| h.len() == 64), "the file's hash is recorded: {r}");+ assert!(x.s.denoise.active.is_some());+ assert_eq!(x.s.execute("denoise.status", &json!({})).unwrap()["model"]["id"], "stand-in");+ // it can be checked again, switched off and on, and removed+ assert_eq!(x.s.execute("denoise.models.test", &json!({"id": "stand-in"})).unwrap()["ok"], true);+ assert_eq!(x.s.execute("denoise.models.select", &json!({"id": null})).unwrap()["model"], Value::Null);+ assert!(x.s.denoise.active.is_none());+ assert!(x.s.execute("denoise.models.select", &json!({"id": "nothing"})).is_err());+ x.s.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap();+ assert!(x.s.denoise.active.is_some());+ let r = x.s.execute("denoise.models.remove", &json!({"id": "stand-in"})).unwrap();+ assert_eq!(r["model"], Value::Null);+ assert!(x.s.denoise.active.is_none());+ assert!(!x.dir.join("denoise-models").join("stand-in").exists());+ // ids that are not folder names, and folders that are not models, are never touched+ for bad in ["../library", "", "a/b", "settings.json"] {+ assert!(x.s.execute("denoise.models.remove", &json!({"id": bad})).is_err(), "{bad}");+ }+ assert!(x.dir.join("library").is_dir());+ // a model that fails its self-test is not left installed+ x.s.denoise.loader = Arc::new(|_, _| Err("it does not load".into()));+ let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();+ assert!(e.contains("it does not load"), "{e}");+ assert!(!x.dir.join("denoise-models").join("stand-in").exists());+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn a_manifest_that_does_not_validate_or_name_the_file_is_refused() {+ let mut x = setup("manifest", false);+ let onnx = model_files(&x.dir, "stand-in");+ let json_path = onnx.with_file_name("denoise-model.json");+ // a tile the model cannot run on+ let mut m = manifest("stand-in");+ m.tile = 50;+ std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();+ let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();+ assert!(e.contains("tile"), "{e}");+ // a hash that is not the file's+ let mut m = manifest("stand-in");+ m.sha256 = Some("0".repeat(64));+ std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();+ let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();+ assert!(e.contains("SHA-256"), "{e}");+ assert!(!x.dir.join("denoise-models").join("stand-in").join("model.onnx").exists(), "nothing half-installed is left");+ // an id that would leave the folder+ let mut m = manifest("../escape");+ m.id = "../escape".into();+ std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();+ assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err());+ assert!(!x.dir.join("escape").exists());+ // garbage instead of a manifest+ std::fs::write(&json_path, b"{ not json").unwrap();+ assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err());+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn a_photo_is_denoised_in_the_background_and_the_amount_mixes_it_in() {+ let mut x = setup("background", true);+ let id = photo(&x.s);+ // nothing is made until a photo has an amount+ let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();+ assert_eq!((r["queued"].clone(), r["running"].clone(), r["ready"].clone()), (json!(0), Value::Null, json!(0)), "{r}");+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);+ let plain = rendered_mean(&mut x.s, id);+ let key_before = x.s.render_job(id, 160, 160, false, true).unwrap().key;++ set_amount(&mut x.s, 100.0);+ // the photo being edited is picked up by itself, and made on another thread+ let r = pump_until_ready(&mut x.s, 1);+ assert_eq!(r["queued"], 0, "{r}");+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);+ assert!(x.runs.load(Ordering::Relaxed) >= 4, "a 192 × 160 mosaic is four 64-cell tiles: {}", x.runs.load(Ordering::Relaxed));+ let products: Vec<_> = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().collect();+ assert_eq!(products.len(), 1, "one cached picture, no DNG or anything else in the library");+ assert!(products[0].path().extension().is_some_and(|e| e == "lcdn"));+ let originals: Vec<_> = std::fs::read_dir(x.dir.join("originals")).unwrap().flatten().collect();+ assert_eq!(originals.len(), 1, "the original is untouched and alone");++ // the render now comes from the denoised picture: darker at 100, in between at 50, the plain one at 0+ let full = rendered_mean(&mut x.s, id);+ assert!(full < plain - 3.0, "denoised {full} vs plain {plain}");+ set_amount(&mut x.s, 50.0);+ let half = rendered_mean(&mut x.s, id);+ assert!(half < plain - 1.0 && half > full + 1.0, "{full} < {half} < {plain}");+ set_amount(&mut x.s, 0.0);+ assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01, "amount 0 is the plain picture again");+ // …and a render that uses it has another identity than one that does not+ assert_ne!(key_before, {+ set_amount(&mut x.s, 100.0);+ x.s.render_job(id, 160, 160, false, true).unwrap().key+ });+ // switching the Detail section off switches the denoise off with it+ set_amount(&mut x.s, 100.0);+ x.s.execute("develop.sectionEnabled", &json!({"section": "detail", "enabled": false})).unwrap();+ assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01);+ assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0);+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn what_was_made_is_found_again_by_a_new_session_and_a_new_model_starts_over() {+ let mut x = setup("again", true);+ let id = photo(&x.s);+ set_amount(&mut x.s, 100.0);+ pump_until_ready(&mut x.s, 1);+ let made = x.runs.load(Ordering::Relaxed);+ x.s.close_library().unwrap();+ drop(x.s);++ // reopened: ready without running the model again+ let mut s2 = Session::new().with_fs();+ s2.set_denoise_models_dir(Some(x.dir.join("denoise-models")));+ s2.denoise.loader = dim_loader(&x.runs);+ s2.open_library(x.dir.join("library"), false).unwrap();+ s2.denoise_pump(crate::denoise::Pace::Full);+ assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready);+ assert_eq!(x.runs.load(Ordering::Relaxed), made, "nothing was made again");++ // another model's pictures are different pictures+ let onnx = model_files(&x.dir, "other");+ s2.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();+ s2.denoise_pump(crate::denoise::Pace::Pause);+ assert!(+ matches!(s2.denoise_photo_state(id), PhotoState::Queued { .. }),+ "the first model's picture is not this model's: it is to be made: {:?}",+ s2.denoise_photo_state(id)+ );+ // and choosing the first again finds its picture where it was+ s2.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap();+ s2.denoise_pump(crate::denoise::Pace::Pause);+ assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready);+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn an_export_makes_its_picture_when_the_queue_has_not_and_never_comes_out_without_it() {+ let mut x = setup("export", true);+ let id = photo(&x.s);+ let o = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160}));+ let decode = |bytes: &[u8]| {+ let d = lightcraft_codecs::decode(bytes, lightcraft_codecs::DecodeOptions::fit(160, 160)).unwrap();+ let w = d.to_working();+ w.data.iter().map(|p| f64::from(p[0] + p[1] + p[2])).sum::<f64>() / (w.data.len() * 3) as f64+ };+ let plain = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);+ assert_eq!(x.runs.load(Ordering::Relaxed), 0, "no amount, no denoise");+ set_amount(&mut x.s, 100.0);+ // no pump: the export makes the picture itself+ let denoised = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);+ assert!(x.runs.load(Ordering::Relaxed) >= 4);+ assert!(denoised < plain * 0.9, "denoised {denoised} vs plain {plain}");+ let products = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().count();+ assert_eq!(products, 1);+ // a second export finds it+ let runs = x.runs.load(Ordering::Relaxed);+ let again = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);+ assert_eq!(x.runs.load(Ordering::Relaxed), runs);+ assert!((again - denoised).abs() < 1e-6);+ // the pump takes in what the export made+ x.s.denoise_pump(crate::denoise::Pace::Pause);+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);++ // no model: the export says so instead of leaving the denoise out+ x.s.execute("denoise.models.select", &json!({"id": null})).unwrap();+ let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();+ assert!(e.contains("no denoise model is chosen") && e.contains("ramp.dng"), "{e}");+ set_amount(&mut x.s, 0.0);+ assert!(export_photo(&mut x.s, id, &o, 1).is_ok(), "without an amount nothing is asked of the model");+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn a_model_that_fails_fails_the_export_and_is_not_retried_by_the_queue() {+ let mut x = setup("fails", true);+ let id = photo(&x.s);+ x.s.denoise.loader = Arc::new(|_, _| Err("the model went away".into()));+ // (the model chosen is read afresh: it was loaded by the install's self-test only)+ set_amount(&mut x.s, 80.0);+ let o = ExportOptions::from_json(&json!({"format": "png", "width": 100, "height": 100}));+ let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();+ assert!(e.contains("ramp.dng") && e.contains("the model went away"), "{e}");+ // the background job fails too, says why once and does not try again every frame+ for _ in 0..200 {+ x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();+ if matches!(x.s.denoise_photo_state(id), PhotoState::Failed { .. }) {+ break;+ }+ std::thread::sleep(std::time::Duration::from_millis(20));+ }+ match x.s.denoise_photo_state(id) {+ PhotoState::Failed { why, unsupported } => assert!(why.contains("the model went away") && !unsupported, "{why}"),+ other => panic!("{other:?}"),+ }+ let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();+ assert_eq!((r["queued"].clone(), r["running"].clone()), (json!(0), Value::Null), "{r}");+ // asking again by hand does retry+ x.s.denoise.loader = dim_loader(&x.runs);+ let r = x.s.execute("denoise.queue", &json!({"retry": true})).unwrap();+ assert_eq!(r["added"], 1, "{r}");+ pump_until_ready(&mut x.s, 1);+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn a_corrupt_cache_with_a_current_header_never_silently_exports_the_plain_photo() {+ let mut x = setup("corrupt-export", true);+ let id = photo(&x.s);+ set_amount(&mut x.s, 100.0);+ let options = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160}));+ export_photo(&mut x.s, id, &options, 1).unwrap();+ x.s.denoise_pump(crate::denoise::Pace::Pause);+ let spec = x.s.media.denoise.spec(id).unwrap().clone();+ let mut bytes = std::fs::read(&spec.product).unwrap();+ for byte in bytes.iter_mut().rev().take(64) {+ *byte ^= 0xff;+ }+ std::fs::write(&spec.product, bytes).unwrap();+ assert!(lightcraft_denoise::product::is_current(&spec.product, &spec.key), "header/length still look current");+ x.s.media.forget(id);+ let error = export_photo(&mut x.s, id, &options, 1).err().unwrap();+ assert!(error.contains("AI Denoise picture could not be read"), "{error}");+ let _ = std::fs::remove_dir_all(x.dir);+}++#[test]+fn only_raw_photos_on_disk_are_denoised_and_nothing_is_queued_without_a_model() {+ let dir = temp_dir("eligible");+ let mut demo = Session::with_demo();+ demo.set_denoise_models_dir(Some(dir.join("models")));+ let id = demo.active().unwrap();+ assert_eq!(demo.denoise_photo_state(id), PhotoState::NotApplicable, "a generated photo has no sensor data");+ let e = demo.execute("denoise.queue", &json!({})).unwrap_err().to_string();+ assert!(e.contains("no denoise model is chosen"), "{e}");+ // no models folder at all (the web build): the commands say so, nothing panics+ let mut web = Session::with_demo();+ for cmd in ["denoise.models.install", "denoise.models.select", "denoise.models.remove", "denoise.settings"] {+ assert!(web.execute(cmd, &json!({"path": "x", "acknowledged": true, "id": "x"})).is_err(), "{cmd}");+ }+ assert_eq!(web.execute("denoise.pump", &json!({})).unwrap()["active"], false);+ assert_eq!(web.execute("denoise.status", &json!({})).unwrap()["enabled"], false);+ // no library open: nowhere to keep pictures, and the queue says so+ let x = setup("eligible2", true);+ let mut bare = Session::new().with_fs();+ bare.set_denoise_models_dir(Some(x.dir.join("denoise-models")));+ assert!(bare.denoise_products_dir().is_none());+ let e = bare.execute("denoise.queue", &json!({})).unwrap_err().to_string();+ assert!(e.contains("open a library on disk first"), "{e}");+ assert_eq!(bare.execute("denoise.pump", &json!({})).unwrap()["queued"], 0);+ let _ = std::fs::remove_dir_all(&dir);+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn files_that_cannot_be_denoised_fail_with_a_reason_and_never_a_panic() {+ let dir = temp_dir("makefails");+ let spec = |path: &Path| crate::denoise::JobSpec {+ path: path.to_string_lossy().into_owned(),+ read: None,+ product: dir.join("p.lcdn"),+ key: "k".into(),+ manifest: manifest("stand-in"),+ onnx: dir.join("model.onnx"),+ model: Default::default(),+ loader: dim_loader(&Arc::new(AtomicUsize::new(0))),+ parallel: 2,+ run_on: RunOn::Auto,+ };+ use crate::denoise::{MakeError, make_product};+ // not a file+ assert!(matches!(make_product(&spec(&dir.join("missing.dng")), None), Err(MakeError::Failed(_))));+ // not a raw+ let junk = dir.join("junk.dng");+ std::fs::write(&junk, b"this is not a raw file at all").unwrap();+ assert!(matches!(make_product(&spec(&junk), None), Err(MakeError::Unsupported(_))));+ // a raw cut short+ let whole = ramp_dng(192, 160, "RGGB");+ let cut = dir.join("cut.dng");+ std::fs::write(&cut, &whole[..whole.len() / 3]).unwrap();+ assert!(make_product(&spec(&cut), None).is_err());+ // a model that gives the wrong number of values is an error from the model, not a crash+ struct Wrong;+ impl TileRunner for Wrong {+ fn run(&self, _: &[f32]) -> Result<Vec<f32>, lightcraft_denoise::Error> {+ Ok(vec![0.0; 7])+ }+ }+ impl Model for Wrong {+ fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) {+ (self, threads)+ }+ fn self_test(&self, _: RunOn) -> Result<Value, String> {+ Err("no".into())+ }+ }+ let good = dir.join("good.dng");+ std::fs::write(&good, &whole).unwrap();+ let mut s = spec(&good);+ s.loader = Arc::new(|_, _| Ok(Arc::new(Wrong) as Arc<dyn Model>));+ match make_product(&s, None) {+ Err(MakeError::Failed(why)) => assert!(why.contains("7 numbers"), "{why}"),+ other => panic!("{other:?}"),+ }+ assert!(!dir.join("p.lcdn").exists(), "a failed job leaves no product");+ // a job told to stop stops before doing anything+ let progress = crate::denoise::Progress::default();+ progress.cancel_now();+ assert_eq!(make_product(&spec(&good), Some(&progress)), Err(MakeError::Cancelled));+ // a Bayer layout of the other kinds is denoised too, and a product that exists is not made again+ for cfa in ["BGGR", "GRBG", "GBRG"] {+ let f = dir.join(format!("{cfa}.dng"));+ std::fs::write(&f, ramp_dng(192, 160, cfa)).unwrap();+ let mut s = spec(&f);+ s.product = dir.join(format!("{cfa}.lcdn"));+ assert_eq!(make_product(&s, None), Ok(true), "{cfa}");+ assert_eq!(make_product(&s, None), Ok(false), "{cfa} is already there");+ }+ let _ = std::fs::remove_dir_all(&dir);+}++#[test]+fn two_threads_making_the_same_picture_make_it_once() {+ let dir = temp_dir("twice");+ let f = dir.join("a.dng");+ std::fs::write(&f, ramp_dng(192, 160, "RGGB")).unwrap();+ let runs = Arc::new(AtomicUsize::new(0));+ let spec = crate::denoise::JobSpec {+ path: f.to_string_lossy().into_owned(),+ read: None,+ product: dir.join("p.lcdn"),+ key: "k".into(),+ manifest: manifest("stand-in"),+ onnx: dir.join("model.onnx"),+ model: Default::default(),+ loader: dim_loader(&runs),+ parallel: 2,+ run_on: RunOn::Auto,+ };+ let made: Vec<bool> = std::thread::scope(|sc| {+ let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| crate::denoise::make_product(&spec, None).unwrap())).collect();+ hs.into_iter().map(|h| h.join().unwrap()).collect()+ });+ assert_eq!(made.iter().filter(|m| **m).count(), 1, "{made:?}");+ assert_eq!(runs.load(Ordering::Relaxed), 4, "one photo's four tiles, once");+ assert!(lightcraft_denoise::product::is_current(&spec.product, "k"));+ let _ = std::fs::remove_dir_all(&dir);+}++#[test]+fn the_cache_is_kept_within_its_limit_and_what_photos_use_goes_last() {+ let mut x = setup("evict", true);+ let id = photo(&x.s);+ set_amount(&mut x.s, 100.0);+ pump_until_ready(&mut x.s, 1);+ let used = x.dir.join("library").join("denoise");+ // two older pictures nobody uses+ for (i, name) in ["aaaa", "bbbb"].iter().enumerate() {+ let f = used.join(format!("{name}.lcdn"));+ std::fs::write(&f, vec![7u8; 1000]).unwrap();+ let t = std::time::SystemTime::now() - std::time::Duration::from_secs(1000 - i as u64);+ std::fs::File::options().write(true).open(&f).unwrap().set_modified(t).unwrap();+ }+ let size_of = |p: &Path| std::fs::metadata(p).unwrap().len();+ let mine: PathBuf = std::fs::read_dir(&used).unwrap().flatten().map(|e| e.path()).find(|p| p.file_stem().is_some_and(|s| s.len() == 32)).unwrap();+ // room for the photo's own picture and one stray: the oldest stray goes, then the other+ x.s.denoise_evict_to(size_of(&mine) + 1500, None);+ assert!(!used.join("aaaa.lcdn").exists() && used.join("bbbb.lcdn").exists() && mine.exists());+ x.s.denoise_evict_to(size_of(&mine) + 10, None);+ assert!(!used.join("bbbb.lcdn").exists() && mine.exists());+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);+ // too small for even that: the photo's own picture is the last thing to go, and the photo knows+ x.s.denoise_evict_to(10, None);+ assert!(!mine.exists());+ assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready);+ // …and it is made again, because the photo still has its amount+ pump_until_ready(&mut x.s, 1);+ assert!(mine.exists());+ // clearing deletes it all+ let r = x.s.execute("denoise.clear", &json!({})).unwrap();+ assert_eq!(r["deleted"], 1, "{r}");+ assert_eq!(std::fs::read_dir(&used).unwrap().flatten().count(), 0);+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn settings_are_checked_saved_and_a_paused_pump_starts_nothing() {+ let mut x = setup("settings", true);+ let r = x.s.execute("denoise.settings", &json!({"auto": false, "cacheGb": 5, "threads": 3})).unwrap();+ assert_eq!((r["auto"].clone(), r["cacheGb"].clone(), r["threads"].clone()), (json!(false), json!(5), json!(3)), "{r}");+ for bad in [+ json!({"cacheGb": 0}),+ json!({"cacheGb": "big"}),+ json!({"threads": 0}),+ json!({"threads": 1000}),+ json!({"auto": "yes"}),+ json!({"runOn": "fastest"}),+ json!({"runOn": true}),+ ] {+ assert!(x.s.execute("denoise.settings", &bad).is_err(), "{bad}");+ }+ assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["cacheGb"], 5, "a refused change changes nothing");+ // the faster of the card and the processor unless the user says otherwise, and the interface can read the choice back+ assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "auto");+ assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap()["runOn"], "cpu");+ assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu");+ let status = x.s.execute("denoise.status", &json!({})).unwrap();+ assert_eq!((status["runOn"].clone(), status["device"]["kind"].clone()), (json!("cpu"), json!("none")), "no model is loaded yet: {status}");+ // the setting before `runOn` still reads as what it meant+ let models = x.s.denoise.models_dir.clone().unwrap();+ let mut st = crate::denoise::read_settings(&models);+ (st.run_on, st.gpu) = (None, Some(false));+ crate::denoise::write_settings(&models, &st).unwrap();+ assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu");+ // choosing where it runs lets the card be tried again after a set-up that closed LightCraft+ let onnx = crate::denoise::installed_models(&models).first().unwrap().onnx.clone();+ let marker = onnx.with_file_name(crate::denoise::GPU_SETUP_MARKER);+ std::fs::write(&marker, "setting up").unwrap();+ assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "auto"})).unwrap()["runOn"], "auto");+ assert!(!marker.exists(), "the crash is forgotten");+ assert_eq!(crate::denoise::read_settings(&models).gpu, None, "the old setting is not kept beside the new one");+ // with auto off nothing is started by itself, and the photo is made when asked+ let id = photo(&x.s);+ set_amount(&mut x.s, 100.0);+ for _ in 0..3 {+ x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();+ }+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);+ x.s.execute("denoise.settings", &json!({"auto": true})).unwrap();+ // paused: it is queued but nothing starts+ for _ in 0..3 {+ let r = x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap();+ assert_eq!(r["running"], Value::Null);+ }+ assert!(matches!(x.s.denoise_photo_state(id), PhotoState::Queued { .. }), "{:?}", x.s.denoise_photo_state(id));+ // …and cancelling forgets it+ assert_eq!(x.s.execute("denoise.cancel", &json!({})).unwrap()["cancelled"], 1);+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);+ // when the pace allows, it is made+ x.s.execute("denoise.queue", &json!({"scope": "withAmount"})).unwrap();+ pump_until_ready(&mut x.s, 1);+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn a_photo_whose_file_changed_has_no_picture_until_it_is_made_again() {+ let mut x = setup("changed", true);+ let id = photo(&x.s);+ set_amount(&mut x.s, 100.0);+ pump_until_ready(&mut x.s, 1);+ // the photo's content is something else now (a replaced file): its old picture is not its picture+ let p = x.s.catalog.photo(id).unwrap().clone();+ x.s.commit(+ "test",+ lightcraft_catalog::Op::SetContent {+ id,+ width: p.width,+ height: p.height,+ file_size: p.file_size + 1,+ content_hash: Some("ffffffffffffffffffffffffffffffff".into()),+ preview_only: None,+ },+ )+ .unwrap();+ x.s.denoise_pump(crate::denoise::Pace::Pause);+ assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready);+ // the old picture is not used by a render either+ assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0);+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn a_photo_waits_for_its_turn_unless_pictures_are_not_made_on_their_own_or_have_nowhere_to_go() {+ let mut x = setup("waits", true);+ let id = photo(&x.s);+ set_amount(&mut x.s, 80.0);+ // automatic off: the open photo is left alone until it is asked for+ x.s.execute("denoise.settings", &json!({"auto": false})).unwrap();+ assert!(!x.s.denoise_auto());+ x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);+ assert!(!x.s.denoise_busy());+ x.s.execute("denoise.queue", &json!({"ids": [id.0]})).unwrap();+ assert!(x.s.denoise_busy(), "queued with a model: the headless runs wait for it");+ pump_until_ready(&mut x.s, 1);+ assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);+ assert!(!x.s.denoise_busy());+ // a session whose library is not on disk has nowhere to keep it: the photo says so instead of waiting for ever+ x.s.denoise.settings.auto = Some(true);+ if let Some(l) = x.s.library.as_mut() {+ l.on_disk = false;+ }+ match x.s.denoise_photo_state(id) {+ PhotoState::Failed { why, unsupported } => assert!(unsupported && why.contains("library folder"), "{why}"),+ other => panic!("{other:?}"),+ }+ let _ = std::fs::remove_dir_all(&x.dir);+}++#[test]+fn the_denoise_switch_validates_arguments_and_accepts_explicit_targets_without_selection() {+ let mut x = setup("toggle", false);+ let id = photo(&x.s);+ for params in [json!({"id": "bad"}), json!({"id": u64::MAX}), json!({"id": id.0, "enabled": "yes"})] {+ assert!(x.s.execute("denoise.toggle", ¶ms).is_err());+ }+ assert!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).is_err());+ assert_eq!(x.s.develop_of(id).unwrap().enhance.denoise, 0.0);+ let model = model_files(&x.dir, "toggle-model");+ x.s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();+ x.s.selection = Default::default();+ let undo = x.s.undo.len();+ let r = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap();+ assert_eq!(r["amount"], 50.0);+ assert_eq!(x.s.undo.len(), undo + 1);+ x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap();+ assert_eq!(x.s.develop_of(id).unwrap().denoise_amount(), 0.0);+ // the switch is its own setting: flipping it keeps the Amount (so it compares with and without), and an Amount of 0+ // does not switch it off+ x.s.execute("library.select", &json!({"ids": [id.0]})).unwrap();+ x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap();+ x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 80.0})).unwrap();+ let off = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap();+ assert_eq!(off["amount"], 80.0);+ let d = x.s.develop_of(id).unwrap();+ assert!(!d.enhance.denoise_enabled() && d.denoise_amount() == 0.0);+ assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 80.0);+ assert!((x.s.develop_of(id).unwrap().denoise_amount() - 0.8).abs() < 1e-6);+ x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0.0})).unwrap();+ assert!(x.s.develop_of(id).unwrap().enhance.denoise_enabled(), "sliding to 0 leaves it on");+ assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 0.0);+ // with no `enabled`, the toggle flips what the switch says+ assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0})).unwrap()["enabled"], false);+ drop(x.s);+ let _ = std::fs::remove_dir_all(x.dir);+}diff --git a/crates/engine/src/tests_xmp.rs b/crates/engine/src/tests_xmp.rsindex fede1fec0..ab801e237 100644--- a/crates/engine/src/tests_xmp.rs+++ b/crates/engine/src/tests_xmp.rs@@ -335,10 +335,15 @@ fn synthetic_dng(xmp: &str) -> Vec<u8> {/// A tiny Bayer DNG written by our own DNG writer (optional embedded XMP, camera metadata).pub(crate) fn synthetic_dng_with(xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec<u8> {- use lightcraft_raw::*;let (w, h) = (32usize, 24usize);- let cfa = Cfa::bayer("RGGB").unwrap();let data: Vec<u16> = (0..w * h).map(|i| 256 + ((i % w) * 300 + (i / w) * 200) as u16).collect();+ synthetic_dng_of(w, h, "RGGB", data, xmp, metadata)+}++/// A synthetic Bayer DNG for denoise and cache tests.+pub(crate) fn synthetic_dng_of(w: usize, h: usize, cfa: &str, data: Vec<u16>, xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec<u8> {+ use lightcraft_raw::*;+ let cfa = Cfa::bayer(cfa).unwrap();let raw = RawImage {format: RawFormat::Dng,width: w,diff --git a/crates/engine/tests/denoise_cpu.rs b/crates/engine/tests/denoise_cpu.rsnew file mode 100644index 000000000..d01b7c5a5--- /dev/null+++ b/crates/engine/tests/denoise_cpu.rs@@ -0,0 +1,101 @@+//! Opt-in same-machine denoise model-stage benchmark on one public Bayer raw.+//! LC_DENOISE_MODEL=<onnx> LC_DENOISE_RAW=<CC0 corpus/raw> cargo test --release+//! -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture+//! LC_DENOISE_ONLY selects a file (default arw-sony-a7m3-compressed.arw).+//! The whole model stage includes packing, scale/clip protection, blending and normalization;+//! file decode, model loading and GPU setup are outside the timer. It uses the application's tile pool convention.+#![cfg(feature = "denoise")]++use lightcraft_denoise::{+ TileRunner,+ bayer::Layout,+ run::{Control, Params, denoise_bayer},+ runtime::{CpuRunner, check_tile},+};+use lightcraft_raster::Rgb32f;+use std::{path::Path, time::Instant};++fn agree(a: &[f32], b: &[f32]) -> f32 {+ assert_eq!(a.len(), b.len());+ assert!(a.iter().chain(b).all(|v| v.is_finite()));+ let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs()));+ let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;+ assert!(error <= 1e-3, "maximum relative error {error:e}");+ error+}+fn tile_time(name: &str, r: &dyn TileRunner, x: &[f32]) {+ r.run(x).unwrap();+ let mut times: Vec<_> = (0..5)+ .map(|_| {+ let t = Instant::now();+ r.run(x).unwrap();+ t.elapsed().as_secs_f64() * 1000.0+ })+ .collect();+ times.sort_by(f64::total_cmp);+ println!("{name}, one 512-cell tile, one thread: {:.1} ms median ({:.1}..{:.1})", times[2], times[0], times[4]);+}+fn photo_time(name: &str, r: &dyn TileRunner, raw: &lightcraft_raw::Normalized, layout: Layout, p: &Params) -> Rgb32f {+ let pool = rayon::ThreadPoolBuilder::new().num_threads(p.parallel).build().unwrap();+ let run = || pool.install(|| denoise_bayer(&raw.data, raw.width, raw.height, layout, r, p, &Control::default())).unwrap();+ drop(run()); // warm the workers and their workspace cache+ let mut times = Vec::new();+ let mut answer = Rgb32f { width: 0, height: 0, data: Vec::new() };+ for i in 0..3 {+ let t = Instant::now();+ answer = run();+ let secs = t.elapsed().as_secs_f64();+ times.push(secs);+ println!("{name}, whole model stage, {} workers, run {i}: {secs:.3} s", p.parallel);+ }+ times.sort_by(f64::total_cmp);+ println!("{name}, whole model stage median {:.3} s", times[1]);+ answer+}+#[test]+#[ignore = "needs LC_DENOISE_MODEL and LC_DENOISE_RAW; release timing and real-model accuracy"]+fn denoise_stage_compares_cpu_and_gpu() {+ let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL");+ let folder = std::env::var_os("LC_DENOISE_RAW").expect("set LC_DENOISE_RAW to public CC0 corpus/raw");+ let name = std::env::var("LC_DENOISE_ONLY").unwrap_or_else(|_| "arw-sony-a7m3-compressed.arw".into());+ let bytes = std::fs::read(Path::new(&folder).join(&name)).unwrap();+ let raw = lightcraft_raw::decode(&bytes).unwrap().normalized().unwrap();+ let cfa = raw.cfa.as_ref().unwrap();+ assert!(cfa.is_bayer());+ let layout = Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)]).unwrap();+ let m = lightcraft_denoise::known::find("rawnind-bayer").unwrap().manifest;+ let cpu = CpuRunner::load(Path::new(&model), &m).unwrap();+ let threads = std::env::var("LC_DENOISE_PARALLEL")+ .ok()+ .and_then(|v| v.parse::<usize>().ok())+ .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get()).clamp(1, 16));+ let p = Params { tile: 512, overlap: 64, gain: m.gain, clip: Some(1.0), parallel: threads };+ println!(+ "{name}: {}x{}, {:.2} MP, {} tiles, {} CPU workers, {:.1} MB/workspace, {:.1} GMAC/tile",+ raw.width,+ raw.height,+ raw.width as f64 * raw.height as f64 / 1e6,+ lightcraft_denoise::run::tile_count(raw.width, raw.height, layout, 512, 64),+ threads,+ cpu.bytes_per_tile() as f64 / 1e6,+ cpu.net().macs(512) as f64 / 1e9+ );+ let x = check_tile(512);+ let want = cpu.run(&x).unwrap();+ tile_time("pure Rust", &cpu, &x);+ let done = photo_time("pure Rust", &cpu, &raw, layout, &p);++ match lightcraft_gpu::nn::runner(cpu.net(), 512) {+ Ok(gpu) => {+ println!("GPU: {}", gpu.adapter());+ println!("CPU/check tile vs GPU: {:e}", agree(&want, &gpu.run(&x).unwrap()));+ tile_time("GPU", &gpu, &x);+ let gp = Params { parallel: 4, ..p };+ let answer = photo_time("GPU", &gpu, &raw, layout, &gp);+ let a: Vec<_> = done.data.iter().flatten().copied().collect();+ let b: Vec<_> = answer.data.iter().flatten().copied().collect();+ println!("CPU/whole photo vs GPU: {:e}", agree(&a, &b));+ }+ Err(e) => println!("GPU unavailable: {e}"),+ }+}diff --git a/crates/engine/tests/denoise_real.rs b/crates/engine/tests/denoise_real.rsnew file mode 100644index 000000000..8d8581753--- /dev/null+++ b/crates/engine/tests/denoise_real.rs@@ -0,0 +1,117 @@+//! Opt-in end-to-end check of AI denoise with the real RawNIND Bayer model, through the engine: install it, make a real+//! raw photo's picture in the background, render and export it with the Amount slider, and see what a file that is not+//! Bayer says.+//!+//! ```text+//! LC_DENOISE_MODEL=<model_bayer.onnx> LC_DENOISE_RAW=<folder with raw files> \+//! cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture+//! ```+//!+//! `LC_DENOISE_RAW` is a folder of CC0 raw files (`cargo xtask corpus --download` → `corpus/raw`); every file in it is+//! tried. Optional: `LC_DENOISE_ONLY` (a file name to try alone), `LC_DENOISE_RUN_ON=cpu|gpu|auto` (where the model runs; `cpu` to compare with the card).+#![cfg(feature = "denoise")]++use std::path::{Path, PathBuf};+use std::time::{Duration, Instant};++use lightcraft_engine::Session;+use serde_json::{Value, json};++fn temp(tag: &str) -> PathBuf {+ let d = std::env::temp_dir().join(format!("lc-denoise-real-{tag}-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&d);+ std::fs::create_dir_all(&d).unwrap();+ d+}++/// Mean absolute second difference of the luma: the fine detail and the noise of a picture.+fn roughness(img: &lightcraft_raster::Rgba8) -> f64 {+ let (w, h) = (img.width, img.height);+ let luma = |x: usize, y: usize| {+ let p = img.data[y * w + x];+ 0.299 * f64::from(p[0]) + 0.587 * f64::from(p[1]) + 0.114 * f64::from(p[2])+ };+ let (mut sum, mut n) = (0.0, 0usize);+ for y in 1..h - 1 {+ for x in 1..w - 1 {+ sum += (4.0 * luma(x, y) - luma(x - 1, y) - luma(x + 1, y) - luma(x, y - 1) - luma(x, y + 1)).abs();+ n += 1;+ }+ }+ sum / n.max(1) as f64+}++fn files(dir: &Path) -> Vec<PathBuf> {+ let only = std::env::var("LC_DENOISE_ONLY").ok();+ let mut v: Vec<PathBuf> = std::fs::read_dir(dir)+ .unwrap()+ .flatten()+ .map(|e| e.path())+ .filter(|p| p.is_file() && only.as_deref().is_none_or(|o| p.file_name().is_some_and(|n| n.to_string_lossy() == o)))+ .collect();+ v.sort();+ v+}++#[test]+#[ignore = "needs the real model and raw files (see the file's header)"]+fn the_real_model_denoises_real_raws_through_the_engine() {+ let (Some(model), Some(raws)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_RAW")) else {+ panic!("set LC_DENOISE_MODEL and LC_DENOISE_RAW");+ };+ let dir = temp("e2e");+ let mut s = Session::new().with_fs();+ s.open_library(dir.join("library"), false).unwrap();+ s.set_denoise_models_dir(Some(dir.join("models")));+ let t = Instant::now();+ let r = s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();+ println!("installed {} in {:.1} s: self-test {}", r["installed"]["id"], t.elapsed().as_secs_f64(), r["installed"]["accepted"]["selfTest"]);+ assert_eq!(r["installed"]["id"], "rawnind-bayer");+ if let Ok(run_on) = std::env::var("LC_DENOISE_RUN_ON") {+ s.execute("denoise.settings", &json!({"runOn": run_on})).unwrap();+ }++ for file in files(Path::new(&raws)) {+ let name = file.file_name().unwrap().to_string_lossy().into_owned();+ let imported = s.execute("library.import", &json!({"paths": [file.to_string_lossy()]})).unwrap();+ let Some(id) = imported["imported"].as_array().and_then(|a| a.first()).and_then(Value::as_u64) else {+ println!("{name}: not imported ({imported})");+ continue;+ };+ s.execute("library.select", &json!({"ids": [id]})).unwrap();+ s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap();+ let photo = lightcraft_engine::catalog::PhotoId(id);+ let started = Instant::now();+ let mut state = Value::Null;+ while started.elapsed() < Duration::from_secs(600) {+ s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();+ state = s.execute("denoise.status", &json!({"id": id})).unwrap()["photo"].clone();+ if matches!(state["state"].as_str(), Some("ready" | "failed")) {+ break;+ }+ std::thread::sleep(Duration::from_millis(250));+ }+ println!("{name}: {} in {:.1} s", state, started.elapsed().as_secs_f64());+ if state["state"] == "failed" {+ // a file that is not a Bayer raw says why, and its export is the plain one+ assert_eq!(state["unsupported"], true, "{name}: {state}");+ continue;+ }+ assert_eq!(state["state"], "ready", "{name}: {state}");+ let denoised = s.render_now(photo, 1600, 1600).unwrap().image;+ s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0})).unwrap();+ let plain = s.render_now(photo, 1600, 1600).unwrap().image;+ let (a, b) = (roughness(&plain), roughness(&denoised));+ println!("{name}: roughness at 1600 px: plain {a:.3}, denoised {b:.3} ({:+.1} %)", 100.0 * (b - a) / a);+ assert!(b < a, "{name}: the denoised picture is not smoother");+ // the export is made from the same picture, without asking the queue+ s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap();+ let o = lightcraft_engine::export::ExportOptions::from_json(&json!({"format": "jpeg", "width": 1200, "height": 1200}));+ let e = lightcraft_engine::export::export_photo(&mut s, photo, &o, 1).unwrap();+ assert!(e.bytes.len() > 1000);+ }+ let status = s.execute("denoise.status", &json!({})).unwrap();+ println!("device: {}", status["device"]);+ println!("cache: {}", status["cache"]);+ let _ = std::fs::remove_dir_all(&dir);+}diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.tomlindex 765e4c6d7..314f69c83 100644--- a/crates/gpu/Cargo.toml+++ b/crates/gpu/Cargo.toml@@ -9,6 +9,7 @@ description = "LightCraft GPU develop pipeline: wgpu compute (WGSL) twin of theworkspace = true[dependencies]+lightcraft-denoise = { workspace = true }lightcraft-geom = { workspace = true }lightcraft-color = { workspace = true }lightcraft-raster = { workspace = true }diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rsindex 312d42635..b4e32ba9c 100644--- a/crates/gpu/src/lib.rs+++ b/crates/gpu/src/lib.rs@@ -30,6 +30,8 @@ pub mod backend;#[cfg(not(target_arch = "wasm32"))]mod ctx;#[cfg(not(target_arch = "wasm32"))]+pub mod nn;+#[cfg(not(target_arch = "wasm32"))]mod params;#[cfg(not(target_arch = "wasm32"))]mod render;@@ -145,6 +147,16 @@ pub fn enabled() -> bool {ENABLED.load(Ordering::Relaxed) && !BROKEN.load(Ordering::Relaxed) && !env_disabled()}+/// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the+/// environment switches and the rendering preference, but not a render-device failure (the devices are separate).+#[cfg(not(target_arch = "wasm32"))]+pub(crate) fn switched_off() -> Option<String> {+ if env_disabled() {+ return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());+ }+ (!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into())+}+/// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process.fn env_disabled() -> bool {static OFF: std::sync::OnceLock<bool> = std::sync::OnceLock::new();diff --git a/crates/gpu/src/nn.rs b/crates/gpu/src/nn.rsnew file mode 100644index 000000000..4231dae8c--- /dev/null+++ b/crates/gpu/src/nn.rs@@ -0,0 +1,1048 @@+//! Denoise networks on the GPU: a [`NetRunner`] executes a [`Net`] (the plain description of a U-Net's layers) with+//! WGSL compute kernels, one tile at a time, behind the same [`TileRunner`] interface the CPU runner has.+//!+//! The runner has a device of its own, so a denoise job and the interactive renders are separate contexts that the+//! driver time-slices, and a failure of one never takes the other down. It respects the same switches as GPU rendering+//! (`LIGHTCRAFT_GPU`, `LIGHTCRAFT_GPU_BACKEND`, the GPU preference) and the same crash sentinel around device creation.+//! `LIGHTCRAFT_GPU_ADAPTER` picks an adapter by (part of) its name, e.g. to try another card.+//!+//! Every failure is an `Err` (never a panic), and a device that errs or is lost stops being used: callers then run the+//! CPU runner. The kernels (`wgsl/nn_conv.wgsl`, `wgsl/nn_pool.wgsl`) are tested against the plain-loop reference+//! interpreter in `lightcraft_denoise::reference` and, with the real model, against the pure-Rust CPU runner.++use std::collections::HashMap;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{Condvar, Mutex, OnceLock};+use std::time::Duration;++use bytemuck::{Pod, Zeroable};+use lightcraft_denoise::net::{Net, Op};+use lightcraft_denoise::run::{Error, TileRunner};+use wgpu::util::DeviceExt;++/// Longest wait for one tile's GPU work before the device is given up on.+const TIMEOUT: Duration = Duration::from_secs(60);+/// The largest tile side (in input cells) the dispatch sizes and 32-bit indexing are checked for.+const MAX_TILE: usize = 1024;+/// The largest single buffer used: indices in the kernels are 32-bit element offsets.+const MAX_BUFFER: u64 = 2 << 30;+/// Tiles that can be on the GPU at once (each has buffers of its own).+const WORKSPACES: usize = 2;++// ---------------------------------------------------------------------------------------------------------------------+// the device++struct Dev {+ device: wgpu::Device,+ queue: wgpu::Queue,+ info: wgpu::AdapterInfo,+ limits: wgpu::Limits,+}++static DEV: OnceLock<Result<Dev, String>> = OnceLock::new();+static BROKEN: AtomicBool = AtomicBool::new(false);+static BROKEN_WHY: Mutex<Option<String>> = Mutex::new(None);++fn mark_broken(why: &str) {+ let mut w = BROKEN_WHY.lock().unwrap_or_else(|e| e.into_inner());+ if w.is_none() {+ *w = Some(why.to_string());+ }+ BROKEN.store(true, Ordering::Relaxed);+}++/// Why the GPU stopped being used for denoise (a device error), if it did.+pub fn broken_reason() -> Option<String> {+ BROKEN.load(Ordering::Relaxed).then(|| BROKEN_WHY.lock().unwrap_or_else(|e| e.into_inner()).clone().unwrap_or_else(|| "device error".into()))+}++fn dev() -> Result<&'static Dev, String> {+ if let Some(why) = broken_reason() {+ return Err(format!("stopped after a GPU failure: {why}"));+ }+ if let Some(why) = crate::switched_off() {+ return Err(why);+ }+ DEV.get_or_init(create_device).as_ref().map_err(|e| e.clone())+}++fn create_device() -> Result<Dev, String> {+ let Some(backends) = crate::backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) };+ crate::backend::with_init_marker(backends, || {+ std::panic::catch_unwind(|| make_device(backends)).unwrap_or_else(|_| Err("device creation panicked".into()))+ })+}++fn make_device(backends: wgpu::Backends) -> Result<Dev, String> {+ let mut desc = wgpu::InstanceDescriptor::new_without_display_handle();+ desc.backends = backends;+ let instance = wgpu::Instance::new(desc);+ let wanted = std::env::var("LIGHTCRAFT_GPU_ADAPTER").ok().map(|s| s.trim().to_lowercase()).filter(|s| !s.is_empty());+ let adapter = match &wanted {+ Some(w) => {+ let all = pollster::block_on(instance.enumerate_adapters(backends));+ let names: Vec<String> = all.iter().map(|a| a.get_info().name).collect();+ all.into_iter()+ .find(|a| {+ let i = a.get_info();+ i.device_type != wgpu::DeviceType::Cpu && i.name.to_lowercase().contains(w.as_str())+ })+ .ok_or_else(|| format!("no GPU adapter named like `{w}` (found: {})", names.join(", ")))?+ }+ None => pollster::block_on(+ instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }),+ )+ .map_err(|e| format!("no GPU adapter among {backends:?} ({e})"))?,+ };+ let info = adapter.get_info();+ if info.device_type == wgpu::DeviceType::Cpu {+ return Err(format!("software adapter ({}) skipped: the CPU is faster", info.name));+ }+ let limits = adapter.limits();+ if limits.max_storage_buffers_per_shader_stage < 6+ || limits.max_compute_invocations_per_workgroup < 256+ || limits.max_compute_workgroup_size_x < 256+ {+ return Err(format!("{} is too limited for the denoise kernels", info.name));+ }+ if limits.max_compute_workgroup_storage_size < 16 * 1024 {+ return Err(format!("{} has too little workgroup memory for the denoise kernels", info.name));+ }+ let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {+ label: Some("lightcraft-denoise"),+ required_limits: limits.clone(),+ ..Default::default()+ }))+ .map_err(|e| format!("{}: device creation failed ({e})", info.name))?;+ device.on_uncaptured_error(std::sync::Arc::new(|e| {+ log::error!("gpu (denoise): {e}");+ mark_broken(&format!("device error: {e}"));+ }));+ device.set_device_lost_callback(|reason, msg| {+ log::error!("gpu (denoise): device lost ({reason:?}): {msg}");+ mark_broken(&format!("device lost ({reason:?}): {msg}"));+ });+ log::info!("gpu (denoise): {} ({:?})", info.name, info.backend);+ Ok(Dev { device, queue, info, limits })+}++/// The adapter denoise would use ("NVIDIA GeForce RTX 4090 Laptop GPU (Dx12)"), or why it cannot use one. Creates the+/// device on first use.+pub fn adapter() -> Result<String, String> {+ dev().map(|d| format!("{} ({:?})", d.info.name, d.info.backend))+}++// ---------------------------------------------------------------------------------------------------------------------+// the plan: what to run, in what buffers++#[repr(C)]+#[derive(Clone, Copy, Pod, Zeroable)]+struct ConvParams {+ h: u32,+ w: u32,+ c0: u32,+ c1: u32,+ n: u32,+ npad: u32,+ k: u32,+ mode: u32,+ cout: u32,+ leaky: u32,+ alpha: f32,+ pad: u32,+}++#[repr(C)]+#[derive(Clone, Copy, Pod, Zeroable)]+struct PoolParams {+ h: u32,+ w: u32,+ c4: u32,+ total: u32,+}++#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]+enum Kernel {+ Conv { bm: u32, bn: u32, bk: u32 },+ Pool,+}++/// Where a step's tensor lives.+#[derive(Clone, Copy, Debug)]+enum Place {+ Input,+ Slot(usize),+ Output,+}++struct Step {+ kernel: Kernel,+ groups: [u32; 3],+ src: Place,+ src2: Option<Place>,+ dst: Place,+ /// Index into the per-layer GPU data (weights, bias), for the convolutions.+ layer: Option<usize>,+ params: Vec<u8>,+}++struct Layout {+ steps: Vec<Step>,+ /// Bytes of each activation buffer.+ slots: Vec<u64>,+ in_floats: usize,+ out_floats: usize,+}++/// Weights arranged for the kernel: `[k][npad]` rows with `k = tap · cin + ci` (columns are output channels, zero past+/// `n`), and the bias padded the same.+struct Layer {+ weight: Vec<f32>,+ bias: Vec<f32>,+}++fn unsupported<T>(why: impl Into<String>) -> Result<T, String> {+ Err(why.into())+}++/// The kernel shape for `n` output columns: 64 × 64 blocks when `n` is a multiple of 64, else 128 × 32.+fn blocks(n: usize, cin: usize) -> (u32, u32, u32) {+ let bn = if n.is_multiple_of(64) { 64 } else { 32 };+ let bk = if cin.is_multiple_of(16) {+ 16+ } else if cin.is_multiple_of(8) {+ 8+ } else {+ 4+ };+ (4096 / bn, bn, bk)+}++fn round_up(v: usize, to: usize) -> usize {+ v.div_ceil(to) * to+}++/// Work out the steps, the buffers and the arranged weights for `net` on `tile × tile` input cells.+fn plan(net: &Net, tile: usize, max_binding: u64) -> Result<(Layout, Vec<Layer>), String> {+ if !net.fits_tile(tile) {+ return unsupported("the tile does not fit the network");+ }+ if tile > MAX_TILE {+ return unsupported("the tile is larger than the kernels' dispatches reach");+ }+ let side = |t: usize| -> usize {+ let level = net.shape(t).map_or(0, |s| s.level);+ if level >= 0 { tile >> level } else { tile << (-level) }+ };+ let chan = |t: usize| net.shape(t).map_or(0, |s| s.channels);+ if !net.in_channels.is_multiple_of(4) {+ return unsupported("the input channel count is not a multiple of 4");+ }+ let ops = net.ops();+ // the network must end in a 1 × 1 convolution and a depth-to-space (they run as one kernel)+ let n_ops = ops.len();+ let ends_right = matches!(+ (n_ops.checked_sub(2).and_then(|i| ops.get(i)), ops.last()),+ (Some(Op::Conv(c)), Some(Op::DepthToSpace2 { src })) if c.k == 1 && c.src2.is_none() && *src == n_ops - 1 && c.cout == 12 && c.leaky.is_none()+ );+ if !ends_right {+ return unsupported("the network does not end in a 1 × 1 convolution to 12 channels and depth-to-space");+ }+ // the last layer that reads each tensor+ let mut last_use = vec![0usize; n_ops + 1];+ for (i, op) in ops.iter().enumerate() {+ let reads: Vec<usize> = match op {+ Op::Conv(c) => std::iter::once(c.src).chain(c.src2).collect(),+ Op::ConvT2(c) => vec![c.src],+ Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => vec![*src],+ };+ for t in reads {+ if let Some(l) = last_use.get_mut(t) {+ *l = i;+ }+ }+ }++ use Place::{Input, Output, Slot};+ let mut slots: Vec<u64> = Vec::new();+ let mut free: Vec<usize> = Vec::new();+ let mut place: Vec<Option<Place>> = vec![None; n_ops + 1];+ if let Some(p) = place.first_mut() {+ *p = Some(Input);+ }+ let mut steps: Vec<Step> = Vec::new();+ let mut layers: Vec<Layer> = Vec::new();+ /// A buffer of at least `bytes`: the smallest free one that is big enough, else the biggest free one grown, else new.+ fn alloc(bytes: u64, slots: &mut Vec<u64>, free: &mut Vec<usize>) -> usize {+ let size = |s: usize| slots.get(s).copied().unwrap_or(0);+ let fits = free.iter().enumerate().filter(|&(_, &s)| size(s) >= bytes).min_by_key(|&(_, &s)| size(s)).map(|(i, _)| i);+ let pick = fits.or_else(|| free.iter().enumerate().max_by_key(|&(_, &s)| size(s)).map(|(i, _)| i));+ match pick {+ Some(i) => {+ let s = free.swap_remove(i);+ if let Some(b) = slots.get_mut(s) {+ *b = (*b).max(bytes);+ }+ s+ }+ None => {+ slots.push(bytes);+ slots.len() - 1+ }+ }+ }+ let get = |place: &[Option<Place>], t: usize| place.get(t).copied().flatten().ok_or_else(|| format!("tensor {t} has no buffer"));++ for (i, op) in ops.iter().enumerate() {+ let out_t = i + 1;+ match op {+ Op::Conv(c) => {+ if !c.cin.is_multiple_of(4) || !c.cout.is_multiple_of(4) {+ return unsupported("a convolution whose channel counts are not multiples of 4");+ }+ let s = side(c.src);+ let last = i + 2 == n_ops;+ let (n, mode) = (c.cout, if last { 2u32 } else { 0u32 });+ let (bm, bn, bk) = blocks(n, c.cin);+ let npad = round_up(n, bn as usize);+ // arranged weights: row (tap · cin + ci), column co+ let taps = c.k * c.k;+ let mut weight = vec![0f32; taps * c.cin * npad];+ for co in 0..c.cout {+ for ci in 0..c.cin {+ for ky in 0..c.k {+ for kx in 0..c.k {+ let v = c.weight.get(((co * c.cin + ci) * c.k + ky) * c.k + kx).copied().unwrap_or(0.0);+ if let Some(w) = weight.get_mut(((ky * c.k + kx) * c.cin + ci) * npad + co) {+ *w = v;+ }+ }+ }+ }+ }+ let mut bias = vec![0f32; npad];+ if let Some(b) = bias.get_mut(..c.cout) {+ b.copy_from_slice(&c.bias);+ }+ let m = s * s;+ let params = ConvParams {+ h: s as u32,+ w: s as u32,+ c0: chan(c.src) as u32,+ c1: c.src2.map_or(0, chan) as u32,+ n: n as u32,+ npad: npad as u32,+ k: c.k as u32,+ mode,+ cout: c.cout as u32,+ leaky: u32::from(c.leaky.is_some()),+ alpha: c.leaky.unwrap_or(0.0),+ pad: 0,+ };+ let src = get(&place, c.src)?;+ let src2 = c.src2.map(|t| get(&place, t)).transpose()?;+ let dst = if last {+ Output+ } else {+ let slot = alloc((m * c.cout * 4) as u64, &mut slots, &mut free);+ Slot(slot)+ };+ if let Some(p) = place.get_mut(out_t) {+ *p = Some(dst);+ }+ layers.push(Layer { weight, bias });+ steps.push(Step {+ kernel: Kernel::Conv { bm, bn, bk },+ groups: [m.div_ceil(bm as usize) as u32, (npad as u32) / bn, 1],+ src,+ src2,+ dst,+ layer: Some(layers.len() - 1),+ params: bytemuck::bytes_of(¶ms).to_vec(),+ });+ }+ Op::ConvT2(c) => {+ if !c.cin.is_multiple_of(4) || !c.cout.is_multiple_of(4) {+ return unsupported("a transposed convolution whose channel counts are not multiples of 4");+ }+ let s = side(c.src);+ let n = 4 * c.cout;+ let (bm, bn, bk) = blocks(n, c.cin);+ let npad = round_up(n, bn as usize);+ // arranged weights: row ci, column (ky · 2 + kx) · cout + co+ let mut weight = vec![0f32; c.cin * npad];+ for ci in 0..c.cin {+ for co in 0..c.cout {+ for ky in 0..2 {+ for kx in 0..2 {+ let v = c.weight.get(((ci * c.cout + co) * 2 + ky) * 2 + kx).copied().unwrap_or(0.0);+ if let Some(w) = weight.get_mut(ci * npad + (ky * 2 + kx) * c.cout + co) {+ *w = v;+ }+ }+ }+ }+ }+ let mut bias = vec![0f32; npad];+ for q in 0..4 {+ for co in 0..c.cout {+ if let (Some(b), Some(v)) = (bias.get_mut(q * c.cout + co), c.bias.get(co)) {+ *b = *v;+ }+ }+ }+ let m = s * s;+ let params = ConvParams {+ h: s as u32,+ w: s as u32,+ c0: c.cin as u32,+ c1: 0,+ n: n as u32,+ npad: npad as u32,+ k: 1,+ mode: 1,+ cout: c.cout as u32,+ leaky: u32::from(c.leaky.is_some()),+ alpha: c.leaky.unwrap_or(0.0),+ pad: 0,+ };+ let src = get(&place, c.src)?;+ let slot = alloc((4 * m * c.cout * 4) as u64, &mut slots, &mut free);+ if let Some(p) = place.get_mut(out_t) {+ *p = Some(Slot(slot));+ }+ layers.push(Layer { weight, bias });+ steps.push(Step {+ kernel: Kernel::Conv { bm, bn, bk },+ groups: [m.div_ceil(bm as usize) as u32, (npad as u32) / bn, 1],+ src,+ src2: None,+ dst: Slot(slot),+ layer: Some(layers.len() - 1),+ params: bytemuck::bytes_of(¶ms).to_vec(),+ });+ }+ Op::MaxPool2 { src } => {+ let (s, c) = (side(*src), chan(*src));+ if !c.is_multiple_of(4) {+ return unsupported("a max-pool over a channel count that is not a multiple of 4");+ }+ let total = (s / 2) * (s / 2) * (c / 4);+ let params = PoolParams { h: s as u32, w: s as u32, c4: (c / 4) as u32, total: total as u32 };+ let from = get(&place, *src)?;+ let slot = alloc((total * 16) as u64, &mut slots, &mut free);+ if let Some(p) = place.get_mut(out_t) {+ *p = Some(Slot(slot));+ }+ let groups = total.div_ceil(64);+ steps.push(Step {+ kernel: Kernel::Pool,+ groups: [groups.min(8192) as u32, groups.div_ceil(8192) as u32, 1],+ src: from,+ src2: None,+ dst: Slot(slot),+ layer: None,+ params: bytemuck::bytes_of(¶ms).to_vec(),+ });+ }+ // run as part of the 1 × 1 convolution before it+ Op::DepthToSpace2 { .. } => {}+ }+ // buffers whose last reader was this layer are free for the next layer's result+ for t in 0..=i {+ if last_use.get(t).copied() == Some(i)+ && let Some(Some(Slot(s))) = place.get(t)+ && !free.contains(s)+ {+ free.push(*s);+ }+ }+ }+ if slots.iter().any(|&b| b > max_binding) {+ return unsupported("an activation buffer is larger than this GPU allows");+ }+ if layers.iter().any(|l| (l.weight.len() as u64).saturating_mul(4) > max_binding) {+ return unsupported("a weight buffer is larger than this GPU allows");+ }+ Ok((Layout { steps, slots, in_floats: net.in_channels * tile * tile, out_floats: 3 * (2 * tile) * (2 * tile) }, layers))+}++// ---------------------------------------------------------------------------------------------------------------------+// the GPU objects++struct Pipelines {+ conv: HashMap<(u32, u32, u32), wgpu::ComputePipeline>,+ conv_layout: wgpu::BindGroupLayout,+ pool: wgpu::ComputePipeline,+ pool_layout: wgpu::BindGroupLayout,+ /// How long each kernel took to build (the driver compiles them), for the log and the tests.+ built: Vec<(String, f64)>,+}++fn entry(binding: u32, ty: wgpu::BufferBindingType) -> wgpu::BindGroupLayoutEntry {+ wgpu::BindGroupLayoutEntry {+ binding,+ visibility: wgpu::ShaderStages::COMPUTE,+ ty: wgpu::BindingType::Buffer { ty, has_dynamic_offset: false, min_binding_size: None },+ count: None,+ }+}++fn pipeline(d: &Dev, label: &str, src: &str, layout: &wgpu::BindGroupLayout) -> wgpu::ComputePipeline {+ let module = d.device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some(label), source: wgpu::ShaderSource::Wgsl(src.into()) });+ let pl = d.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: None, bind_group_layouts: &[Some(layout)], immediate_size: 0 });+ d.device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {+ label: Some(label),+ layout: Some(&pl),+ module: &module,+ entry_point: Some("main"),+ // The kernels write all of their workgroup memory before reading any of it, so the zeroing wgpu adds by default+ // is not needed; on DX12 it is written out as one assignment per element, which makes the shader compiler take+ // a minute over a 2k-element array.+ compilation_options: wgpu::PipelineCompilationOptions { zero_initialize_workgroup_memory: false, ..Default::default() },+ cache: None,+ })+}++fn conv_source(bm: u32, bn: u32, bk: u32) -> String {+ include_str!("wgsl/nn_conv.wgsl")+ .replace("{{BM}}", &bm.to_string())+ .replace("{{BN}}", &bn.to_string())+ .replace("{{BK}}", &bk.to_string())+ .replace("{{A_VECS}}", &(bk * bm / 4).to_string())+ .replace("{{B_VECS}}", &(bk * bn / 4).to_string())+}++fn conv_bgl(d: &Dev) -> wgpu::BindGroupLayout {+ use wgpu::BufferBindingType::{Storage, Uniform};+ d.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {+ label: None,+ entries: &[+ entry(0, Uniform),+ entry(1, Storage { read_only: true }),+ entry(2, Storage { read_only: true }),+ entry(3, Storage { read_only: true }),+ entry(4, Storage { read_only: true }),+ entry(5, Storage { read_only: false }),+ ],+ })+}++fn pool_bgl(d: &Dev) -> wgpu::BindGroupLayout {+ use wgpu::BufferBindingType::{Storage, Uniform};+ d.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {+ label: None,+ entries: &[entry(0, Uniform), entry(1, Storage { read_only: true }), entry(2, Storage { read_only: false })],+ })+}++fn build_pipelines(d: &Dev, layout: &Layout) -> Pipelines {+ let (conv_layout, pool_layout) = (conv_bgl(d), pool_bgl(d));+ let mut conv = HashMap::new();+ let mut built = Vec::new();+ for s in &layout.steps {+ if let Kernel::Conv { bm, bn, bk } = s.kernel {+ conv.entry((bm, bn, bk)).or_insert_with(|| {+ let started = std::time::Instant::now();+ let p = pipeline(d, "nn_conv", &conv_source(bm, bn, bk), &conv_layout);+ built.push((format!("conv {bm}x{bn}x{bk}"), started.elapsed().as_secs_f64() * 1000.0));+ p+ });+ }+ }+ let started = std::time::Instant::now();+ let pool = pipeline(d, "nn_pool", include_str!("wgsl/nn_pool.wgsl"), &pool_layout);+ built.push(("pool".into(), started.elapsed().as_secs_f64() * 1000.0));+ Pipelines { conv, conv_layout, pool, pool_layout, built }+}++/// One tile's buffers and the bound steps.+struct Workspace {+ input: wgpu::Buffer,+ output: wgpu::Buffer,+ staging: wgpu::Buffer,+ bound: Vec<(Kernel, [u32; 3], wgpu::BindGroup)>,+ _slots: Vec<wgpu::Buffer>,+}++struct Shared {+ dev: &'static Dev,+ layout: Layout,+ pipes: Pipelines,+ weights: Vec<(wgpu::Buffer, wgpu::Buffer)>,+ uniforms: Vec<wgpu::Buffer>,+ dummy: wgpu::Buffer,+}++fn storage(d: &Dev, label: &str, bytes: u64, extra: wgpu::BufferUsages) -> wgpu::Buffer {+ d.device.create_buffer(&wgpu::BufferDescriptor {+ label: Some(label),+ size: bytes.max(16),+ usage: wgpu::BufferUsages::STORAGE | extra,+ mapped_at_creation: false,+ })+}++impl Shared {+ fn workspace(&self) -> Workspace {+ let d = self.dev;+ let slots: Vec<wgpu::Buffer> = self.layout.slots.iter().map(|&b| storage(d, "activations", b, wgpu::BufferUsages::empty())).collect();+ let input = storage(d, "input", (self.layout.in_floats * 4) as u64, wgpu::BufferUsages::COPY_DST);+ let output = storage(d, "output", (self.layout.out_floats * 4) as u64, wgpu::BufferUsages::COPY_SRC);+ let staging = d.device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("readback"),+ size: (self.layout.out_floats * 4) as u64,+ usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,+ mapped_at_creation: false,+ });+ let buf = |p: Place| -> &wgpu::Buffer {+ match p {+ Place::Input => &input,+ Place::Output => &output,+ Place::Slot(i) => slots.get(i).unwrap_or(&self.dummy),+ }+ };+ let mut bound = Vec::new();+ for (i, s) in self.layout.steps.iter().enumerate() {+ let uniform = self.uniforms.get(i).unwrap_or(&self.dummy);+ let bg = match s.kernel {+ Kernel::Conv { .. } => {+ let (w, b) = s.layer.and_then(|l| self.weights.get(l)).map_or((&self.dummy, &self.dummy), |(w, b)| (w, b));+ d.device.create_bind_group(&wgpu::BindGroupDescriptor {+ label: None,+ layout: &self.pipes.conv_layout,+ entries: &[+ wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() },+ wgpu::BindGroupEntry { binding: 1, resource: buf(s.src).as_entire_binding() },+ wgpu::BindGroupEntry { binding: 2, resource: s.src2.map_or(&self.dummy, buf).as_entire_binding() },+ wgpu::BindGroupEntry { binding: 3, resource: w.as_entire_binding() },+ wgpu::BindGroupEntry { binding: 4, resource: b.as_entire_binding() },+ wgpu::BindGroupEntry { binding: 5, resource: buf(s.dst).as_entire_binding() },+ ],+ })+ }+ Kernel::Pool => d.device.create_bind_group(&wgpu::BindGroupDescriptor {+ label: None,+ layout: &self.pipes.pool_layout,+ entries: &[+ wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() },+ wgpu::BindGroupEntry { binding: 1, resource: buf(s.src).as_entire_binding() },+ wgpu::BindGroupEntry { binding: 2, resource: buf(s.dst).as_entire_binding() },+ ],+ }),+ };+ bound.push((s.kernel, s.groups, bg));+ }+ Workspace { input, output, staging, bound, _slots: slots }+ }+}++/// A network on the GPU for one tile size. Cheap to share between threads: each call takes a free workspace.+pub struct NetRunner {+ shared: Shared,+ tile: usize,+ in_channels: usize,+ free: Mutex<Vec<Workspace>>,+ /// Workspaces made so far (they are made when a call finds none free, up to [`WORKSPACES`]).+ made: Mutex<usize>,+ wake: Condvar,+ bytes: u64,+}++/// Put `net` on the GPU for `tile × tile` input cells, or say why it cannot be (no GPU, a layer or a size the kernels+/// do not do, the device refusing the buffers). Creates the device on first use and compiles the kernels.+pub fn runner(net: &Net, tile: usize) -> Result<NetRunner, String> {+ let d = dev()?;+ let max_binding = d.limits.max_storage_buffer_binding_size.min(d.limits.max_buffer_size).min(MAX_BUFFER);+ let (layout, layers) = plan(net, tile, max_binding)?;+ let scope = [+ d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory),+ d.device.push_error_scope(wgpu::ErrorFilter::Validation),+ d.device.push_error_scope(wgpu::ErrorFilter::Internal),+ ];+ let pipes = build_pipelines(d, &layout);+ let weights: Vec<(wgpu::Buffer, wgpu::Buffer)> = layers+ .iter()+ .map(|l| {+ let make = |v: &[f32]| {+ d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {+ label: Some("weights"),+ contents: bytemuck::cast_slice(v),+ usage: wgpu::BufferUsages::STORAGE,+ })+ };+ (make(&l.weight), make(&l.bias))+ })+ .collect();+ let uniforms: Vec<wgpu::Buffer> = layout+ .steps+ .iter()+ .map(|s| {+ d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {+ label: Some("params"),+ contents: &s.params,+ usage: wgpu::BufferUsages::UNIFORM,+ })+ })+ .collect();+ let dummy = storage(d, "dummy", 16, wgpu::BufferUsages::empty());+ let bytes: u64 = layout.slots.iter().sum::<u64>() + ((layout.in_floats + layout.out_floats) * 8) as u64;+ let shared = Shared { dev: d, layout, pipes, weights, uniforms, dummy };+ let first = shared.workspace();+ let [oom, validation, internal] = scope;+ let errors = [pollster::block_on(internal.pop()), pollster::block_on(validation.pop()), pollster::block_on(oom.pop())];+ if let Some(e) = errors.into_iter().flatten().next() {+ return Err(format!("{}: the denoise kernels could not be set up ({e})", d.info.name));+ }+ Ok(NetRunner { shared, tile, in_channels: net.in_channels, free: Mutex::new(vec![first]), made: Mutex::new(1), wake: Condvar::new(), bytes })+}++impl NetRunner {+ /// The adapter this runner uses.+ pub fn adapter(&self) -> String {+ format!("{} ({:?})", self.shared.dev.info.name, self.shared.dev.info.backend)+ }++ /// How long each kernel took to build, e.g. `conv 64x64x16: 812 ms`.+ pub fn build_report(&self) -> String {+ self.shared.pipes.built.iter().map(|(k, ms)| format!("{k}: {ms:.0} ms")).collect::<Vec<_>>().join(", ")+ }++ /// Device memory one tile in flight uses (bytes), without the weights.+ pub fn bytes_per_tile(&self) -> u64 {+ self.bytes+ }++ /// A free workspace, a new one when fewer than [`WORKSPACES`] exist, else wait for one to come back.+ fn take(&self) -> Result<Workspace, Error> {+ let mut free = self.free.lock().unwrap_or_else(|e| e.into_inner());+ loop {+ if let Some(w) = free.pop() {+ return Ok(w);+ }+ {+ let mut made = self.made.lock().unwrap_or_else(|e| e.into_inner());+ if *made < WORKSPACES {+ *made += 1;+ drop(made);+ drop(free);+ let d = self.shared.dev;+ let scopes =+ [d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), d.device.push_error_scope(wgpu::ErrorFilter::Validation)];+ let w = self.shared.workspace();+ let [oom, validation] = scopes;+ let (validation, oom) = (pollster::block_on(validation.pop()), pollster::block_on(oom.pop()));+ if let Some(e) = oom.or(validation) {+ self.forget();+ return Err(Error::Runtime(format!("the GPU could not make room for another tile: {e}")));+ }+ return Ok(w);+ }+ }+ free = self.wake.wait(free).unwrap_or_else(|e| e.into_inner());+ }+ }++ fn give(&self, w: Workspace) {+ self.free.lock().unwrap_or_else(|e| e.into_inner()).push(w);+ self.wake.notify_one();+ }++ /// A workspace is gone (dropped after a failure): room for a new one, and a waiting caller may make it.+ fn forget(&self) {+ let mut made = self.made.lock().unwrap_or_else(|e| e.into_inner());+ *made = made.saturating_sub(1);+ drop(made);+ self.wake.notify_one();+ }++ fn execute(&self, w: &Workspace, input: &[f32]) -> Result<Vec<f32>, Error> {+ let d = self.shared.dev;+ let (t, c) = (self.tile, self.in_channels);+ // planar → channels innermost+ let mut nhwc = vec![0f32; input.len()];+ for (ch, plane) in input.chunks_exact(t * t).enumerate() {+ for (i, v) in plane.iter().enumerate() {+ if let Some(o) = nhwc.get_mut(i * c + ch) {+ *o = *v;+ }+ }+ }+ let mut calls = Vec::with_capacity(w.bound.len());+ for (kernel, groups, bg) in &w.bound {+ let pipe = match kernel {+ Kernel::Conv { bm, bn, bk } => self.shared.pipes.conv.get(&(*bm, *bn, *bk)),+ Kernel::Pool => Some(&self.shared.pipes.pool),+ };+ let Some(pipe) = pipe else { return Err(Error::Runtime("a kernel is missing".into())) };+ calls.push((pipe, groups, bg));+ }+ let scopes = [+ d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory),+ d.device.push_error_scope(wgpu::ErrorFilter::Validation),+ d.device.push_error_scope(wgpu::ErrorFilter::Internal),+ ];+ d.queue.write_buffer(&w.input, 0, bytemuck::cast_slice(&nhwc));+ let mut enc = d.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None });+ {+ let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("denoise tile"), timestamp_writes: None });+ for (pipe, groups, bg) in calls {+ pass.set_pipeline(pipe);+ pass.set_bind_group(0, bg, &[]);+ pass.dispatch_workgroups(groups[0], groups[1], groups[2]);+ }+ }+ let bytes = (self.shared.layout.out_floats * 4) as u64;+ enc.copy_buffer_to_buffer(&w.output, 0, &w.staging, 0, bytes);+ d.queue.submit([enc.finish()]);+ let slice = w.staging.slice(..bytes);+ let (tx, rx) = std::sync::mpsc::channel();+ slice.map_async(wgpu::MapMode::Read, move |r| {+ let _ = tx.send(r);+ });+ let waited = d.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: Some(TIMEOUT) });+ let [oom, validation, internal] = scopes;+ let errors = [pollster::block_on(internal.pop()), pollster::block_on(validation.pop()), pollster::block_on(oom.pop())];+ let [internal, validation, oom] = errors;+ if let Some(e) = oom {+ // not a broken device: this tile runs on the CPU, and the next may fit+ return Err(Error::Runtime(format!("the GPU ran out of memory: {e}")));+ }+ if let Some(e) = internal.or(validation) {+ mark_broken(&format!("device error: {e}"));+ return Err(Error::Runtime(format!("the GPU reported an error: {e}")));+ }+ if let Err(e) = waited {+ mark_broken(&format!("the GPU did not finish a tile: {e}"));+ return Err(Error::Runtime(format!("the GPU did not finish: {e}")));+ }+ match rx.recv_timeout(Duration::from_secs(10)) {+ Ok(Ok(())) => {}+ Ok(Err(e)) => return Err(Error::Runtime(format!("reading the result back failed: {e}"))),+ Err(e) => return Err(Error::Runtime(format!("reading the result back failed: {e}"))),+ }+ let out = {+ let data = slice.get_mapped_range().map_err(|e| Error::Runtime(format!("reading the result back failed: {e:?}")))?;+ data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect::<Vec<f32>>()+ };+ w.staging.unmap();+ Ok(out)+ }+}++impl TileRunner for NetRunner {+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {+ if let Some(why) = broken_reason() {+ return Err(Error::Runtime(format!("the GPU is not used any more: {why}")));+ }+ if input.len() != self.shared.layout.in_floats {+ return Err(Error::Input("the tile is not the size the model wants".into()));+ }+ let w = self.take()?;+ let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.execute(&w, input)));+ match r {+ Ok(Ok(out)) => {+ self.give(w);+ Ok(out)+ }+ Ok(Err(e)) => {+ // its readback buffer may be left half-mapped: a fresh workspace is made when one is next needed+ drop(w);+ self.forget();+ Err(e)+ }+ Err(_) => {+ // a panic inside wgpu: this workspace is not given back, and the GPU is not used again+ mark_broken("a denoise tile panicked");+ drop(w);+ self.forget();+ Err(Error::Runtime("the GPU runner gave up".into()))+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use lightcraft_denoise::{onnx, reference, synthetic};++ fn net(tile: u64, ch: u64, depth: usize, seed: u64) -> Net {+ let path = std::env::temp_dir().join(format!("lc-nn-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id()));+ std::fs::write(&path, synthetic::unet_onnx(tile, ch, depth, seed)).unwrap();+ let n = onnx::read(&path).unwrap();+ let _ = std::fs::remove_file(&path);+ n+ }++ fn picture(len: usize, seed: u64) -> Vec<f32> {+ let mut s = seed | 1;+ (0..len)+ .map(|_| {+ s ^= s >> 12;+ s ^= s << 25;+ s ^= s >> 27;+ (s.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32+ })+ .collect()+ }++ /// The runner, or `None` (with a note) on a machine with no usable GPU.+ fn gpu(net: &Net, tile: usize) -> Option<NetRunner> {+ match runner(net, tile) {+ Ok(r) => Some(r),+ Err(e) => {+ eprintln!("skipped: no GPU runner here ({e})");+ None+ }+ }+ }++ fn worst(a: &[f32], b: &[f32]) -> f32 {+ assert_eq!(a.len(), b.len());+ let scale = b.iter().fold(1e-3f32, |m, v| m.max(v.abs()));+ a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) / scale+ }++ #[test]+ fn the_gpu_gives_what_the_reference_gives() {+ // (tile, channels, depth): a tile that is not a multiple of the block size, odd channel counts, a deeper net+ for (tile, ch, depth, seed) in+ [(1u64, 4u64, 0usize, 1u64), (3, 4, 0, 8), (6, 4, 1, 9), (20, 12, 2, 2), (32, 8, 2, 3), (48, 12, 2, 4), (64, 16, 3, 5)]+ {+ let n = net(tile, ch, depth, seed);+ let Some(g) = gpu(&n, tile as usize) else { return };+ let x = picture(4 * (tile * tile) as usize, seed);+ let want = reference::run(&n, tile as usize, &x).unwrap();+ let got = g.run(&x).unwrap();+ let cpu = lightcraft_denoise::cpu::NetRunner::new(&n, tile as usize).unwrap().run(&x).unwrap();+ assert!(worst(&got, &cpu) < 1e-3, "GPU and pure-Rust CPU differ");+ let e = worst(&got, &want);+ eprintln!("{}: tile {tile}, {ch} channels, depth {depth}: off by {e:e} of the output's size", g.adapter());+ assert!(e < 2e-4, "tile {tile}, {ch} channels, depth {depth}: the GPU is off by {e:e} of the output's size");+ }+ }++ #[test]+ fn a_runner_serves_many_tiles_from_many_threads() {+ let n = net(32, 8, 2, 7);+ let Some(g) = gpu(&n, 32) else { return };+ let inputs: Vec<Vec<f32>> = (0..12).map(|i| picture(4 * 32 * 32, 100 + i)).collect();+ let wants: Vec<Vec<f32>> = inputs.iter().map(|x| reference::run(&n, 32, x).unwrap()).collect();+ std::thread::scope(|s| {+ for chunk in inputs.chunks(3).zip(wants.chunks(3)) {+ let g = &g;+ s.spawn(move || {+ for (x, want) in chunk.0.iter().zip(chunk.1) {+ let e = worst(&g.run(x).unwrap(), want);+ assert!(e < 2e-4, "off by {e:e}");+ }+ });+ }+ });+ }++ /// The real model on the GPU against the pure-Rust CPU runner, with timings (the numbers in docs/denoise.md come from here):+ /// `LC_DENOISE_MODEL=<model_bayer.onnx> cargo test --release -p lightcraft-gpu --lib real_model -- --ignored --nocapture`+ #[test]+ #[ignore = "needs the real model: see the doc comment"]+ fn real_model_on_the_gpu_matches_cpu() {+ use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};+ use lightcraft_denoise::runtime::CpuRunner;+ use std::time::Instant;+ let Some(model) = std::env::var_os("LC_DENOISE_MODEL") else { return };+ let net = onnx::read(std::path::Path::new(&model)).unwrap();+ let tile = 512usize;+ let started = Instant::now();+ let g = runner(&net, tile).unwrap();+ println!("{}: set up in {:.2} s, {:.0} MB per tile in flight", g.adapter(), started.elapsed().as_secs_f64(), g.bytes_per_tile() as f64 / 1e6);+ println!("kernels: {}", g.build_report());+ let manifest = DenoiserManifest {+ id: "rawnind-bayer".into(),+ name: "RawNIND Bayer".into(),+ version: "1".into(),+ licence: Default::default(),+ source: None,+ sha256: None,+ size_bytes: None,+ provenance: String::new(),+ domain: Domain::BayerToRgb,+ tile: 512,+ overlap: 64,+ gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },+ };+ let cpu = CpuRunner::load(std::path::Path::new(&model), &manifest).unwrap();+ // a gradient with noise, in the model's input range+ let noise = picture(4 * tile * tile, 42);+ let input: Vec<f32> =+ (0..4 * tile * tile).map(|i| (0.2 + 0.3 * ((i % tile) as f32 / tile as f32) + 0.1 * (noise[i] - 0.5)).clamp(0.0, 1.0)).collect();+ let t = Instant::now();+ let want = cpu.run(&input).unwrap();+ println!("CPU: {:.0} ms for one tile (one call runs on one core)", t.elapsed().as_secs_f64() * 1000.0);+ let t = Instant::now();+ let got = g.run(&input).unwrap();+ println!("gpu, first tile: {:.0} ms", t.elapsed().as_secs_f64() * 1000.0);+ let e = worst(&got, &want);+ println!("worst difference {e:e} of the output's size");+ assert!(e < 1e-3, "the GPU does not give the CPU answer ({e:e})");+ let mut times: Vec<f64> = (0..8)+ .map(|_| {+ let t = Instant::now();+ g.run(&input).unwrap();+ t.elapsed().as_secs_f64() * 1000.0+ })+ .collect();+ times.sort_by(|a, b| a.total_cmp(b));+ println!("gpu, one tile at a time: median {:.0} ms (best {:.0}, worst {:.0})", times[times.len() / 2], times[0], times[times.len() - 1]);+ // two threads, as the denoiser feeds it+ let t = Instant::now();+ std::thread::scope(|s| {+ for _ in 0..2 {+ s.spawn(|| {+ for _ in 0..8 {+ g.run(&input).unwrap();+ }+ });+ }+ });+ println!("gpu, 16 tiles from 2 threads: {:.0} ms per tile", t.elapsed().as_secs_f64() * 1000.0 / 16.0);+ }++ /// How long the driver takes to build one conv kernel variant: `LC_SHADER=<file.wgsl> LC_BLOCKS=128,32,16`.+ #[test]+ #[ignore = "a probe for shader compile times"]+ fn time_a_shader_build() {+ let Some(path) = std::env::var_os("LC_SHADER") else { return };+ let blocks: Vec<u32> = std::env::var("LC_BLOCKS").unwrap_or_else(|_| "128,32,16".into()).split(',').filter_map(|v| v.parse().ok()).collect();+ let (bm, bn, bk) = (blocks[0], blocks[1], blocks[2]);+ let src = std::fs::read_to_string(path)+ .unwrap()+ .replace("{{BM}}", &bm.to_string())+ .replace("{{BN}}", &bn.to_string())+ .replace("{{BK}}", &bk.to_string())+ .replace("{{A_VECS}}", &(bk * bm / 4).to_string())+ .replace("{{B_VECS}}", &(bk * bn / 4).to_string());+ let d = dev().unwrap();+ let layout = conv_bgl(d);+ let started = std::time::Instant::now();+ let _ = pipeline(d, "probe", &src, &layout);+ println!("{} {bm}x{bn}x{bk}: built in {:.0} ms", d.info.backend.to_str(), started.elapsed().as_secs_f64() * 1000.0);+ }++ #[test]+ fn a_tile_of_the_wrong_size_is_an_error_not_a_crash() {+ let n = net(16, 4, 1, 9);+ let Some(g) = gpu(&n, 16) else { return };+ assert!(g.run(&[0.0; 10]).is_err());+ assert!(g.run(&[]).is_err());+ }++ #[test]+ fn networks_the_kernels_cannot_run_are_refused() {+ let n = net(16, 4, 1, 11);+ // a tile the network does not fit+ assert!(plan(&n, 15, u64::MAX).is_err());+ // a buffer limit smaller than the activations+ assert!(plan(&n, 16, 64).is_err());+ }+}diff --git a/crates/gpu/src/wgsl/nn_conv.wgsl b/crates/gpu/src/wgsl/nn_conv.wgslnew file mode 100644index 000000000..d991e9e76--- /dev/null+++ b/crates/gpu/src/wgsl/nn_conv.wgsl@@ -0,0 +1,162 @@+// Convolution for the denoise networks, as an implicit matrix product: every output pixel is a row, every input+// channel of every kernel tap is a column of K, and the output channels are the columns of N.+//+// out[m][n] = bias[n] + sum_k A[m][k] * W[k][n] A[m][k] = input pixel (m + tap offset), channel ci+//+// Tensors are NHWC, channels innermost, read four at a time (every channel count is a multiple of 4). A workgroup of+// 256 threads makes a BM x BN block of the output, 4 x 4 values per thread, and walks K in steps of BK through+// workgroup memory. The source can be two tensors whose channels follow each other (a skip connection's join, so it+// is never made), and a leaky ReLU is applied on the way out. Specialised by the host: BM, BN, BK (BM * BN = 4096).+//+// Output modes:+// 0 NHWC at the input's size.+// 1 a 2x2 stride-2 transposed convolution: the columns are (dy, dx, co) and each lands in the output pixel+// (2y + dy, 2x + dx) of an output twice the size.+// 2 the last 1x1 convolution followed by depth-to-space (CRD): columns 4c + 2dy + dx land in plane c at+// (2y + dy, 2x + dx) of a planar (CHW) output twice the size.++struct P {+ h: u32,+ w: u32,+ c0: u32,+ c1: u32,+ n: u32,+ npad: u32,+ k: u32,+ mode: u32,+ cout: u32,+ leaky: u32,+ alpha: f32,+ pad: u32,+}++@group(0) @binding(0) var<uniform> p: P;+@group(0) @binding(1) var<storage, read> src0: array<vec4<f32>>;+@group(0) @binding(2) var<storage, read> src1: array<vec4<f32>>;+@group(0) @binding(3) var<storage, read> wgt: array<vec4<f32>>;+@group(0) @binding(4) var<storage, read> bias: array<vec4<f32>>;+@group(0) @binding(5) var<storage, read_write> dst: array<f32>;++const BM: u32 = {{BM}}u;+const BN: u32 = {{BN}}u;+const BK: u32 = {{BK}}u;+const TX: u32 = BN / 4u;+const A_ITEMS: u32 = BM * (BK / 4u);+const B_ITEMS: u32 = BK * (BN / 4u);++// A[k][m] four pixels to a vec4, B[k][n] four channels to a vec4+var<workgroup> As: array<vec4<f32>, {{A_VECS}}>;+var<workgroup> Bs: array<vec4<f32>, {{B_VECS}}>;++@compute @workgroup_size(256)+fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {+ let m0 = wg.x * BM;+ let n0 = wg.y * BN;+ let tx = tid % TX;+ let ty = tid / TX;+ let hw = p.h * p.w;+ let cin = p.c0 + p.c1;+ let taps = p.k * p.k;+ let pad = (p.k - 1u) / 2u;+ let chunks = cin / BK;++ var acc0 = vec4<f32>(0.0);+ var acc1 = vec4<f32>(0.0);+ var acc2 = vec4<f32>(0.0);+ var acc3 = vec4<f32>(0.0);++ for (var tap = 0u; tap < taps; tap++) {+ let ty_off = i32(tap / p.k) - i32(pad);+ let tx_off = i32(tap % p.k) - i32(pad);+ for (var chunk = 0u; chunk < chunks; chunk++) {+ let ci0 = chunk * BK;+ for (var item = tid; item < A_ITEMS; item += 256u) {+ let ml = item / (BK / 4u);+ let q = item % (BK / 4u);+ let m = m0 + ml;+ var v = vec4<f32>(0.0);+ if (m < hw) {+ let y = i32(m / p.w) + ty_off;+ let x = i32(m % p.w) + tx_off;+ if (y >= 0 && y < i32(p.h) && x >= 0 && x < i32(p.w)) {+ let pix = u32(y) * p.w + u32(x);+ let ci = ci0 + q * 4u;+ if (ci < p.c0) {+ v = src0[pix * (p.c0 / 4u) + ci / 4u];+ } else {+ v = src1[pix * (p.c1 / 4u) + (ci - p.c0) / 4u];+ }+ }+ }+ let base = (q * 4u) * (BM / 4u) + ml / 4u;+ let lane = ml % 4u;+ As[base][lane] = v.x;+ As[base + BM / 4u][lane] = v.y;+ As[base + 2u * (BM / 4u)][lane] = v.z;+ As[base + 3u * (BM / 4u)][lane] = v.w;+ }+ for (var item = tid; item < B_ITEMS; item += 256u) {+ let r = item / (BN / 4u);+ let c4 = item % (BN / 4u);+ let krow = tap * cin + ci0 + r;+ Bs[item] = wgt[krow * (p.npad / 4u) + n0 / 4u + c4];+ }+ workgroupBarrier();+ for (var kk = 0u; kk < BK; kk++) {+ let a = As[kk * (BM / 4u) + ty];+ let b = Bs[kk * TX + tx];+ acc0 += a.x * b;+ acc1 += a.y * b;+ acc2 += a.z * b;+ acc3 += a.w * b;+ }+ workgroupBarrier();+ }+ }++ let n = n0 + tx * 4u;+ if (n >= p.n) {+ return;+ }+ let bv = bias[n / 4u];+ for (var i = 0u; i < 4u; i++) {+ let m = m0 + ty * 4u + i;+ if (m >= hw) {+ continue;+ }+ var v = bv;+ if (i == 0u) { v += acc0; } else if (i == 1u) { v += acc1; } else if (i == 2u) { v += acc2; } else { v += acc3; }+ if (p.leaky != 0u) {+ v = select(v * p.alpha, v, v >= vec4<f32>(0.0));+ }+ if (p.mode == 0u) {+ let o = m * p.n + n;+ dst[o] = v.x;+ dst[o + 1u] = v.y;+ dst[o + 2u] = v.z;+ dst[o + 3u] = v.w;+ } else if (p.mode == 1u) {+ let q = n / p.cout;+ let co = n % p.cout;+ let y = m / p.w;+ let x = m % p.w;+ let opix = (2u * y + q / 2u) * (2u * p.w) + 2u * x + q % 2u;+ let o = opix * p.cout + co;+ dst[o] = v.x;+ dst[o + 1u] = v.y;+ dst[o + 2u] = v.z;+ dst[o + 3u] = v.w;+ } else {+ let c = n / 4u;+ let y = m / p.w;+ let x = m % p.w;+ let ow = 2u * p.w;+ let plane = c * (2u * p.h) * ow;+ let o = plane + (2u * y) * ow + 2u * x;+ dst[o] = v.x;+ dst[o + 1u] = v.y;+ dst[o + ow] = v.z;+ dst[o + ow + 1u] = v.w;+ }+ }+}diff --git a/crates/gpu/src/wgsl/nn_pool.wgsl b/crates/gpu/src/wgsl/nn_pool.wgslnew file mode 100644index 000000000..6117eb069--- /dev/null+++ b/crates/gpu/src/wgsl/nn_pool.wgsl@@ -0,0 +1,28 @@+// 2x2 maximum with stride 2 over an NHWC tensor read and written four channels at a time.++struct P {+ h: u32,+ w: u32,+ c4: u32,+ total: u32,+}++@group(0) @binding(0) var<uniform> p: P;+@group(0) @binding(1) var<storage, read> src: array<vec4<f32>>;+@group(0) @binding(2) var<storage, read_write> dst: array<vec4<f32>>;++@compute @workgroup_size(64)+fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {+ let o = gid.x + gid.y * nwg.x * 64u;+ if (o >= p.total) {+ return;+ }+ let c = o % p.c4;+ let pix = o / p.c4;+ let ow = p.w / 2u;+ let oy = pix / ow;+ let ox = pix % ow;+ let i00 = ((2u * oy) * p.w + 2u * ox) * p.c4 + c;+ let i10 = i00 + p.w * p.c4;+ dst[o] = max(max(src[i00], src[i00 + p.c4]), max(src[i10], src[i10 + p.c4]));+}diff --git a/crates/mcp/src/headless.rs b/crates/mcp/src/headless.rsindex bdebec60a..7d2d929c7 100644--- a/crates/mcp/src/headless.rs+++ b/crates/mcp/src/headless.rs@@ -31,7 +31,7 @@ impl Drop for Headless {impl Default for Headless {fn default() -> Self {- Self::new(Session::new().with_fs())+ Self::new(Session::new().with_fs().with_default_denoise_models())}}@@ -42,7 +42,7 @@ impl Headless {/// A headless session with the procedurally generated demo library.pub fn demo() -> Self {- Self::new(Session::with_demo().with_fs())+ Self::new(Session::with_demo().with_fs().with_default_denoise_models())}fn photo_or_active(&self, p: &Value) -> Result<PhotoId, String> {diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rsindex d63e52544..ec52b3ddf 100644--- a/crates/raw/src/lib.rs+++ b/crates/raw/src/lib.rs@@ -677,14 +677,27 @@ impl RawImage {/// The result is camera RGB (not white balanced), white level = 1.0, not oriented.pub fn develop(&self, method: Method) -> Result<Rgb32f> {let n = self.normalized()?;- let mut rgb = demosaic(&n, method);+ let rgb = demosaic(&n, method);drop(n);+ Ok(self.finish_demosaiced(rgb))+ }++ /// The last steps of [`Self::develop`] for camera RGB made some other way (a denoiser that demosaics too) from+ /// [`Self::normalized`]'s mosaic: apply `OpcodeList3`, then the default crop.+ pub fn finish_demosaiced(&self, mut rgb: Rgb32f) -> Rgb32f {opcodes::apply_list3(&self.opcodes.list3, &mut rgb);- let c = self.crop.clipped(rgb.width, rgb.height);- if c.width == 0 || c.height == 0 || (c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height) {- return Ok(rgb);+ let c = self.develop_crop(rgb.width, rgb.height);+ if c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height {+ return rgb;}- Ok(rgb.into_crop(c.x, c.y, c.width, c.height))+ rgb.into_crop(c.x, c.y, c.width, c.height)+ }++ /// The part of a `width × height` demosaiced picture (the active area) that [`Self::develop`] keeps: the default+ /// crop, or all of it when there is none.+ pub fn develop_crop(&self, width: usize, height: usize) -> Rect {+ let c = self.crop.clipped(width, height);+ if c.width == 0 || c.height == 0 { Rect::new(0, 0, width, height) } else { c }}}@@ -717,6 +730,43 @@ mod tests {}}+ #[test]+ fn a_picture_made_elsewhere_is_finished_like_develop_does() {+ let (w, h) = (12usize, 10usize);+ let raw = RawImage {+ format: RawFormat::Dng,+ width: w,+ height: h,+ cpp: 1,+ data: RawData::U16((0..w * h).map(|i| (100 + (i * 37) % 900) as u16).collect()),+ cfa: Cfa::bayer("RGGB"),+ bits: 12,+ black: BlackLevel::uniform(64.0),+ white: vec![1023.0],+ active_area: Rect::new(2, 2, 8, 6),+ crop: Rect::new(1, 1, 5, 4),+ orientation: Orientation::Normal,+ color: ColorData::default(),+ wb_multipliers: None,+ linearized: false,+ opcodes: OpcodeLists::default(),+ metadata: Metadata::default(),+ };+ let direct = raw.develop(Method::Bilinear).unwrap();+ let by_hand = raw.finish_demosaiced(demosaic(&raw.normalized().unwrap(), Method::Bilinear));+ assert_eq!((direct.width, direct.height), (5, 4));+ assert_eq!(direct, by_hand);+ // the crop is what develop keeps; no usable crop keeps everything+ assert_eq!(raw.develop_crop(8, 6), Rect::new(1, 1, 5, 4));+ let mut whole = raw.clone();+ whole.crop = Rect::default();+ assert_eq!(whole.develop_crop(8, 6), Rect::new(0, 0, 8, 6));+ assert_eq!(whole.develop(Method::Bilinear).unwrap().width, 8);+ // a crop that pokes out is clipped to the picture+ whole.crop = Rect::new(6, 4, 10, 10);+ assert_eq!(whole.develop_crop(8, 6), Rect::new(6, 4, 2, 2));+ }+#[test]fn black_levels() {let b = BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![1.0, 2.0, 3.0, 4.0], delta_h: vec![0.5, 0.0], delta_v: vec![0.0, 10.0] };diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.tomlindex 49bdfc901..7836d96c5 100644--- a/crates/ui-egui/Cargo.toml+++ b/crates/ui-egui/Cargo.toml@@ -26,6 +26,7 @@ log = { workspace = true }[dev-dependencies]lightcraft-codecs = { workspace = true }+lightcraft-denoise = { workspace = true }[build-dependencies]serde_json = { workspace = true }diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rsindex 206d75b18..48688b6a8 100644--- a/crates/ui-egui/src/headless.rs+++ b/crates/ui-egui/src/headless.rs@@ -194,6 +194,7 @@ impl Headless {|| self.app.scan.is_some()|| self.app.import.is_some()|| self.app.export.is_some()+ || self.app.session.denoise_busy()|| !self.app.tasks.is_empty()|| !self.app.synthetic.is_empty()|| !self.events.is_empty()@@ -291,6 +292,61 @@ impl Headless {#[cfg(test)]mod tests {+ #[test]+ fn detail_offers_ai_denoise_for_raw_photos_and_leads_to_its_settings() {+ use crate::state::Dialog;+ use lightcraft_catalog::{MediaKind, Op, Photo, PhotoId, Source};+ let mut h = demo([1400.0, 900.0]);+ let t = Duration::from_secs(10);+ let dir = std::env::temp_dir().join(format!("lc-ui-denoise-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ h.app.session.set_denoise_models_dir(Some(dir.join("denoise-models")));+ // a raw photo (its file need not exist: nothing here reads it)+ let id = PhotoId(h.app.session.catalog.photos().map(|p| p.id.0).max().unwrap_or(0) + 1);+ let mut raw = Photo::new(+ id,+ Source::File { path: dir.join("IMG_1.dng").to_string_lossy().into_owned() },+ "IMG_1.dng",+ "DNG",+ 6000,+ 4000,+ "2026-10-01T00:00:00",+ );+ raw.kind = MediaKind::Raw;+ h.app.session.commit("setup", Op::AddPhoto { photo: Box::new(raw) }).unwrap();+ let widgets = |h: &mut Headless| h.request("ui.widgets", json!({}), t).to_string();+ h.request("engine.execute", json!({"command": "library.select", "params": {"ids": [id.0]}}), t);+ h.request("ui.set", json!({"view": "detail"}), t);+ h.app.ui.open_sections = vec!["detail".to_string()];+ h.settle(SETTLE);+ h.step();+ h.step();+ let w = widgets(&mut h);+ assert!(w.contains("\"denoise:setup\""), "a raw photo with no model is offered the setup: {w}");+ // one click lands in Settings ▸ AI Denoise, where the model LightCraft knows is listed+ let r = h.request("ui.clickWidget", json!({"id": "denoise:setup"}), t);+ assert_eq!(r["ok"], true, "{r}");+ h.step();+ h.step();+ assert_eq!(h.app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() }));+ h.settle(SETTLE);+ let listed = h.app.session.execute("denoise.models.list", &json!({})).unwrap();+ assert!(listed["models"].as_array().unwrap().iter().any(|m| m["id"] == "rawnind-bayer" && m["installed"] == false), "{listed}");+ h.app.ui.dialog = None;+ // a photo that is not raw: no offer+ h.request(+ "engine.execute",+ json!({"command": "library.select", "params": {"ids": [h.app.session.catalog.photos().find(|p| p.id != id).unwrap().id.0]}}),+ t,+ );+ h.settle(SETTLE);+ h.step();+ h.step();+ assert!(!widgets(&mut h).contains("\"denoise:setup\""), "nothing is offered for a photo denoise does not apply to");+ let _ = std::fs::remove_dir_all(&dir);+ }+use super::*;/// Generous: renders are slow when the machine is loaded (parallel builds), and a timed-outdiff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rsindex aeab2794b..be2095fcd 100644--- a/crates/ui-egui/src/lib.rs+++ b/crates/ui-egui/src/lib.rs@@ -19,6 +19,7 @@ pub mod menu_level;pub mod menubar;pub mod menus;pub mod merge;+mod model_setup;pub mod panels;pub mod pick;pub mod region;@@ -157,6 +158,7 @@ pub struct Perf {}pub struct LightcraftApp {+ pub(crate) model_setup: model_setup::Pending,/// Per-catalog-revision caches of library-wide results the panels show every frame/// (expensive on big libraries).pub caches: Caches,@@ -312,6 +314,7 @@ impl LightcraftApp {gpu_applied: None,memory_applied: None,library_problem: None,+ model_setup: Default::default(),}}@@ -324,6 +327,9 @@ impl LightcraftApp {/// Run a UI or engine command by id. The single entry point for every frontend path.pub fn run(&mut self, id: &str, params: Value) -> Result<Value, String> {+ if let Some(result) = model_setup::intercept(self, id, ¶ms) {+ return result;+ }if let Some(r) = menus::run_ui_command(self, id, ¶ms) {return r;}@@ -751,6 +757,8 @@ impl LightcraftApp {ctx.request_repaint_after(std::time::Duration::from_secs(3));}self.session.persist_if_dirty();+ panels::denoise::pump(self, ctx);+ model_setup::pump(self, ctx);self.collect_screenshots(ctx);self.issue_screenshots(ctx);if self.fonts_ready {@@ -988,6 +996,11 @@ pub struct Caches {album_counts: Option<(u64, std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>>)>,/// How often the album counts were recomputed (tests check that unchanged frames don't).pub album_count_scans: usize,+ /// AI denoise: what the pump last saw, the model list and the downloads being watched.+ pub denoise: panels::denoise::Ui,+ /// When the user last dragged, typed or scrolled, and when they last moved the pointer (egui time).+ pub last_input: f64,+ pub last_move: f64,/// The grid's date runs, layout and indexes (by the visible list's generation).pub grid: panels::grid::GridCache,/// What the grid did on its frames (benchmarks and tests check unchanged frames stay cheap).@@ -1213,3 +1226,6 @@ mod cache_tests {assert_eq!(c.album_count_scans, 1);}}++#[cfg(test)]+mod tests_model_setup;diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rsindex 5ca01cb40..96fc1dc97 100644--- a/crates/ui-egui/src/menus.rs+++ b/crates/ui-egui/src/menus.rs@@ -47,6 +47,7 @@ pub fn language_from_command(id: &str) -> Option<crate::i18n::Locale> {}pub const UI_COMMANDS: &[UiCommand] = &[+ ("modelSetup.cancel", "Cancel Pending Photo Action", None, ""),("view.photoGrid", "Photo Grid", None, "View"),("view.squareGrid", "Square Grid", None, "View"),// G: Photo Grid ↔ Square Grid (from other views: the photo grid)diff --git a/crates/ui-egui/src/model_setup.rs b/crates/ui-egui/src/model_setup.rsnew file mode 100644index 000000000..238a4afea--- /dev/null+++ b/crates/ui-egui/src/model_setup.rs@@ -0,0 +1,133 @@+//! Resume a denoise request after model setup, bound to its original photo and library.++use crate::{LightcraftApp, state::Dialog};+use lightcraft_catalog::{PhotoId, Source};+use lightcraft_engine::denoise::PhotoState;+use serde_json::{Value, json};++struct Request {+ library: u64,+ photo: PhotoId,+ source: Source,+}++impl Request {+ fn capture(app: &LightcraftApp, id: PhotoId) -> Result<Self, String> {+ let photo = app.session.catalog.photo(id).filter(|p| !p.deleted).ok_or("the requested photo is no longer available")?;+ Ok(Self { library: app.session.library_generation(), photo: id, source: photo.source.clone() })+ }++ fn valid(&self, app: &LightcraftApp) -> bool {+ self.library == app.session.library_generation()+ && app.session.catalog.photo(self.photo).is_some_and(|p| !p.deleted && p.source == self.source)+ }+}++#[derive(Default)]+pub(crate) struct Pending {+ denoise: Option<Request>,+ next_check: f64,+}++pub(crate) fn denoise_requested(app: &LightcraftApp, id: PhotoId) -> bool {+ app.model_setup.denoise.as_ref().is_some_and(|r| r.photo == id && r.valid(app))+}++pub(crate) fn intercept(app: &mut LightcraftApp, command: &str, params: &Value) -> Option<Result<Value, String>> {+ if matches!(command, "app.openLibrary" | "file.restoreLibrary") {+ app.model_setup = Pending::default();+ return None;+ }+ if command == "modelSetup.cancel" {+ if params.get("kind").and_then(Value::as_str) != Some("denoise") {+ return Some(Err("modelSetup.cancel: use kind denoise".into()));+ }+ app.model_setup.denoise = None;+ app.ui.status = "Pending photo action cancelled".into();+ return Some(Ok(json!({"cancelled": true})));+ }+ if command == "denoise.models.downloadCancel" {+ app.model_setup.denoise = None;+ }+ if command != "denoise.toggle" {+ return None;+ }+ let id = match params.get("id") {+ Some(v) => match v.as_u64() {+ Some(id) => PhotoId(id),+ None => return Some(Err("denoise.toggle: id must be a photo id".into())),+ },+ None => match app.session.active() {+ Some(id) => id,+ None => return Some(Err("select a raw photo".into())),+ },+ };+ let enabled = app.session.develop_of(id).is_some_and(|d| d.enhance.denoise_enabled());+ let on = match params.get("enabled") {+ Some(v) => match v.as_bool() {+ Some(on) => on,+ None => return Some(Err("denoise.toggle: enabled must be true or false".into())),+ },+ None => !enabled && !denoise_requested(app, id),+ };+ if !on {+ app.model_setup.denoise = None;+ return Some(app.session.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).map_err(|e| e.to_string()));+ }+ let _ = app.session.execute("denoise.status", &json!({"id": id.0}));+ if !matches!(app.session.denoise_photo_state(id), PhotoState::NoModel) {+ return None;+ }+ if app.session.execute("denoise.models.list", &json!({})).ok().is_none_or(|v| v["dir"].is_null()) {+ return None;+ }+ Some(Request::capture(app, id).map(|request| {+ app.model_setup.denoise = Some(request);+ app.model_setup.next_check = 0.0;+ app.ui.dialog = Some(Dialog::Settings { tab: "denoise".into() });+ app.ui.status = "Install a denoise model to enable AI Denoise on this photo".into();+ json!({"setupRequired": true, "photo": id.0})+ }))+}++pub(crate) fn notice(app: &mut LightcraftApp, ui: &mut egui::Ui, kind: &str) {+ if kind != "denoise" || !app.model_setup.denoise.as_ref().is_some_and(|r| r.valid(app)) {+ return;+ }+ ui.add(egui::Label::new("After installation, AI Denoise will turn on for the requesting photo.").wrap());+ let r = ui.small_button("Cancel pending action");+ crate::widgets::register(ui.ctx(), "modelSetup:cancel:denoise", r.rect);+ if r.clicked() {+ let _ = app.run("modelSetup.cancel", json!({"kind": "denoise"}));+ }+ ui.add_space(6.0);+}++pub(crate) fn pump(app: &mut LightcraftApp, ctx: &egui::Context) {+ if app.model_setup.denoise.is_none() {+ return;+ }+ if app.session.interaction.is_some() {+ ctx.request_repaint_after(std::time::Duration::from_millis(250));+ return;+ }+ let now = ctx.input(|i| i.time);+ if now < app.model_setup.next_check {+ return;+ }+ app.model_setup.next_check = now + 0.25;+ let Some(request) = app.model_setup.denoise.take() else { return };+ if !request.valid(app) {+ app.ui.status = "Pending photo action cancelled because its library or photo changed".into();+ return;+ }+ if matches!(app.session.denoise_photo_state(request.photo), PhotoState::NoModel | PhotoState::NotApplicable) {+ app.model_setup.denoise = Some(request);+ ctx.request_repaint_after(std::time::Duration::from_millis(250));+ return;+ }+ match app.run("denoise.toggle", json!({"id": request.photo.0, "enabled": true})) {+ Ok(_) => app.ui.status = "AI Denoise is enabled on the requesting photo".into(),+ Err(error) => app.ui.status = error,+ }+}diff --git a/crates/ui-egui/src/panels/denoise.rs b/crates/ui-egui/src/panels/denoise.rsnew file mode 100644index 000000000..bd69b0cfa--- /dev/null+++ b/crates/ui-egui/src/panels/denoise.rs@@ -0,0 +1,647 @@+//! AI denoise in the interface: the line under the Detail panel's Denoise slider (what the picture is doing), the+//! Settings ▸ Denoise tab (the model and the cache) and the per-frame pump that keeps the engine's background work going.+//!+//! The Amount slider is an ordinary develop control (`enhance.denoise`); everything here is about the cleaned picture the+//! slider mixes in. It is cached data made in the background for the photos being looked at, and never a file in the+//! library. A model is fetched only when the user presses Download and accepts its terms.++use egui::RichText;+use lightcraft_catalog::PhotoId;+use lightcraft_engine::denoise::PhotoState;+use serde_json::{Value, json};++use super::settings::{check, choices, heading, hint};+use crate::LightcraftApp;+use crate::theme::Tokens;+use crate::widgets::register;++/// How long a read of the model list is reused (it is a few small files, but not for every frame).+const REFRESH_SECS: f64 = 1.5;+/// Cache limits offered (GB).+const CACHE_LIMITS: [u32; 6] = [5, 10, 20, 50, 100, 250];++/// What the interface keeps between frames.+#[derive(Default)]+pub struct Ui {+ next_pump: f64,+ generation: u64,+ active: bool,+ /// (tiles done, tiles in all) of the photo being made.+ running: Option<(u64, u64)>,+ queued: u64,+ /// Bumped when a model is installed, removed or chosen, so Settings re-reads the list at once.+ pub epoch: u64,+ /// Models the user pressed Download for that have not been installed yet.+ dl_watch: Vec<String>,+ list: Option<(u64, f64, Value)>,+}++/// Called every frame: keeps the engine's denoise work going and asks for a redraw when a photo's picture appears (the+/// loupe and the thumbnails then render from it) or while one is being made (the progress).+pub fn pump(app: &mut LightcraftApp, ctx: &egui::Context) {+ watch_downloads(app, ctx);+ let now = ctx.input(|i| i.time);+ let (working, moving) = ctx.input(|i| {+ let keys = i.events.iter().any(|e| matches!(e, egui::Event::Key { .. } | egui::Event::Text(_) | egui::Event::MouseWheel { .. }));+ (i.pointer.any_down() || keys, i.pointer.is_moving())+ });+ if working {+ app.caches.last_input = now;+ }+ if moving || working {+ app.caches.last_move = now;+ }+ if now < app.caches.denoise.next_pump {+ return;+ }+ let watching = matches!(&app.ui.dialog, Some(crate::state::Dialog::Settings { tab }) if tab == "denoise");+ let pace = window_pace(app, ctx, now, watching);+ let Ok(v) = app.session.execute("denoise.pump", &json!({"pace": pace})) else { return };+ let c = &mut app.caches.denoise;+ let generation = v["generation"].as_u64().unwrap_or(0);+ if generation != c.generation {+ c.generation = generation;+ ctx.request_repaint();+ }+ c.active = v["active"] == true;+ c.queued = v["queued"].as_u64().unwrap_or(0);+ c.running = v["running"].is_object().then(|| (v["running"]["done"].as_u64().unwrap_or(0), v["running"]["total"].as_u64().unwrap_or(0)));+ let busy = c.running.is_some() || c.queued > 0;+ c.next_pump = now+ + if busy {+ 0.1+ } else if c.active {+ 0.5+ } else {+ 1.0+ };+ // frames are drawn only when something asks for one: while a photo is being made, ask again for its progress+ if busy {+ ctx.request_repaint_after(std::time::Duration::from_millis(100));+ }+}++/// The engine's model list, re-read at most every [`REFRESH_SECS`], or at once after an action changed it.+fn models(app: &mut LightcraftApp, ctx: &egui::Context) -> Value {+ let now = ctx.input(|i| i.time);+ let epoch = app.caches.denoise.epoch;+ if let Some((e, at, v)) = &app.caches.denoise.list+ && *e == epoch+ && now - at < REFRESH_SECS+ {+ return v.clone();+ }+ let v = app.run("denoise.models.list", json!({})).unwrap_or(Value::Null);+ app.caches.denoise.list = Some((epoch, now, v.clone()));+ v+}++/// Follows the downloads the user started: keeps redrawing while one runs (the progress bar), and says so when a+/// model has been installed and is in use (the engine does all of that by itself).+fn watch_downloads(app: &mut LightcraftApp, ctx: &egui::Context) {+ if app.caches.denoise.dl_watch.is_empty() {+ return;+ }+ let Ok(v) = app.session.execute("denoise.models.downloads", &json!({})) else {+ app.caches.denoise.dl_watch.clear();+ return;+ };+ let rows: Vec<Value> = v["downloads"].as_array().cloned().unwrap_or_default();+ let (mut keep, mut installed) = (Vec::new(), Vec::new());+ for id in std::mem::take(&mut app.caches.denoise.dl_watch) {+ match rows.iter().find(|r| r["id"] == id.as_str()).and_then(|r| r["state"].as_str()) {+ Some("running" | "done") => keep.push(id),+ Some("installed") => installed.push(id),+ _ => {}+ }+ }+ app.caches.denoise.dl_watch = keep;+ for id in installed {+ let _ = app.session.execute("denoise.models.downloadCancel", &json!({"id": id}));+ app.caches.denoise.epoch += 1;+ let text = "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works";+ app.ui.status = text.into();+ app.toast(ctx, text);+ }+ ctx.request_repaint_after(std::time::Duration::from_millis(120));+}++/// Per-photo AI Denoise switch, including a request waiting for model installation.+pub fn detail_toggle(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, on: bool, applicable: bool) {+ let mut enabled = on || crate::model_setup::denoise_requested(app, id);+ egui::Frame::NONE.inner_margin(egui::Margin { left: 24, right: 22, top: 6, bottom: 2 }).show(ui, |ui| {+ let r = ui.add_enabled(applicable, egui::Checkbox::new(&mut enabled, crate::i18n::tr("AI Denoise")));+ register(ui.ctx(), "denoise:enabled", r.rect);+ if r.changed() {+ let _ = app.run("denoise.toggle", json!({"id": id.0, "enabled": enabled}));+ }+ });+}++/// The line under the Denoise slider for the photo `id`, whose Amount is `amount`.+pub fn detail_status(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, amount: f64) {+ let t = Tokens::get(ui.ctx());+ let state = app.session.denoise_photo_state(id);+ let pad = egui::Margin { left: 24, right: 22, top: 0, bottom: 6 };+ let note = |ui: &mut egui::Ui, text: &str, color: egui::Color32| {+ ui.add(egui::Label::new(RichText::new(crate::i18n::tr(text)).size(11.0).color(color)).wrap());+ };+ match state {+ PhotoState::NotApplicable | PhotoState::Ready => {}+ PhotoState::NoModel => {+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {+ note(ui, "AI Denoise needs a model: a one-time download, in Settings.", t.text_dim);+ ui.add_space(2.0);+ let r = ui.small_button(crate::i18n::tr("Set up AI Denoise…"));+ register(ui.ctx(), "denoise:setup", r.rect);+ if r.clicked() {+ let _ = app.run("denoise.toggle", json!({"id": id.0, "enabled": true}));+ }+ });+ }+ PhotoState::Idle => {+ if amount > 0.0 && app.session.denoise_auto() {+ // the pump has not met this photo yet: let it come round soon instead of waiting out its pause+ let now = ui.ctx().input(|i| i.time);+ app.caches.denoise.next_pump = app.caches.denoise.next_pump.min(now + 0.05);+ ui.ctx().request_repaint_after(std::time::Duration::from_millis(60));+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| note(ui, "Waiting to start…", t.text_dim));+ } else if amount > 0.0 {+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {+ note(ui, "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", t.text_dim);+ ui.add_space(2.0);+ let r = ui.small_button(crate::i18n::tr("Make it now"));+ register(ui.ctx(), "denoise:make", r.rect);+ if r.clicked() {+ let _ = app.run("denoise.queue", json!({"ids": [id.0]}));+ }+ });+ }+ }+ PhotoState::Queued { ahead } => {+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {+ let text = if ahead == 0 { "Next up for AI Denoise…".to_string() } else { format!("In line for AI Denoise ({ahead} ahead)…") };+ note(ui, &text, t.text_dim);+ });+ }+ PhotoState::Running { done, total } => {+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {+ let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 };+ let text =+ if total > 0 { format!("Making the denoised picture… {}%", (frac * 100.0).round() as u32) } else { "Starting…".to_string() };+ let bar = ui.add(egui::ProgressBar::new(frac).desired_width(ui.available_width().min(240.0)).text(RichText::new(text).size(11.0)));+ register(ui.ctx(), "denoise:progress", bar.rect);+ });+ }+ PhotoState::Failed { why, unsupported } => {+ egui::Frame::NONE.inner_margin(pad).show(ui, |ui| {+ let color = if unsupported { t.text_dim } else { t.caution };+ note(ui, &sentence(&why), color);+ if !unsupported {+ let r = ui.small_button(crate::i18n::tr("Try again"));+ register(ui.ctx(), "denoise:retry", r.rect);+ if r.clicked() {+ let _ = app.run("denoise.queue", json!({"ids": [id.0], "retry": true}));+ }+ }+ });+ }+ }+}++/// The Settings ▸ Denoise tab.+pub fn settings_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) {+ let list = models(app, ui.ctx());+ heading(ui, t, "AI Denoise");+ crate::model_setup::notice(app, ui, "denoise");+ if list["dir"].is_null() {+ hint(ui, t, "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)");+ return;+ }+ hint(+ ui,+ t,+ "Cleans noise out of raw photos with an AI model, before the picture is built: the AI Denoise Amount slider under Detail sets how much of it you see. \+ Your files are never changed and no extra files are made in your library: the cleaned picture is cached data that is made again when needed.",+ );+ let runtime = list["runtime"].as_bool() == Some(true);+ if !runtime {+ hint(ui, t, "This build cannot run denoise models.");+ }+ let all: Vec<Value> = list["models"].as_array().cloned().unwrap_or_default();+ let downloads = app+ .session+ .execute("denoise.models.downloads", &json!({}))+ .map(|v| v["downloads"].as_array().cloned().unwrap_or_default())+ .unwrap_or_default();+ heading(ui, t, "Models");+ for m in &all {+ let dl = downloads.iter().find(|d| d["id"] == m["id"]);+ model_row(app, ui, t, m, dl, runtime);+ }+ if all.iter().all(|m| m["selected"] != true) && all.iter().any(|m| m["installed"] == true) {+ hint(ui, t, "AI Denoise is off: press Use on a model.");+ }+ hint(ui, t, "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.");+ work_section(app, ui, t, &list);+}++/// What the cache holds, its limit, and what is being made.+fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &Value) {+ let status = app.session.execute("denoise.status", &json!({})).unwrap_or(Value::Null);+ if status["enabled"] != true {+ return;+ }+ heading(ui, t, "Speed");+ let current = list["runOn"].as_str().unwrap_or("auto").to_string();+ let mut run_on = current.as_str();+ let options = [("auto", "Automatic"), ("gpu", "Graphics card"), ("cpu", "Processor")];+ if ui.horizontal(|ui| choices(ui, "denoiseRunOn", &options, &mut run_on)).inner {+ let _ = app.run("denoise.settings", json!({"runOn": run_on}));+ app.caches.denoise.epoch += 1;+ }+ hint(+ ui,+ t,+ "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.",+ );+ hint(ui, t, &device_line(&status["device"], run_on));+ if run_on != "cpu" && status["device"]["retry"] == true {+ let r = ui.small_button(crate::i18n::tr("Try the graphics card again"));+ register(ui.ctx(), "denoise:retryGpu", r.rect);+ if r.clicked() {+ // choosing where it runs, even the same again, forgets the failed set-up+ let _ = app.run("denoise.settings", json!({"runOn": run_on}));+ app.caches.denoise.epoch += 1;+ }+ }+ heading(ui, t, "Photos");+ let mut auto = list["auto"].as_bool().unwrap_or(true);+ if check(ui, "settings.denoiseAuto", &mut auto, "Make the denoised picture of the photos I am looking at") {+ let _ = app.run("denoise.settings", json!({"auto": auto}));+ app.caches.denoise.epoch += 1;+ }+ hint(ui, t, "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.");+ let (running, queued) = (app.caches.denoise.running, app.caches.denoise.queued);+ match running {+ Some((done, total)) => {+ let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 };+ let name = status["running"]["photo"]+ .as_u64()+ .and_then(|p| app.session.catalog.photo(PhotoId(p)).map(|p| p.file_name.clone()))+ .unwrap_or_default();+ let text =+ format!("Denoising {name}: {done} of {total} tiles{}", if queued > 0 { format!(" · {queued} waiting") } else { String::new() });+ let bar = ui.add(egui::ProgressBar::new(frac).desired_width(380.0).text(RichText::new(text).size(11.5)));+ register(ui.ctx(), "denoise:settingsProgress", bar.rect);+ }+ None if queued > 0 => hint(ui, t, &count(queued, "photo waiting", "photos waiting")),+ None => {+ let ready = status["ready"].as_u64().unwrap_or(0);+ hint(ui, t, &count(ready, "photo has its denoised picture.", "photos have their denoised picture."))+ }+ }+ ui.horizontal(|ui| {+ let r = ui.button("Denoise all photos with an Amount");+ register(ui.ctx(), "denoise:queueAll", r.rect);+ if r.clicked() {+ let _ = app.run("denoise.queue", json!({"scope": "withAmount"}));+ }+ if running.is_some() || queued > 0 {+ let r = ui.button("Stop");+ register(ui.ctx(), "denoise:stop", r.rect);+ if r.clicked() {+ let _ = app.run("denoise.cancel", json!({}));+ }+ }+ });+ heading(ui, t, "Cache");+ let (files, bytes) = (status["cache"]["files"].as_u64().unwrap_or(0), status["cache"]["bytes"].as_u64().unwrap_or(0));+ hint(+ ui,+ t,+ &format!(+ "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.",+ count(files, "cached picture", "cached pictures"),+ mb(Some(bytes))+ ),+ );+ ui.horizontal(|ui| {+ ui.label(RichText::new(crate::i18n::tr("Limit")).color(t.text_label));+ let current = list["cacheGb"].as_u64().unwrap_or(20) as u32;+ let mut choice = current;+ egui::ComboBox::from_id_salt("denoise-cache-limit").selected_text(format!("{current} GB")).show_ui(ui, |ui| {+ for gb in CACHE_LIMITS {+ ui.selectable_value(&mut choice, gb, format!("{gb} GB"));+ }+ });+ if choice != current {+ let _ = app.run("denoise.settings", json!({"cacheGb": choice}));+ app.caches.denoise.epoch += 1;+ }+ let r = ui.button("Clear cache");+ register(ui.ctx(), "denoise:clear", r.rect);+ if r.clicked() {+ let _ = app.run("denoise.clear", json!({}));+ }+ });+}++/// Where the work runs, in a sentence (`denoise.status` → `device`), for the setting `run_on`. No timings: they are this+/// computer's, and only the comparison matters.+fn device_line(d: &Value, run_on: &str) -> String {+ match d["kind"].as_str() {+ _ if run_on == "cpu" => "The processor does the work.".into(),+ Some("gpu") => format!("Running on {}.", d["adapter"].as_str().unwrap_or("the graphics card")),+ Some("cpu") => {+ format!("Running on the processor: {}.", d["reason"].as_str().unwrap_or("the graphics card is not used").trim_end_matches('.'))+ }+ _ => "The graphics card is set up when the next photo is made.".into(),+ }+}++fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, dl: Option<&Value>, can_run: bool) {+ let id = m["id"].as_str().unwrap_or("").to_string();+ let (installed, selected) = (m["installed"].as_bool() == Some(true), m["selected"].as_bool() == Some(true));+ let dl_state = if installed { None } else { dl.and_then(|d| d["state"].as_str()) };+ let host = if can_run { m["downloadHost"].as_str().unwrap_or("").to_string() } else { String::new() };+ egui::Frame::NONE.inner_margin(egui::Margin::symmetric(0, 3)).show(ui, |ui| {+ ui.horizontal(|ui| {+ ui.vertical(|ui| {+ ui.horizontal(|ui| {+ ui.label(RichText::new(m["name"].as_str().unwrap_or("")).color(t.text));+ let size = mb(m["sizeBytes"].as_u64());+ if !size.is_empty() {+ ui.label(RichText::new(size).color(t.text_dim));+ }+ if selected {+ ui.label(RichText::new("in use").color(t.accent));+ } else if installed {+ ui.label(RichText::new("installed").color(t.text_dim));+ }+ });+ ui.label(RichText::new(licence_line(m)).font(t.font(11.5)).color(if m["licence"]["commercial"] == "yes" {+ t.text_dim+ } else {+ t.caution+ }));+ match m["accepted"]["selfTest"].as_object() {+ Some(test) if test.get("ok").and_then(Value::as_bool) == Some(true) => {+ ui.label(RichText::new("Tested and works on this computer").font(t.font(11.5)).color(t.text_dim));+ }+ Some(_) => {+ ui.label(RichText::new("Failed its last test").font(t.font(11.5)).color(t.caution));+ }+ None => {}+ }+ match (dl_state, dl) {+ (Some("running"), Some(d)) => {+ let (got, total) = (d["bytes"].as_u64().unwrap_or(0), d["total"].as_u64().unwrap_or(0));+ let frac = if total > 0 { (got as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 };+ let text = if total > 0 && got >= total {+ "Checking…".to_string()+ } else {+ format!("{} of {} from {host}", mb(Some(got)), mb(Some(total)))+ };+ let bar = ui.add(egui::ProgressBar::new(frac).desired_width(260.0).text(RichText::new(text).font(t.font(11.5))));+ register(ui.ctx(), format!("denoise:progress:{id}"), bar.rect);+ }+ (Some("done"), _) => {+ ui.label(RichText::new("Downloaded and checked. Installing…").font(t.font(11.5)).color(t.text_dim));+ }+ (Some("failed"), Some(d)) => {+ let why = sentence(d["error"].as_str().unwrap_or("The download failed"));+ ui.add(egui::Label::new(RichText::new(why).font(t.font(11.5)).color(t.caution)).wrap());+ }+ _ => {}+ }+ });+ ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {+ if installed {+ let r = ui.button("Remove");+ register(ui.ctx(), format!("denoise:remove:{id}"), r.rect);+ if r.clicked() {+ let _ = app.run("denoise.models.remove", json!({"id": id}));+ app.caches.denoise.epoch += 1;+ }+ if selected {+ let r = ui.button("Turn off");+ register(ui.ctx(), format!("denoise:off:{id}"), r.rect);+ if r.clicked() {+ let _ = app.run("denoise.models.select", json!({"id": null}));+ app.caches.denoise.epoch += 1;+ }+ } else {+ let r = ui.button("Use");+ register(ui.ctx(), format!("denoise:use:{id}"), r.rect);+ if r.clicked() {+ let _ = app.run("denoise.models.select", json!({"id": id}));+ app.caches.denoise.epoch += 1;+ }+ }+ } else {+ match dl_state {+ Some("running") => {+ let r = ui.button("Cancel");+ register(ui.ctx(), format!("denoise:cancelDownload:{id}"), r.rect);+ if r.clicked() {+ let _ = app.run("denoise.models.downloadCancel", json!({"id": id}));+ app.caches.denoise.dl_watch.retain(|w| w != &id);+ }+ }+ // the engine installs it within a moment; nothing to press+ Some("done") => {}+ _ => {+ if let Some(url) = m["source"].as_str() {+ let r = ui+ .button("Open page")+ .on_hover_text("Opens the model's own page in your browser, to read about it or get the file yourself.");+ register(ui.ctx(), format!("denoise:get:{id}"), r.rect);+ if r.clicked() {+ open_page(app, url);+ }+ }+ if !host.is_empty() {+ let label = if dl_state == Some("failed") { "Try again" } else { "Download" };+ let tip = format!(+ "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.",+ mb(m["sizeBytes"].as_u64())+ );+ let r = ui.button(label).on_hover_text(tip);+ register(ui.ctx(), format!("denoise:download:{id}"), r.rect);+ if r.clicked() {+ open_download_dialog(app, m);+ }+ }+ }+ }+ }+ });+ });+ });+}++/// Pressing Download: the model's terms first, in the licence dialog. Accepting them starts the download;+/// nothing is fetched before.+fn open_download_dialog(app: &mut LightcraftApp, m: &Value) {+ let info = json!({+ "kind": "known",+ "domain": "denoise",+ "fileName": m["id"],+ "sizeBytes": m["sizeBytes"],+ "model": m,+ "alreadyInstalled": false,+ "download": m["id"],+ "host": m["downloadHost"],+ "assumptions": [],+ "reason": null,+ });+ app.ui.dialog = Some(crate::state::Dialog::DenoiseModel { info, accepted: false });+}++/// The dialog's OK for a denoise model: start the download and watch it.+pub fn install(app: &mut LightcraftApp, info: &Value, accepted: bool) -> Result<Value, String> {+ if !accepted {+ return Err("Tick the box to accept the model's terms first".into());+ }+ app.caches.denoise.epoch += 1;+ let id = info["download"].as_str().ok_or("There is no download for this model")?;+ let r = app.run("denoise.models.download", json!({"id": id, "acknowledged": true}));+ if r.is_ok() {+ app.caches.denoise.dl_watch.push(id.to_string());+ // the progress is in Settings: stay there+ app.ui.dialog = Some(crate::state::Dialog::Settings { tab: "denoise".into() });+ }+ r+}++/// "1 photo" / "3 photos": the count with the right noun.+fn count(n: u64, one: &str, many: &str) -> String {+ format!("{n} {}", if n == 1 { one } else { many })+}++pub(super) fn window_pace(app: &LightcraftApp, ctx: &egui::Context, now: f64, watching: bool) -> &'static str {+ let (focused, minimized) = ctx.input(|i| (i.focused, i.raw.viewports.get(&i.raw.viewport_id).and_then(|v| v.minimized).unwrap_or(false)));+ pace_for(focused, minimized, now - app.caches.last_input, now - app.caches.last_move, watching)+}++fn pace_for(focused: bool, minimized: bool, worked: f64, moved: f64, watching: bool) -> &'static str {+ if minimized {+ "light"+ } else if !focused {+ // still on screen, but the user is working in another app: no input here to get in the way of+ "normal"+ } else if worked < 0.4 {+ "pause"+ } else if moved < 3.0 {+ "light"+ } else if watching {+ "full"+ } else {+ "normal"+ }+}++pub(super) fn mb(bytes: Option<u64>) -> String {+ match bytes {+ Some(b) if b >= 10_000_000 => format!("{} MB", (b as f64 / 1e6).round() as u64),+ Some(b) if b >= 1_000_000 => format!("{:.1} MB", b as f64 / 1e6),+ Some(b) => format!("{} KB", (b as f64 / 1e3).round().max(1.0) as u64),+ None => String::new(),+ }+}++pub(super) fn sentence(s: &str) -> String {+ let s = s.trim();+ let mut c = s.chars();+ let mut out: String = c.next().map(|f| f.to_uppercase().collect()).unwrap_or_default();+ out.push_str(c.as_str());+ if !out.is_empty() && !out.ends_with(['.', '!', '?']) {+ out.push('.');+ }+ out+}++pub(super) fn licence_line(m: &Value) -> String {+ let name = m["licence"]["name"].as_str().filter(|s| !s.is_empty()).unwrap_or("Unknown licence");+ let terms = match m["licence"]["commercial"].as_str() {+ Some("yes") => "commercial use allowed",+ Some("no") => "non-commercial use only",+ _ => "terms unclear",+ };+ format!("{name} · {terms}")+}++pub(super) fn open_page(app: &mut LightcraftApp, url: &str) {+ if let Some(f) = app.services.open_url.as_mut() {+ let _ = f(url);+ }+}++pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info: &Value, accepted: &mut bool) {+ let kind = info["kind"].as_str().unwrap_or("unsupported");+ let file = info["fileName"].as_str().unwrap_or("");+ if kind == "unsupported" {+ ui.label(RichText::new("This file cannot be used yet").font(t.semibold(13.5)).color(t.caution));+ ui.add(+ egui::Label::new(+ RichText::new(sentence(info["reason"].as_str().unwrap_or("It is not a denoise model LightCraft understands"))).color(t.text_label),+ )+ .wrap(),+ );+ ui.label(RichText::new(format!("{file} · {}", mb(info["sizeBytes"].as_u64()))).color(t.text_dim));+ return;+ }+ let m = &info["model"];+ ui.horizontal(|ui| {+ ui.label(RichText::new(m["name"].as_str().unwrap_or(file)).font(t.semibold(14.0)).color(t.text));+ ui.label(RichText::new(mb(info["sizeBytes"].as_u64())).color(t.text_dim));+ });+ if info["alreadyInstalled"] == true {+ ui.label(RichText::new("Already installed. Installing again is harmless.").color(t.text_dim));+ }+ let download = info["download"].is_string();+ if download {+ let host = info["host"].as_str().unwrap_or("its own repository");+ let after = if info["domain"] == "denoise" { "starts using it for AI Denoise" } else { "makes AI Denoise available" };+ ui.add(+ egui::Label::new(+ RichText::new(format!("Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}."))+ .color(t.text_label),+ )+ .wrap(),+ );+ }+ let commercial = m["licence"]["commercial"].as_str().unwrap_or("unknown");+ ui.label(RichText::new(licence_line(m)).font(t.semibold(12.5)).color(if commercial == "yes" { t.text } else { t.caution }));+ let notice = m["licence"]["notice"].as_str().unwrap_or("");+ if !notice.is_empty() {+ ui.add(egui::Label::new(RichText::new(notice).color(t.text_label)).wrap());+ }+ let provenance = m["provenance"].as_str().unwrap_or("");+ if !provenance.is_empty() {+ ui.add(egui::Label::new(RichText::new(format!("Trained on: {provenance}")).color(t.text_dim)).wrap());+ }+ if kind == "draft" {+ ui.add_space(2.0);+ ui.label(RichText::new("LightCraft does not know this model, so it assumed:").color(t.text_label));+ for a in info["assumptions"].as_array().into_iter().flatten().filter_map(Value::as_str) {+ ui.add(egui::Label::new(RichText::new(format!("• {a}")).font(t.font(12.0)).color(t.text_dim)).wrap());+ }+ }+ if let Some(url) = m["source"].as_str().or(m["licence"]["url"].as_str()) {+ let r = ui.link("Open the model's page");+ register(ui.ctx(), "denoiseModel:page", r.rect);+ if r.clicked() {+ open_page(app, url);+ }+ }+ ui.add_space(4.0);+ check(ui, "denoiseModel.accept", accepted, "I have read these terms and accept them for my own use");+ let closing = if download {+ "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it."+ } else {+ "The model is kept on this computer only. LightCraft never uploads or shares it."+ };+ ui.label(RichText::new(closing).font(t.font(11.5)).color(t.text_dim));+}diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rsindex cd087776d..da400bc2b 100644--- a/crates/ui-egui/src/panels/dialogs.rs+++ b/crates/ui-egui/src/panels/dialogs.rs@@ -85,6 +85,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {Dialog::NewSmartAlbum { .. } => "Create Smart Album",Dialog::AllMetadata { .. } => "All Metadata",Dialog::SystemInfo { .. } => "System Info",+ Dialog::DenoiseModel { .. } => "AI Denoise Model",Dialog::WhatsNew => "What's New",Dialog::Cull { .. } => "Assisted Culling",Dialog::SmartRules { id: None, .. } => "New Smart Album",@@ -174,6 +175,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {}});}+ Dialog::DenoiseModel { info, accepted } => crate::panels::denoise::model_dialog(app, ui, &t, info, accepted),Dialog::SystemInfo { rows } => {egui::Grid::new("sysinfo").num_columns(2).spacing([16.0, 4.0]).striped(true).show(ui, |ui| {for (k, v) in rows.iter() {@@ -784,7 +786,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {}ui.add_space(4.0);ui.horizontal(|ui| {- let informational = matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. });+ let unusable_model = matches!(&dlg, Dialog::DenoiseModel { info, .. } if info["kind"] == "unsupported");+ let informational = unusable_model || matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. });let sam = &app.session.segmenter;let (sam_installed, sam_running, sam_failed) = (sam.installed(), sam.download_status().running, sam.download_status().error.is_some());// no download location in this build: nothing to offer but the manual install@@ -810,6 +813,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {add_label.as_str()}Dialog::Merge { .. } => "Merge",+ Dialog::DenoiseModel { info, .. } if info["download"].is_string() => "Accept & Download",+ Dialog::DenoiseModel { .. } => "Install",Dialog::ConfirmDelete { .. } => "Delete",Dialog::RemoveFolder { .. } => "Remove",Dialog::SamModel { then: Some(_), .. } if sam_installed => "Continue",@@ -819,7 +824,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {_ if informational => "Close",_ => "OK",};- let r = (!ok.is_empty()).then(|| ui.button(crate::i18n::tr(ok)));+ let can_confirm = informational || !matches!(&dlg, Dialog::DenoiseModel { accepted: false, .. });+ let r = (!ok.is_empty()).then(|| ui.add_enabled(can_confirm, egui::Button::new(crate::i18n::tr(ok))));if let Some(r) = &r {crate::widgets::register(ui.ctx(), "button:dialogOk", r.rect);}@@ -1025,6 +1031,7 @@ pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result<serde_jsoDialog::RenameKeyword { from, to } => app.run("keyword.rename", json!({"from": from, "to": to})),Dialog::MergeKeywords { from, into } => app.run("keyword.merge", json!({"from": from, "into": into})),Dialog::AutoStack { gap } => app.run("stack.auto", json!({"gap": gap})),+ Dialog::DenoiseModel { info, accepted } => crate::panels::denoise::install(app, info, *accepted),Dialog::AllMetadata { .. } | Dialog::SystemInfo { .. } | Dialog::WhatsNew => Ok(serde_json::Value::Null),Dialog::Cull { reject_below, pick_best } => {let mut p = json!({"pickBest": pick_best});diff --git a/crates/ui-egui/src/panels/edit.rs b/crates/ui-egui/src/panels/edit.rsindex 2188afa66..51a54bc93 100644--- a/crates/ui-egui/src/panels/edit.rs+++ b/crates/ui-egui/src/panels/edit.rs@@ -272,7 +272,15 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {for c in ["detail.sharpenAmount", "detail.sharpenRadius", "detail.sharpenDetail", "detail.sharpenMasking"] {control(app, ui, d, c, c == "detail.sharpenAmount" || d.detail.sharpen_amount > 0.0);}- sub_title(ui, crate::i18n::tr("Noise Reduction"));+ // AI Denoise is its own tool, apart from the manual noise reduction sliders below it+ sub_title(ui, crate::i18n::tr("AI Denoise"));+ // AI Denoise: how much of the photo's cleaned picture to mix in (raw files only), and what that picture is doing+ let applicable = !matches!(app.session.denoise_photo_state(id), lightcraft_engine::denoise::PhotoState::NotApplicable);+ let on = d.enhance.denoise_enabled();+ super::denoise::detail_toggle(app, ui, id, on, applicable);+ control(app, ui, d, "enhance.denoise", applicable && on);+ super::denoise::detail_status(app, ui, id, if on { d.enhance.denoise } else { 0.0 });+ sub_title(ui, crate::i18n::tr("Manual Noise Reduction"));for c in ["detail.nrLuminance", "detail.nrDetail", "detail.nrContrast", "detail.nrColor", "detail.nrColorDetail", "detail.nrColorSmoothness"]{control(app, ui, d, c, true);diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rsindex 300756485..1d2285385 100644--- a/crates/ui-egui/src/panels/mod.rs+++ b/crates/ui-egui/src/panels/mod.rs@@ -4,6 +4,7 @@ pub mod bottombar;pub mod chips;pub mod compare;pub mod crop_overlay;+pub mod denoise;pub mod detail;pub mod dialogs;pub mod edit;diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rsindex db5ca5a1a..112857767 100644--- a/crates/ui-egui/src/panels/settings.rs+++ b/crates/ui-egui/src/panels/settings.rs@@ -1,4 +1,4 @@-//! The Settings dialog (⌘,): General, Import, Performance, Interface.+//! The Settings dialog (⌘,): General, Import, Performance, Interface, AI Denoise.//!//! Changes apply immediately (no OK/Cancel). Where they are stored://! - **app settings** ([`crate::state::AppSettings`]: startup view, delete confirmation, GPU,@@ -17,7 +17,8 @@ use crate::theme::Tokens;use crate::widgets::register;/// (id, label) of the tabs, in order.-pub const TABS: &[(&str, &str)] = &[("general", "General"), ("import", "Import"), ("performance", "Performance"), ("interface", "Interface")];+pub const TABS: &[(&str, &str)] =+ &[("general", "General"), ("import", "Import"), ("performance", "Performance"), ("interface", "Interface"), ("denoise", "AI Denoise")];/// Thumbnail cache sizes offered (MB).const CACHE_SIZES: [u32; 5] = [512, 1024, 2048, 4096, 8192];@@ -43,11 +44,12 @@ pub fn body(app: &mut LightcraftApp, ui: &mut egui::Ui, tab: &mut String) {"import" => import_tab(app, ui, &t),"performance" => performance_tab(app, ui, &t),"interface" => interface_tab(app, ui, &t),+ "denoise" => super::denoise::settings_tab(app, ui, &t),_ => general_tab(app, ui, &t),}}-fn heading(ui: &mut egui::Ui, t: &Tokens, text: &str) {+pub(super) fn heading(ui: &mut egui::Ui, t: &Tokens, text: &str) {ui.add_space(4.0);ui.label(RichText::new(crate::i18n::tr(text)).font(t.semibold(12.5)).color(t.text));}@@ -63,19 +65,19 @@ fn row<R>(ui: &mut egui::Ui, t: &Tokens, label: &str, add: impl FnOnce(&mut egui.inner}-fn hint(ui: &mut egui::Ui, t: &Tokens, text: &str) {+pub(super) fn hint(ui: &mut egui::Ui, t: &Tokens, text: &str) {ui.label(RichText::new(crate::i18n::tr(text)).size(11.0).color(t.text_dim));}/// A checkbox addressable as `check:{id}`; true when toggled.-fn check(ui: &mut egui::Ui, id: &str, value: &mut bool, label: &str) -> bool {+pub(super) fn check(ui: &mut egui::Ui, id: &str, value: &mut bool, label: &str) -> bool {let r = ui.checkbox(value, crate::i18n::tr(label));register(ui.ctx(), format!("check:{id}"), r.rect);r.changed()}/// Mutually exclusive buttons (`button:{id}-{index}`).-fn choices<V: PartialEq + Copy>(ui: &mut egui::Ui, id: &str, options: &[(V, &str)], value: &mut V) -> bool {+pub(super) fn choices<V: PartialEq + Copy>(ui: &mut egui::Ui, id: &str, options: &[(V, &str)], value: &mut V) -> bool {let mut changed = false;ui.spacing_mut().item_spacing.x = 4.0;for (i, (v, l)) in options.iter().enumerate() {diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rsindex 17f5c58b7..45949bb8c 100644--- a/crates/ui-egui/src/state.rs+++ b/crates/ui-egui/src/state.rs@@ -535,6 +535,10 @@ pub enum Dialog {pick_best: bool,},/// Help ▸ System Info: (label, value) rows.+ DenoiseModel {+ info: serde_json::Value,+ accepted: bool,+ },SystemInfo {rows: Vec<(String, String)>,},diff --git a/crates/ui-egui/src/tests_model_setup.rs b/crates/ui-egui/src/tests_model_setup.rsnew file mode 100644index 000000000..892df22db--- /dev/null+++ b/crates/ui-egui/src/tests_model_setup.rs@@ -0,0 +1,171 @@+//! Synthetic setup flows: model installation must resume the requested action on its original targets.+use std::path::{Path, PathBuf};++use lightcraft_catalog::{MediaKind, Op, Photo, PhotoId, Source};+use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};+use lightcraft_engine::Session;+use serde_json::json;++use crate::{LightcraftApp, Services, model_setup, state::Dialog};++fn setup(name: &str) -> (LightcraftApp, PathBuf, PhotoId) {+ let dir = std::env::temp_dir().join(format!("lc-model-flow-{name}-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&dir);+ let mut s = Session::new().with_fs();+ s.open_library(dir.join("library"), true).unwrap();+ s.set_denoise_models_dir(Some(dir.join("denoise")));+ s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap();+ let id = PhotoId(s.catalog.photos().map(|p| p.id.0).max().unwrap() + 1);+ let mut raw = Photo::new(+ id,+ Source::File { path: dir.join("synthetic.dng").to_string_lossy().into_owned() },+ "synthetic.dng",+ "DNG",+ 128,+ 128,+ "2026-10-07",+ );+ raw.kind = MediaKind::Raw;+ s.commit("fixture", Op::AddPhoto { photo: Box::new(raw) }).unwrap();+ (LightcraftApp::new(s, Services::default()), dir, id)+}++fn install_denoise(app: &mut LightcraftApp, dir: &Path) {+ let source = dir.join("source");+ std::fs::create_dir_all(&source).unwrap();+ let bytes = lightcraft_denoise::synthetic::smoothing_onnx();+ std::fs::write(source.join("model.onnx"), &bytes).unwrap();+ let m = DenoiserManifest {+ id: "test-smoothing".into(),+ name: "Synthetic smoothing".into(),+ version: "1".into(),+ licence: Default::default(),+ source: None,+ sha256: None,+ size_bytes: Some(bytes.len() as u64),+ provenance: "Synthetic tests".into(),+ domain: Domain::BayerToRgb,+ tile: 64,+ overlap: 16,+ gain: Gain::None,+ };+ std::fs::write(source.join("denoise-model.json"), serde_json::to_vec(&m).unwrap()).unwrap();+ app.run("denoise.models.install", json!({"path": source.join("model.onnx").to_string_lossy(), "acknowledged": true})).unwrap();+}++fn amount(app: &LightcraftApp, id: PhotoId) -> f64 {+ app.session.develop_of(id).unwrap().enhance.denoise+}+fn resume(app: &mut LightcraftApp) {+ model_setup::pump(app, &egui::Context::default());+}++#[test]+fn denoise_installation_enables_only_the_requesting_photo_once_and_is_undoable() {+ let (mut app, dir, raw) = setup("denoise");+ let other = app.session.active().unwrap();+ app.session.auto_sync = true;+ let r = app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();+ assert_eq!(r["setupRequired"], true);+ assert_eq!(app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() }));+ assert_eq!(amount(&app, raw), 0.0, "setup alone never changes the photo");+ app.run("library.select", json!({"ids": [other.0]})).unwrap();+ let undo = app.session.undo.len();+ install_denoise(&mut app, &dir);+ resume(&mut app);+ assert_eq!(amount(&app, raw), 50.0);+ assert_eq!(amount(&app, other), 0.0);+ assert_eq!(app.session.active(), Some(other));+ assert_eq!(app.session.undo.len(), undo + 1);+ resume(&mut app);+ assert_eq!(app.session.undo.len(), undo + 1, "a later frame cannot apply it again");+ app.run("edit.undo", json!({})).unwrap();+ assert_eq!(amount(&app, raw), 0.0);+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();+ app.run("develop.set", json!({"ids": [raw.0], "control": "enhance.denoise", "value": 75})).unwrap();+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();+ assert_eq!(amount(&app, raw), 75.0, "enabling an already chosen amount keeps it");+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": false})).unwrap();+ assert_eq!(amount(&app, raw), 75.0, "switching off keeps the Amount");+ assert_eq!(app.session.develop_of(raw).unwrap().denoise_amount(), 0.0, "but mixes nothing in");+ drop(app);+ let _ = std::fs::remove_dir_all(dir);+}++#[test]+fn cancelled_denoise_requests_do_not_apply_after_installation() {+ for (name, command, params) in [+ ("cancel-action", "modelSetup.cancel", json!({"kind": "denoise"})),+ ("cancel-toggle", "denoise.toggle", json!({"enabled": false})),+ ("cancel-download", "denoise.models.downloadCancel", json!({"id": "rawnind-bayer"})),+ ] {+ let (mut app, dir, raw) = setup(name);+ app.run("library.select", json!({"ids": [raw.0]})).unwrap();+ app.run("denoise.toggle", json!({"enabled": true})).unwrap();+ app.run(command, params).unwrap();+ install_denoise(&mut app, &dir);+ resume(&mut app);+ assert_eq!(amount(&app, raw), 0.0, "{name}");+ drop(app);+ let _ = std::fs::remove_dir_all(dir);+ }+}++#[test]+fn pending_actions_do_not_cross_libraries_or_relinked_sources() {+ for changed in ["library", "source"] {+ let (mut app, dir, raw) = setup(changed);+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();+ if changed == "library" {+ app.session.open_library(dir.join("other-library"), true).unwrap();+ } else {+ app.session+ .commit(+ "relink",+ Op::Relink {+ id: raw,+ file_name: "another.dng".into(),+ source: Source::File { path: dir.join("another.dng").to_string_lossy().into_owned() },+ format: Some("DNG".into()),+ },+ )+ .unwrap();+ }+ install_denoise(&mut app, &dir);+ resume(&mut app);+ assert!(!model_setup::denoise_requested(&app, raw));+ assert!(app.session.catalog.photos().all(|p| p.develop.enhance.denoise == 0.0));+ assert!(app.ui.status.contains("cancelled"));+ drop(app);+ let _ = std::fs::remove_dir_all(dir);+ }+}++#[test]+fn hostile_or_inapplicable_toggle_arguments_are_errors_without_changes() {+ let (mut app, dir, raw) = setup("hostile");+ let jpeg = app.session.active().unwrap();+ for params in [json!({"id": "wrong"}), json!({"id": u64::MAX}), json!({"id": raw.0, "enabled": 1}), json!({"id": jpeg.0, "enabled": true})] {+ assert!(app.run("denoise.toggle", params).is_err());+ assert_eq!(amount(&app, raw), 0.0);+ }+ assert!(app.run("modelSetup.cancel", json!({"kind": "nonsense"})).is_err());+ drop(app);+ let _ = std::fs::remove_dir_all(dir);+}++#[test]+fn model_installation_waits_for_an_active_slider_gesture_to_finish() {+ let (mut app, dir, raw) = setup("interaction");+ app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap();+ app.session.begin_interaction("Exposure").unwrap();+ install_denoise(&mut app, &dir);+ resume(&mut app);+ assert_eq!(amount(&app, raw), 0.0);+ assert!(app.session.interaction.is_some());+ app.session.end_interaction().unwrap();+ resume(&mut app);+ assert_eq!(amount(&app, raw), 50.0);+ drop(app);+ let _ = std::fs::remove_dir_all(dir);+}diff --git a/docs/denoise.md b/docs/denoise.mdnew file mode 100644index 000000000..b8fd2901e--- /dev/null+++ b/docs/denoise.md@@ -0,0 +1,118 @@+# AI Denoise++Edit ▸ Detail ▸ **AI Denoise** is a per-photo switch above **Amount** (0–100).+Turning it on starts at 50, or preserves an already positive Amount. Turning it off+preserves Amount and mixes none of the cleaned picture. It affects only the requesting+photo, independently of Auto Sync, and is undoable. Manual luminance and colour noise+reduction remain separate controls.++If no model is selected, turning it on opens **Settings ▸ AI Denoise** without changing+the photo. After installation, the pending request enables the original photo even if+the selection changed. Cancel the pending action in Settings. Cancelled downloads,+deleted or relinked photos, switching libraries and quitting cancel the request.+An active slider gesture finishes before the pending action resumes.++## Models and terms++**No model weights are bundled.** Models live in the per-user `denoise-models` folder+(override with `LIGHTCRAFT_DENOISE_MODELS`). Settings offers a pinned, verified download+of RawNIND UtNet2 for Bayer sensors, or installation of a user-supplied ONNX file with+a `denoise-model.json` beside it. Terms must be acknowledged before installation or+download. Downloads use the shared pure-Rust `lightcraft-fetch` client, with resume,+progress, cancellation, size limits and SHA-256 verification.++RawNIND's offered ONNX weights are **GPL-3.0**, as recorded in the pinned model catalog.+They are downloaded separately after consent and read by our own Rust executor.+Whether releases should offer this download remains a maintainer policy decision;+permissively licensed or independently trained weights remain a gap. This is not a+claim that optional downloading settles licensing questions. No GPL decoder or neural+runtime code is included. Model metadata is in `crates/denoise/src/known.rs`.++Installation checks the manifest, model hash and size, then runs a synthetic self-test:+finite output of the promised shape, repeatability, the declared scale and reduced+noise. Unsupported operators, malformed files and a failed self-test return errors.++The current contract is `bayerToRgb`: normalized, black-subtracted, non-white-balanced+RGGB planes `[1, 4, tile, tile]` produce camera RGB `[1, 3, 2*tile, 2*tile]`.+RawNIND uses a 512-cell tile, 64-cell overlap and a mean-matched nominal gain of 1e6.+See `DenoiserManifest` for the validated JSON schema.++## Photos, previews and exports++Original RAW files stay untouched. A cleaned picture is cached under+`<library>/denoise/<key>.lcdn`, keyed by source content, model and algorithm version.+The compressed half-float cache has checked headers, bounded windows and checksums.+It can be cleared or recreated; it is not a new catalog photo or a duplicate DNG.++Plain and cleaned camera RGB go through the same highlight reconstruction, white+balance, colour matrix, DNG profile tables, camera hue/saturation look, crop and+orientation. Amount blends the two before the remaining develop pipeline, so edits+and exports use the same data. At zero Amount or with Detail disabled, the ordinary+render path remains in use.++The queue prioritizes the open and selected photos, pauses after input, and adjusts+its worker count while the user works. Progress, cancellation and retry are available+in Settings and through commands. The default cache limit is 20 GB; unused products+are evicted oldest first. A failed file is not continuously retried.++An export with a nonzero effective Amount creates a missing product synchronously.+It returns a clear error if it cannot produce the denoised result; it never silently+exports the plain image instead. Unsupported sensor layouts retain manual noise+reduction and report their limitation.++## Pure-Rust CPU and GPU execution++The CPU executor reads a bounded subset of ONNX into checked network data. Convolutions+use bounded im2col blocks and `faer-core 0.17.1` with its Rust-intrinsic GEMM kernels,+without BLAS, generated assembly or an inner thread pool. Tiles share the outer worker+pool. A scalar implementation checks the optimized operators on synthetic networks.+Tract is not a workspace dependency or a product feature.++The GPU executor uses our WGSL kernels through wgpu. It checks a real model's output+against the CPU and falls back on errors, unsupported networks or a slower device.+Setup has a time limit and a crash marker. Automatic selection estimates throughput+from a check tile; this estimate can choose imperfectly on unmeasured hardware.+Settings offers Automatic, Graphics card and Processor. GPU tests cover supported+synthetic networks against the scalar executor; a real-model test compares CPU/GPU.++### Historical performance, measured 2026-10-07++These are the contributor's measurements **before the current upstream integration**,+on a Ryzen 9 7945HX and RTX 4090 Laptop GPU (DX12), release build, model SHA-256+`da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901`.+The 24.34 MP Sony a7 III photo is 35 tiles. Numbers exclude model loading, source+decode, GPU setup, cache encoding and export, but include packing and blending.++| Work | Pure-Rust CPU | Prior Tract reference | GPU |+|---|---:|---:|---:|+| One warm 512-cell tile, one caller | 1148 ms | 1463 ms | 15.1 ms |+| Whole photo, 16 CPU / 4 GPU feeding workers | 5.185 s | 9.534 s | 0.582 s |++The largest relative differences were 1.68e-6 CPU/reference and 1.57e-6 CPU/GPU for+the check tile, below the required 1e-3 tolerance. Reference comparisons were made+on the older contributor branch; that reference runtime is intentionally excluded+from the current workspace. These numbers are not a fresh benchmark of this rebase.++Fresh checks can be run using an already installed model and CC0 raw samples:++```text+LC_DENOISE_MODEL=<model.onnx> cargo test --release -p lightcraft-gpu --lib real_model_on_the_gpu_matches_cpu -- --ignored --nocapture+LC_DENOISE_MODEL=<model.onnx> LC_DENOISE_RAW=<corpus/raw> cargo test --release -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture+LC_DENOISE_MODEL=<model.onnx> LC_DENOISE_RAW=<corpus/raw> cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture+```++## Commands and limits++UI, CLI, control channel and MCP dispatch `denoise.toggle`, `denoise.settings`,+`denoise.queue`, `.cancel`, `.status`, `.pump`, `.clear`, and `denoise.models.*`+(`list`, `install`, `download`, `downloads`, `downloadCancel`, `test`, `remove`,+`select`). The develop control is `enhance.denoise`.++Native apps enable inference through their `denoise` feature. A build without it and+the web build report inference unavailable. Manifest/cache primitives still compile+for WASM. Settings and models remain opt-in; nothing is downloaded on launch.++This implementation supports **Bayer RAW only**. X-Trans, Foveon, demosaiced DNG and+non-RAW images remain unsupported by this model contract. No measured quality parity+with Lightroom is claimed. Apple/Metal and wider hardware verification, model policy,+a linear-RGB contract and a side-by-side fidelity suite remain future work.diff --git a/docs/parity.md b/docs/parity.mdindex 90dc8a748..a47189493 100644--- a/docs/parity.md+++ b/docs/parity.md@@ -79,7 +79,7 @@ Take the first one nobody is working on.Object and Describe masks run SAM 3 in pure Rust (`crates/segment`); the weights (SAM License) are never bundled:the app offers a consented, verified download, but **LightCraft's CDN mirrors are not configured yet** (thebuilt-in list is empty, so today users need their own mirror or a manual install). Subject / Sky / People could- use the same model with fixed prompts; denoise / super-resolution models remain a maintainer decision.+ use the same model with fixed prompts. Bayer RAW denoise now runs on pure-Rust CPU/GPU with separately installed weights; its opt-in GPL model policy, linear RGB and quality comparison remain open. Super-resolution models remain a maintainer decision.7. **HDR** (Q. HDR, LR-EXP-HDR), **video** (R. Video), **Classic output modules** (Map view, Book, Slideshow module,Print, publish): large, well understood, lower priority than 1–5.8. **LR-VIEW-ZOOM** (P0): pinch zoom and two-finger pan now work, with stable image cursors between gesture events.@@ -262,7 +262,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.| LR-EDIT-DETAIL-SHARPEN | Sharpening | P0 | ✅ | `ctl:detail.sharpenAmount`, `ctl:detail.sharpenRadius`, `ctl:detail.sharpenDetail`, `ctl:detail.sharpenMasking` | no Alt-drag mask preview || LR-EDIT-DETAIL-NR | Luminance noise reduction | P0 | ✅ | `ctl:detail.nrLuminance`, `ctl:detail.nrDetail`, `ctl:detail.nrContrast` | || LR-EDIT-DETAIL-CNR | Colour noise reduction | P0 | ✅ | `ctl:detail.nrColor`, `ctl:detail.nrColorDetail`, `ctl:detail.nrColorSmoothness` | |-| LR-EDIT-DETAIL-DENOISE | AI denoise | P2 | ⬜ | | settings field reserved, not rendered |+| LR-EDIT-DETAIL-DENOISE | AI denoise | P2 | 🟡 | cmd:denoise.toggle, cmd:denoise.models.install, cmd:denoise.queue, ctl:enhance.denoise, `crates/denoise/`, `docs/denoise.md` | non-destructive Bayer RAW, opt-in weights, pure-Rust CPU/GPU; X-Trans/linear RGB, model policy and Lightroom fidelity remain || LR-EDIT-DETAIL-RAWDETAILS | Improved demosaic toggle | P2 | ⬜ | | || LR-EDIT-DETAIL-SUPERRES | Super resolution | P2 | ⬜ | | || LR-EDIT-DETAIL-AISHARPEN | AI sharpen | OOS | 🚫 | | |diff --git a/docs/whats-new.md b/docs/whats-new.mdindex f9b295bde..b172faaae 100644--- a/docs/whats-new.md+++ b/docs/whats-new.md@@ -2,6 +2,10 @@## October 2026+### AI RAW denoise+- Detail has a per-photo AI Denoise switch and Amount. Bayer RAW photos use a disposable cache, with matching previews and exports and pure-Rust CPU/GPU inference.+- Models are installed separately after accepting their terms; no weights are bundled. See [setup and limitations](denoise.md).+### Library keyboard culling- Colour labels tint thumbnail surrounds in Square Grid and the Detail filmstrip, and a translucent footer along the bottom of the photo in Photo Grid. Label confirmations use a matching pale colour.- Setting or clearing a colour label shows a brief bottom confirmation, like rating a photo; custom label names appear in the message too.diff --git a/xtask/src/layers.rs b/xtask/src/layers.rsindex 28d52677c..e330a8e1b 100644--- a/xtask/src/layers.rs+++ b/xtask/src/layers.rs@@ -39,6 +39,7 @@ pub const TABLE: &[(&str, Class)] = &[("tiff", Class::Layer(0)),("sysmem", Class::Layer(0)),("fetch", Class::Layer(0)),+ ("denoise", Class::Layer(1)),("raw", Class::Layer(1)),("codecs", Class::Layer(1)),("meta", Class::Layer(1)),