The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
perf(enhance): run the model on the GPU in compute shaders of our own - 4.2 s a 1080p frame where ONNX Runtime took 27
Jareeraa38b9d519fa
diff --git a/src/Cargo.lock b/src/Cargo.lockindex 096b8b59..a89ae64a 100644--- a/src/Cargo.lock+++ b/src/Cargo.lock@@ -1424,6 +1424,7 @@ version = "0.2.8"dependencies = ["concat-core","concat-effects",+ "concat-vision","half","log","pollster",diff --git a/src/crates/concat-host/src/enhance.rs b/src/crates/concat-host/src/enhance.rsindex 8d0e9396..dfcdfa16 100644--- a/src/crates/concat-host/src/enhance.rs+++ b/src/crates/concat-host/src/enhance.rs@@ -237,7 +237,7 @@ impl Enhancers {if !self.installed() {crate::cutout::fetch(ModelId::Enhance, &file, cancel, progress)?;}- let enhancer = Arc::new(Enhancer::load(&file)?);+ let enhancer = Arc::new(on_gpu(&file).map_or_else(|| Enhancer::load(&file), Ok)?);*self.model.lock().map_err(|_| "model slot poisoned")? = Some(Arc::clone(&enhancer));Ok(enhancer)}@@ -471,10 +471,104 @@ pub(crate) fn crop(frame: &concat_core::Frame, width: u32, height: u32) -> concaout}+/// The model on the GPU, in compute shaders of our own+/// (`concat_render::upscale`), when its file reads as the compact network+/// and a GPU opens: about six times what ONNX Runtime does on this+/// machine's CPU, and the CPU is where ONNX Runtime ran nearly everywhere -+/// Windows' runtime has no GPU provider at all. `None` sends Enhance the+/// slow way rather than not at all.+fn on_gpu(file: &Path) -> Option<Enhancer> {+ let bytes = std::fs::read(file).ok()?;+ let net = match concat_vision::compact::CompactNet::read(&bytes) {+ Ok(net) => net,+ Err(error) => {+ log::warn!("enhance: {error}; on the CPU instead");+ return None;+ }+ };+ let layers: Vec<concat_render::upscale::ConvLayer> = net+ .layers+ .into_iter()+ .map(|layer| concat_render::upscale::ConvLayer {+ in_channels: layer.in_channels,+ out_channels: layer.out_channels,+ weight: layer.weight,+ bias: layer.bias,+ prelu: layer.prelu,+ })+ .collect();+ let upscaler =+ concat_render::upscale::Upscaler::new(&layers, net.scale as u32, vision::WINDOW)?;+ log::info!(+ "enhance: on the GPU{}",+ if upscaler.half() { ", in half floats" } else { "" }+ );+ Some(Enhancer::with(Box::new(GpuWindows(upscaler))))+}++/// The GPU upscaler as the model `Enhancer` runs window by window.+struct GpuWindows(concat_render::upscale::Upscaler);++impl vision::WindowRunner for GpuWindows {+ fn run(&mut self, planes: Vec<f32>) -> Result<Vec<f32>, String> {+ self.0.run(&planes)+ }+}+#[cfg(test)]mod tests {use super::*;+ /// The model on the GPU and through ONNX Runtime give the same picture,+ /// tiles, margins and pasting included: a few levels at the worst pixel+ /// and well under one on average, the+ /// GPU working in half floats. Needs the fetched model and a GPU.+ #[test]+ fn the_gpu_enhances_as_onnx_runtime_does() {+ let Some(file) = ["org.standalones.concat", "app.concat.editor"]+ .iter()+ .filter_map(|id| {+ let home = std::env::var_os("HOME")?;+ let file = PathBuf::from(home)+ .join("Library/Application Support")+ .join(id)+ .join("cutout-models/realesr-general-x4v3.onnx");+ file.is_file().then_some(file)+ })+ .next()+ else {+ eprintln!("no Enhance model here; skipped");+ return;+ };+ let Some(gpu) = on_gpu(&file) else {+ eprintln!("no GPU here; skipped");+ return;+ };+ let cpu = Enhancer::load(&file).expect("loads");+ // Wider than one tile, so a seam between two is in the comparison.+ let (width, height) = (300u32, 120u32);+ let pixels: Vec<u8> = (0..width * height)+ .flat_map(|at| {+ let (x, y) = (at % width, at / width);+ [(x * 255 / width) as u8, (y * 2) as u8, ((x ^ y) & 0xff) as u8, 255]+ })+ .collect();+ let frame = Frame::from_rgba(width, height, pixels).expect("a frame");+ let a = gpu.enhance(&frame, 2).expect("the GPU enhances");+ let b = cpu.enhance(&frame, 2).expect("the CPU enhances");+ assert_eq!((a.width(), a.height()), (b.width(), b.height()));+ let diffs: Vec<u32> = a+ .pixels()+ .iter()+ .zip(b.pixels())+ .map(|(x, y)| u32::from(x.abs_diff(*y)))+ .collect();+ let worst = diffs.iter().max().copied().unwrap_or(0);+ let mean = diffs.iter().sum::<u32>() as f64 / diffs.len() as f64;+ eprintln!("GPU against ONNX Runtime: worst {worst}, mean {mean:.3} levels");+ assert!(worst <= 6 && mean < 0.5, "worst {worst}, mean {mean}");+ }+#[test]fn the_copy_is_named_after_the_file_and_the_factor() {let dir = std::env::temp_dir().join(format!("concat-enhance-{}", std::process::id()));diff --git a/src/crates/concat-render/Cargo.toml b/src/crates/concat-render/Cargo.tomlindex 0b6262f0..9a8110d1 100644--- a/src/crates/concat-render/Cargo.toml+++ b/src/crates/concat-render/Cargo.toml@@ -24,6 +24,8 @@ half = "2"# The pass test compiles a real package shader through the catalogue's# stitching, so what the test runs is what a package would run.concat-effects.workspace = true+# The upscaler's answer is held to the network's plain CPU reference.+concat-vision.workspace = true[lints]workspace = truediff --git a/src/crates/concat-render/src/lib.rs b/src/crates/concat-render/src/lib.rsindex ce8dcb6e..8e86e372 100644--- a/src/crates/concat-render/src/lib.rs+++ b/src/crates/concat-render/src/lib.rs@@ -22,6 +22,7 @@ pub mod compositor;pub mod gpu;pub mod metrics;pub mod plan;+pub mod upscale;#[cfg(test)]mod reference;#[cfg(test)]diff --git a/src/crates/concat-render/src/upscale.rs b/src/crates/concat-render/src/upscale.rsnew file mode 100644index 00000000..5665102f--- /dev/null+++ b/src/crates/concat-render/src/upscale.rs@@ -0,0 +1,724 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer Abdullah and Concat contributors++//! Enhance's network on the GPU, in compute shaders of our own.+//!+//! The network is a stack of 3x3 convolutions with a PReLU after each but+//! the last, then a pixel shuffle with the input, enlarged, added (see+//! `concat_vision::compact`). Run by ONNX Runtime it ran on the CPU almost+//! everywhere - Windows' bundled runtime has no GPU provider at all, and+//! CoreML hands back to the CPU what it will not take - at about 27+//! seconds a 1080p frame: 1.2 million multiply-adds a pixel is a CPU's+//! afternoon and a GPU's second. Here each layer is one dispatch on a+//! device of the upscaler's own, so a long job never holds the window's.+//!+//! Activations live in storage buffers, four channels to a `vec4` and the+//! channels of a pixel together: `[(y * width + x) * groups + group]`.+//! Each invocation works out sixteen output channels of a 2x2 block of+//! pixels, a `mat4x4` of weights to each input `vec4` under each of the+//! nine taps; a padded edge reads as zero, as the model's padding is. The last pass shuffles+//! the final layer's channels into the larger picture and adds the input+//! to it, writing three planes, which is the ONNX model's answer exactly:+//! the tiling and pasting around it do not know which ran.++use std::sync::mpsc;++use wgpu::util::DeviceExt;++/// One 3x3 convolution and its PReLU, as the network file holds them; see+/// `concat_vision::compact::Conv`.+#[derive(Clone, Debug)]+pub struct ConvLayer {+ /// Channels read.+ pub in_channels: usize,+ /// Channels written.+ pub out_channels: usize,+ /// `[out][in][3][3]`.+ pub weight: Vec<f32>,+ /// One per output channel.+ pub bias: Vec<f32>,+ /// The PReLU's slope per output channel, or `None` for none.+ pub prelu: Option<Vec<f32>>,+}++const SHADER_COMMON: &str = r#"+struct Dims {+ width: u32,+ height: u32,+ in_groups: u32,+ out_groups: u32,+ prelu: u32,+ scale: u32,+}++@group(0) @binding(0) var<uniform> dims: Dims;+@group(0) @binding(1) var<storage, read> src: array<vec4<T>>;+@group(0) @binding(2) var<storage, read_write> dst: array<vec4<T>>;+@group(0) @binding(3) var<storage, read> weights: array<mat4x4<T>>;+@group(0) @binding(4) var<storage, read> bias: array<vec4<f32>>;+@group(0) @binding(5) var<storage, read> slope: array<vec4<f32>>;++// `conv` is generated (see `conv_shader`): every tap and channel group+// unrolled so the accumulators and the neighbourhood are plain registers.++@group(0) @binding(6) var<storage, read> picture: array<vec4<T>>;+@group(0) @binding(7) var<storage, read_write> planes: array<f32>;++// The last layer's channels, `3 * scale * scale` of them, laid out as the+// picture `scale` times larger - channel `c * scale * scale + dy * scale + dx`+// is colour `c` at `(x * scale + dx, y * scale + dy)`, PyTorch's order - with+// the input added, enlarged by nearest neighbour.+@compute @workgroup_size(8, 8, 1)+fn shuffle(@builtin(global_invocation_id) id: vec3<u32>) {+ let s = dims.scale;+ let big_w = dims.width * s;+ let big_h = dims.height * s;+ if (id.x >= big_w || id.y >= big_h) {+ return;+ }+ let x = id.x / s;+ let y = id.y / s;+ let dx = id.x % s;+ let dy = id.y % s;+ let at = y * dims.width + x;+ let input = picture[at];+ for (var c = 0u; c < 3u; c++) {+ let channel = c * s * s + dy * s + dx;+ let value = f32(src[at * dims.in_groups + channel / 4u][channel % 4u]);+ planes[c * big_w * big_h + id.y * big_w + id.x] = value + f32(input[c]);+ }+}+"#;++/// The whole module, in half floats where the device has them: the+/// activations and weights stored at sixteen bits, which halves both the+/// memory read and the registers a block of pixels takes, and the sums+/// kept at thirty-two so 576 of them a channel do not drift.+fn shader(half: bool) -> String {+ let (head, ty) = if half { ("enable f16;\n", "f16") } else { ("", "f32") };+ let body = SHADER_COMMON.to_owned() + &conv_shader();+ format!("{head}alias T = {ty};\n{body}")+}++/// The convolution kernel. Each invocation works out a 2x2 block of pixels+/// and sixteen output channels (four groups) of each, so every weight+/// matrix read serves four pixels and every input read up to sixteen+/// channels under up to four taps. Written out with every tap and group+/// unrolled: a local array indexed by a loop variable cannot stay in+/// registers, and the first version, which did that, spilled to memory+/// and ran at a third of the GPU.+fn conv_shader() -> String {+ conv_shader_for(BLOCK_W, BLOCK_H)+}++/// The block of pixels one invocation works out, across and down.+const BLOCK_W: u32 = 2;+const BLOCK_H: u32 = 2;++fn conv_shader_for(bw: u32, bh: u32) -> String {+ // The workgroup's patch of pixels, and the patch with its one-pixel+ // rim that its taps read.+ let (pw, ph) = (8 * bw, 8 * bh);+ let (tw, th) = (pw + 2, ph + 2);+ let tile = tw * th;+ let mut code = format!(+ "var<workgroup> tile: array<vec4<T>, {tile}>;\n\+ var<workgroup> taps: array<mat4x4<T>, 36>;\n\n\+ fn near(sx: i32, sy: i32, i: u32) -> vec4<T> {{\n\+ \x20 if (sx < 0 || sy < 0 || sx >= i32(dims.width) || sy >= i32(dims.height)) {{\n\+ \x20 return vec4<T>(0.0);\n\+ \x20 }}\n\+ \x20 return src[(u32(sy) * dims.width + u32(sx)) * dims.in_groups + i];\n\+ }}\n\n\+ @compute @workgroup_size(8, 8, 1)\n\+ fn conv(@builtin(global_invocation_id) id: vec3<u32>,\n\+ \x20 @builtin(local_invocation_id) local: vec3<u32>,\n\+ \x20 @builtin(local_invocation_index) lid: u32,\n\+ \x20 @builtin(workgroup_id) group: vec3<u32>) {{\n\+ \x20 let x0 = id.x * {bw}u;\n\+ \x20 let y0 = id.y * {bh}u;\n\+ \x20 let o0 = group.z * 4u;\n\+ \x20 let px = i32(group.x * {pw}u) - 1;\n\+ \x20 let py = i32(group.y * {ph}u) - 1;\n\+ \x20 let base = local.y * {bh}u * {tw}u + local.x * {bw}u;\n"+ );+ let pixels: Vec<(u32, u32)> = (0..bh).flat_map(|dy| (0..bw).map(move |dx| (dy, dx))).collect();+ for g in 0..4 {+ code += &format!(" let b{g} = bias[o0 + {g}u];\n");+ for p in 0..pixels.len() {+ code += &format!(" var a{p}_{g} = b{g};\n");+ }+ }+ // Every invocation stays to the barriers, past the edge or not: the+ // loads are shared, and only the writes are held to the picture.+ code += &format!(+ " for (var i = 0u; i < dims.in_groups; i++) {{\n\+ \x20 workgroupBarrier();\n\+ \x20 for (var k = lid; k < {tile}u; k += 64u) {{\n\+ \x20 tile[k] = near(px + i32(k % {tw}u), py + i32(k / {tw}u), i);\n\+ \x20 }}\n\+ \x20 if (lid < 36u) {{\n\+ \x20 taps[lid] = weights[((o0 + lid / 9u) * 9u + lid % 9u) * dims.in_groups + i];\n\+ \x20 }}\n\+ \x20 workgroupBarrier();\n"+ );+ for ny in 0..bh + 2 {+ for nx in 0..bw + 2 {+ code += &format!(" let n{ny}_{nx} = tile[base + {}u];\n", ny * tw + nx);+ }+ }+ for g in 0..4u32 {+ for tap in 0..9u32 {+ code += &format!(" let m{g}_{tap} = taps[{}u];\n", g * 9 + tap);+ }+ // A group's nine taps summed in the stored type, then the partial+ // carried into the f32 sum: one conversion a group rather than one+ // a product, and only 36 terms ever added at sixteen bits.+ for (p, (dy, dx)) in pixels.iter().enumerate() {+ let terms: Vec<String> = (0..9u32)+ .map(|tap| format!("m{g}_{tap} * n{}_{}", tap / 3 + dy, tap % 3 + dx))+ .collect();+ code += &format!(" a{p}_{g} += vec4<f32>({});\n", terms.join(" + "));+ }+ }+ code += " }\n";+ for (p, (dy, dx)) in pixels.iter().enumerate() {+ code += &format!(+ " if (x0 + {dx}u < dims.width && y0 + {dy}u < dims.height) {{\n\+ \x20 let at = ((y0 + {dy}u) * dims.width + x0 + {dx}u) * dims.out_groups + o0;\n"+ );+ for g in 0..4 {+ code += &format!(+ " var v{g} = a{p}_{g};\n\+ \x20 if (dims.prelu == 1u) {{ v{g} = select(v{g}, v{g} * slope[o0 + {g}u], v{g} < vec4<f32>(0.0)); }}\n\+ \x20 dst[at + {g}u] = vec4<T>(v{g});\n"+ );+ }+ code += " }\n";+ }+ code += "}\n";+ code+}++/// A layer as the GPU holds it.+struct GpuLayer {+ out_groups: u32,+ bind_group: wgpu::BindGroup,+}++/// The network on a device of its own, for windows of one size.+pub struct Upscaler {+ device: wgpu::Device,+ queue: wgpu::Queue,+ conv: wgpu::ComputePipeline,+ shuffle: wgpu::ComputePipeline,+ layers: Vec<GpuLayer>,+ shuffle_group: wgpu::BindGroup,+ picture: wgpu::Buffer,+ planes: wgpu::Buffer,+ staging: wgpu::Buffer,+ side: u32,+ scale: u32,+ /// Activations and weights are stored as f16.+ half: bool,+}++/// `values` as the shader stores them: little-endian f32, or f16.+fn stored(values: &[f32], half: bool) -> Vec<u8> {+ if half {+ values+ .iter()+ .flat_map(|v| half::f16::from_f32(*v).to_le_bytes())+ .collect()+ } else {+ values.iter().flat_map(|v| v.to_le_bytes()).collect()+ }+}++/// Four channels to a group.+fn groups(channels: usize) -> usize {+ channels.div_ceil(4)+}++impl Upscaler {+ /// The network `layers`, enlarging `scale` times, for square windows+ /// `side` pixels across, on the first GPU that opens - `None` where+ /// there is none, and the caller runs the model the slow way.+ pub fn new(layers: &[ConvLayer], scale: u32, side: u32) -> Option<Upscaler> {+ let instance = wgpu::Instance::default();+ let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {+ power_preference: wgpu::PowerPreference::HighPerformance,+ ..Default::default()+ }))+ .ok()?;+ if adapter.get_info().device_type == wgpu::DeviceType::Cpu {+ return None;+ }+ let half = adapter.features().contains(wgpu::Features::SHADER_F16);+ let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {+ label: Some("concat upscaler"),+ required_features: if half {+ wgpu::Features::SHADER_F16+ } else {+ wgpu::Features::empty()+ },+ required_limits: wgpu::Limits::default()+ .or_worse_values_from(&adapter.limits())+ .using_resolution(adapter.limits()),+ ..Default::default()+ }))+ .ok()?;+ Self::on(device, queue, layers, scale, side, half)+ }++ fn on(+ device: wgpu::Device,+ queue: wgpu::Queue,+ layers: &[ConvLayer],+ scale: u32,+ side: u32,+ half: bool,+ ) -> Option<Upscaler> {+ // The kernel works out four channel groups at a time, so a layer's+ // outputs are padded up to a multiple of sixteen with zero weights,+ // and the next layer reads the padding as inputs it weighs by zero.+ let padded_groups = |channels: usize| groups(channels).div_ceil(4) * 4;+ let widest = layers.iter().map(|l| padded_groups(l.out_channels)).max()?;+ let pixels = (side * side) as u64;+ // A vec4 of the stored type: sixteen bytes of f32, eight of f16.+ let vec4 = if half { 8u64 } else { 16u64 };+ let activation = |label| {+ device.create_buffer(&wgpu::BufferDescriptor {+ label: Some(label),+ size: pixels * widest as u64 * vec4,+ usage: wgpu::BufferUsages::STORAGE,+ mapped_at_creation: false,+ })+ };+ let ping = activation("upscaler ping");+ let pong = activation("upscaler pong");+ let picture = device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("upscaler picture"),+ size: pixels * vec4,+ usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,+ mapped_at_creation: false,+ });+ let planes_size = 3 * pixels * u64::from(scale * scale) * 4;+ let planes = device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("upscaler planes"),+ size: planes_size,+ usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,+ mapped_at_creation: false,+ });+ let staging = device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("upscaler staging"),+ size: planes_size,+ usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,+ mapped_at_creation: false,+ });++ let descriptor = wgpu::ShaderModuleDescriptor {+ label: Some("upscaler"),+ source: wgpu::ShaderSource::Wgsl(shader(half).into()),+ };+ let module = device.create_shader_module(descriptor);+ let entry = |binding: u32, kind: wgpu::BufferBindingType| wgpu::BindGroupLayoutEntry {+ binding,+ visibility: wgpu::ShaderStages::COMPUTE,+ ty: wgpu::BindingType::Buffer {+ ty: kind,+ has_dynamic_offset: false,+ min_binding_size: None,+ },+ count: None,+ };+ let read = wgpu::BufferBindingType::Storage { read_only: true };+ let write = wgpu::BufferBindingType::Storage { read_only: false };+ let conv_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {+ label: Some("upscaler conv"),+ entries: &[+ entry(0, wgpu::BufferBindingType::Uniform),+ entry(1, read),+ entry(2, write),+ entry(3, read),+ entry(4, read),+ entry(5, read),+ ],+ });+ let shuffle_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {+ label: Some("upscaler shuffle"),+ entries: &[+ entry(0, wgpu::BufferBindingType::Uniform),+ entry(1, read),+ entry(6, read),+ entry(7, write),+ ],+ });+ let pipeline = |layout: &wgpu::BindGroupLayout, entry_point: &str| {+ device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {+ label: Some(entry_point),+ layout: Some(&device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {+ label: Some(entry_point),+ bind_group_layouts: &[Some(layout)],+ immediate_size: 0,+ })),+ module: &module,+ entry_point: Some(entry_point),+ compilation_options: Default::default(),+ cache: None,+ })+ };+ let conv = pipeline(&conv_layout, "conv");+ let shuffle = pipeline(&shuffle_layout, "shuffle");++ let dims = |in_groups: usize, out_groups: usize, prelu: bool| {+ let words = [+ side,+ side,+ in_groups as u32,+ out_groups as u32,+ u32::from(prelu),+ scale,+ 0,+ 0,+ ];+ device.create_buffer_init(&wgpu::util::BufferInitDescriptor {+ label: Some("upscaler dims"),+ contents: &words.iter().flat_map(|w| w.to_le_bytes()).collect::<Vec<u8>>(),+ usage: wgpu::BufferUsages::UNIFORM,+ })+ };+ let floats = |label, values: &[f32]| {+ device.create_buffer_init(&wgpu::util::BufferInitDescriptor {+ label: Some(label),+ contents: &values.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>(),+ usage: wgpu::BufferUsages::STORAGE,+ })+ };++ let mut gpu_layers = Vec::new();+ for (index, layer) in layers.iter().enumerate() {+ let in_groups = if index == 0 {+ groups(layer.in_channels)+ } else {+ padded_groups(layers[index - 1].out_channels)+ };+ let out_groups = padded_groups(layer.out_channels);+ // Each mat4x4 is four input channels into four outputs, kept+ // column by column: column `j` is input `4i + j`'s weights to+ // outputs `4o..4o + 4`, so `matrix * input` is the four sums.+ let mut packed = vec![0.0f32; out_groups * 9 * in_groups * 16];+ for o in 0..layer.out_channels {+ for i in 0..layer.in_channels {+ for tap in 0..9 {+ let w = layer.weight[(o * layer.in_channels + i) * 9 + tap];+ let matrix = ((o / 4) * 9 + tap) * in_groups + i / 4;+ packed[matrix * 16 + (i % 4) * 4 + o % 4] = w;+ }+ }+ }+ let padded = |values: &[f32]| {+ let mut out = values.to_vec();+ out.resize(out_groups * 4, 0.0);+ out+ };+ let slopes = padded(layer.prelu.as_deref().unwrap_or(&[]));+ let (src, dst) = if index % 2 == 0 {+ (if index == 0 { &picture } else { &pong }, &ping)+ } else {+ (&ping, &pong)+ };+ let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {+ label: Some("upscaler layer"),+ layout: &conv_layout,+ entries: &[+ wgpu::BindGroupEntry {+ binding: 0,+ resource: dims(in_groups, out_groups, layer.prelu.is_some())+ .as_entire_binding(),+ },+ wgpu::BindGroupEntry {+ binding: 1,+ resource: src.as_entire_binding(),+ },+ wgpu::BindGroupEntry {+ binding: 2,+ resource: dst.as_entire_binding(),+ },+ wgpu::BindGroupEntry {+ binding: 3,+ resource: device+ .create_buffer_init(&wgpu::util::BufferInitDescriptor {+ label: Some("upscaler weights"),+ contents: &stored(&packed, half),+ usage: wgpu::BufferUsages::STORAGE,+ })+ .as_entire_binding(),+ },+ wgpu::BindGroupEntry {+ binding: 4,+ resource: floats("upscaler bias", &padded(&layer.bias)).as_entire_binding(),+ },+ wgpu::BindGroupEntry {+ binding: 5,+ resource: floats("upscaler slope", &slopes).as_entire_binding(),+ },+ ],+ });+ gpu_layers.push(GpuLayer {+ out_groups: out_groups as u32,+ bind_group,+ });+ }+ // The last layer wrote ping when there is an odd number of layers.+ let last = if layers.len() % 2 == 1 { &ping } else { &pong };+ let shuffle_group = device.create_bind_group(&wgpu::BindGroupDescriptor {+ label: Some("upscaler shuffle"),+ layout: &shuffle_layout,+ entries: &[+ wgpu::BindGroupEntry {+ binding: 0,+ resource: dims(padded_groups(layers.last()?.out_channels), 0, false)+ .as_entire_binding(),+ },+ wgpu::BindGroupEntry {+ binding: 1,+ resource: last.as_entire_binding(),+ },+ wgpu::BindGroupEntry {+ binding: 6,+ resource: picture.as_entire_binding(),+ },+ wgpu::BindGroupEntry {+ binding: 7,+ resource: planes.as_entire_binding(),+ },+ ],+ });+ Some(Upscaler {+ device,+ queue,+ conv,+ shuffle,+ layers: gpu_layers,+ shuffle_group,+ picture,+ planes,+ half,+ staging,+ side,+ scale,+ })+ }++ /// Whether it works in half floats; see [`shader`].+ pub fn half(&self) -> bool {+ self.half+ }++ /// The side of the square windows this runs.+ pub fn side(&self) -> u32 {+ self.side+ }++ /// The network over one window: three planes of `side` by `side`,+ /// values `0..=1`, in; three planes `scale` times the side out - the+ /// ONNX model's input and answer, value for value.+ pub fn run(&self, planes: &[f32]) -> Result<Vec<f32>, String> {+ let side = self.side as usize;+ let area = side * side;+ if planes.len() != 3 * area {+ return Err(format!(+ "upscaler: {} values for a {side}-pixel window",+ planes.len()+ ));+ }+ let mut packed = vec![0.0f32; area * 4];+ for at in 0..area {+ for c in 0..3 {+ packed[at * 4 + c] = planes[c * area + at];+ }+ }+ self.queue.write_buffer(+ &self.picture,+ 0,+ &stored(&packed, self.half),+ );+ let mut encoder = self+ .device+ .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("upscale") });+ {+ let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {+ label: Some("upscale"),+ timestamp_writes: None,+ });+ pass.set_pipeline(&self.conv);+ let across = self.side.div_ceil(8 * BLOCK_W);+ let down = self.side.div_ceil(8 * BLOCK_H);+ for layer in &self.layers {+ pass.set_bind_group(0, &layer.bind_group, &[]);+ pass.dispatch_workgroups(across, down, layer.out_groups.div_ceil(4));+ }+ pass.set_pipeline(&self.shuffle);+ pass.set_bind_group(0, &self.shuffle_group, &[]);+ let big = (self.side * self.scale).div_ceil(8);+ pass.dispatch_workgroups(big, big, 1);+ }+ encoder.copy_buffer_to_buffer(&self.planes, 0, &self.staging, 0, self.planes.size());+ self.queue.submit([encoder.finish()]);++ let slice = self.staging.slice(..);+ let (sent, mapped) = mpsc::channel();+ slice.map_async(wgpu::MapMode::Read, move |result| {+ let _ = sent.send(result);+ });+ self.device+ .poll(wgpu::PollType::wait_indefinitely())+ .map_err(|error| format!("upscaler: the device did not finish: {error}"))?;+ mapped+ .recv()+ .map_err(|_| "upscaler: the device never answered".to_owned())?+ .map_err(|error| format!("upscaler: the answer would not map: {error}"))?;+ let answer = {+ let data = slice.get_mapped_range().map_err(|e| format!("upscaler: {e}"))?;+ data.chunks_exact(4)+ .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))+ .collect()+ };+ self.staging.unmap();+ Ok(answer)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use concat_vision::compact::{CompactNet, Conv};++ /// A small network of made-up weights, run on the GPU and on the CPU+ /// reference: the two agree to rounding, edges and shuffle included.+ #[test]+ fn the_gpu_answers_as_the_reference_does() {+ let mut seed = 0x2545_f491_4f6c_dd1du64;+ let mut next = || {+ seed ^= seed << 13;+ seed ^= seed >> 7;+ seed ^= seed << 17;+ (seed % 2001) as f32 / 1000.0 - 1.0+ };+ let mut conv = |inputs: usize, outputs: usize, prelu: bool| Conv {+ in_channels: inputs,+ out_channels: outputs,+ weight: (0..outputs * inputs * 9).map(|_| next() * 0.3).collect(),+ bias: (0..outputs).map(|_| next() * 0.1).collect(),+ prelu: prelu.then(|| (0..outputs).map(|_| next() * 0.5).collect()),+ };+ let net = CompactNet {+ layers: vec![conv(3, 8, true), conv(8, 8, true), conv(8, 12, false)],+ scale: 2,+ };+ let side = 20usize;+ let mut seed = 7u64;+ let planes: Vec<f32> = (0..3 * side * side)+ .map(|_| {+ seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);+ (seed >> 40) as f32 / (1u64 << 24) as f32+ })+ .collect();+ let layers: Vec<ConvLayer> = net+ .layers+ .iter()+ .map(|l| ConvLayer {+ in_channels: l.in_channels,+ out_channels: l.out_channels,+ weight: l.weight.clone(),+ bias: l.bias.clone(),+ prelu: l.prelu.clone(),+ })+ .collect();+ let Some(gpu) = Upscaler::new(&layers, 2, side as u32) else {+ eprintln!("no GPU here; skipped");+ return;+ };+ let got = gpu.run(&planes).expect("runs");+ let want = net.run_reference(&planes, side);+ assert_eq!(got.len(), want.len());+ let worst = got+ .iter()+ .zip(&want)+ .map(|(a, b)| (a - b).abs())+ .fold(0.0f32, f32::max);+ // Half floats keep about three decimal digits.+ let allowed = if gpu.half() { 2e-2 } else { 1e-4 };+ assert!(worst < allowed, "the GPU is off by {worst}");+ }++ /// The real model, when this machine has fetched it, timed over a few+ /// windows of the size Enhance cuts a frame into, and the 1080p frame+ /// that comes to. `cargo test -p concat-render --release upscale_speed+ /// -- --ignored --nocapture`.+ #[test]+ #[ignore = "a benchmark, and it needs the fetched model"]+ fn upscale_speed() {+ let Some(home) = std::env::var_os("HOME") else { return };+ let support = std::path::Path::new(&home).join("Library/Application Support");+ let Some(bytes) = ["org.standalones.concat", "app.concat.editor"]+ .iter()+ .map(|id| support.join(id).join("cutout-models/realesr-general-x4v3.onnx"))+ .find_map(|file| std::fs::read(file).ok())+ else {+ eprintln!("no Enhance model on this machine");+ return;+ };+ let net = CompactNet::read(&bytes).expect("reads");+ let layers: Vec<ConvLayer> = net+ .layers+ .iter()+ .map(|l| ConvLayer {+ in_channels: l.in_channels,+ out_channels: l.out_channels,+ weight: l.weight.clone(),+ bias: l.bias.clone(),+ prelu: l.prelu.clone(),+ })+ .collect();+ // The real model on a small window against the CPU reference, in+ // eight-bit levels, which is what the file is written in.+ let small = 40u32;+ let gpu = Upscaler::new(&layers, net.scale as u32, small).expect("a GPU");+ let mut seed = 11u64;+ let planes: Vec<f32> = (0..3 * (small * small) as usize)+ .map(|_| {+ seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);+ (seed >> 40) as f32 / (1u64 << 24) as f32+ })+ .collect();+ let got = gpu.run(&planes).expect("runs");+ let want = net.run_reference(&planes, small as usize);+ let worst = got+ .iter()+ .zip(&want)+ .map(|(a, b)| (a.clamp(0.0, 1.0) - b.clamp(0.0, 1.0)).abs() * 255.0)+ .fold(0.0f32, f32::max);+ eprintln!("half floats: {}, worst error {worst:.2} levels of 255", gpu.half());+ for side in [288u32, 544] {+ let gpu = Upscaler::new(&layers, net.scale as u32, side).expect("a GPU");+ let planes = vec![0.5f32; 3 * (side * side) as usize];+ gpu.run(&planes).expect("warms up");+ let runs = 5;+ let started = std::time::Instant::now();+ for _ in 0..runs {+ gpu.run(&planes).expect("runs");+ }+ let each = started.elapsed().as_secs_f64() / runs as f64;+ let core = f64::from(side - 32);+ let windows = (1920.0 / core).ceil() * (1080.0 / core).ceil();+ eprintln!(+ "window {side}: {:.1} ms each, a 1080p frame in {:.0} windows = {:.2} s",+ each * 1000.0,+ windows,+ each * windows+ );+ }+ }+}diff --git a/src/crates/concat-vision/src/compact.rs b/src/crates/concat-vision/src/compact.rsnew file mode 100644index 00000000..425e537f--- /dev/null+++ b/src/crates/concat-vision/src/compact.rs@@ -0,0 +1,450 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer Abdullah and Concat contributors++//! The Enhance model's weights, read out of its ONNX file for a GPU of our+//! own to run.+//!+//! Real-ESRGAN's compact network (SRVGGNetCompact) is a plain stack: a 3x3+//! convolution from the picture's three channels to 64, a PReLU, thirty-two+//! more 3x3 convolutions of 64 to 64 each with its PReLU, one last 3x3 to+//! `3 * scale * scale` channels, those channels rearranged into a picture+//! `scale` times larger (PyTorch's pixel shuffle, ONNX's `DepthToSpace` in+//! CRD order), and the input, enlarged by nearest neighbour, added to it.+//! Nothing in it needs a runtime: [`CompactNet::read`] takes the layers'+//! numbers out of the file and checks the graph is that shape, so a file+//! that is anything else is refused rather than run wrong.+//!+//! ONNX is protocol buffers. The handful of fields read here are decoded by+//! hand (see [`fields`]) rather than through a protobuf crate: a model+//! file is the only one this reads, and only its graph's nodes and+//! initializers.++use std::collections::HashMap;++/// One 3x3 convolution, padded by one, stride one, and the PReLU after it+/// when there is one.+#[derive(Clone, Debug, PartialEq)]+pub struct Conv {+ /// Channels read.+ pub in_channels: usize,+ /// Channels written.+ pub out_channels: usize,+ /// `[out][in][3][3]`, row-major, as ONNX and PyTorch keep it.+ pub weight: Vec<f32>,+ /// One per output channel.+ pub bias: Vec<f32>,+ /// One slope per output channel, or `None` for the last layer, whose+ /// answer goes to the pixel shuffle as it is.+ pub prelu: Option<Vec<f32>>,+}++/// The whole network: its convolutions in order, and how much larger the+/// picture comes out.+#[derive(Clone, Debug, PartialEq)]+pub struct CompactNet {+ /// The convolutions, input first.+ pub layers: Vec<Conv>,+ /// How many times larger the picture comes out.+ pub scale: usize,+}++impl CompactNet {+ /// The network in the ONNX file `bytes`.+ pub fn read(bytes: &[u8]) -> Result<CompactNet, String> {+ let graph = fields(bytes)?+ .into_iter()+ .find_map(|(field, value)| (field == 7).then_some(value))+ .and_then(Value::bytes)+ .ok_or("enhance: the model file has no graph")?;+ let mut tensors: HashMap<String, (Vec<usize>, Vec<f32>)> = HashMap::new();+ let mut nodes = Vec::new();+ for (field, value) in fields(graph)? {+ match (field, value.bytes()) {+ (1, Some(node)) => nodes.push(Node::read(node)?),+ (5, Some(tensor)) => {+ let (name, dims, data) = tensor_of(tensor)?;+ tensors.insert(name, (dims, data));+ }+ _ => {}+ }+ }++ let mut layers = Vec::new();+ let mut scale = None;+ for (at, node) in nodes.iter().enumerate() {+ match node.op.as_str() {+ "Conv" => {+ if node.ints("kernel_shape") != Some(&[3, 3][..])+ || node.ints("pads").is_some_and(|pads| pads != [1, 1, 1, 1])+ || node.ints("strides").is_some_and(|s| s != [1, 1])+ || node.ints("group").is_some_and(|g| g != [1])+ {+ return Err(format!(+ "enhance: layer {} is not a plain 3x3 convolution",+ layers.len()+ ));+ }+ let tensor = |index: usize| {+ node.inputs+ .get(index)+ .and_then(|name| tensors.get(name))+ .ok_or_else(|| format!("enhance: layer {} has no weights", layers.len()))+ };+ let (dims, weight) = tensor(1)?;+ let (_, bias) = tensor(2)?;+ let [out_channels, in_channels, 3, 3] = dims[..] else {+ return Err(format!("enhance: layer {} is shaped {dims:?}", layers.len()));+ };+ if bias.len() != out_channels {+ return Err(format!("enhance: layer {}'s bias is the wrong length", layers.len()));+ }+ // The PReLU after it, when the next node is one.+ let prelu = match nodes.get(at + 1) {+ Some(next) if next.op == "PRelu" => {+ let (_, slopes) = next+ .inputs+ .get(1)+ .and_then(|name| tensors.get(name))+ .ok_or("enhance: a PReLU has no slopes")?;+ if slopes.len() != out_channels {+ return Err("enhance: a PReLU's slopes are the wrong length".into());+ }+ Some(slopes.clone())+ }+ _ => None,+ };+ layers.push(Conv {+ in_channels,+ out_channels,+ weight: weight.clone(),+ bias: bias.clone(),+ prelu,+ });+ }+ "DepthToSpace" => {+ if node.string("mode").is_some_and(|mode| mode != "CRD") {+ return Err("enhance: the pixel shuffle is not in CRD order".into());+ }+ scale = node.ints("blocksize").and_then(|b| b.first().copied());+ }+ "Resize" => {+ if node.string("mode").is_some_and(|mode| mode != "nearest") {+ return Err("enhance: the input is not enlarged by nearest neighbour".into());+ }+ }+ "PRelu" | "Constant" | "Add" => {}+ other => return Err(format!("enhance: the model has a {other} this cannot run")),+ }+ }+ let scale = scale.ok_or("enhance: the model has no pixel shuffle")? as usize;+ // The chain must join up: three channels in, 64s through, and the+ // last layer's channels exactly the shuffle's.+ let chained = layers.first().is_some_and(|first| first.in_channels == 3)+ && layers.windows(2).all(|pair| pair[0].out_channels == pair[1].in_channels)+ && layers.last().is_some_and(|last| {+ last.out_channels == 3 * scale * scale && last.prelu.is_none()+ })+ && layers[..layers.len() - 1].iter().all(|layer| layer.prelu.is_some());+ if !chained {+ return Err("enhance: the model's layers do not chain as a compact network".into());+ }+ Ok(CompactNet { layers, scale })+ }++ /// The network run on the CPU, plainly, over `planes` - three planes of+ /// `side` by `side`, values `0..=1` - into three planes `scale` times+ /// the side: the reference the GPU's answer is held to. Far too slow+ /// for a frame; a small square is what a test gives it.+ pub fn run_reference(&self, planes: &[f32], side: usize) -> Vec<f32> {+ let area = side * side;+ let mut x = planes.to_vec();+ for layer in &self.layers {+ let mut y = vec![0.0f32; layer.out_channels * area];+ for o in 0..layer.out_channels {+ for py in 0..side {+ for px in 0..side {+ let mut acc = layer.bias[o];+ for i in 0..layer.in_channels {+ for ky in 0..3 {+ let sy = py as isize + ky as isize - 1;+ if sy < 0 || sy >= side as isize {+ continue;+ }+ for kx in 0..3 {+ let sx = px as isize + kx as isize - 1;+ if sx < 0 || sx >= side as isize {+ continue;+ }+ let w = layer.weight[((o * layer.in_channels + i) * 3 + ky) * 3 + kx];+ acc += w * x[i * area + sy as usize * side + sx as usize];+ }+ }+ }+ if let Some(slopes) = &layer.prelu+ && acc < 0.0+ {+ acc *= slopes[o];+ }+ y[o * area + py * side + px] = acc;+ }+ }+ }+ x = y;+ }+ let s = self.scale;+ let big = side * s;+ let mut out = vec![0.0f32; 3 * big * big];+ for c in 0..3 {+ for oy in 0..big {+ for ox in 0..big {+ let (y, dy, xx, dx) = (oy / s, oy % s, ox / s, ox % s);+ let shuffled = x[(c * s * s + dy * s + dx) * area + y * side + xx];+ out[c * big * big + oy * big + ox] = shuffled + planes[c * area + y * side + xx];+ }+ }+ }+ out+ }+}++/// One node of the graph: what it does, what it reads, and its attributes.+struct Node {+ op: String,+ inputs: Vec<String>,+ attributes: HashMap<String, Attribute>,+}++enum Attribute {+ Ints(Vec<i64>),+ Text(String),+}++impl Node {+ fn read(bytes: &[u8]) -> Result<Node, String> {+ let mut node = Node {+ op: String::new(),+ inputs: Vec::new(),+ attributes: HashMap::new(),+ };+ for (field, value) in fields(bytes)? {+ match (field, value) {+ (1, Value::Bytes(name)) => node.inputs.push(text(name)),+ (4, Value::Bytes(op)) => node.op = text(op),+ (5, Value::Bytes(attribute)) => {+ let mut name = String::new();+ let mut ints = Vec::new();+ let mut string = None;+ for (field, value) in fields(attribute)? {+ match (field, value) {+ (1, Value::Bytes(n)) => name = text(n),+ // `i`, and `ints` written one at a time.+ (3 | 8, Value::Varint(v)) => ints.push(v as i64),+ // `ints` packed.+ (8, Value::Bytes(packed)) => {+ let mut at = 0;+ while at < packed.len() {+ let (v, next) = varint(packed, at)?;+ ints.push(v as i64);+ at = next;+ }+ }+ (4, Value::Bytes(s)) => string = Some(text(s)),+ _ => {}+ }+ }+ let value = match string {+ Some(s) => Attribute::Text(s),+ None => Attribute::Ints(ints),+ };+ node.attributes.insert(name, value);+ }+ _ => {}+ }+ }+ Ok(node)+ }++ fn ints(&self, name: &str) -> Option<&[i64]> {+ match self.attributes.get(name)? {+ Attribute::Ints(ints) => Some(ints),+ Attribute::Text(_) => None,+ }+ }++ fn string(&self, name: &str) -> Option<&str> {+ match self.attributes.get(name)? {+ Attribute::Text(s) => Some(s),+ Attribute::Ints(_) => None,+ }+ }+}++/// A float tensor's name, shape and numbers, from a `TensorProto`: kept as+/// `raw_data` little-endian, or as packed `float_data`.+fn tensor_of(bytes: &[u8]) -> Result<(String, Vec<usize>, Vec<f32>), String> {+ let mut name = String::new();+ let mut dims = Vec::new();+ let mut data = Vec::new();+ let mut kind = 1;+ for (field, value) in fields(bytes)? {+ match (field, value) {+ (1, Value::Varint(d)) => dims.push(d as usize),+ (1, Value::Bytes(packed)) => {+ let mut at = 0;+ while at < packed.len() {+ let (d, next) = varint(packed, at)?;+ dims.push(d as usize);+ at = next;+ }+ }+ (2, Value::Varint(k)) => kind = k,+ (8, Value::Bytes(n)) => name = text(n),+ (9, Value::Bytes(raw)) | (4, Value::Bytes(raw)) => data.extend(+ raw.chunks_exact(4)+ .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])),+ ),+ _ => {}+ }+ }+ if kind != 1 && !data.is_empty() {+ return Err(format!("enhance: {name} is not 32-bit floats"));+ }+ let count: usize = dims.iter().product();+ if !data.is_empty() && data.len() != count.max(1) {+ return Err(format!("enhance: {name} holds {} numbers, not {count}", data.len()));+ }+ Ok((name, dims, data))+}++/// A protobuf field's value: the two wire types a model file's fields use,+/// and the fixed-width ones skipped over.+enum Value<'a> {+ Varint(u64),+ Bytes(&'a [u8]),+ Fixed,+}++impl<'a> Value<'a> {+ fn bytes(self) -> Option<&'a [u8]> {+ match self {+ Value::Bytes(bytes) => Some(bytes),+ _ => None,+ }+ }+}++/// A protobuf message's fields, in order, as `(number, value)`.+fn fields(bytes: &[u8]) -> Result<Vec<(u64, Value<'_>)>, String> {+ let mut out = Vec::new();+ let mut at = 0;+ while at < bytes.len() {+ let (key, next) = varint(bytes, at)?;+ at = next;+ let value = match key & 7 {+ 0 => {+ let (v, next) = varint(bytes, at)?;+ at = next;+ Value::Varint(v)+ }+ 1 => {+ at += 8;+ Value::Fixed+ }+ 2 => {+ let (length, next) = varint(bytes, at)?;+ let end = next+ .checked_add(length as usize)+ .filter(|end| *end <= bytes.len())+ .ok_or("enhance: the model file is cut short")?;+ at = end;+ Value::Bytes(&bytes[next..end])+ }+ 5 => {+ at += 4;+ Value::Fixed+ }+ other => return Err(format!("enhance: the model file has wire type {other}")),+ };+ if at > bytes.len() {+ return Err("enhance: the model file is cut short".into());+ }+ out.push((key >> 3, value));+ }+ Ok(out)+}++fn varint(bytes: &[u8], mut at: usize) -> Result<(u64, usize), String> {+ let mut value = 0u64;+ for shift in (0..64).step_by(7) {+ let byte = *bytes.get(at).ok_or("enhance: the model file is cut short")?;+ at += 1;+ value |= u64::from(byte & 0x7f) << shift;+ if byte < 0x80 {+ return Ok((value, at));+ }+ }+ Err("enhance: the model file has a number too long to be one".into())+}++fn text(bytes: &[u8]) -> String {+ String::from_utf8_lossy(bytes).into_owned()+}++#[cfg(test)]+mod tests {+ use super::*;++ /// The real model, when this machine has fetched it: read whole, the+ /// shape Real-ESRGAN's compact network has.+ #[test]+ fn the_fetched_model_reads_as_a_compact_network() {+ let Some(file) = fetched_model() else {+ eprintln!("no Enhance model on this machine; skipped");+ return;+ };+ let net = CompactNet::read(&std::fs::read(file).expect("reads")).expect("parses");+ assert_eq!(net.scale, 4);+ assert_eq!(net.layers.len(), 34);+ assert_eq!(net.layers[0].in_channels, 3);+ assert_eq!(net.layers[33].out_channels, 48);+ let params: usize = net+ .layers+ .iter()+ .map(|l| l.weight.len() + l.bias.len() + l.prelu.as_ref().map_or(0, Vec::len))+ .sum();+ assert_eq!(params, 1_213_296);+ }++ /// Anything but the shape it knows is refused, not run wrong.+ #[test]+ fn a_file_that_is_not_a_model_is_refused() {+ assert!(CompactNet::read(b"not a model").is_err());+ assert!(CompactNet::read(&[]).is_err());+ }++ /// Where the app keeps the model on this machine, if it has it: under+ /// the app's identifier, or the one before it on a machine whose app+ /// has not started since the rename.+ pub(crate) fn fetched_model() -> Option<std::path::PathBuf> {+ let bases: Vec<std::path::PathBuf> = if cfg!(target_os = "macos") {+ std::env::var_os("HOME")+ .map(|home| std::path::PathBuf::from(home).join("Library/Application Support"))+ .into_iter()+ .collect()+ } else if cfg!(windows) {+ std::env::var_os("LOCALAPPDATA").map(std::path::PathBuf::from).into_iter().collect()+ } else {+ std::env::var_os("HOME")+ .map(|home| std::path::PathBuf::from(home).join(".local/share"))+ .into_iter()+ .collect()+ };+ bases+ .iter()+ .flat_map(|base| {+ ["org.standalones.concat", "app.concat.editor"]+ .map(|id| base.join(id).join("cutout-models").join("realesr-general-x4v3.onnx"))+ })+ .find(|file| file.is_file())+ }+}diff --git a/src/crates/concat-vision/src/enhance.rs b/src/crates/concat-vision/src/enhance.rsindex 9e8a6869..59033c9e 100644--- a/src/crates/concat-vision/src/enhance.rs+++ b/src/crates/concat-vision/src/enhance.rs@@ -347,19 +347,56 @@ pub fn downscale(frame: &Frame, by: u32) -> Frame {Frame::from_rgba(ow as u32, oh as u32, pixels).expect("sides and bytes agree")}-/// The model, loaded once and run tile by tile.+/// What runs the model over one window: the [`WINDOW`]-square planes+/// [`window_planes`] cuts, in; the planes [`paste`] takes, [`SCALE`] times+/// the side, out. ONNX Runtime is one, on the CPU almost everywhere; the+/// host's GPU upscaler, the same network in compute shaders of our own, is+/// the other, and the one used wherever a GPU opens.+pub trait WindowRunner: Send {+ /// The model's answer for one window.+ fn run(&mut self, planes: Vec<f32>) -> Result<Vec<f32>, String>;+}++/// The model through ONNX Runtime.+#[cfg(feature = "infer")]+struct Runtime(crate::runtime::Model);+#[cfg(feature = "infer")]+impl WindowRunner for Runtime {+ fn run(&mut self, planes: Vec<f32>) -> Result<Vec<f32>, String> {+ let side = WINDOW as usize;+ let mut answer = self.0.run(+ vec![crate::runtime::Input {+ name: "input",+ dims: vec![1, 3, side, side],+ data: planes.into(),+ }],+ &["output"],+ )?;+ Ok(std::mem::take(&mut answer[0].data))+ }+}++/// The model, loaded once and run tile by tile.pub struct Enhancer {- model: std::sync::Mutex<crate::runtime::Model>,+ model: std::sync::Mutex<Box<dyn WindowRunner>>,}-#[cfg(feature = "infer")]impl Enhancer {- /// Loads the model from its file; see [`crate::models::ModelId::Enhance`].+ /// Loads the model from its file into ONNX Runtime; see+ /// [`crate::models::ModelId::Enhance`].+ #[cfg(feature = "infer")]pub fn load(file: &std::path::Path) -> Result<Enhancer, String> {- Ok(Enhancer {- model: std::sync::Mutex::new(crate::runtime::Model::from_file(file)?),- })+ Ok(Enhancer::with(Box::new(Runtime(+ crate::runtime::Model::from_file(file)?,+ ))))+ }++ /// The model run by `runner`.+ pub fn with(runner: Box<dyn WindowRunner>) -> Enhancer {+ Enhancer {+ model: std::sync::Mutex::new(runner),+ }}/// `frame` restored at [`SCALE`] times its size.@@ -386,7 +423,6 @@ impl Enhancer {return Err("enhance: an empty picture".to_owned());}let mut out = Frame::black(region.width * SCALE, region.height * SCALE);- let side = WINDOW as usize;let mut model = self.model.lock().map_err(|_| "enhance: model poisoned")?;let mut times = Vec::new();for tile in tiles(region.width, region.height) {@@ -397,16 +433,9 @@ impl Enhancer {};let planes = window_planes(frame, placed);let started = std::time::Instant::now();- let answer = model.run(- vec![crate::runtime::Input {- name: "input",- dims: vec![1, 3, side, side],- data: planes.into(),- }],- &["output"],- )?;+ let answer = model.run(planes)?;times.push(started.elapsed());- paste(&mut out, tile, &answer[0].data)?;+ paste(&mut out, tile, &answer)?;}Ok((out, times))}diff --git a/src/crates/concat-vision/src/lib.rs b/src/crates/concat-vision/src/lib.rsindex 485626ca..45689077 100644--- a/src/crates/concat-vision/src/lib.rs+++ b/src/crates/concat-vision/src/lib.rs@@ -47,6 +47,7 @@pub mod apply;#[cfg(feature = "infer")]pub mod brush;+pub mod compact;pub mod enhance;pub mod mask;pub mod models;