Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

perf(enhance): run the model on the GPU in compute shaders of our own - 4.2 s a 1080p frame where ONNX Runtime took 27

Jareeraa38b9d519fa
diff --git a/src/Cargo.lock b/src/Cargo.lock
index 096b8b59..a89ae64a 100644
--- a/src/Cargo.lock
+++ b/src/Cargo.lock
@@ -1424,6 +1424,7 @@ version = "0.2.8"
dependencies = [
"concat-core",
"concat-effects",
+ "concat-vision",
"half",
"log",
"pollster",
diff --git a/src/crates/concat-host/src/enhance.rs b/src/crates/concat-host/src/enhance.rs
index 8d0e9396..dfcdfa16 100644
--- a/src/crates/concat-host/src/enhance.rs
+++ b/src/crates/concat-host/src/enhance.rs
@@ -237,7 +237,7 @@ impl Enhancers {
if !self.installed() {
crate::cutout::fetch(ModelId::Enhance, &file, cancel, progress)?;
}
- let enhancer = Arc::new(Enhancer::load(&file)?);
+ let enhancer = Arc::new(on_gpu(&file).map_or_else(|| Enhancer::load(&file), Ok)?);
*self.model.lock().map_err(|_| "model slot poisoned")? = Some(Arc::clone(&enhancer));
Ok(enhancer)
}
@@ -471,10 +471,104 @@ pub(crate) fn crop(frame: &concat_core::Frame, width: u32, height: u32) -> conca
out
}
+/// The model on the GPU, in compute shaders of our own
+/// (`concat_render::upscale`), when its file reads as the compact network
+/// and a GPU opens: about six times what ONNX Runtime does on this
+/// machine's CPU, and the CPU is where ONNX Runtime ran nearly everywhere -
+/// Windows' runtime has no GPU provider at all. `None` sends Enhance the
+/// slow way rather than not at all.
+fn on_gpu(file: &Path) -> Option<Enhancer> {
+ let bytes = std::fs::read(file).ok()?;
+ let net = match concat_vision::compact::CompactNet::read(&bytes) {
+ Ok(net) => net,
+ Err(error) => {
+ log::warn!("enhance: {error}; on the CPU instead");
+ return None;
+ }
+ };
+ let layers: Vec<concat_render::upscale::ConvLayer> = net
+ .layers
+ .into_iter()
+ .map(|layer| concat_render::upscale::ConvLayer {
+ in_channels: layer.in_channels,
+ out_channels: layer.out_channels,
+ weight: layer.weight,
+ bias: layer.bias,
+ prelu: layer.prelu,
+ })
+ .collect();
+ let upscaler =
+ concat_render::upscale::Upscaler::new(&layers, net.scale as u32, vision::WINDOW)?;
+ log::info!(
+ "enhance: on the GPU{}",
+ if upscaler.half() { ", in half floats" } else { "" }
+ );
+ Some(Enhancer::with(Box::new(GpuWindows(upscaler))))
+}
+
+/// The GPU upscaler as the model `Enhancer` runs window by window.
+struct GpuWindows(concat_render::upscale::Upscaler);
+
+impl vision::WindowRunner for GpuWindows {
+ fn run(&mut self, planes: Vec<f32>) -> Result<Vec<f32>, String> {
+ self.0.run(&planes)
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
+ /// The model on the GPU and through ONNX Runtime give the same picture,
+ /// tiles, margins and pasting included: a few levels at the worst pixel
+ /// and well under one on average, the
+ /// GPU working in half floats. Needs the fetched model and a GPU.
+ #[test]
+ fn the_gpu_enhances_as_onnx_runtime_does() {
+ let Some(file) = ["org.standalones.concat", "app.concat.editor"]
+ .iter()
+ .filter_map(|id| {
+ let home = std::env::var_os("HOME")?;
+ let file = PathBuf::from(home)
+ .join("Library/Application Support")
+ .join(id)
+ .join("cutout-models/realesr-general-x4v3.onnx");
+ file.is_file().then_some(file)
+ })
+ .next()
+ else {
+ eprintln!("no Enhance model here; skipped");
+ return;
+ };
+ let Some(gpu) = on_gpu(&file) else {
+ eprintln!("no GPU here; skipped");
+ return;
+ };
+ let cpu = Enhancer::load(&file).expect("loads");
+ // Wider than one tile, so a seam between two is in the comparison.
+ let (width, height) = (300u32, 120u32);
+ let pixels: Vec<u8> = (0..width * height)
+ .flat_map(|at| {
+ let (x, y) = (at % width, at / width);
+ [(x * 255 / width) as u8, (y * 2) as u8, ((x ^ y) & 0xff) as u8, 255]
+ })
+ .collect();
+ let frame = Frame::from_rgba(width, height, pixels).expect("a frame");
+ let a = gpu.enhance(&frame, 2).expect("the GPU enhances");
+ let b = cpu.enhance(&frame, 2).expect("the CPU enhances");
+ assert_eq!((a.width(), a.height()), (b.width(), b.height()));
+ let diffs: Vec<u32> = a
+ .pixels()
+ .iter()
+ .zip(b.pixels())
+ .map(|(x, y)| u32::from(x.abs_diff(*y)))
+ .collect();
+ let worst = diffs.iter().max().copied().unwrap_or(0);
+ let mean = diffs.iter().sum::<u32>() as f64 / diffs.len() as f64;
+ eprintln!("GPU against ONNX Runtime: worst {worst}, mean {mean:.3} levels");
+ assert!(worst <= 6 && mean < 0.5, "worst {worst}, mean {mean}");
+ }
+
#[test]
fn the_copy_is_named_after_the_file_and_the_factor() {
let dir = std::env::temp_dir().join(format!("concat-enhance-{}", std::process::id()));
diff --git a/src/crates/concat-render/Cargo.toml b/src/crates/concat-render/Cargo.toml
index 0b6262f0..9a8110d1 100644
--- a/src/crates/concat-render/Cargo.toml
+++ b/src/crates/concat-render/Cargo.toml
@@ -24,6 +24,8 @@ half = "2"
# The pass test compiles a real package shader through the catalogue's
# stitching, so what the test runs is what a package would run.
concat-effects.workspace = true
+# The upscaler's answer is held to the network's plain CPU reference.
+concat-vision.workspace = true
[lints]
workspace = true
diff --git a/src/crates/concat-render/src/lib.rs b/src/crates/concat-render/src/lib.rs
index ce8dcb6e..8e86e372 100644
--- a/src/crates/concat-render/src/lib.rs
+++ b/src/crates/concat-render/src/lib.rs
@@ -22,6 +22,7 @@ pub mod compositor;
pub mod gpu;
pub mod metrics;
pub mod plan;
+pub mod upscale;
#[cfg(test)]
mod reference;
#[cfg(test)]
diff --git a/src/crates/concat-render/src/upscale.rs b/src/crates/concat-render/src/upscale.rs
new file mode 100644
index 00000000..5665102f
--- /dev/null
+++ b/src/crates/concat-render/src/upscale.rs
@@ -0,0 +1,724 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer Abdullah and Concat contributors
+
+//! Enhance's network on the GPU, in compute shaders of our own.
+//!
+//! The network is a stack of 3x3 convolutions with a PReLU after each but
+//! the last, then a pixel shuffle with the input, enlarged, added (see
+//! `concat_vision::compact`). Run by ONNX Runtime it ran on the CPU almost
+//! everywhere - Windows' bundled runtime has no GPU provider at all, and
+//! CoreML hands back to the CPU what it will not take - at about 27
+//! seconds a 1080p frame: 1.2 million multiply-adds a pixel is a CPU's
+//! afternoon and a GPU's second. Here each layer is one dispatch on a
+//! device of the upscaler's own, so a long job never holds the window's.
+//!
+//! Activations live in storage buffers, four channels to a `vec4` and the
+//! channels of a pixel together: `[(y * width + x) * groups + group]`.
+//! Each invocation works out sixteen output channels of a 2x2 block of
+//! pixels, a `mat4x4` of weights to each input `vec4` under each of the
+//! nine taps; a padded edge reads as zero, as the model's padding is. The last pass shuffles
+//! the final layer's channels into the larger picture and adds the input
+//! to it, writing three planes, which is the ONNX model's answer exactly:
+//! the tiling and pasting around it do not know which ran.
+
+use std::sync::mpsc;
+
+use wgpu::util::DeviceExt;
+
+/// One 3x3 convolution and its PReLU, as the network file holds them; see
+/// `concat_vision::compact::Conv`.
+#[derive(Clone, Debug)]
+pub struct ConvLayer {
+ /// Channels read.
+ pub in_channels: usize,
+ /// Channels written.
+ pub out_channels: usize,
+ /// `[out][in][3][3]`.
+ pub weight: Vec<f32>,
+ /// One per output channel.
+ pub bias: Vec<f32>,
+ /// The PReLU's slope per output channel, or `None` for none.
+ pub prelu: Option<Vec<f32>>,
+}
+
+const SHADER_COMMON: &str = r#"
+struct Dims {
+ width: u32,
+ height: u32,
+ in_groups: u32,
+ out_groups: u32,
+ prelu: u32,
+ scale: u32,
+}
+
+@group(0) @binding(0) var<uniform> dims: Dims;
+@group(0) @binding(1) var<storage, read> src: array<vec4<T>>;
+@group(0) @binding(2) var<storage, read_write> dst: array<vec4<T>>;
+@group(0) @binding(3) var<storage, read> weights: array<mat4x4<T>>;
+@group(0) @binding(4) var<storage, read> bias: array<vec4<f32>>;
+@group(0) @binding(5) var<storage, read> slope: array<vec4<f32>>;
+
+// `conv` is generated (see `conv_shader`): every tap and channel group
+// unrolled so the accumulators and the neighbourhood are plain registers.
+
+@group(0) @binding(6) var<storage, read> picture: array<vec4<T>>;
+@group(0) @binding(7) var<storage, read_write> planes: array<f32>;
+
+// The last layer's channels, `3 * scale * scale` of them, laid out as the
+// picture `scale` times larger - channel `c * scale * scale + dy * scale + dx`
+// is colour `c` at `(x * scale + dx, y * scale + dy)`, PyTorch's order - with
+// the input added, enlarged by nearest neighbour.
+@compute @workgroup_size(8, 8, 1)
+fn shuffle(@builtin(global_invocation_id) id: vec3<u32>) {
+ let s = dims.scale;
+ let big_w = dims.width * s;
+ let big_h = dims.height * s;
+ if (id.x >= big_w || id.y >= big_h) {
+ return;
+ }
+ let x = id.x / s;
+ let y = id.y / s;
+ let dx = id.x % s;
+ let dy = id.y % s;
+ let at = y * dims.width + x;
+ let input = picture[at];
+ for (var c = 0u; c < 3u; c++) {
+ let channel = c * s * s + dy * s + dx;
+ let value = f32(src[at * dims.in_groups + channel / 4u][channel % 4u]);
+ planes[c * big_w * big_h + id.y * big_w + id.x] = value + f32(input[c]);
+ }
+}
+"#;
+
+/// The whole module, in half floats where the device has them: the
+/// activations and weights stored at sixteen bits, which halves both the
+/// memory read and the registers a block of pixels takes, and the sums
+/// kept at thirty-two so 576 of them a channel do not drift.
+fn shader(half: bool) -> String {
+ let (head, ty) = if half { ("enable f16;\n", "f16") } else { ("", "f32") };
+ let body = SHADER_COMMON.to_owned() + &conv_shader();
+ format!("{head}alias T = {ty};\n{body}")
+}
+
+/// The convolution kernel. Each invocation works out a 2x2 block of pixels
+/// and sixteen output channels (four groups) of each, so every weight
+/// matrix read serves four pixels and every input read up to sixteen
+/// channels under up to four taps. Written out with every tap and group
+/// unrolled: a local array indexed by a loop variable cannot stay in
+/// registers, and the first version, which did that, spilled to memory
+/// and ran at a third of the GPU.
+fn conv_shader() -> String {
+ conv_shader_for(BLOCK_W, BLOCK_H)
+}
+
+/// The block of pixels one invocation works out, across and down.
+const BLOCK_W: u32 = 2;
+const BLOCK_H: u32 = 2;
+
+fn conv_shader_for(bw: u32, bh: u32) -> String {
+ // The workgroup's patch of pixels, and the patch with its one-pixel
+ // rim that its taps read.
+ let (pw, ph) = (8 * bw, 8 * bh);
+ let (tw, th) = (pw + 2, ph + 2);
+ let tile = tw * th;
+ let mut code = format!(
+ "var<workgroup> tile: array<vec4<T>, {tile}>;\n\
+ var<workgroup> taps: array<mat4x4<T>, 36>;\n\n\
+ fn near(sx: i32, sy: i32, i: u32) -> vec4<T> {{\n\
+ \x20 if (sx < 0 || sy < 0 || sx >= i32(dims.width) || sy >= i32(dims.height)) {{\n\
+ \x20 return vec4<T>(0.0);\n\
+ \x20 }}\n\
+ \x20 return src[(u32(sy) * dims.width + u32(sx)) * dims.in_groups + i];\n\
+ }}\n\n\
+ @compute @workgroup_size(8, 8, 1)\n\
+ fn conv(@builtin(global_invocation_id) id: vec3<u32>,\n\
+ \x20 @builtin(local_invocation_id) local: vec3<u32>,\n\
+ \x20 @builtin(local_invocation_index) lid: u32,\n\
+ \x20 @builtin(workgroup_id) group: vec3<u32>) {{\n\
+ \x20 let x0 = id.x * {bw}u;\n\
+ \x20 let y0 = id.y * {bh}u;\n\
+ \x20 let o0 = group.z * 4u;\n\
+ \x20 let px = i32(group.x * {pw}u) - 1;\n\
+ \x20 let py = i32(group.y * {ph}u) - 1;\n\
+ \x20 let base = local.y * {bh}u * {tw}u + local.x * {bw}u;\n"
+ );
+ let pixels: Vec<(u32, u32)> = (0..bh).flat_map(|dy| (0..bw).map(move |dx| (dy, dx))).collect();
+ for g in 0..4 {
+ code += &format!(" let b{g} = bias[o0 + {g}u];\n");
+ for p in 0..pixels.len() {
+ code += &format!(" var a{p}_{g} = b{g};\n");
+ }
+ }
+ // Every invocation stays to the barriers, past the edge or not: the
+ // loads are shared, and only the writes are held to the picture.
+ code += &format!(
+ " for (var i = 0u; i < dims.in_groups; i++) {{\n\
+ \x20 workgroupBarrier();\n\
+ \x20 for (var k = lid; k < {tile}u; k += 64u) {{\n\
+ \x20 tile[k] = near(px + i32(k % {tw}u), py + i32(k / {tw}u), i);\n\
+ \x20 }}\n\
+ \x20 if (lid < 36u) {{\n\
+ \x20 taps[lid] = weights[((o0 + lid / 9u) * 9u + lid % 9u) * dims.in_groups + i];\n\
+ \x20 }}\n\
+ \x20 workgroupBarrier();\n"
+ );
+ for ny in 0..bh + 2 {
+ for nx in 0..bw + 2 {
+ code += &format!(" let n{ny}_{nx} = tile[base + {}u];\n", ny * tw + nx);
+ }
+ }
+ for g in 0..4u32 {
+ for tap in 0..9u32 {
+ code += &format!(" let m{g}_{tap} = taps[{}u];\n", g * 9 + tap);
+ }
+ // A group's nine taps summed in the stored type, then the partial
+ // carried into the f32 sum: one conversion a group rather than one
+ // a product, and only 36 terms ever added at sixteen bits.
+ for (p, (dy, dx)) in pixels.iter().enumerate() {
+ let terms: Vec<String> = (0..9u32)
+ .map(|tap| format!("m{g}_{tap} * n{}_{}", tap / 3 + dy, tap % 3 + dx))
+ .collect();
+ code += &format!(" a{p}_{g} += vec4<f32>({});\n", terms.join(" + "));
+ }
+ }
+ code += " }\n";
+ for (p, (dy, dx)) in pixels.iter().enumerate() {
+ code += &format!(
+ " if (x0 + {dx}u < dims.width && y0 + {dy}u < dims.height) {{\n\
+ \x20 let at = ((y0 + {dy}u) * dims.width + x0 + {dx}u) * dims.out_groups + o0;\n"
+ );
+ for g in 0..4 {
+ code += &format!(
+ " var v{g} = a{p}_{g};\n\
+ \x20 if (dims.prelu == 1u) {{ v{g} = select(v{g}, v{g} * slope[o0 + {g}u], v{g} < vec4<f32>(0.0)); }}\n\
+ \x20 dst[at + {g}u] = vec4<T>(v{g});\n"
+ );
+ }
+ code += " }\n";
+ }
+ code += "}\n";
+ code
+}
+
+/// A layer as the GPU holds it.
+struct GpuLayer {
+ out_groups: u32,
+ bind_group: wgpu::BindGroup,
+}
+
+/// The network on a device of its own, for windows of one size.
+pub struct Upscaler {
+ device: wgpu::Device,
+ queue: wgpu::Queue,
+ conv: wgpu::ComputePipeline,
+ shuffle: wgpu::ComputePipeline,
+ layers: Vec<GpuLayer>,
+ shuffle_group: wgpu::BindGroup,
+ picture: wgpu::Buffer,
+ planes: wgpu::Buffer,
+ staging: wgpu::Buffer,
+ side: u32,
+ scale: u32,
+ /// Activations and weights are stored as f16.
+ half: bool,
+}
+
+/// `values` as the shader stores them: little-endian f32, or f16.
+fn stored(values: &[f32], half: bool) -> Vec<u8> {
+ if half {
+ values
+ .iter()
+ .flat_map(|v| half::f16::from_f32(*v).to_le_bytes())
+ .collect()
+ } else {
+ values.iter().flat_map(|v| v.to_le_bytes()).collect()
+ }
+}
+
+/// Four channels to a group.
+fn groups(channels: usize) -> usize {
+ channels.div_ceil(4)
+}
+
+impl Upscaler {
+ /// The network `layers`, enlarging `scale` times, for square windows
+ /// `side` pixels across, on the first GPU that opens - `None` where
+ /// there is none, and the caller runs the model the slow way.
+ pub fn new(layers: &[ConvLayer], scale: u32, side: u32) -> Option<Upscaler> {
+ let instance = wgpu::Instance::default();
+ let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
+ power_preference: wgpu::PowerPreference::HighPerformance,
+ ..Default::default()
+ }))
+ .ok()?;
+ if adapter.get_info().device_type == wgpu::DeviceType::Cpu {
+ return None;
+ }
+ let half = adapter.features().contains(wgpu::Features::SHADER_F16);
+ let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {
+ label: Some("concat upscaler"),
+ required_features: if half {
+ wgpu::Features::SHADER_F16
+ } else {
+ wgpu::Features::empty()
+ },
+ required_limits: wgpu::Limits::default()
+ .or_worse_values_from(&adapter.limits())
+ .using_resolution(adapter.limits()),
+ ..Default::default()
+ }))
+ .ok()?;
+ Self::on(device, queue, layers, scale, side, half)
+ }
+
+ fn on(
+ device: wgpu::Device,
+ queue: wgpu::Queue,
+ layers: &[ConvLayer],
+ scale: u32,
+ side: u32,
+ half: bool,
+ ) -> Option<Upscaler> {
+ // The kernel works out four channel groups at a time, so a layer's
+ // outputs are padded up to a multiple of sixteen with zero weights,
+ // and the next layer reads the padding as inputs it weighs by zero.
+ let padded_groups = |channels: usize| groups(channels).div_ceil(4) * 4;
+ let widest = layers.iter().map(|l| padded_groups(l.out_channels)).max()?;
+ let pixels = (side * side) as u64;
+ // A vec4 of the stored type: sixteen bytes of f32, eight of f16.
+ let vec4 = if half { 8u64 } else { 16u64 };
+ let activation = |label| {
+ device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some(label),
+ size: pixels * widest as u64 * vec4,
+ usage: wgpu::BufferUsages::STORAGE,
+ mapped_at_creation: false,
+ })
+ };
+ let ping = activation("upscaler ping");
+ let pong = activation("upscaler pong");
+ let picture = device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("upscaler picture"),
+ size: pixels * vec4,
+ usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,
+ mapped_at_creation: false,
+ });
+ let planes_size = 3 * pixels * u64::from(scale * scale) * 4;
+ let planes = device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("upscaler planes"),
+ size: planes_size,
+ usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
+ mapped_at_creation: false,
+ });
+ let staging = device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("upscaler staging"),
+ size: planes_size,
+ usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
+ mapped_at_creation: false,
+ });
+
+ let descriptor = wgpu::ShaderModuleDescriptor {
+ label: Some("upscaler"),
+ source: wgpu::ShaderSource::Wgsl(shader(half).into()),
+ };
+ let module = device.create_shader_module(descriptor);
+ let entry = |binding: u32, kind: wgpu::BufferBindingType| wgpu::BindGroupLayoutEntry {
+ binding,
+ visibility: wgpu::ShaderStages::COMPUTE,
+ ty: wgpu::BindingType::Buffer {
+ ty: kind,
+ has_dynamic_offset: false,
+ min_binding_size: None,
+ },
+ count: None,
+ };
+ let read = wgpu::BufferBindingType::Storage { read_only: true };
+ let write = wgpu::BufferBindingType::Storage { read_only: false };
+ let conv_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+ label: Some("upscaler conv"),
+ entries: &[
+ entry(0, wgpu::BufferBindingType::Uniform),
+ entry(1, read),
+ entry(2, write),
+ entry(3, read),
+ entry(4, read),
+ entry(5, read),
+ ],
+ });
+ let shuffle_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+ label: Some("upscaler shuffle"),
+ entries: &[
+ entry(0, wgpu::BufferBindingType::Uniform),
+ entry(1, read),
+ entry(6, read),
+ entry(7, write),
+ ],
+ });
+ let pipeline = |layout: &wgpu::BindGroupLayout, entry_point: &str| {
+ device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
+ label: Some(entry_point),
+ layout: Some(&device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
+ label: Some(entry_point),
+ bind_group_layouts: &[Some(layout)],
+ immediate_size: 0,
+ })),
+ module: &module,
+ entry_point: Some(entry_point),
+ compilation_options: Default::default(),
+ cache: None,
+ })
+ };
+ let conv = pipeline(&conv_layout, "conv");
+ let shuffle = pipeline(&shuffle_layout, "shuffle");
+
+ let dims = |in_groups: usize, out_groups: usize, prelu: bool| {
+ let words = [
+ side,
+ side,
+ in_groups as u32,
+ out_groups as u32,
+ u32::from(prelu),
+ scale,
+ 0,
+ 0,
+ ];
+ device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("upscaler dims"),
+ contents: &words.iter().flat_map(|w| w.to_le_bytes()).collect::<Vec<u8>>(),
+ usage: wgpu::BufferUsages::UNIFORM,
+ })
+ };
+ let floats = |label, values: &[f32]| {
+ device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some(label),
+ contents: &values.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>(),
+ usage: wgpu::BufferUsages::STORAGE,
+ })
+ };
+
+ let mut gpu_layers = Vec::new();
+ for (index, layer) in layers.iter().enumerate() {
+ let in_groups = if index == 0 {
+ groups(layer.in_channels)
+ } else {
+ padded_groups(layers[index - 1].out_channels)
+ };
+ let out_groups = padded_groups(layer.out_channels);
+ // Each mat4x4 is four input channels into four outputs, kept
+ // column by column: column `j` is input `4i + j`'s weights to
+ // outputs `4o..4o + 4`, so `matrix * input` is the four sums.
+ let mut packed = vec![0.0f32; out_groups * 9 * in_groups * 16];
+ for o in 0..layer.out_channels {
+ for i in 0..layer.in_channels {
+ for tap in 0..9 {
+ let w = layer.weight[(o * layer.in_channels + i) * 9 + tap];
+ let matrix = ((o / 4) * 9 + tap) * in_groups + i / 4;
+ packed[matrix * 16 + (i % 4) * 4 + o % 4] = w;
+ }
+ }
+ }
+ let padded = |values: &[f32]| {
+ let mut out = values.to_vec();
+ out.resize(out_groups * 4, 0.0);
+ out
+ };
+ let slopes = padded(layer.prelu.as_deref().unwrap_or(&[]));
+ let (src, dst) = if index % 2 == 0 {
+ (if index == 0 { &picture } else { &pong }, &ping)
+ } else {
+ (&ping, &pong)
+ };
+ let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: Some("upscaler layer"),
+ layout: &conv_layout,
+ entries: &[
+ wgpu::BindGroupEntry {
+ binding: 0,
+ resource: dims(in_groups, out_groups, layer.prelu.is_some())
+ .as_entire_binding(),
+ },
+ wgpu::BindGroupEntry {
+ binding: 1,
+ resource: src.as_entire_binding(),
+ },
+ wgpu::BindGroupEntry {
+ binding: 2,
+ resource: dst.as_entire_binding(),
+ },
+ wgpu::BindGroupEntry {
+ binding: 3,
+ resource: device
+ .create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("upscaler weights"),
+ contents: &stored(&packed, half),
+ usage: wgpu::BufferUsages::STORAGE,
+ })
+ .as_entire_binding(),
+ },
+ wgpu::BindGroupEntry {
+ binding: 4,
+ resource: floats("upscaler bias", &padded(&layer.bias)).as_entire_binding(),
+ },
+ wgpu::BindGroupEntry {
+ binding: 5,
+ resource: floats("upscaler slope", &slopes).as_entire_binding(),
+ },
+ ],
+ });
+ gpu_layers.push(GpuLayer {
+ out_groups: out_groups as u32,
+ bind_group,
+ });
+ }
+ // The last layer wrote ping when there is an odd number of layers.
+ let last = if layers.len() % 2 == 1 { &ping } else { &pong };
+ let shuffle_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: Some("upscaler shuffle"),
+ layout: &shuffle_layout,
+ entries: &[
+ wgpu::BindGroupEntry {
+ binding: 0,
+ resource: dims(padded_groups(layers.last()?.out_channels), 0, false)
+ .as_entire_binding(),
+ },
+ wgpu::BindGroupEntry {
+ binding: 1,
+ resource: last.as_entire_binding(),
+ },
+ wgpu::BindGroupEntry {
+ binding: 6,
+ resource: picture.as_entire_binding(),
+ },
+ wgpu::BindGroupEntry {
+ binding: 7,
+ resource: planes.as_entire_binding(),
+ },
+ ],
+ });
+ Some(Upscaler {
+ device,
+ queue,
+ conv,
+ shuffle,
+ layers: gpu_layers,
+ shuffle_group,
+ picture,
+ planes,
+ half,
+ staging,
+ side,
+ scale,
+ })
+ }
+
+ /// Whether it works in half floats; see [`shader`].
+ pub fn half(&self) -> bool {
+ self.half
+ }
+
+ /// The side of the square windows this runs.
+ pub fn side(&self) -> u32 {
+ self.side
+ }
+
+ /// The network over one window: three planes of `side` by `side`,
+ /// values `0..=1`, in; three planes `scale` times the side out - the
+ /// ONNX model's input and answer, value for value.
+ pub fn run(&self, planes: &[f32]) -> Result<Vec<f32>, String> {
+ let side = self.side as usize;
+ let area = side * side;
+ if planes.len() != 3 * area {
+ return Err(format!(
+ "upscaler: {} values for a {side}-pixel window",
+ planes.len()
+ ));
+ }
+ let mut packed = vec![0.0f32; area * 4];
+ for at in 0..area {
+ for c in 0..3 {
+ packed[at * 4 + c] = planes[c * area + at];
+ }
+ }
+ self.queue.write_buffer(
+ &self.picture,
+ 0,
+ &stored(&packed, self.half),
+ );
+ let mut encoder = self
+ .device
+ .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("upscale") });
+ {
+ let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
+ label: Some("upscale"),
+ timestamp_writes: None,
+ });
+ pass.set_pipeline(&self.conv);
+ let across = self.side.div_ceil(8 * BLOCK_W);
+ let down = self.side.div_ceil(8 * BLOCK_H);
+ for layer in &self.layers {
+ pass.set_bind_group(0, &layer.bind_group, &[]);
+ pass.dispatch_workgroups(across, down, layer.out_groups.div_ceil(4));
+ }
+ pass.set_pipeline(&self.shuffle);
+ pass.set_bind_group(0, &self.shuffle_group, &[]);
+ let big = (self.side * self.scale).div_ceil(8);
+ pass.dispatch_workgroups(big, big, 1);
+ }
+ encoder.copy_buffer_to_buffer(&self.planes, 0, &self.staging, 0, self.planes.size());
+ self.queue.submit([encoder.finish()]);
+
+ let slice = self.staging.slice(..);
+ let (sent, mapped) = mpsc::channel();
+ slice.map_async(wgpu::MapMode::Read, move |result| {
+ let _ = sent.send(result);
+ });
+ self.device
+ .poll(wgpu::PollType::wait_indefinitely())
+ .map_err(|error| format!("upscaler: the device did not finish: {error}"))?;
+ mapped
+ .recv()
+ .map_err(|_| "upscaler: the device never answered".to_owned())?
+ .map_err(|error| format!("upscaler: the answer would not map: {error}"))?;
+ let answer = {
+ let data = slice.get_mapped_range().map_err(|e| format!("upscaler: {e}"))?;
+ data.chunks_exact(4)
+ .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
+ .collect()
+ };
+ self.staging.unmap();
+ Ok(answer)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use concat_vision::compact::{CompactNet, Conv};
+
+ /// A small network of made-up weights, run on the GPU and on the CPU
+ /// reference: the two agree to rounding, edges and shuffle included.
+ #[test]
+ fn the_gpu_answers_as_the_reference_does() {
+ let mut seed = 0x2545_f491_4f6c_dd1du64;
+ let mut next = || {
+ seed ^= seed << 13;
+ seed ^= seed >> 7;
+ seed ^= seed << 17;
+ (seed % 2001) as f32 / 1000.0 - 1.0
+ };
+ let mut conv = |inputs: usize, outputs: usize, prelu: bool| Conv {
+ in_channels: inputs,
+ out_channels: outputs,
+ weight: (0..outputs * inputs * 9).map(|_| next() * 0.3).collect(),
+ bias: (0..outputs).map(|_| next() * 0.1).collect(),
+ prelu: prelu.then(|| (0..outputs).map(|_| next() * 0.5).collect()),
+ };
+ let net = CompactNet {
+ layers: vec![conv(3, 8, true), conv(8, 8, true), conv(8, 12, false)],
+ scale: 2,
+ };
+ let side = 20usize;
+ let mut seed = 7u64;
+ let planes: Vec<f32> = (0..3 * side * side)
+ .map(|_| {
+ seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
+ (seed >> 40) as f32 / (1u64 << 24) as f32
+ })
+ .collect();
+ let layers: Vec<ConvLayer> = net
+ .layers
+ .iter()
+ .map(|l| ConvLayer {
+ in_channels: l.in_channels,
+ out_channels: l.out_channels,
+ weight: l.weight.clone(),
+ bias: l.bias.clone(),
+ prelu: l.prelu.clone(),
+ })
+ .collect();
+ let Some(gpu) = Upscaler::new(&layers, 2, side as u32) else {
+ eprintln!("no GPU here; skipped");
+ return;
+ };
+ let got = gpu.run(&planes).expect("runs");
+ let want = net.run_reference(&planes, side);
+ assert_eq!(got.len(), want.len());
+ let worst = got
+ .iter()
+ .zip(&want)
+ .map(|(a, b)| (a - b).abs())
+ .fold(0.0f32, f32::max);
+ // Half floats keep about three decimal digits.
+ let allowed = if gpu.half() { 2e-2 } else { 1e-4 };
+ assert!(worst < allowed, "the GPU is off by {worst}");
+ }
+
+ /// The real model, when this machine has fetched it, timed over a few
+ /// windows of the size Enhance cuts a frame into, and the 1080p frame
+ /// that comes to. `cargo test -p concat-render --release upscale_speed
+ /// -- --ignored --nocapture`.
+ #[test]
+ #[ignore = "a benchmark, and it needs the fetched model"]
+ fn upscale_speed() {
+ let Some(home) = std::env::var_os("HOME") else { return };
+ let support = std::path::Path::new(&home).join("Library/Application Support");
+ let Some(bytes) = ["org.standalones.concat", "app.concat.editor"]
+ .iter()
+ .map(|id| support.join(id).join("cutout-models/realesr-general-x4v3.onnx"))
+ .find_map(|file| std::fs::read(file).ok())
+ else {
+ eprintln!("no Enhance model on this machine");
+ return;
+ };
+ let net = CompactNet::read(&bytes).expect("reads");
+ let layers: Vec<ConvLayer> = net
+ .layers
+ .iter()
+ .map(|l| ConvLayer {
+ in_channels: l.in_channels,
+ out_channels: l.out_channels,
+ weight: l.weight.clone(),
+ bias: l.bias.clone(),
+ prelu: l.prelu.clone(),
+ })
+ .collect();
+ // The real model on a small window against the CPU reference, in
+ // eight-bit levels, which is what the file is written in.
+ let small = 40u32;
+ let gpu = Upscaler::new(&layers, net.scale as u32, small).expect("a GPU");
+ let mut seed = 11u64;
+ let planes: Vec<f32> = (0..3 * (small * small) as usize)
+ .map(|_| {
+ seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
+ (seed >> 40) as f32 / (1u64 << 24) as f32
+ })
+ .collect();
+ let got = gpu.run(&planes).expect("runs");
+ let want = net.run_reference(&planes, small as usize);
+ let worst = got
+ .iter()
+ .zip(&want)
+ .map(|(a, b)| (a.clamp(0.0, 1.0) - b.clamp(0.0, 1.0)).abs() * 255.0)
+ .fold(0.0f32, f32::max);
+ eprintln!("half floats: {}, worst error {worst:.2} levels of 255", gpu.half());
+ for side in [288u32, 544] {
+ let gpu = Upscaler::new(&layers, net.scale as u32, side).expect("a GPU");
+ let planes = vec![0.5f32; 3 * (side * side) as usize];
+ gpu.run(&planes).expect("warms up");
+ let runs = 5;
+ let started = std::time::Instant::now();
+ for _ in 0..runs {
+ gpu.run(&planes).expect("runs");
+ }
+ let each = started.elapsed().as_secs_f64() / runs as f64;
+ let core = f64::from(side - 32);
+ let windows = (1920.0 / core).ceil() * (1080.0 / core).ceil();
+ eprintln!(
+ "window {side}: {:.1} ms each, a 1080p frame in {:.0} windows = {:.2} s",
+ each * 1000.0,
+ windows,
+ each * windows
+ );
+ }
+ }
+}
diff --git a/src/crates/concat-vision/src/compact.rs b/src/crates/concat-vision/src/compact.rs
new file mode 100644
index 00000000..425e537f
--- /dev/null
+++ b/src/crates/concat-vision/src/compact.rs
@@ -0,0 +1,450 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer Abdullah and Concat contributors
+
+//! The Enhance model's weights, read out of its ONNX file for a GPU of our
+//! own to run.
+//!
+//! Real-ESRGAN's compact network (SRVGGNetCompact) is a plain stack: a 3x3
+//! convolution from the picture's three channels to 64, a PReLU, thirty-two
+//! more 3x3 convolutions of 64 to 64 each with its PReLU, one last 3x3 to
+//! `3 * scale * scale` channels, those channels rearranged into a picture
+//! `scale` times larger (PyTorch's pixel shuffle, ONNX's `DepthToSpace` in
+//! CRD order), and the input, enlarged by nearest neighbour, added to it.
+//! Nothing in it needs a runtime: [`CompactNet::read`] takes the layers'
+//! numbers out of the file and checks the graph is that shape, so a file
+//! that is anything else is refused rather than run wrong.
+//!
+//! ONNX is protocol buffers. The handful of fields read here are decoded by
+//! hand (see [`fields`]) rather than through a protobuf crate: a model
+//! file is the only one this reads, and only its graph's nodes and
+//! initializers.
+
+use std::collections::HashMap;
+
+/// One 3x3 convolution, padded by one, stride one, and the PReLU after it
+/// when there is one.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Conv {
+ /// Channels read.
+ pub in_channels: usize,
+ /// Channels written.
+ pub out_channels: usize,
+ /// `[out][in][3][3]`, row-major, as ONNX and PyTorch keep it.
+ pub weight: Vec<f32>,
+ /// One per output channel.
+ pub bias: Vec<f32>,
+ /// One slope per output channel, or `None` for the last layer, whose
+ /// answer goes to the pixel shuffle as it is.
+ pub prelu: Option<Vec<f32>>,
+}
+
+/// The whole network: its convolutions in order, and how much larger the
+/// picture comes out.
+#[derive(Clone, Debug, PartialEq)]
+pub struct CompactNet {
+ /// The convolutions, input first.
+ pub layers: Vec<Conv>,
+ /// How many times larger the picture comes out.
+ pub scale: usize,
+}
+
+impl CompactNet {
+ /// The network in the ONNX file `bytes`.
+ pub fn read(bytes: &[u8]) -> Result<CompactNet, String> {
+ let graph = fields(bytes)?
+ .into_iter()
+ .find_map(|(field, value)| (field == 7).then_some(value))
+ .and_then(Value::bytes)
+ .ok_or("enhance: the model file has no graph")?;
+ let mut tensors: HashMap<String, (Vec<usize>, Vec<f32>)> = HashMap::new();
+ let mut nodes = Vec::new();
+ for (field, value) in fields(graph)? {
+ match (field, value.bytes()) {
+ (1, Some(node)) => nodes.push(Node::read(node)?),
+ (5, Some(tensor)) => {
+ let (name, dims, data) = tensor_of(tensor)?;
+ tensors.insert(name, (dims, data));
+ }
+ _ => {}
+ }
+ }
+
+ let mut layers = Vec::new();
+ let mut scale = None;
+ for (at, node) in nodes.iter().enumerate() {
+ match node.op.as_str() {
+ "Conv" => {
+ if node.ints("kernel_shape") != Some(&[3, 3][..])
+ || node.ints("pads").is_some_and(|pads| pads != [1, 1, 1, 1])
+ || node.ints("strides").is_some_and(|s| s != [1, 1])
+ || node.ints("group").is_some_and(|g| g != [1])
+ {
+ return Err(format!(
+ "enhance: layer {} is not a plain 3x3 convolution",
+ layers.len()
+ ));
+ }
+ let tensor = |index: usize| {
+ node.inputs
+ .get(index)
+ .and_then(|name| tensors.get(name))
+ .ok_or_else(|| format!("enhance: layer {} has no weights", layers.len()))
+ };
+ let (dims, weight) = tensor(1)?;
+ let (_, bias) = tensor(2)?;
+ let [out_channels, in_channels, 3, 3] = dims[..] else {
+ return Err(format!("enhance: layer {} is shaped {dims:?}", layers.len()));
+ };
+ if bias.len() != out_channels {
+ return Err(format!("enhance: layer {}'s bias is the wrong length", layers.len()));
+ }
+ // The PReLU after it, when the next node is one.
+ let prelu = match nodes.get(at + 1) {
+ Some(next) if next.op == "PRelu" => {
+ let (_, slopes) = next
+ .inputs
+ .get(1)
+ .and_then(|name| tensors.get(name))
+ .ok_or("enhance: a PReLU has no slopes")?;
+ if slopes.len() != out_channels {
+ return Err("enhance: a PReLU's slopes are the wrong length".into());
+ }
+ Some(slopes.clone())
+ }
+ _ => None,
+ };
+ layers.push(Conv {
+ in_channels,
+ out_channels,
+ weight: weight.clone(),
+ bias: bias.clone(),
+ prelu,
+ });
+ }
+ "DepthToSpace" => {
+ if node.string("mode").is_some_and(|mode| mode != "CRD") {
+ return Err("enhance: the pixel shuffle is not in CRD order".into());
+ }
+ scale = node.ints("blocksize").and_then(|b| b.first().copied());
+ }
+ "Resize" => {
+ if node.string("mode").is_some_and(|mode| mode != "nearest") {
+ return Err("enhance: the input is not enlarged by nearest neighbour".into());
+ }
+ }
+ "PRelu" | "Constant" | "Add" => {}
+ other => return Err(format!("enhance: the model has a {other} this cannot run")),
+ }
+ }
+ let scale = scale.ok_or("enhance: the model has no pixel shuffle")? as usize;
+ // The chain must join up: three channels in, 64s through, and the
+ // last layer's channels exactly the shuffle's.
+ let chained = layers.first().is_some_and(|first| first.in_channels == 3)
+ && layers.windows(2).all(|pair| pair[0].out_channels == pair[1].in_channels)
+ && layers.last().is_some_and(|last| {
+ last.out_channels == 3 * scale * scale && last.prelu.is_none()
+ })
+ && layers[..layers.len() - 1].iter().all(|layer| layer.prelu.is_some());
+ if !chained {
+ return Err("enhance: the model's layers do not chain as a compact network".into());
+ }
+ Ok(CompactNet { layers, scale })
+ }
+
+ /// The network run on the CPU, plainly, over `planes` - three planes of
+ /// `side` by `side`, values `0..=1` - into three planes `scale` times
+ /// the side: the reference the GPU's answer is held to. Far too slow
+ /// for a frame; a small square is what a test gives it.
+ pub fn run_reference(&self, planes: &[f32], side: usize) -> Vec<f32> {
+ let area = side * side;
+ let mut x = planes.to_vec();
+ for layer in &self.layers {
+ let mut y = vec![0.0f32; layer.out_channels * area];
+ for o in 0..layer.out_channels {
+ for py in 0..side {
+ for px in 0..side {
+ let mut acc = layer.bias[o];
+ for i in 0..layer.in_channels {
+ for ky in 0..3 {
+ let sy = py as isize + ky as isize - 1;
+ if sy < 0 || sy >= side as isize {
+ continue;
+ }
+ for kx in 0..3 {
+ let sx = px as isize + kx as isize - 1;
+ if sx < 0 || sx >= side as isize {
+ continue;
+ }
+ let w = layer.weight[((o * layer.in_channels + i) * 3 + ky) * 3 + kx];
+ acc += w * x[i * area + sy as usize * side + sx as usize];
+ }
+ }
+ }
+ if let Some(slopes) = &layer.prelu
+ && acc < 0.0
+ {
+ acc *= slopes[o];
+ }
+ y[o * area + py * side + px] = acc;
+ }
+ }
+ }
+ x = y;
+ }
+ let s = self.scale;
+ let big = side * s;
+ let mut out = vec![0.0f32; 3 * big * big];
+ for c in 0..3 {
+ for oy in 0..big {
+ for ox in 0..big {
+ let (y, dy, xx, dx) = (oy / s, oy % s, ox / s, ox % s);
+ let shuffled = x[(c * s * s + dy * s + dx) * area + y * side + xx];
+ out[c * big * big + oy * big + ox] = shuffled + planes[c * area + y * side + xx];
+ }
+ }
+ }
+ out
+ }
+}
+
+/// One node of the graph: what it does, what it reads, and its attributes.
+struct Node {
+ op: String,
+ inputs: Vec<String>,
+ attributes: HashMap<String, Attribute>,
+}
+
+enum Attribute {
+ Ints(Vec<i64>),
+ Text(String),
+}
+
+impl Node {
+ fn read(bytes: &[u8]) -> Result<Node, String> {
+ let mut node = Node {
+ op: String::new(),
+ inputs: Vec::new(),
+ attributes: HashMap::new(),
+ };
+ for (field, value) in fields(bytes)? {
+ match (field, value) {
+ (1, Value::Bytes(name)) => node.inputs.push(text(name)),
+ (4, Value::Bytes(op)) => node.op = text(op),
+ (5, Value::Bytes(attribute)) => {
+ let mut name = String::new();
+ let mut ints = Vec::new();
+ let mut string = None;
+ for (field, value) in fields(attribute)? {
+ match (field, value) {
+ (1, Value::Bytes(n)) => name = text(n),
+ // `i`, and `ints` written one at a time.
+ (3 | 8, Value::Varint(v)) => ints.push(v as i64),
+ // `ints` packed.
+ (8, Value::Bytes(packed)) => {
+ let mut at = 0;
+ while at < packed.len() {
+ let (v, next) = varint(packed, at)?;
+ ints.push(v as i64);
+ at = next;
+ }
+ }
+ (4, Value::Bytes(s)) => string = Some(text(s)),
+ _ => {}
+ }
+ }
+ let value = match string {
+ Some(s) => Attribute::Text(s),
+ None => Attribute::Ints(ints),
+ };
+ node.attributes.insert(name, value);
+ }
+ _ => {}
+ }
+ }
+ Ok(node)
+ }
+
+ fn ints(&self, name: &str) -> Option<&[i64]> {
+ match self.attributes.get(name)? {
+ Attribute::Ints(ints) => Some(ints),
+ Attribute::Text(_) => None,
+ }
+ }
+
+ fn string(&self, name: &str) -> Option<&str> {
+ match self.attributes.get(name)? {
+ Attribute::Text(s) => Some(s),
+ Attribute::Ints(_) => None,
+ }
+ }
+}
+
+/// A float tensor's name, shape and numbers, from a `TensorProto`: kept as
+/// `raw_data` little-endian, or as packed `float_data`.
+fn tensor_of(bytes: &[u8]) -> Result<(String, Vec<usize>, Vec<f32>), String> {
+ let mut name = String::new();
+ let mut dims = Vec::new();
+ let mut data = Vec::new();
+ let mut kind = 1;
+ for (field, value) in fields(bytes)? {
+ match (field, value) {
+ (1, Value::Varint(d)) => dims.push(d as usize),
+ (1, Value::Bytes(packed)) => {
+ let mut at = 0;
+ while at < packed.len() {
+ let (d, next) = varint(packed, at)?;
+ dims.push(d as usize);
+ at = next;
+ }
+ }
+ (2, Value::Varint(k)) => kind = k,
+ (8, Value::Bytes(n)) => name = text(n),
+ (9, Value::Bytes(raw)) | (4, Value::Bytes(raw)) => data.extend(
+ raw.chunks_exact(4)
+ .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])),
+ ),
+ _ => {}
+ }
+ }
+ if kind != 1 && !data.is_empty() {
+ return Err(format!("enhance: {name} is not 32-bit floats"));
+ }
+ let count: usize = dims.iter().product();
+ if !data.is_empty() && data.len() != count.max(1) {
+ return Err(format!("enhance: {name} holds {} numbers, not {count}", data.len()));
+ }
+ Ok((name, dims, data))
+}
+
+/// A protobuf field's value: the two wire types a model file's fields use,
+/// and the fixed-width ones skipped over.
+enum Value<'a> {
+ Varint(u64),
+ Bytes(&'a [u8]),
+ Fixed,
+}
+
+impl<'a> Value<'a> {
+ fn bytes(self) -> Option<&'a [u8]> {
+ match self {
+ Value::Bytes(bytes) => Some(bytes),
+ _ => None,
+ }
+ }
+}
+
+/// A protobuf message's fields, in order, as `(number, value)`.
+fn fields(bytes: &[u8]) -> Result<Vec<(u64, Value<'_>)>, String> {
+ let mut out = Vec::new();
+ let mut at = 0;
+ while at < bytes.len() {
+ let (key, next) = varint(bytes, at)?;
+ at = next;
+ let value = match key & 7 {
+ 0 => {
+ let (v, next) = varint(bytes, at)?;
+ at = next;
+ Value::Varint(v)
+ }
+ 1 => {
+ at += 8;
+ Value::Fixed
+ }
+ 2 => {
+ let (length, next) = varint(bytes, at)?;
+ let end = next
+ .checked_add(length as usize)
+ .filter(|end| *end <= bytes.len())
+ .ok_or("enhance: the model file is cut short")?;
+ at = end;
+ Value::Bytes(&bytes[next..end])
+ }
+ 5 => {
+ at += 4;
+ Value::Fixed
+ }
+ other => return Err(format!("enhance: the model file has wire type {other}")),
+ };
+ if at > bytes.len() {
+ return Err("enhance: the model file is cut short".into());
+ }
+ out.push((key >> 3, value));
+ }
+ Ok(out)
+}
+
+fn varint(bytes: &[u8], mut at: usize) -> Result<(u64, usize), String> {
+ let mut value = 0u64;
+ for shift in (0..64).step_by(7) {
+ let byte = *bytes.get(at).ok_or("enhance: the model file is cut short")?;
+ at += 1;
+ value |= u64::from(byte & 0x7f) << shift;
+ if byte < 0x80 {
+ return Ok((value, at));
+ }
+ }
+ Err("enhance: the model file has a number too long to be one".into())
+}
+
+fn text(bytes: &[u8]) -> String {
+ String::from_utf8_lossy(bytes).into_owned()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// The real model, when this machine has fetched it: read whole, the
+ /// shape Real-ESRGAN's compact network has.
+ #[test]
+ fn the_fetched_model_reads_as_a_compact_network() {
+ let Some(file) = fetched_model() else {
+ eprintln!("no Enhance model on this machine; skipped");
+ return;
+ };
+ let net = CompactNet::read(&std::fs::read(file).expect("reads")).expect("parses");
+ assert_eq!(net.scale, 4);
+ assert_eq!(net.layers.len(), 34);
+ assert_eq!(net.layers[0].in_channels, 3);
+ assert_eq!(net.layers[33].out_channels, 48);
+ let params: usize = net
+ .layers
+ .iter()
+ .map(|l| l.weight.len() + l.bias.len() + l.prelu.as_ref().map_or(0, Vec::len))
+ .sum();
+ assert_eq!(params, 1_213_296);
+ }
+
+ /// Anything but the shape it knows is refused, not run wrong.
+ #[test]
+ fn a_file_that_is_not_a_model_is_refused() {
+ assert!(CompactNet::read(b"not a model").is_err());
+ assert!(CompactNet::read(&[]).is_err());
+ }
+
+ /// Where the app keeps the model on this machine, if it has it: under
+ /// the app's identifier, or the one before it on a machine whose app
+ /// has not started since the rename.
+ pub(crate) fn fetched_model() -> Option<std::path::PathBuf> {
+ let bases: Vec<std::path::PathBuf> = if cfg!(target_os = "macos") {
+ std::env::var_os("HOME")
+ .map(|home| std::path::PathBuf::from(home).join("Library/Application Support"))
+ .into_iter()
+ .collect()
+ } else if cfg!(windows) {
+ std::env::var_os("LOCALAPPDATA").map(std::path::PathBuf::from).into_iter().collect()
+ } else {
+ std::env::var_os("HOME")
+ .map(|home| std::path::PathBuf::from(home).join(".local/share"))
+ .into_iter()
+ .collect()
+ };
+ bases
+ .iter()
+ .flat_map(|base| {
+ ["org.standalones.concat", "app.concat.editor"]
+ .map(|id| base.join(id).join("cutout-models").join("realesr-general-x4v3.onnx"))
+ })
+ .find(|file| file.is_file())
+ }
+}
diff --git a/src/crates/concat-vision/src/enhance.rs b/src/crates/concat-vision/src/enhance.rs
index 9e8a6869..59033c9e 100644
--- a/src/crates/concat-vision/src/enhance.rs
+++ b/src/crates/concat-vision/src/enhance.rs
@@ -347,19 +347,56 @@ pub fn downscale(frame: &Frame, by: u32) -> Frame {
Frame::from_rgba(ow as u32, oh as u32, pixels).expect("sides and bytes agree")
}
-/// The model, loaded once and run tile by tile.
+/// What runs the model over one window: the [`WINDOW`]-square planes
+/// [`window_planes`] cuts, in; the planes [`paste`] takes, [`SCALE`] times
+/// the side, out. ONNX Runtime is one, on the CPU almost everywhere; the
+/// host's GPU upscaler, the same network in compute shaders of our own, is
+/// the other, and the one used wherever a GPU opens.
+pub trait WindowRunner: Send {
+ /// The model's answer for one window.
+ fn run(&mut self, planes: Vec<f32>) -> Result<Vec<f32>, String>;
+}
+
+/// The model through ONNX Runtime.
+#[cfg(feature = "infer")]
+struct Runtime(crate::runtime::Model);
+
#[cfg(feature = "infer")]
+impl WindowRunner for Runtime {
+ fn run(&mut self, planes: Vec<f32>) -> Result<Vec<f32>, String> {
+ let side = WINDOW as usize;
+ let mut answer = self.0.run(
+ vec![crate::runtime::Input {
+ name: "input",
+ dims: vec![1, 3, side, side],
+ data: planes.into(),
+ }],
+ &["output"],
+ )?;
+ Ok(std::mem::take(&mut answer[0].data))
+ }
+}
+
+/// The model, loaded once and run tile by tile.
pub struct Enhancer {
- model: std::sync::Mutex<crate::runtime::Model>,
+ model: std::sync::Mutex<Box<dyn WindowRunner>>,
}
-#[cfg(feature = "infer")]
impl Enhancer {
- /// Loads the model from its file; see [`crate::models::ModelId::Enhance`].
+ /// Loads the model from its file into ONNX Runtime; see
+ /// [`crate::models::ModelId::Enhance`].
+ #[cfg(feature = "infer")]
pub fn load(file: &std::path::Path) -> Result<Enhancer, String> {
- Ok(Enhancer {
- model: std::sync::Mutex::new(crate::runtime::Model::from_file(file)?),
- })
+ Ok(Enhancer::with(Box::new(Runtime(
+ crate::runtime::Model::from_file(file)?,
+ ))))
+ }
+
+ /// The model run by `runner`.
+ pub fn with(runner: Box<dyn WindowRunner>) -> Enhancer {
+ Enhancer {
+ model: std::sync::Mutex::new(runner),
+ }
}
/// `frame` restored at [`SCALE`] times its size.
@@ -386,7 +423,6 @@ impl Enhancer {
return Err("enhance: an empty picture".to_owned());
}
let mut out = Frame::black(region.width * SCALE, region.height * SCALE);
- let side = WINDOW as usize;
let mut model = self.model.lock().map_err(|_| "enhance: model poisoned")?;
let mut times = Vec::new();
for tile in tiles(region.width, region.height) {
@@ -397,16 +433,9 @@ impl Enhancer {
};
let planes = window_planes(frame, placed);
let started = std::time::Instant::now();
- let answer = model.run(
- vec![crate::runtime::Input {
- name: "input",
- dims: vec![1, 3, side, side],
- data: planes.into(),
- }],
- &["output"],
- )?;
+ let answer = model.run(planes)?;
times.push(started.elapsed());
- paste(&mut out, tile, &answer[0].data)?;
+ paste(&mut out, tile, &answer)?;
}
Ok((out, times))
}
diff --git a/src/crates/concat-vision/src/lib.rs b/src/crates/concat-vision/src/lib.rs
index 485626ca..45689077 100644
--- a/src/crates/concat-vision/src/lib.rs
+++ b/src/crates/concat-vision/src/lib.rs
@@ -47,6 +47,7 @@
pub mod apply;
#[cfg(feature = "infer")]
pub mod brush;
+pub mod compact;
pub mod enhance;
pub mod mask;
pub mod models;