Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.
TypeScript ai-photo-editorcomfyui-alternativegenerative-fillimage-editingimage-generationinpaintinglinuxlocal-aimacosmcpmcp-servernano-banana-alternativeofflinephoto-editorprivacyqwen-imageqwen-image-2-1stable-diffusion-cppvulkanwindows
Mac engine: the VAE encoder, checked against diffusers
Gabriele Lozuponedfd6c5011a13
diff --git a/engine-mac/README.md b/engine-mac/README.mdindex 8cd2365..c82241d 100644--- a/engine-mac/README.md+++ b/engine-mac/README.md@@ -44,6 +44,7 @@ Windows and Linux.| `AttentionSplit.swift`, `CPUAttention.swift` | the target's attention cut by heads: GPU, CPU (AMX + cores), optionally the Neural Engine || `AttentionProbe.swift` | `probe-attention`: the target's attention on each unit alone || `QwenVAE.swift`, `Safetensors.swift` | the VAE decoder as one MPSGraph, from Kokon's VAE file (Wan 2.2 names) |+| `QwenVAEEncoder.swift` | the VAE encoder (edits' reference images), checked against diffusers || `QwenTokenizer.swift` | Qwen's byte-level BPE tokenizer, read from the encoder GGUF's metadata || `QwenTextEncoder.swift`, `Kernels/text.metal` | the text encoder: Qwen3-VL-8B's text decoder on the GPU, diffusers' prompt template || `QwenPipeline.swift` | text to image: noise, Turbo's 8 flow-Euler steps (`planner::TURBO_SIGMAS`), VAE, PNG; models kept loaded for the server |@@ -378,5 +379,13 @@ them on the Neural Engine (optimization level 1, its default); at level 0 they sgain little: all five 2.43 s against 2.77 at 1024², for 2.7e-3 of diffusers instead of 1.1e-5 (up to ~19 levels off).The VAE stays fp32 (2026-10-25).+**VAE encoder** (`QwenVAEEncoder.swift`; for the reference images of edits): the down blocks as single operations as+the decoder's up blocks (its convolution graphs, one with stride 2 and the zero row and column after the input; the+AvgDown shortcut as a kernel, with the zero frame the single image is padded by counted in its average), the mid block+and the head as one graph at 1/16 of the image, then the posterior's mode normalized as the edit pipeline does.+`check-vae-encoder` against `tools/reference_vae_encoder.py` (diffusers fp32, a smooth random RGBA image, 256 × 192,+texture-fix VAE, M2 Pro, 2026-10-25): the latent within 6.5e-6. 1024²: 0.59 s an encode, peak 2.4 GB, 0.87 GB kept after+(its weights); in its own autorelease pool (without one, a loop on a thread without a pool kept 1.5 GB per encode).+Credits: the Neural Engine program technique and the GPU + AMX + Neural Engine split come from[Siliconed](https://github.com/gwenn-ha-dev/Siliconed) (Apache-2.0), written again here.diff --git a/engine-mac/Sources/KokonEngine/Checks.swift b/engine-mac/Sources/KokonEngine/Checks.swiftindex 9be0f87..58c8e0b 100644--- a/engine-mac/Sources/KokonEngine/Checks.swift+++ b/engine-mac/Sources/KokonEngine/Checks.swift@@ -384,3 +384,36 @@ extension Checks {print(String(format: "int8 weights, fp16 activations %d×%d×%d: %.2f ms (%.1f TFLOP/s), relative error %.2e", rows, k, n, tw * 1e3, flops / tw / 1e12, ew))}}++extension Checks {+ /// The VAE encoder against `tools/reference_vae_encoder.py` (diffusers fp32): the normalized latent of its image;+ /// then the time and memory of an encode at `timeSize` (a smooth random image, as the reference's).+ public static func vaeEncoder(path: String, reference directory: String, timeSize: (width: Int, height: Int) = (1024, 1024)) throws {+ let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory)+ .appendingPathComponent("meta.json"))) as! [String: Any]+ let size = meta["size"] as! [Int]+ let (w, h) = (size[0], size[1])+ let context = try MetalContext()+ let vae = try QwenVAE(path: path, context: context)+ let image = try reference(directory, "image"), want = try reference(directory, "latent")+ var started = DispatchTime.now().uptimeNanoseconds+ let got = try vae.encode(image, height: h, width: w)+ var seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9+ let r = got.withUnsafeBufferPointer { compare($0.baseAddress!, want) }+ print(String(format: "VAE encoder %d×%d → %d×%d latent: rel %.2e, max |Δ| %.3g%@ (%.2f s with the graphs' build)",+ w, h, w / 16, h / 16, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE", seconds))+ let (tw, th) = timeSize+ let random = (0..<(4 * th * tw)).map { i -> Float in+ let (y, x) = ((i / tw) % th, i % tw)+ return sinf(Float(x) * 0.013 + Float(i / (tw * th))) * cosf(Float(y) * 0.021) * 0.9+ }+ for run in 1...3 {+ started = DispatchTime.now().uptimeNanoseconds+ _ = try vae.encode(random, height: th, width: tw)+ seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9+ Thread.sleep(forTimeInterval: 1.5) // buffers the GPU used leave the footprint up to a second after their release+ let m = MemoryUsage.now()+ print(String(format: "encode %d at %d×%d: %.2f s, footprint after %.2f GB (peak %.2f GB)", run, tw, th, seconds, m.footprint, m.footprintPeak))+ }+ }+}diff --git a/engine-mac/Sources/KokonEngine/Kernels/vae.metal b/engine-mac/Sources/KokonEngine/Kernels/vae.metalindex 5e2d0d1..e483980 100644--- a/engine-mac/Sources/KokonEngine/Kernels/vae.metal+++ b/engine-mac/Sources/KokonEngine/Kernels/vae.metal@@ -50,3 +50,23 @@ kernel void vae_dup_up_add(device float *out [[buffer(0)]], device const float *uint q = ((gid.z * p.ft + p.ft - 1) * 2 + (gid.y & 1)) * 2 + (gid.x & 1);out[(ulong(gid.z) * 2 * p.h + gid.y) * 2 * p.w + gid.x] += x[(ulong(q / repeats) * p.h + gid.y / 2) * p.w + gid.x / 2];}++struct AvgDown { uint inC, outC, ft, fs, h, w; };++/// QwenImage21AvgDown3D on a first chunk, added to the down block's output `out` [outC, h, w]: the input [inC, fs·h,+/// fs·w] seen as inC · ft · fs² channels ((c · ft + t) · fs + i) · fs + j, of which `group` consecutive ones average+/// into each output channel. With ft = 2 the single frame is padded by a zero frame before it (t = 0), which counts in+/// the average.+kernel void vae_avg_down_add(device float *out [[buffer(0)]], device const float *x [[buffer(1)]],+ constant AvgDown &p [[buffer(2)]], uint3 gid [[thread_position_in_grid]]) {+ if (gid.x >= p.w || gid.y >= p.h || gid.z >= p.outC) return;+ uint group = p.inC * p.ft * p.fs * p.fs / p.outC;+ float sum = 0;+ for (uint k = 0; k < group; k++) {+ uint index = gid.z * group + k;+ uint j = index % p.fs, i = (index / p.fs) % p.fs, t = (index / (p.fs * p.fs)) % p.ft, c = index / (p.fs * p.fs * p.ft);+ if (t + 1 < p.ft) continue; // the padded frame+ sum += x[(ulong(c) * p.fs * p.h + gid.y * p.fs + i) * p.fs * p.w + gid.x * p.fs + j];+ }+ out[(ulong(gid.z) * p.h + gid.y) * p.w + gid.x] += sum / float(group);+}diff --git a/engine-mac/Sources/KokonEngine/QwenVAE.swift b/engine-mac/Sources/KokonEngine/QwenVAE.swiftindex c4d83dc..3215d3c 100644--- a/engine-mac/Sources/KokonEngine/QwenVAE.swift+++ b/engine-mac/Sources/KokonEngine/QwenVAE.swift@@ -46,9 +46,9 @@ public final class QwenVAE {/// Engine; at level 0 they stay on the GPU and are right, but save little: all five in fp16 2.43 s against 2.77 at/// 1024², the first two 2.65, for 2.2–2.7e-3 of diffusers instead of 1.1e-5 (M2 Pro, `check-vae`, 2026-10-25). The/// VAE stays fp32.- private var executables: [String: MPSGraphExecutable] = [:]+ var executables: [String: MPSGraphExecutable] = [:]private var feedOrders: [String: [Int]] = [:]- private let lock = NSLock()+ let lock = NSLock()public init(path: String, context: MetalContext) throws {file = try SafetensorsFile(path: path)@@ -163,7 +163,7 @@ public final class QwenVAE {func give(_ buffer: MTLBuffer) { free.append(buffer) }}- private func data(_ buffer: MTLBuffer, _ shape: [Int]) -> MPSGraphTensorData {+ func data(_ buffer: MTLBuffer, _ shape: [Int]) -> MPSGraphTensorData {MPSGraphTensorData(buffer, shape: shape.map { NSNumber(value: $0) }, dataType: .float32)}@@ -255,7 +255,7 @@ public final class QwenVAE {/// Weights as fp32 buffers, each read from the file once.private var weightBuffers: [String: MTLBuffer] = [:]- private func weight(_ name: String) throws -> MTLBuffer {+ func weight(_ name: String) throws -> MTLBuffer {try lock.withLock {if let b = weightBuffers[name] { return b }let values = try file.floats(name)@@ -266,7 +266,7 @@ public final class QwenVAE {}}- private func dispatch(_ kernel: String, _ buffers: [MTLBuffer], _ parameters: [UInt32], grid: MTLSize,+ func dispatch(_ kernel: String, _ buffers: [MTLBuffer], _ parameters: [UInt32], grid: MTLSize,commands: MPSCommandBuffer) throws {let pipeline = try context.pipeline(kernel)var p = parameters@@ -280,7 +280,7 @@ public final class QwenVAE {}/// QwenImage21RMS_norm then SiLU, per pixel over `channels` planes (`x` and `out` may be the same).- private func normSilu(_ x: MTLBuffer, to out: MTLBuffer, gamma: String, channels: Int, pixels: Int, commands: MPSCommandBuffer) throws {+ func normSilu(_ x: MTLBuffer, to out: MTLBuffer, gamma: String, channels: Int, pixels: Int, commands: MPSCommandBuffer) throws {try dispatch("vae_norm_silu", [x, out, try weight(gamma)], [UInt32(channels), UInt32(pixels)],grid: MTLSize(width: pixels, height: 1, depth: 1), commands: commands)}@@ -294,12 +294,14 @@ public final class QwenVAE {}/// `name` (a k × k convolution and its bias) on `[inC, h, w]`, plus `skip` (the residual block's input, through- /// its 1×1 shortcut when named), clamped to [−1, 1] for the head.- private func convolution(_ x: MTLBuffer, _ name: String, inC: Int, outC: Int, k: Int, height h: Int, width w: Int,- to out: MTLBuffer, skip: (buffer: MTLBuffer, shortcut: String?, channels: Int)? = nil,- clamp: Bool = false, commands: MPSCommandBuffer) throws {+ /// its 1×1 shortcut when named), clamped to [−1, 1] for the head. `downsample`: stride 2 with one zero row and+ /// column after the input (the encoder's `ZeroPad2d((0, 1, 0, 1))` then `Conv2d(stride=2)`), `[outC, h/2, w/2]`.+ func convolution(_ x: MTLBuffer, _ name: String, inC: Int, outC: Int, k: Int, height h: Int, width w: Int,+ to out: MTLBuffer, skip: (buffer: MTLBuffer, shortcut: String?, channels: Int)? = nil,+ clamp: Bool = false, downsample: Bool = false, commands: MPSCommandBuffer) throws {let kind = skip == nil ? "" : skip!.shortcut == nil ? "+x" : "+1x1(\(skip!.channels))"- let key = "conv \(inC)→\(outC) \(k)×\(k)\(kind)\(clamp ? " clamp" : "") \(h)×\(w)"+ let key = "conv \(inC)→\(outC) \(k)×\(k)\(kind)\(clamp ? " clamp" : "")\(downsample ? " /2" : "") \(h)×\(w)"+ let (oh, ow) = downsample ? (h / 2, w / 2) : (h, w)let (executable, order) = try lock.withLock { () throws -> (MPSGraphExecutable, [Int]) inif let e = executables[key], let order = feedOrders[key] { return (e, order) }let g = MPSGraph()@@ -312,9 +314,10 @@ public final class QwenVAE {return t}func conv(_ x: MPSGraphTensor, _ inC: Int, _ k: Int) -> MPSGraphTensor {- let d = MPSGraphConvolution2DOpDescriptor(strideInX: 1, strideInY: 1, dilationRateInX: 1, dilationRateInY: 1,- groups: 1, paddingLeft: k / 2, paddingRight: k / 2, paddingTop: k / 2,- paddingBottom: k / 2, paddingStyle: .explicit, dataLayout: .NCHW,+ let (stride, before, after) = downsample ? (2, 0, 1) : (1, k / 2, k / 2)+ let d = MPSGraphConvolution2DOpDescriptor(strideInX: stride, strideInY: stride, dilationRateInX: 1, dilationRateInY: 1,+ groups: 1, paddingLeft: before, paddingRight: after, paddingTop: before,+ paddingBottom: after, paddingStyle: .explicit, dataLayout: .NCHW,weightsLayout: .OIHW)!let y = g.convolution2D(x, weights: input([outC, inC, k, k]), descriptor: d, name: nil)return g.addition(y, input([1, outC, 1, 1]), name: nil)@@ -323,7 +326,7 @@ public final class QwenVAE {let x = input([1, inC, h, w])var y = conv(x, inC, k)if let skip {- let s = input([1, skip.channels, h, w])+ let s = input([1, skip.channels, oh, ow])y = g.addition(y, skip.shortcut == nil ? s : conv(s, skip.channels, 1), name: nil)}if clamp {@@ -342,13 +345,13 @@ public final class QwenVAE {var inputs = [data(x, [1, inC, h, w]), data(try weight(name + ".weight"), [outC, inC, k, k]),data(try weight(name + ".bias"), [1, outC, 1, 1])]if let skip {- inputs.append(data(skip.buffer, [1, skip.channels, h, w]))+ inputs.append(data(skip.buffer, [1, skip.channels, oh, ow]))if let shortcut = skip.shortcut {inputs += [data(try weight(shortcut + ".weight"), [outC, skip.channels, 1, 1]), data(try weight(shortcut + ".bias"), [1, outC, 1, 1])]}}_ = executable.encode(to: commands, inputs: order.map { inputs[$0] },- results: [data(out, [1, outC, h, w])], executionDescriptor: nil)+ results: [data(out, [1, outC, oh, ow])], executionDescriptor: nil)}/// Tiles of side `tile` over `n` pixels: each one's start, and the share it gives (`core`, `length`) at least@@ -384,105 +387,113 @@ public final class QwenVAE {/// The graph of `stage` for an `h` × `w` input.private func build(_ stage: Stage, height h: Int, width w: Int) throws -> (graph: MPSGraph, input: MPSGraphTensor, output: MPSGraphTensor) {let g = MPSGraph()- func n(_ values: [Int]) -> [NSNumber] { values.map { NSNumber(value: $0) } }- func constant(_ values: [Float], _ shape: [Int]) -> MPSGraphTensor {- g.constant(values.withUnsafeBytes { Data($0) }, shape: n(shape), dataType: .float32)- }- func weight(_ name: String, _ shape: [Int]) throws -> MPSGraphTensor {- let values = try file.floats(name)- guard values.count == shape.reduce(1, *) else {- throw GGUFFile.Failure.unsupported("\(name): \(values.count) values, \(shape) expected")- }- return constant(values, shape)- }- /// Conv (OIHW weights, the file's 5D `[O, I, 1, k, k]` squeezed) + bias, padding k / 2.- func conv(_ x: MPSGraphTensor, _ name: String, _ inC: Int, _ outC: Int, _ k: Int) throws -> MPSGraphTensor {- let pad = k / 2- let d = MPSGraphConvolution2DOpDescriptor(strideInX: 1, strideInY: 1, dilationRateInX: 1, dilationRateInY: 1,- groups: 1, paddingLeft: pad, paddingRight: pad, paddingTop: pad,- paddingBottom: pad, paddingStyle: .explicit, dataLayout: .NCHW,- weightsLayout: .OIHW)!- let y = g.convolution2D(x, weights: try weight(name + ".weight", [outC, inC, k, k]), descriptor: d, name: nil)- return g.addition(y, try weight(name + ".bias", [1, outC, 1, 1]), name: nil)- }- /// QwenImage21RMS_norm: F.normalize over channels (x / max(‖x‖, 1e-12)) · √C · γ.- func norm(_ x: MPSGraphTensor, _ name: String, _ c: Int) throws -> MPSGraphTensor {- let length = g.squareRoot(with: g.reductionSum(with: g.square(with: x, name: nil), axes: [1], name: nil), name: nil)- let normalized = g.division(x, g.maximum(length, g.constant(1e-12, dataType: .float32), name: nil), name: nil)- let scaled = g.multiplication(normalized, g.constant(Double(Float(c).squareRoot()), dataType: .float32), name: nil)- return g.multiplication(scaled, try weight(name, [1, c, 1, 1]), name: nil)- }- func silu(_ x: MPSGraphTensor) -> MPSGraphTensor { g.multiplication(x, g.sigmoid(with: x, name: nil), name: nil) }- /// QwenImage21ResidualBlock (Wan `residual.{0,2,3,6}`, `shortcut`).- func residual(_ x: MPSGraphTensor, _ name: String, _ inC: Int, _ outC: Int) throws -> MPSGraphTensor {- let skip = inC == outC ? x : try conv(x, name + ".shortcut", inC, outC, 1)- var y = silu(try norm(x, name + ".residual.0.gamma", inC))- y = try conv(y, name + ".residual.2", inC, outC, 3)- y = silu(try norm(y, name + ".residual.3.gamma", outC))- y = try conv(y, name + ".residual.6", outC, outC, 3)- return g.addition(y, skip, name: nil)- }- /// QwenImage21AttentionBlock: one head over all `hw` pixels.- func attention(_ x: MPSGraphTensor, _ name: String, _ c: Int, _ hw: Int, _ hh: Int, _ ww: Int) throws -> MPSGraphTensor {- let qkv = try conv(try norm(x, name + ".norm.gamma", c), name + ".to_qkv", c, 3 * c, 1) // [1, 3C, H, W]- let rows = g.transpose(g.reshape(qkv, shape: n([3 * c, hw]), name: nil), permutation: n([1, 0]), name: nil)- func part(_ i: Int) -> MPSGraphTensor {- g.reshape(g.sliceTensor(rows, dimension: 1, start: i * c, length: c, name: nil), shape: n([1, 1, hw, c]), name: nil)- }- let o = g.scaledDotProductAttention(query: part(0), key: part(1), value: part(2), mask: nil,- scale: 1 / Float(c).squareRoot(), name: nil)- let back = g.reshape(g.transpose(g.reshape(o, shape: n([hw, c]), name: nil), permutation: n([1, 0]), name: nil),- shape: n([1, c, hh, ww]), name: nil)- return g.addition(try conv(back, name + ".proj", c, c, 1), x, name: nil)- }- /// QwenImage21DupUp3D on a first chunk: each output channel's 2×2 pixels from the input channels it repeats.- func dupUp(_ x: MPSGraphTensor, _ inC: Int, _ outC: Int, _ ft: Int, _ hh: Int, _ ww: Int) -> MPSGraphTensor {- let repeats = outC * ft * 4 / inC- var y = g.broadcast(g.reshape(x, shape: n([inC, 1, hh, ww]), name: nil), shape: n([inC, repeats, hh, ww]), name: nil)- y = g.reshape(y, shape: n([outC, ft, 2, 2, hh, ww]), name: nil)- y = g.sliceTensor(y, dimension: 1, start: ft - 1, length: 1, name: nil)- y = g.reshape(y, shape: n([outC, 2, 2, hh, ww]), name: nil)- y = g.transpose(y, permutation: n([0, 3, 1, 4, 2]), name: nil) // [C, H, 2, W, 2]- return g.reshape(y, shape: n([1, outC, 2 * hh, 2 * ww]), name: nil)- }-+ let parts = VAEGraph(g: g, file: file)let input: MPSGraphTensorvar x: MPSGraphTensorlet dims = QwenVAE.dimsvar (hh, ww) = (h, w)if stage.first >= 0 {- input = g.placeholder(shape: n([1, Stage.channels(stage.first), h, w]), dataType: .float32, name: "input")+ input = g.placeholder(shape: parts.n([1, Stage.channels(stage.first), h, w]), dataType: .float32, name: "input")x = input} else {- input = g.placeholder(shape: n([1, 64, h, w]), dataType: .float32, name: "latent")- x = g.addition(g.multiplication(input, constant(QwenVAE.latentsStd, [1, 64, 1, 1]), name: nil),- constant(QwenVAE.latentsMean, [1, 64, 1, 1]), name: nil)- x = try conv(x, "conv2", 64, 64, 1) // post_quant_conv- x = try conv(x, "decoder.conv1", 64, dims[0], 3) // conv_in- x = try residual(x, "decoder.middle.0", dims[0], dims[0]) // mid_block.resnets.0- x = try attention(x, "decoder.middle.1", dims[0], h * w, h, w) // mid_block.attentions.0- x = try residual(x, "decoder.middle.2", dims[0], dims[0]) // mid_block.resnets.1+ input = g.placeholder(shape: parts.n([1, 64, h, w]), dataType: .float32, name: "latent")+ x = g.addition(g.multiplication(input, parts.constant(QwenVAE.latentsStd, [1, 64, 1, 1]), name: nil),+ parts.constant(QwenVAE.latentsMean, [1, 64, 1, 1]), name: nil)+ x = try parts.conv(x, "conv2", 64, 64, 1) // post_quant_conv+ x = try parts.conv(x, "decoder.conv1", 64, dims[0], 3) // conv_in+ x = try parts.residual(x, "decoder.middle.0", dims[0], dims[0]) // mid_block.resnets.0+ x = try parts.attention(x, "decoder.middle.1", dims[0], h * w, h, w) // mid_block.attentions.0+ x = try parts.residual(x, "decoder.middle.2", dims[0], dims[0]) // mid_block.resnets.1}for i in max(stage.first, 0)..<stage.end { // up_blocks.ilet (inC, outC) = (dims[i], dims[i + 1])let blockInput = xvar c = inCfor j in 0..<3 {- x = try residual(x, "decoder.upsamples.\(i).upsamples.\(j)", c, outC)+ x = try parts.residual(x, "decoder.upsamples.\(i).upsamples.\(j)", c, outC)c = outC}if i < 4 {- x = g.resize(x, size: n([2 * hh, 2 * ww]), mode: .nearest, centerResult: true, alignCorners: false,+ x = g.resize(x, size: parts.n([2 * hh, 2 * ww]), mode: .nearest, centerResult: true, alignCorners: false,layout: .NCHW, name: nil)- x = try conv(x, "decoder.upsamples.\(i).upsamples.3.resample.1", outC, outC, 3) // upsampler- x = g.addition(x, dupUp(blockInput, inC, outC, QwenVAE.temporal[i], hh, ww), name: nil) // avg_shortcut+ x = try parts.conv(x, "decoder.upsamples.\(i).upsamples.3.resample.1", outC, outC, 3) // upsampler+ x = g.addition(x, parts.dupUp(blockInput, inC, outC, QwenVAE.temporal[i], hh, ww), name: nil) // avg_shortcut(hh, ww) = (2 * hh, 2 * ww)}}if stage.end < 5 { return (g, input, x) }- x = silu(try norm(x, "decoder.head.0.gamma", dims[5])) // norm_out- x = try conv(x, "decoder.head.2", dims[5], 4, 3) // conv_out+ x = parts.silu(try parts.norm(x, "decoder.head.0.gamma", dims[5])) // norm_out+ x = try parts.conv(x, "decoder.head.2", dims[5], 4, 3) // conv_outx = g.clamp(x, min: g.constant(-1, dataType: .float32), max: g.constant(1, dataType: .float32), name: nil)return (g, input, x)}}++/// The VAE's pieces as MPSGraph operations, the file's weights as constants: the decoder's graphs (`build`) and the+/// encoder's mid block and head (`QwenVAEEncoder.swift`).+struct VAEGraph {+ let g: MPSGraph+ let file: SafetensorsFile++ func n(_ values: [Int]) -> [NSNumber] { values.map { NSNumber(value: $0) } }+ func constant(_ values: [Float], _ shape: [Int]) -> MPSGraphTensor {+ g.constant(values.withUnsafeBytes { Data($0) }, shape: n(shape), dataType: .float32)+ }+ func weight(_ name: String, _ shape: [Int]) throws -> MPSGraphTensor {+ let values = try file.floats(name)+ guard values.count == shape.reduce(1, *) else {+ throw GGUFFile.Failure.unsupported("\(name): \(values.count) values, \(shape) expected")+ }+ return constant(values, shape)+ }+ /// Conv (OIHW weights, the file's 5D `[O, I, 1, k, k]` squeezed) + bias, padding k / 2.+ func conv(_ x: MPSGraphTensor, _ name: String, _ inC: Int, _ outC: Int, _ k: Int) throws -> MPSGraphTensor {+ let pad = k / 2+ let d = MPSGraphConvolution2DOpDescriptor(strideInX: 1, strideInY: 1, dilationRateInX: 1, dilationRateInY: 1,+ groups: 1, paddingLeft: pad, paddingRight: pad, paddingTop: pad,+ paddingBottom: pad, paddingStyle: .explicit, dataLayout: .NCHW,+ weightsLayout: .OIHW)!+ let y = g.convolution2D(x, weights: try weight(name + ".weight", [outC, inC, k, k]), descriptor: d, name: nil)+ return g.addition(y, try weight(name + ".bias", [1, outC, 1, 1]), name: nil)+ }+ /// QwenImage21RMS_norm: F.normalize over channels (x / max(‖x‖, 1e-12)) · √C · γ.+ func norm(_ x: MPSGraphTensor, _ name: String, _ c: Int) throws -> MPSGraphTensor {+ let length = g.squareRoot(with: g.reductionSum(with: g.square(with: x, name: nil), axes: [1], name: nil), name: nil)+ let normalized = g.division(x, g.maximum(length, g.constant(1e-12, dataType: .float32), name: nil), name: nil)+ let scaled = g.multiplication(normalized, g.constant(Double(Float(c).squareRoot()), dataType: .float32), name: nil)+ return g.multiplication(scaled, try weight(name, [1, c, 1, 1]), name: nil)+ }+ func silu(_ x: MPSGraphTensor) -> MPSGraphTensor { g.multiplication(x, g.sigmoid(with: x, name: nil), name: nil) }+ /// QwenImage21ResidualBlock (Wan `residual.{0,2,3,6}`, `shortcut`).+ func residual(_ x: MPSGraphTensor, _ name: String, _ inC: Int, _ outC: Int) throws -> MPSGraphTensor {+ let skip = inC == outC ? x : try conv(x, name + ".shortcut", inC, outC, 1)+ var y = silu(try norm(x, name + ".residual.0.gamma", inC))+ y = try conv(y, name + ".residual.2", inC, outC, 3)+ y = silu(try norm(y, name + ".residual.3.gamma", outC))+ y = try conv(y, name + ".residual.6", outC, outC, 3)+ return g.addition(y, skip, name: nil)+ }+ /// QwenImage21AttentionBlock: one head over all `hw` pixels.+ func attention(_ x: MPSGraphTensor, _ name: String, _ c: Int, _ hw: Int, _ hh: Int, _ ww: Int) throws -> MPSGraphTensor {+ let qkv = try conv(try norm(x, name + ".norm.gamma", c), name + ".to_qkv", c, 3 * c, 1) // [1, 3C, H, W]+ let rows = g.transpose(g.reshape(qkv, shape: n([3 * c, hw]), name: nil), permutation: n([1, 0]), name: nil)+ func part(_ i: Int) -> MPSGraphTensor {+ g.reshape(g.sliceTensor(rows, dimension: 1, start: i * c, length: c, name: nil), shape: n([1, 1, hw, c]), name: nil)+ }+ let o = g.scaledDotProductAttention(query: part(0), key: part(1), value: part(2), mask: nil,+ scale: 1 / Float(c).squareRoot(), name: nil)+ let back = g.reshape(g.transpose(g.reshape(o, shape: n([hw, c]), name: nil), permutation: n([1, 0]), name: nil),+ shape: n([1, c, hh, ww]), name: nil)+ return g.addition(try conv(back, name + ".proj", c, c, 1), x, name: nil)+ }+ /// QwenImage21DupUp3D on a first chunk: each output channel's 2×2 pixels from the input channels it repeats.+ func dupUp(_ x: MPSGraphTensor, _ inC: Int, _ outC: Int, _ ft: Int, _ hh: Int, _ ww: Int) -> MPSGraphTensor {+ let repeats = outC * ft * 4 / inC+ var y = g.broadcast(g.reshape(x, shape: n([inC, 1, hh, ww]), name: nil), shape: n([inC, repeats, hh, ww]), name: nil)+ y = g.reshape(y, shape: n([outC, ft, 2, 2, hh, ww]), name: nil)+ y = g.sliceTensor(y, dimension: 1, start: ft - 1, length: 1, name: nil)+ y = g.reshape(y, shape: n([outC, 2, 2, hh, ww]), name: nil)+ y = g.transpose(y, permutation: n([0, 3, 1, 4, 2]), name: nil) // [C, H, 2, W, 2]+ return g.reshape(y, shape: n([1, outC, 2 * hh, 2 * ww]), name: nil)+ }+}diff --git a/engine-mac/Sources/KokonEngine/QwenVAEEncoder.swift b/engine-mac/Sources/KokonEngine/QwenVAEEncoder.swiftnew file mode 100644index 0000000..d8830e8--- /dev/null+++ b/engine-mac/Sources/KokonEngine/QwenVAEEncoder.swift@@ -0,0 +1,116 @@+import Foundation+import Metal+import MetalPerformanceShaders+import MetalPerformanceShadersGraph++/// **The Qwen-Image 2.1 VAE encoder** (`AutoencoderKLQwenImage21.encode`, diffusers 0.42), fp32, for the reference+/// images of edits: the latent the edit pipeline uses (`_encode_vae_image`: the posterior's mode, `(z − mean) / std`).+/// For a single image the Wan 2.2 design is a 2D network, as the decoder (`QwenVAE`):+///+/// RGBA in [−1, 1] ─ conv_in (3×3, 4 → 96)+/// 5 down blocks (96 → 96, 96 → 192, 192 → 384, 384 → 768, 768 → 768): 2 residual blocks, then (the first four)+/// zero pad right/bottom + conv 3×3 stride 2, plus the block input averaged down (AvgDown3D)+/// mid: residual ─ single-head attention over all pixels ─ residual+/// RMS norm ─ SiLU ─ conv_out (3×3, 768 → 128) ─ quant_conv (1×1) ─ the first 64 channels (the mode)+///+/// The down blocks run as single operations (the decoder's convolution graphs and `vae.metal` kernels, on pooled+/// buffers), the mid block and the head as one graph at 1/16 of the image (its attention sees every pixel).+extension QwenVAE {+ /// Encoder channels (diffusers' `dims`: base 96 × [1, 1, 2, 4, 8, 8]) and each down block's temporal factor+ /// (`temperal_downsample` [False, True, True, True]; the last block does not downsample).+ static let encoderDims = [96, 96, 192, 384, 768, 768]+ static let encoderTemporal = [1, 2, 2, 2, 1]++ /// The normalized latent `[64, H/16, W/16]` of an RGBA image `[4, H, W]` in [−1, 1] (sides multiples of 16).+ /// In its own autorelease pool: the command buffer and the pooled buffers go when it returns, whoever calls it (a+ /// loop on a thread without a pool kept 1.5 GB per 1024² encode, M2 Pro, 2026-10-25).+ public func encode(_ image: [Float], height H: Int, width W: Int) throws -> [Float] {+ try autoreleasepool { try encodeImage(image, height: H, width: W) }+ }++ private func encodeImage(_ image: [Float], height H: Int, width W: Int) throws -> [Float] {+ guard H % 16 == 0, W % 16 == 0, image.count == 4 * H * W else {+ throw QwenSequence.Failure.layout("VAE encoder: \(image.count) values for 4 × \(H) × \(W) (sides multiples of 16)")+ }+ guard file.tensors["encoder.conv1.weight"] != nil else { throw GGUFFile.Failure.unsupported("the VAE file has no encoder") }+ let commands = MPSCommandBuffer(from: context.queue)+ let pool = BufferPool(context: context)+ let dims = QwenVAE.encoderDims+ let input = try pool.take(image.count)+ input.contents().copyMemory(from: image, byteCount: 4 * image.count)+ var (hh, ww) = (H, W)+ var x = try pool.take(dims[0] * hh * ww)+ try convolution(input, "encoder.conv1", inC: 4, outC: dims[0], k: 3, height: hh, width: ww, to: x, commands: commands)+ pool.give(input)+ for i in 0..<5 { // down_blocks.i+ let (inC, outC) = (dims[i], dims[i + 1])+ let blockInput = x+ var c = inC+ for j in 0..<2 {+ let name = "encoder.downsamples.\(i).downsamples.\(j)"+ let t = try pool.take(c * hh * ww)+ try normSilu(x, to: t, gamma: name + ".residual.0.gamma", channels: c, pixels: hh * ww, commands: commands)+ let u = try pool.take(outC * hh * ww)+ try convolution(t, name + ".residual.2", inC: c, outC: outC, k: 3, height: hh, width: ww, to: u, commands: commands)+ pool.give(t)+ try normSilu(u, to: u, gamma: name + ".residual.3.gamma", channels: outC, pixels: hh * ww, commands: commands)+ let y = try pool.take(outC * hh * ww)+ try convolution(u, name + ".residual.6", inC: outC, outC: outC, k: 3, height: hh, width: ww, to: y,+ skip: (x, c == outC ? nil : name + ".shortcut", c), commands: commands)+ pool.give(u)+ if x !== blockInput { pool.give(x) }+ (x, c) = (y, outC)+ }+ let fs = i < 4 ? 2 : 1+ if i < 4 { // downsampler+ let y = try pool.take(outC * (hh / 2) * (ww / 2))+ try convolution(x, "encoder.downsamples.\(i).downsamples.2.resample.1", inC: outC, outC: outC, k: 3,+ height: hh, width: ww, to: y, downsample: true, commands: commands)+ pool.give(x)+ (x, hh, ww) = (y, hh / 2, ww / 2)+ }+ try dispatch("vae_avg_down_add", [x, blockInput], // avg_shortcut+ [UInt32(inC), UInt32(outC), UInt32(QwenVAE.encoderTemporal[i]), UInt32(fs), UInt32(hh), UInt32(ww)],+ grid: MTLSize(width: ww, height: hh, depth: outC), commands: commands)+ pool.give(blockInput)+ }+ let latent = try pool.take(64 * hh * ww)+ _ = try prepareEncoderTail(height: hh, width: ww)+ .encode(to: commands, inputs: [data(x, [1, dims[5], hh, ww])], results: [data(latent, [1, 64, hh, ww])],+ executionDescriptor: nil)+ commands.commit()+ commands.waitUntilCompleted()+ if let error = commands.error { throw MetalContext.Failure.kernel("VAE encode: \(error.localizedDescription)") }+ var out = [Float](repeating: 0, count: 64 * hh * ww)+ out.withUnsafeMutableBytes { $0.copyMemory(from: UnsafeRawBufferPointer(start: latent.contents(), count: $0.count)) }+ return out+ }++ /// The mid block, the head, quant_conv and the normalization as one graph for an `h` × `w` input, once per size.+ func prepareEncoderTail(height h: Int, width w: Int) throws -> MPSGraphExecutable {+ try lock.withLock {+ let key = "encoder tail \(h)×\(w)"+ if let e = executables[key] { return e }+ let g = MPSGraph()+ let parts = VAEGraph(g: g, file: file)+ let c = QwenVAE.encoderDims[5]+ let input = g.placeholder(shape: parts.n([1, c, h, w]), dataType: .float32, name: "input")+ var x = try parts.residual(input, "encoder.middle.0", c, c) // mid_block.resnets.0+ x = try parts.attention(x, "encoder.middle.1", c, h * w, h, w) // mid_block.attentions.0+ x = try parts.residual(x, "encoder.middle.2", c, c) // mid_block.resnets.1+ x = parts.silu(try parts.norm(x, "encoder.head.0.gamma", c)) // norm_out+ x = try parts.conv(x, "encoder.head.2", c, 128, 3) // conv_out+ x = try parts.conv(x, "conv1", 128, 128, 1) // quant_conv+ x = g.sliceTensor(x, dimension: 1, start: 0, length: 64, name: nil) // the mode: the mean+ x = g.division(g.subtraction(x, parts.constant(QwenVAE.latentsMean, [1, 64, 1, 1]), name: nil),+ parts.constant(QwenVAE.latentsStd, [1, 64, 1, 1]), name: nil)+ let descriptor = MPSGraphCompilationDescriptor()+ descriptor.waitForCompilationCompletion = true+ let e = g.compile(with: MPSGraphDevice(mtlDevice: context.device),+ feeds: [input: MPSGraphShapedType(shape: parts.n([1, c, h, w]), dataType: .float32)],+ targetTensors: [x], targetOperations: nil, compilationDescriptor: descriptor)+ executables[key] = e+ return e+ }+ }+}diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swiftindex 0bfed05..445bf47 100644--- a/engine-mac/Sources/kokon-engine-mac/main.swift+++ b/engine-mac/Sources/kokon-engine-mac/main.swift@@ -9,6 +9,7 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)check-dit <dit.gguf> <reference dir> one DiT evaluation against tools/reference_dit.pytime-dit <dit.gguf> [grid] [text] [runs] DiT evaluations on random inputs (grid 64 = 1024²)check-vae <vae.safetensors> <reference dir> [grid] the VAE decoder against tools/reference_vae.py, tiles against the whole graph+ check-vae-encoder <vae.safetensors> <reference dir> [WxH] the VAE encoder against tools/reference_vae_encoder.py, then its timetime-vae <vae.safetensors> [grid] the VAE decoder's time and memory (64 = 1024²; KOKON_VAE_TILE)check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.pycheck-text <encoder.gguf> <reference dir> the text encoder against tools/reference_text_encoder.py@@ -79,6 +80,11 @@ do {case "check-vae":guard arguments.count >= 3 else { fail(usage) }try Checks.vae(path: arguments[1], reference: arguments[2], timeGrid: arguments.count > 3 ? Int(arguments[3]) : nil)+ case "check-vae-encoder":+ guard arguments.count >= 3 else { fail(usage) }+ let size = arguments.count > 3 ? arguments[3].split(separator: "x").compactMap { Int($0) } : [1024, 1024]+ guard size.count == 2 else { fail("size as WxH") }+ try Checks.vaeEncoder(path: arguments[1], reference: arguments[2], timeSize: (size[0], size[1]))case "time-vae":guard arguments.count >= 2 else { fail(usage) }try Checks.timeVAE(path: arguments[1], grid: arguments.count > 2 ? Int(arguments[2]) ?? 64 : 64)diff --git a/engine-mac/tools/reference_vae_encoder.py b/engine-mac/tools/reference_vae_encoder.pynew file mode 100644index 0000000..2e17a08--- /dev/null+++ b/engine-mac/tools/reference_vae_encoder.py@@ -0,0 +1,105 @@+"""The Qwen-Image 2.1 VAE encoder reference for the Mac engine: diffusers' `AutoencoderKLQwenImage21.encode` in fp32 on+the CPU, on Kokon's VAE file, as the edit pipeline uses it (`QwenImage21Pipeline._encode_vae_image`: the posterior's+mode, then `(latent − mean) / std`). The input is a smooth random RGBA image in [−1, 1] (random low-resolution noise,+upsampled bilinearly, so it has edges and flat areas as photos do). Saves the input, every down block's output, the+mid block's and the normalized latent for `kokon-engine-mac check-vae-encoder`.++ python -I engine-mac/tools/reference_vae_encoder.py <vae.safetensors> <out dir> [--size 256x256]+"""+import argparse+import json+import os+import sys++import torch+import torch.nn.functional as F+from safetensors.torch import load_file++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from reference_vae import decoder_name # noqa: E402++RESIDUAL = {"residual.0.gamma": "norm1.gamma", "residual.2.weight": "conv1.weight", "residual.2.bias": "conv1.bias",+ "residual.3.gamma": "norm2.gamma", "residual.6.weight": "conv2.weight", "residual.6.bias": "conv2.bias",+ "shortcut.weight": "conv_shortcut.weight", "shortcut.bias": "conv_shortcut.bias"}+++def encoder_name(key):+ """Wan 2.2 VAE name -> diffusers AutoencoderKLQwenImage21 name, encoder side; None for the decoder."""+ if key.startswith("conv1."):+ return "quant_conv." + key[6:]+ if not key.startswith("encoder."):+ return None+ parts = key[len("encoder."):].split(".")+ if parts[0] == "conv1":+ return "encoder.conv_in." + parts[1]+ if parts[0] == "head":+ return "encoder.norm_out.gamma" if parts[1] == "0" else "encoder.conv_out." + parts[2]+ if parts[0] == "middle":+ rest = ".".join(parts[2:])+ if parts[1] == "1":+ return "encoder.mid_block.attentions.0." + rest+ return f"encoder.mid_block.resnets.{int(parts[1]) // 2}." + RESIDUAL[rest]+ if parts[0] == "downsamples":+ i, j, rest = parts[1], int(parts[3]), ".".join(parts[4:])+ if j == 2:+ return f"encoder.down_blocks.{i}.downsampler." + rest+ return f"encoder.down_blocks.{i}.resnets.{j}." + RESIDUAL[rest]+ raise KeyError(key)+++def main():+ parser = argparse.ArgumentParser()+ parser.add_argument("vae")+ parser.add_argument("out")+ parser.add_argument("--size", default="256x256", help="WxH, multiples of 16")+ parser.add_argument("--seed", type=int, default=11)+ args = parser.parse_args()+ from diffusers import AutoencoderKLQwenImage21+ torch.manual_seed(args.seed)+ torch.set_grad_enabled(False)+ os.makedirs(args.out, exist_ok=True)+ width, height = (int(v) for v in args.size.split("x"))++ vae = AutoencoderKLQwenImage21() # the defaults are Qwen-Image 2.1's config+ model = vae.state_dict()+ converted = {}+ for key, value in load_file(args.vae).items():+ name = encoder_name(key) or decoder_name(key)+ if name is not None:+ converted[name] = value.float().reshape(model[name].shape)+ missing, unexpected = vae.load_state_dict(converted, strict=False)+ missing = [k for k in missing if k.startswith("encoder.") or k.startswith("quant_conv")]+ assert not missing and not unexpected, (missing[:10], unexpected[:10])+ vae.eval()++ image = torch.rand(1, 4, 1, height // 16, width // 16) * 2 - 1+ image = F.interpolate(image[:, :, 0], size=(height, width), mode="bilinear", align_corners=False)[:, :, None]+ image = image + 0.05 * torch.randn_like(image) # some fine texture+ image = image.clamp(-1, 1)++ saved = {}++ def save(name, tensor):+ array = tensor.detach().to(torch.float32).contiguous().numpy()+ array.tofile(os.path.join(args.out, name + ".f32"))+ saved[name] = list(array.shape)++ hooks = [block.register_forward_hook(lambda m, a, o, i=i: save(f"down{i}", o[0, :, 0]))+ for i, block in enumerate(vae.encoder.down_blocks)]+ hooks.append(vae.encoder.conv_in.register_forward_hook(lambda m, a, o: save("conv_in", o[0, :, 0])))+ hooks.append(vae.encoder.mid_block.register_forward_hook(lambda m, a, o: save("mid", o[0, :, 0])))+ latent = vae.encode(image).latent_dist.mode()+ for h in hooks:+ h.remove()+ mean = torch.tensor(vae.config.latents_mean).view(1, 64, 1, 1, 1)+ std = torch.tensor(vae.config.latents_std).view(1, 64, 1, 1, 1)+ save("image", image[0, :, 0])+ save("latent", ((latent - mean) / std)[0, :, 0])+ with open(os.path.join(args.out, "meta.json"), "w") as f:+ json.dump({"size": [width, height], "seed": args.seed, "tensors": saved,+ "torch": torch.__version__}, f, indent=1)+ print(f"{width}×{height}: latent {saved['latent']}")+++if __name__ == "__main__":+ main()