Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: the VAE encoder, checked against diffusers

Gabriele Lozuponedfd6c5011a13
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 8cd2365..c82241d 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -44,6 +44,7 @@ Windows and Linux.
| `AttentionSplit.swift`, `CPUAttention.swift` | the target's attention cut by heads: GPU, CPU (AMX + cores), optionally the Neural Engine |
| `AttentionProbe.swift` | `probe-attention`: the target's attention on each unit alone |
| `QwenVAE.swift`, `Safetensors.swift` | the VAE decoder as one MPSGraph, from Kokon's VAE file (Wan 2.2 names) |
+| `QwenVAEEncoder.swift` | the VAE encoder (edits' reference images), checked against diffusers |
| `QwenTokenizer.swift` | Qwen's byte-level BPE tokenizer, read from the encoder GGUF's metadata |
| `QwenTextEncoder.swift`, `Kernels/text.metal` | the text encoder: Qwen3-VL-8B's text decoder on the GPU, diffusers' prompt template |
| `QwenPipeline.swift` | text to image: noise, Turbo's 8 flow-Euler steps (`planner::TURBO_SIGMAS`), VAE, PNG; models kept loaded for the server |
@@ -378,5 +379,13 @@ them on the Neural Engine (optimization level 1, its default); at level 0 they s
gain little: all five 2.43 s against 2.77 at 1024², for 2.7e-3 of diffusers instead of 1.1e-5 (up to ~19 levels off).
The VAE stays fp32 (2026-10-25).
+**VAE encoder** (`QwenVAEEncoder.swift`; for the reference images of edits): the down blocks as single operations as
+the decoder's up blocks (its convolution graphs, one with stride 2 and the zero row and column after the input; the
+AvgDown shortcut as a kernel, with the zero frame the single image is padded by counted in its average), the mid block
+and the head as one graph at 1/16 of the image, then the posterior's mode normalized as the edit pipeline does.
+`check-vae-encoder` against `tools/reference_vae_encoder.py` (diffusers fp32, a smooth random RGBA image, 256 × 192,
+texture-fix VAE, M2 Pro, 2026-10-25): the latent within 6.5e-6. 1024²: 0.59 s an encode, peak 2.4 GB, 0.87 GB kept after
+(its weights); in its own autorelease pool (without one, a loop on a thread without a pool kept 1.5 GB per encode).
+
Credits: the Neural Engine program technique and the GPU + AMX + Neural Engine split come from
[Siliconed](https://github.com/gwenn-ha-dev/Siliconed) (Apache-2.0), written again here.
diff --git a/engine-mac/Sources/KokonEngine/Checks.swift b/engine-mac/Sources/KokonEngine/Checks.swift
index 9be0f87..58c8e0b 100644
--- a/engine-mac/Sources/KokonEngine/Checks.swift
+++ b/engine-mac/Sources/KokonEngine/Checks.swift
@@ -384,3 +384,36 @@ extension Checks {
print(String(format: "int8 weights, fp16 activations %d×%d×%d: %.2f ms (%.1f TFLOP/s), relative error %.2e", rows, k, n, tw * 1e3, flops / tw / 1e12, ew))
}
}
+
+extension Checks {
+ /// The VAE encoder against `tools/reference_vae_encoder.py` (diffusers fp32): the normalized latent of its image;
+ /// then the time and memory of an encode at `timeSize` (a smooth random image, as the reference's).
+ public static func vaeEncoder(path: String, reference directory: String, timeSize: (width: Int, height: Int) = (1024, 1024)) throws {
+ let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory)
+ .appendingPathComponent("meta.json"))) as! [String: Any]
+ let size = meta["size"] as! [Int]
+ let (w, h) = (size[0], size[1])
+ let context = try MetalContext()
+ let vae = try QwenVAE(path: path, context: context)
+ let image = try reference(directory, "image"), want = try reference(directory, "latent")
+ var started = DispatchTime.now().uptimeNanoseconds
+ let got = try vae.encode(image, height: h, width: w)
+ var seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
+ let r = got.withUnsafeBufferPointer { compare($0.baseAddress!, want) }
+ print(String(format: "VAE encoder %d×%d → %d×%d latent: rel %.2e, max |Δ| %.3g%@ (%.2f s with the graphs' build)",
+ w, h, w / 16, h / 16, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE", seconds))
+ let (tw, th) = timeSize
+ let random = (0..<(4 * th * tw)).map { i -> Float in
+ let (y, x) = ((i / tw) % th, i % tw)
+ return sinf(Float(x) * 0.013 + Float(i / (tw * th))) * cosf(Float(y) * 0.021) * 0.9
+ }
+ for run in 1...3 {
+ started = DispatchTime.now().uptimeNanoseconds
+ _ = try vae.encode(random, height: th, width: tw)
+ seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
+ Thread.sleep(forTimeInterval: 1.5) // buffers the GPU used leave the footprint up to a second after their release
+ let m = MemoryUsage.now()
+ print(String(format: "encode %d at %d×%d: %.2f s, footprint after %.2f GB (peak %.2f GB)", run, tw, th, seconds, m.footprint, m.footprintPeak))
+ }
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/Kernels/vae.metal b/engine-mac/Sources/KokonEngine/Kernels/vae.metal
index 5e2d0d1..e483980 100644
--- a/engine-mac/Sources/KokonEngine/Kernels/vae.metal
+++ b/engine-mac/Sources/KokonEngine/Kernels/vae.metal
@@ -50,3 +50,23 @@ kernel void vae_dup_up_add(device float *out [[buffer(0)]], device const float *
uint q = ((gid.z * p.ft + p.ft - 1) * 2 + (gid.y & 1)) * 2 + (gid.x & 1);
out[(ulong(gid.z) * 2 * p.h + gid.y) * 2 * p.w + gid.x] += x[(ulong(q / repeats) * p.h + gid.y / 2) * p.w + gid.x / 2];
}
+
+struct AvgDown { uint inC, outC, ft, fs, h, w; };
+
+/// QwenImage21AvgDown3D on a first chunk, added to the down block's output `out` [outC, h, w]: the input [inC, fs·h,
+/// fs·w] seen as inC · ft · fs² channels ((c · ft + t) · fs + i) · fs + j, of which `group` consecutive ones average
+/// into each output channel. With ft = 2 the single frame is padded by a zero frame before it (t = 0), which counts in
+/// the average.
+kernel void vae_avg_down_add(device float *out [[buffer(0)]], device const float *x [[buffer(1)]],
+ constant AvgDown &p [[buffer(2)]], uint3 gid [[thread_position_in_grid]]) {
+ if (gid.x >= p.w || gid.y >= p.h || gid.z >= p.outC) return;
+ uint group = p.inC * p.ft * p.fs * p.fs / p.outC;
+ float sum = 0;
+ for (uint k = 0; k < group; k++) {
+ uint index = gid.z * group + k;
+ uint j = index % p.fs, i = (index / p.fs) % p.fs, t = (index / (p.fs * p.fs)) % p.ft, c = index / (p.fs * p.fs * p.ft);
+ if (t + 1 < p.ft) continue; // the padded frame
+ sum += x[(ulong(c) * p.fs * p.h + gid.y * p.fs + i) * p.fs * p.w + gid.x * p.fs + j];
+ }
+ out[(ulong(gid.z) * p.h + gid.y) * p.w + gid.x] += sum / float(group);
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenVAE.swift b/engine-mac/Sources/KokonEngine/QwenVAE.swift
index c4d83dc..3215d3c 100644
--- a/engine-mac/Sources/KokonEngine/QwenVAE.swift
+++ b/engine-mac/Sources/KokonEngine/QwenVAE.swift
@@ -46,9 +46,9 @@ public final class QwenVAE {
/// Engine; at level 0 they stay on the GPU and are right, but save little: all five in fp16 2.43 s against 2.77 at
/// 1024², the first two 2.65, for 2.2–2.7e-3 of diffusers instead of 1.1e-5 (M2 Pro, `check-vae`, 2026-10-25). The
/// VAE stays fp32.
- private var executables: [String: MPSGraphExecutable] = [:]
+ var executables: [String: MPSGraphExecutable] = [:]
private var feedOrders: [String: [Int]] = [:]
- private let lock = NSLock()
+ let lock = NSLock()
public init(path: String, context: MetalContext) throws {
file = try SafetensorsFile(path: path)
@@ -163,7 +163,7 @@ public final class QwenVAE {
func give(_ buffer: MTLBuffer) { free.append(buffer) }
}
- private func data(_ buffer: MTLBuffer, _ shape: [Int]) -> MPSGraphTensorData {
+ func data(_ buffer: MTLBuffer, _ shape: [Int]) -> MPSGraphTensorData {
MPSGraphTensorData(buffer, shape: shape.map { NSNumber(value: $0) }, dataType: .float32)
}
@@ -255,7 +255,7 @@ public final class QwenVAE {
/// Weights as fp32 buffers, each read from the file once.
private var weightBuffers: [String: MTLBuffer] = [:]
- private func weight(_ name: String) throws -> MTLBuffer {
+ func weight(_ name: String) throws -> MTLBuffer {
try lock.withLock {
if let b = weightBuffers[name] { return b }
let values = try file.floats(name)
@@ -266,7 +266,7 @@ public final class QwenVAE {
}
}
- private func dispatch(_ kernel: String, _ buffers: [MTLBuffer], _ parameters: [UInt32], grid: MTLSize,
+ func dispatch(_ kernel: String, _ buffers: [MTLBuffer], _ parameters: [UInt32], grid: MTLSize,
commands: MPSCommandBuffer) throws {
let pipeline = try context.pipeline(kernel)
var p = parameters
@@ -280,7 +280,7 @@ public final class QwenVAE {
}
/// QwenImage21RMS_norm then SiLU, per pixel over `channels` planes (`x` and `out` may be the same).
- private func normSilu(_ x: MTLBuffer, to out: MTLBuffer, gamma: String, channels: Int, pixels: Int, commands: MPSCommandBuffer) throws {
+ func normSilu(_ x: MTLBuffer, to out: MTLBuffer, gamma: String, channels: Int, pixels: Int, commands: MPSCommandBuffer) throws {
try dispatch("vae_norm_silu", [x, out, try weight(gamma)], [UInt32(channels), UInt32(pixels)],
grid: MTLSize(width: pixels, height: 1, depth: 1), commands: commands)
}
@@ -294,12 +294,14 @@ public final class QwenVAE {
}
/// `name` (a k × k convolution and its bias) on `[inC, h, w]`, plus `skip` (the residual block's input, through
- /// its 1×1 shortcut when named), clamped to [−1, 1] for the head.
- private func convolution(_ x: MTLBuffer, _ name: String, inC: Int, outC: Int, k: Int, height h: Int, width w: Int,
- to out: MTLBuffer, skip: (buffer: MTLBuffer, shortcut: String?, channels: Int)? = nil,
- clamp: Bool = false, commands: MPSCommandBuffer) throws {
+ /// its 1×1 shortcut when named), clamped to [−1, 1] for the head. `downsample`: stride 2 with one zero row and
+ /// column after the input (the encoder's `ZeroPad2d((0, 1, 0, 1))` then `Conv2d(stride=2)`), `[outC, h/2, w/2]`.
+ func convolution(_ x: MTLBuffer, _ name: String, inC: Int, outC: Int, k: Int, height h: Int, width w: Int,
+ to out: MTLBuffer, skip: (buffer: MTLBuffer, shortcut: String?, channels: Int)? = nil,
+ clamp: Bool = false, downsample: Bool = false, commands: MPSCommandBuffer) throws {
let kind = skip == nil ? "" : skip!.shortcut == nil ? "+x" : "+1x1(\(skip!.channels))"
- let key = "conv \(inC)→\(outC) \(k)×\(k)\(kind)\(clamp ? " clamp" : "") \(h)×\(w)"
+ let key = "conv \(inC)→\(outC) \(k)×\(k)\(kind)\(clamp ? " clamp" : "")\(downsample ? " /2" : "") \(h)×\(w)"
+ let (oh, ow) = downsample ? (h / 2, w / 2) : (h, w)
let (executable, order) = try lock.withLock { () throws -> (MPSGraphExecutable, [Int]) in
if let e = executables[key], let order = feedOrders[key] { return (e, order) }
let g = MPSGraph()
@@ -312,9 +314,10 @@ public final class QwenVAE {
return t
}
func conv(_ x: MPSGraphTensor, _ inC: Int, _ k: Int) -> MPSGraphTensor {
- let d = MPSGraphConvolution2DOpDescriptor(strideInX: 1, strideInY: 1, dilationRateInX: 1, dilationRateInY: 1,
- groups: 1, paddingLeft: k / 2, paddingRight: k / 2, paddingTop: k / 2,
- paddingBottom: k / 2, paddingStyle: .explicit, dataLayout: .NCHW,
+ let (stride, before, after) = downsample ? (2, 0, 1) : (1, k / 2, k / 2)
+ let d = MPSGraphConvolution2DOpDescriptor(strideInX: stride, strideInY: stride, dilationRateInX: 1, dilationRateInY: 1,
+ groups: 1, paddingLeft: before, paddingRight: after, paddingTop: before,
+ paddingBottom: after, paddingStyle: .explicit, dataLayout: .NCHW,
weightsLayout: .OIHW)!
let y = g.convolution2D(x, weights: input([outC, inC, k, k]), descriptor: d, name: nil)
return g.addition(y, input([1, outC, 1, 1]), name: nil)
@@ -323,7 +326,7 @@ public final class QwenVAE {
let x = input([1, inC, h, w])
var y = conv(x, inC, k)
if let skip {
- let s = input([1, skip.channels, h, w])
+ let s = input([1, skip.channels, oh, ow])
y = g.addition(y, skip.shortcut == nil ? s : conv(s, skip.channels, 1), name: nil)
}
if clamp {
@@ -342,13 +345,13 @@ public final class QwenVAE {
var inputs = [data(x, [1, inC, h, w]), data(try weight(name + ".weight"), [outC, inC, k, k]),
data(try weight(name + ".bias"), [1, outC, 1, 1])]
if let skip {
- inputs.append(data(skip.buffer, [1, skip.channels, h, w]))
+ inputs.append(data(skip.buffer, [1, skip.channels, oh, ow]))
if let shortcut = skip.shortcut {
inputs += [data(try weight(shortcut + ".weight"), [outC, skip.channels, 1, 1]), data(try weight(shortcut + ".bias"), [1, outC, 1, 1])]
}
}
_ = executable.encode(to: commands, inputs: order.map { inputs[$0] },
- results: [data(out, [1, outC, h, w])], executionDescriptor: nil)
+ results: [data(out, [1, outC, oh, ow])], executionDescriptor: nil)
}
/// Tiles of side `tile` over `n` pixels: each one's start, and the share it gives (`core`, `length`) at least
@@ -384,105 +387,113 @@ public final class QwenVAE {
/// The graph of `stage` for an `h` × `w` input.
private func build(_ stage: Stage, height h: Int, width w: Int) throws -> (graph: MPSGraph, input: MPSGraphTensor, output: MPSGraphTensor) {
let g = MPSGraph()
- func n(_ values: [Int]) -> [NSNumber] { values.map { NSNumber(value: $0) } }
- func constant(_ values: [Float], _ shape: [Int]) -> MPSGraphTensor {
- g.constant(values.withUnsafeBytes { Data($0) }, shape: n(shape), dataType: .float32)
- }
- func weight(_ name: String, _ shape: [Int]) throws -> MPSGraphTensor {
- let values = try file.floats(name)
- guard values.count == shape.reduce(1, *) else {
- throw GGUFFile.Failure.unsupported("\(name): \(values.count) values, \(shape) expected")
- }
- return constant(values, shape)
- }
- /// Conv (OIHW weights, the file's 5D `[O, I, 1, k, k]` squeezed) + bias, padding k / 2.
- func conv(_ x: MPSGraphTensor, _ name: String, _ inC: Int, _ outC: Int, _ k: Int) throws -> MPSGraphTensor {
- let pad = k / 2
- let d = MPSGraphConvolution2DOpDescriptor(strideInX: 1, strideInY: 1, dilationRateInX: 1, dilationRateInY: 1,
- groups: 1, paddingLeft: pad, paddingRight: pad, paddingTop: pad,
- paddingBottom: pad, paddingStyle: .explicit, dataLayout: .NCHW,
- weightsLayout: .OIHW)!
- let y = g.convolution2D(x, weights: try weight(name + ".weight", [outC, inC, k, k]), descriptor: d, name: nil)
- return g.addition(y, try weight(name + ".bias", [1, outC, 1, 1]), name: nil)
- }
- /// QwenImage21RMS_norm: F.normalize over channels (x / max(‖x‖, 1e-12)) · √C · γ.
- func norm(_ x: MPSGraphTensor, _ name: String, _ c: Int) throws -> MPSGraphTensor {
- let length = g.squareRoot(with: g.reductionSum(with: g.square(with: x, name: nil), axes: [1], name: nil), name: nil)
- let normalized = g.division(x, g.maximum(length, g.constant(1e-12, dataType: .float32), name: nil), name: nil)
- let scaled = g.multiplication(normalized, g.constant(Double(Float(c).squareRoot()), dataType: .float32), name: nil)
- return g.multiplication(scaled, try weight(name, [1, c, 1, 1]), name: nil)
- }
- func silu(_ x: MPSGraphTensor) -> MPSGraphTensor { g.multiplication(x, g.sigmoid(with: x, name: nil), name: nil) }
- /// QwenImage21ResidualBlock (Wan `residual.{0,2,3,6}`, `shortcut`).
- func residual(_ x: MPSGraphTensor, _ name: String, _ inC: Int, _ outC: Int) throws -> MPSGraphTensor {
- let skip = inC == outC ? x : try conv(x, name + ".shortcut", inC, outC, 1)
- var y = silu(try norm(x, name + ".residual.0.gamma", inC))
- y = try conv(y, name + ".residual.2", inC, outC, 3)
- y = silu(try norm(y, name + ".residual.3.gamma", outC))
- y = try conv(y, name + ".residual.6", outC, outC, 3)
- return g.addition(y, skip, name: nil)
- }
- /// QwenImage21AttentionBlock: one head over all `hw` pixels.
- func attention(_ x: MPSGraphTensor, _ name: String, _ c: Int, _ hw: Int, _ hh: Int, _ ww: Int) throws -> MPSGraphTensor {
- let qkv = try conv(try norm(x, name + ".norm.gamma", c), name + ".to_qkv", c, 3 * c, 1) // [1, 3C, H, W]
- let rows = g.transpose(g.reshape(qkv, shape: n([3 * c, hw]), name: nil), permutation: n([1, 0]), name: nil)
- func part(_ i: Int) -> MPSGraphTensor {
- g.reshape(g.sliceTensor(rows, dimension: 1, start: i * c, length: c, name: nil), shape: n([1, 1, hw, c]), name: nil)
- }
- let o = g.scaledDotProductAttention(query: part(0), key: part(1), value: part(2), mask: nil,
- scale: 1 / Float(c).squareRoot(), name: nil)
- let back = g.reshape(g.transpose(g.reshape(o, shape: n([hw, c]), name: nil), permutation: n([1, 0]), name: nil),
- shape: n([1, c, hh, ww]), name: nil)
- return g.addition(try conv(back, name + ".proj", c, c, 1), x, name: nil)
- }
- /// QwenImage21DupUp3D on a first chunk: each output channel's 2×2 pixels from the input channels it repeats.
- func dupUp(_ x: MPSGraphTensor, _ inC: Int, _ outC: Int, _ ft: Int, _ hh: Int, _ ww: Int) -> MPSGraphTensor {
- let repeats = outC * ft * 4 / inC
- var y = g.broadcast(g.reshape(x, shape: n([inC, 1, hh, ww]), name: nil), shape: n([inC, repeats, hh, ww]), name: nil)
- y = g.reshape(y, shape: n([outC, ft, 2, 2, hh, ww]), name: nil)
- y = g.sliceTensor(y, dimension: 1, start: ft - 1, length: 1, name: nil)
- y = g.reshape(y, shape: n([outC, 2, 2, hh, ww]), name: nil)
- y = g.transpose(y, permutation: n([0, 3, 1, 4, 2]), name: nil) // [C, H, 2, W, 2]
- return g.reshape(y, shape: n([1, outC, 2 * hh, 2 * ww]), name: nil)
- }
-
+ let parts = VAEGraph(g: g, file: file)
let input: MPSGraphTensor
var x: MPSGraphTensor
let dims = QwenVAE.dims
var (hh, ww) = (h, w)
if stage.first >= 0 {
- input = g.placeholder(shape: n([1, Stage.channels(stage.first), h, w]), dataType: .float32, name: "input")
+ input = g.placeholder(shape: parts.n([1, Stage.channels(stage.first), h, w]), dataType: .float32, name: "input")
x = input
} else {
- input = g.placeholder(shape: n([1, 64, h, w]), dataType: .float32, name: "latent")
- x = g.addition(g.multiplication(input, constant(QwenVAE.latentsStd, [1, 64, 1, 1]), name: nil),
- constant(QwenVAE.latentsMean, [1, 64, 1, 1]), name: nil)
- x = try conv(x, "conv2", 64, 64, 1) // post_quant_conv
- x = try conv(x, "decoder.conv1", 64, dims[0], 3) // conv_in
- x = try residual(x, "decoder.middle.0", dims[0], dims[0]) // mid_block.resnets.0
- x = try attention(x, "decoder.middle.1", dims[0], h * w, h, w) // mid_block.attentions.0
- x = try residual(x, "decoder.middle.2", dims[0], dims[0]) // mid_block.resnets.1
+ input = g.placeholder(shape: parts.n([1, 64, h, w]), dataType: .float32, name: "latent")
+ x = g.addition(g.multiplication(input, parts.constant(QwenVAE.latentsStd, [1, 64, 1, 1]), name: nil),
+ parts.constant(QwenVAE.latentsMean, [1, 64, 1, 1]), name: nil)
+ x = try parts.conv(x, "conv2", 64, 64, 1) // post_quant_conv
+ x = try parts.conv(x, "decoder.conv1", 64, dims[0], 3) // conv_in
+ x = try parts.residual(x, "decoder.middle.0", dims[0], dims[0]) // mid_block.resnets.0
+ x = try parts.attention(x, "decoder.middle.1", dims[0], h * w, h, w) // mid_block.attentions.0
+ x = try parts.residual(x, "decoder.middle.2", dims[0], dims[0]) // mid_block.resnets.1
}
for i in max(stage.first, 0)..<stage.end { // up_blocks.i
let (inC, outC) = (dims[i], dims[i + 1])
let blockInput = x
var c = inC
for j in 0..<3 {
- x = try residual(x, "decoder.upsamples.\(i).upsamples.\(j)", c, outC)
+ x = try parts.residual(x, "decoder.upsamples.\(i).upsamples.\(j)", c, outC)
c = outC
}
if i < 4 {
- x = g.resize(x, size: n([2 * hh, 2 * ww]), mode: .nearest, centerResult: true, alignCorners: false,
+ x = g.resize(x, size: parts.n([2 * hh, 2 * ww]), mode: .nearest, centerResult: true, alignCorners: false,
layout: .NCHW, name: nil)
- x = try conv(x, "decoder.upsamples.\(i).upsamples.3.resample.1", outC, outC, 3) // upsampler
- x = g.addition(x, dupUp(blockInput, inC, outC, QwenVAE.temporal[i], hh, ww), name: nil) // avg_shortcut
+ x = try parts.conv(x, "decoder.upsamples.\(i).upsamples.3.resample.1", outC, outC, 3) // upsampler
+ x = g.addition(x, parts.dupUp(blockInput, inC, outC, QwenVAE.temporal[i], hh, ww), name: nil) // avg_shortcut
(hh, ww) = (2 * hh, 2 * ww)
}
}
if stage.end < 5 { return (g, input, x) }
- x = silu(try norm(x, "decoder.head.0.gamma", dims[5])) // norm_out
- x = try conv(x, "decoder.head.2", dims[5], 4, 3) // conv_out
+ x = parts.silu(try parts.norm(x, "decoder.head.0.gamma", dims[5])) // norm_out
+ x = try parts.conv(x, "decoder.head.2", dims[5], 4, 3) // conv_out
x = g.clamp(x, min: g.constant(-1, dataType: .float32), max: g.constant(1, dataType: .float32), name: nil)
return (g, input, x)
}
}
+
+/// The VAE's pieces as MPSGraph operations, the file's weights as constants: the decoder's graphs (`build`) and the
+/// encoder's mid block and head (`QwenVAEEncoder.swift`).
+struct VAEGraph {
+ let g: MPSGraph
+ let file: SafetensorsFile
+
+ func n(_ values: [Int]) -> [NSNumber] { values.map { NSNumber(value: $0) } }
+ func constant(_ values: [Float], _ shape: [Int]) -> MPSGraphTensor {
+ g.constant(values.withUnsafeBytes { Data($0) }, shape: n(shape), dataType: .float32)
+ }
+ func weight(_ name: String, _ shape: [Int]) throws -> MPSGraphTensor {
+ let values = try file.floats(name)
+ guard values.count == shape.reduce(1, *) else {
+ throw GGUFFile.Failure.unsupported("\(name): \(values.count) values, \(shape) expected")
+ }
+ return constant(values, shape)
+ }
+ /// Conv (OIHW weights, the file's 5D `[O, I, 1, k, k]` squeezed) + bias, padding k / 2.
+ func conv(_ x: MPSGraphTensor, _ name: String, _ inC: Int, _ outC: Int, _ k: Int) throws -> MPSGraphTensor {
+ let pad = k / 2
+ let d = MPSGraphConvolution2DOpDescriptor(strideInX: 1, strideInY: 1, dilationRateInX: 1, dilationRateInY: 1,
+ groups: 1, paddingLeft: pad, paddingRight: pad, paddingTop: pad,
+ paddingBottom: pad, paddingStyle: .explicit, dataLayout: .NCHW,
+ weightsLayout: .OIHW)!
+ let y = g.convolution2D(x, weights: try weight(name + ".weight", [outC, inC, k, k]), descriptor: d, name: nil)
+ return g.addition(y, try weight(name + ".bias", [1, outC, 1, 1]), name: nil)
+ }
+ /// QwenImage21RMS_norm: F.normalize over channels (x / max(‖x‖, 1e-12)) · √C · γ.
+ func norm(_ x: MPSGraphTensor, _ name: String, _ c: Int) throws -> MPSGraphTensor {
+ let length = g.squareRoot(with: g.reductionSum(with: g.square(with: x, name: nil), axes: [1], name: nil), name: nil)
+ let normalized = g.division(x, g.maximum(length, g.constant(1e-12, dataType: .float32), name: nil), name: nil)
+ let scaled = g.multiplication(normalized, g.constant(Double(Float(c).squareRoot()), dataType: .float32), name: nil)
+ return g.multiplication(scaled, try weight(name, [1, c, 1, 1]), name: nil)
+ }
+ func silu(_ x: MPSGraphTensor) -> MPSGraphTensor { g.multiplication(x, g.sigmoid(with: x, name: nil), name: nil) }
+ /// QwenImage21ResidualBlock (Wan `residual.{0,2,3,6}`, `shortcut`).
+ func residual(_ x: MPSGraphTensor, _ name: String, _ inC: Int, _ outC: Int) throws -> MPSGraphTensor {
+ let skip = inC == outC ? x : try conv(x, name + ".shortcut", inC, outC, 1)
+ var y = silu(try norm(x, name + ".residual.0.gamma", inC))
+ y = try conv(y, name + ".residual.2", inC, outC, 3)
+ y = silu(try norm(y, name + ".residual.3.gamma", outC))
+ y = try conv(y, name + ".residual.6", outC, outC, 3)
+ return g.addition(y, skip, name: nil)
+ }
+ /// QwenImage21AttentionBlock: one head over all `hw` pixels.
+ func attention(_ x: MPSGraphTensor, _ name: String, _ c: Int, _ hw: Int, _ hh: Int, _ ww: Int) throws -> MPSGraphTensor {
+ let qkv = try conv(try norm(x, name + ".norm.gamma", c), name + ".to_qkv", c, 3 * c, 1) // [1, 3C, H, W]
+ let rows = g.transpose(g.reshape(qkv, shape: n([3 * c, hw]), name: nil), permutation: n([1, 0]), name: nil)
+ func part(_ i: Int) -> MPSGraphTensor {
+ g.reshape(g.sliceTensor(rows, dimension: 1, start: i * c, length: c, name: nil), shape: n([1, 1, hw, c]), name: nil)
+ }
+ let o = g.scaledDotProductAttention(query: part(0), key: part(1), value: part(2), mask: nil,
+ scale: 1 / Float(c).squareRoot(), name: nil)
+ let back = g.reshape(g.transpose(g.reshape(o, shape: n([hw, c]), name: nil), permutation: n([1, 0]), name: nil),
+ shape: n([1, c, hh, ww]), name: nil)
+ return g.addition(try conv(back, name + ".proj", c, c, 1), x, name: nil)
+ }
+ /// QwenImage21DupUp3D on a first chunk: each output channel's 2×2 pixels from the input channels it repeats.
+ func dupUp(_ x: MPSGraphTensor, _ inC: Int, _ outC: Int, _ ft: Int, _ hh: Int, _ ww: Int) -> MPSGraphTensor {
+ let repeats = outC * ft * 4 / inC
+ var y = g.broadcast(g.reshape(x, shape: n([inC, 1, hh, ww]), name: nil), shape: n([inC, repeats, hh, ww]), name: nil)
+ y = g.reshape(y, shape: n([outC, ft, 2, 2, hh, ww]), name: nil)
+ y = g.sliceTensor(y, dimension: 1, start: ft - 1, length: 1, name: nil)
+ y = g.reshape(y, shape: n([outC, 2, 2, hh, ww]), name: nil)
+ y = g.transpose(y, permutation: n([0, 3, 1, 4, 2]), name: nil) // [C, H, 2, W, 2]
+ return g.reshape(y, shape: n([1, outC, 2 * hh, 2 * ww]), name: nil)
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenVAEEncoder.swift b/engine-mac/Sources/KokonEngine/QwenVAEEncoder.swift
new file mode 100644
index 0000000..d8830e8
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/QwenVAEEncoder.swift
@@ -0,0 +1,116 @@
+import Foundation
+import Metal
+import MetalPerformanceShaders
+import MetalPerformanceShadersGraph
+
+/// **The Qwen-Image 2.1 VAE encoder** (`AutoencoderKLQwenImage21.encode`, diffusers 0.42), fp32, for the reference
+/// images of edits: the latent the edit pipeline uses (`_encode_vae_image`: the posterior's mode, `(z − mean) / std`).
+/// For a single image the Wan 2.2 design is a 2D network, as the decoder (`QwenVAE`):
+///
+/// RGBA in [−1, 1] ─ conv_in (3×3, 4 → 96)
+/// 5 down blocks (96 → 96, 96 → 192, 192 → 384, 384 → 768, 768 → 768): 2 residual blocks, then (the first four)
+/// zero pad right/bottom + conv 3×3 stride 2, plus the block input averaged down (AvgDown3D)
+/// mid: residual ─ single-head attention over all pixels ─ residual
+/// RMS norm ─ SiLU ─ conv_out (3×3, 768 → 128) ─ quant_conv (1×1) ─ the first 64 channels (the mode)
+///
+/// The down blocks run as single operations (the decoder's convolution graphs and `vae.metal` kernels, on pooled
+/// buffers), the mid block and the head as one graph at 1/16 of the image (its attention sees every pixel).
+extension QwenVAE {
+ /// Encoder channels (diffusers' `dims`: base 96 × [1, 1, 2, 4, 8, 8]) and each down block's temporal factor
+ /// (`temperal_downsample` [False, True, True, True]; the last block does not downsample).
+ static let encoderDims = [96, 96, 192, 384, 768, 768]
+ static let encoderTemporal = [1, 2, 2, 2, 1]
+
+ /// The normalized latent `[64, H/16, W/16]` of an RGBA image `[4, H, W]` in [−1, 1] (sides multiples of 16).
+ /// In its own autorelease pool: the command buffer and the pooled buffers go when it returns, whoever calls it (a
+ /// loop on a thread without a pool kept 1.5 GB per 1024² encode, M2 Pro, 2026-10-25).
+ public func encode(_ image: [Float], height H: Int, width W: Int) throws -> [Float] {
+ try autoreleasepool { try encodeImage(image, height: H, width: W) }
+ }
+
+ private func encodeImage(_ image: [Float], height H: Int, width W: Int) throws -> [Float] {
+ guard H % 16 == 0, W % 16 == 0, image.count == 4 * H * W else {
+ throw QwenSequence.Failure.layout("VAE encoder: \(image.count) values for 4 × \(H) × \(W) (sides multiples of 16)")
+ }
+ guard file.tensors["encoder.conv1.weight"] != nil else { throw GGUFFile.Failure.unsupported("the VAE file has no encoder") }
+ let commands = MPSCommandBuffer(from: context.queue)
+ let pool = BufferPool(context: context)
+ let dims = QwenVAE.encoderDims
+ let input = try pool.take(image.count)
+ input.contents().copyMemory(from: image, byteCount: 4 * image.count)
+ var (hh, ww) = (H, W)
+ var x = try pool.take(dims[0] * hh * ww)
+ try convolution(input, "encoder.conv1", inC: 4, outC: dims[0], k: 3, height: hh, width: ww, to: x, commands: commands)
+ pool.give(input)
+ for i in 0..<5 { // down_blocks.i
+ let (inC, outC) = (dims[i], dims[i + 1])
+ let blockInput = x
+ var c = inC
+ for j in 0..<2 {
+ let name = "encoder.downsamples.\(i).downsamples.\(j)"
+ let t = try pool.take(c * hh * ww)
+ try normSilu(x, to: t, gamma: name + ".residual.0.gamma", channels: c, pixels: hh * ww, commands: commands)
+ let u = try pool.take(outC * hh * ww)
+ try convolution(t, name + ".residual.2", inC: c, outC: outC, k: 3, height: hh, width: ww, to: u, commands: commands)
+ pool.give(t)
+ try normSilu(u, to: u, gamma: name + ".residual.3.gamma", channels: outC, pixels: hh * ww, commands: commands)
+ let y = try pool.take(outC * hh * ww)
+ try convolution(u, name + ".residual.6", inC: outC, outC: outC, k: 3, height: hh, width: ww, to: y,
+ skip: (x, c == outC ? nil : name + ".shortcut", c), commands: commands)
+ pool.give(u)
+ if x !== blockInput { pool.give(x) }
+ (x, c) = (y, outC)
+ }
+ let fs = i < 4 ? 2 : 1
+ if i < 4 { // downsampler
+ let y = try pool.take(outC * (hh / 2) * (ww / 2))
+ try convolution(x, "encoder.downsamples.\(i).downsamples.2.resample.1", inC: outC, outC: outC, k: 3,
+ height: hh, width: ww, to: y, downsample: true, commands: commands)
+ pool.give(x)
+ (x, hh, ww) = (y, hh / 2, ww / 2)
+ }
+ try dispatch("vae_avg_down_add", [x, blockInput], // avg_shortcut
+ [UInt32(inC), UInt32(outC), UInt32(QwenVAE.encoderTemporal[i]), UInt32(fs), UInt32(hh), UInt32(ww)],
+ grid: MTLSize(width: ww, height: hh, depth: outC), commands: commands)
+ pool.give(blockInput)
+ }
+ let latent = try pool.take(64 * hh * ww)
+ _ = try prepareEncoderTail(height: hh, width: ww)
+ .encode(to: commands, inputs: [data(x, [1, dims[5], hh, ww])], results: [data(latent, [1, 64, hh, ww])],
+ executionDescriptor: nil)
+ commands.commit()
+ commands.waitUntilCompleted()
+ if let error = commands.error { throw MetalContext.Failure.kernel("VAE encode: \(error.localizedDescription)") }
+ var out = [Float](repeating: 0, count: 64 * hh * ww)
+ out.withUnsafeMutableBytes { $0.copyMemory(from: UnsafeRawBufferPointer(start: latent.contents(), count: $0.count)) }
+ return out
+ }
+
+ /// The mid block, the head, quant_conv and the normalization as one graph for an `h` × `w` input, once per size.
+ func prepareEncoderTail(height h: Int, width w: Int) throws -> MPSGraphExecutable {
+ try lock.withLock {
+ let key = "encoder tail \(h)×\(w)"
+ if let e = executables[key] { return e }
+ let g = MPSGraph()
+ let parts = VAEGraph(g: g, file: file)
+ let c = QwenVAE.encoderDims[5]
+ let input = g.placeholder(shape: parts.n([1, c, h, w]), dataType: .float32, name: "input")
+ var x = try parts.residual(input, "encoder.middle.0", c, c) // mid_block.resnets.0
+ x = try parts.attention(x, "encoder.middle.1", c, h * w, h, w) // mid_block.attentions.0
+ x = try parts.residual(x, "encoder.middle.2", c, c) // mid_block.resnets.1
+ x = parts.silu(try parts.norm(x, "encoder.head.0.gamma", c)) // norm_out
+ x = try parts.conv(x, "encoder.head.2", c, 128, 3) // conv_out
+ x = try parts.conv(x, "conv1", 128, 128, 1) // quant_conv
+ x = g.sliceTensor(x, dimension: 1, start: 0, length: 64, name: nil) // the mode: the mean
+ x = g.division(g.subtraction(x, parts.constant(QwenVAE.latentsMean, [1, 64, 1, 1]), name: nil),
+ parts.constant(QwenVAE.latentsStd, [1, 64, 1, 1]), name: nil)
+ let descriptor = MPSGraphCompilationDescriptor()
+ descriptor.waitForCompilationCompletion = true
+ let e = g.compile(with: MPSGraphDevice(mtlDevice: context.device),
+ feeds: [input: MPSGraphShapedType(shape: parts.n([1, c, h, w]), dataType: .float32)],
+ targetTensors: [x], targetOperations: nil, compilationDescriptor: descriptor)
+ executables[key] = e
+ return e
+ }
+ }
+}
diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swift
index 0bfed05..445bf47 100644
--- a/engine-mac/Sources/kokon-engine-mac/main.swift
+++ b/engine-mac/Sources/kokon-engine-mac/main.swift
@@ -9,6 +9,7 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)
check-dit <dit.gguf> <reference dir> one DiT evaluation against tools/reference_dit.py
time-dit <dit.gguf> [grid] [text] [runs] DiT evaluations on random inputs (grid 64 = 1024²)
check-vae <vae.safetensors> <reference dir> [grid] the VAE decoder against tools/reference_vae.py, tiles against the whole graph
+ check-vae-encoder <vae.safetensors> <reference dir> [WxH] the VAE encoder against tools/reference_vae_encoder.py, then its time
time-vae <vae.safetensors> [grid] the VAE decoder's time and memory (64 = 1024²; KOKON_VAE_TILE)
check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.py
check-text <encoder.gguf> <reference dir> the text encoder against tools/reference_text_encoder.py
@@ -79,6 +80,11 @@ do {
case "check-vae":
guard arguments.count >= 3 else { fail(usage) }
try Checks.vae(path: arguments[1], reference: arguments[2], timeGrid: arguments.count > 3 ? Int(arguments[3]) : nil)
+ case "check-vae-encoder":
+ guard arguments.count >= 3 else { fail(usage) }
+ let size = arguments.count > 3 ? arguments[3].split(separator: "x").compactMap { Int($0) } : [1024, 1024]
+ guard size.count == 2 else { fail("size as WxH") }
+ try Checks.vaeEncoder(path: arguments[1], reference: arguments[2], timeSize: (size[0], size[1]))
case "time-vae":
guard arguments.count >= 2 else { fail(usage) }
try Checks.timeVAE(path: arguments[1], grid: arguments.count > 2 ? Int(arguments[2]) ?? 64 : 64)
diff --git a/engine-mac/tools/reference_vae_encoder.py b/engine-mac/tools/reference_vae_encoder.py
new file mode 100644
index 0000000..2e17a08
--- /dev/null
+++ b/engine-mac/tools/reference_vae_encoder.py
@@ -0,0 +1,105 @@
+"""The Qwen-Image 2.1 VAE encoder reference for the Mac engine: diffusers' `AutoencoderKLQwenImage21.encode` in fp32 on
+the CPU, on Kokon's VAE file, as the edit pipeline uses it (`QwenImage21Pipeline._encode_vae_image`: the posterior's
+mode, then `(latent − mean) / std`). The input is a smooth random RGBA image in [−1, 1] (random low-resolution noise,
+upsampled bilinearly, so it has edges and flat areas as photos do). Saves the input, every down block's output, the
+mid block's and the normalized latent for `kokon-engine-mac check-vae-encoder`.
+
+ python -I engine-mac/tools/reference_vae_encoder.py <vae.safetensors> <out dir> [--size 256x256]
+"""
+import argparse
+import json
+import os
+import sys
+
+import torch
+import torch.nn.functional as F
+from safetensors.torch import load_file
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from reference_vae import decoder_name # noqa: E402
+
+RESIDUAL = {"residual.0.gamma": "norm1.gamma", "residual.2.weight": "conv1.weight", "residual.2.bias": "conv1.bias",
+ "residual.3.gamma": "norm2.gamma", "residual.6.weight": "conv2.weight", "residual.6.bias": "conv2.bias",
+ "shortcut.weight": "conv_shortcut.weight", "shortcut.bias": "conv_shortcut.bias"}
+
+
+def encoder_name(key):
+ """Wan 2.2 VAE name -> diffusers AutoencoderKLQwenImage21 name, encoder side; None for the decoder."""
+ if key.startswith("conv1."):
+ return "quant_conv." + key[6:]
+ if not key.startswith("encoder."):
+ return None
+ parts = key[len("encoder."):].split(".")
+ if parts[0] == "conv1":
+ return "encoder.conv_in." + parts[1]
+ if parts[0] == "head":
+ return "encoder.norm_out.gamma" if parts[1] == "0" else "encoder.conv_out." + parts[2]
+ if parts[0] == "middle":
+ rest = ".".join(parts[2:])
+ if parts[1] == "1":
+ return "encoder.mid_block.attentions.0." + rest
+ return f"encoder.mid_block.resnets.{int(parts[1]) // 2}." + RESIDUAL[rest]
+ if parts[0] == "downsamples":
+ i, j, rest = parts[1], int(parts[3]), ".".join(parts[4:])
+ if j == 2:
+ return f"encoder.down_blocks.{i}.downsampler." + rest
+ return f"encoder.down_blocks.{i}.resnets.{j}." + RESIDUAL[rest]
+ raise KeyError(key)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("vae")
+ parser.add_argument("out")
+ parser.add_argument("--size", default="256x256", help="WxH, multiples of 16")
+ parser.add_argument("--seed", type=int, default=11)
+ args = parser.parse_args()
+ from diffusers import AutoencoderKLQwenImage21
+ torch.manual_seed(args.seed)
+ torch.set_grad_enabled(False)
+ os.makedirs(args.out, exist_ok=True)
+ width, height = (int(v) for v in args.size.split("x"))
+
+ vae = AutoencoderKLQwenImage21() # the defaults are Qwen-Image 2.1's config
+ model = vae.state_dict()
+ converted = {}
+ for key, value in load_file(args.vae).items():
+ name = encoder_name(key) or decoder_name(key)
+ if name is not None:
+ converted[name] = value.float().reshape(model[name].shape)
+ missing, unexpected = vae.load_state_dict(converted, strict=False)
+ missing = [k for k in missing if k.startswith("encoder.") or k.startswith("quant_conv")]
+ assert not missing and not unexpected, (missing[:10], unexpected[:10])
+ vae.eval()
+
+ image = torch.rand(1, 4, 1, height // 16, width // 16) * 2 - 1
+ image = F.interpolate(image[:, :, 0], size=(height, width), mode="bilinear", align_corners=False)[:, :, None]
+ image = image + 0.05 * torch.randn_like(image) # some fine texture
+ image = image.clamp(-1, 1)
+
+ saved = {}
+
+ def save(name, tensor):
+ array = tensor.detach().to(torch.float32).contiguous().numpy()
+ array.tofile(os.path.join(args.out, name + ".f32"))
+ saved[name] = list(array.shape)
+
+ hooks = [block.register_forward_hook(lambda m, a, o, i=i: save(f"down{i}", o[0, :, 0]))
+ for i, block in enumerate(vae.encoder.down_blocks)]
+ hooks.append(vae.encoder.conv_in.register_forward_hook(lambda m, a, o: save("conv_in", o[0, :, 0])))
+ hooks.append(vae.encoder.mid_block.register_forward_hook(lambda m, a, o: save("mid", o[0, :, 0])))
+ latent = vae.encode(image).latent_dist.mode()
+ for h in hooks:
+ h.remove()
+ mean = torch.tensor(vae.config.latents_mean).view(1, 64, 1, 1, 1)
+ std = torch.tensor(vae.config.latents_std).view(1, 64, 1, 1, 1)
+ save("image", image[0, :, 0])
+ save("latent", ((latent - mean) / std)[0, :, 0])
+ with open(os.path.join(args.out, "meta.json"), "w") as f:
+ json.dump({"size": [width, height], "seed": args.seed, "tensors": saved,
+ "torch": torch.__version__}, f, indent=1)
+ print(f"{width}×{height}: latent {saved['latent']}")
+
+
+if __name__ == "__main__":
+ main()