Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: the vision tower, checked against transformers

Gabriele Lozupone76d0604c8bfa
diff --git a/engine-mac/README.md b/engine-mac/README.md
index c82241d..4e58c89 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -45,6 +45,7 @@ Windows and Linux.
| `AttentionProbe.swift` | `probe-attention`: the target's attention on each unit alone |
| `QwenVAE.swift`, `Safetensors.swift` | the VAE decoder as one MPSGraph, from Kokon's VAE file (Wan 2.2 names) |
| `QwenVAEEncoder.swift` | the VAE encoder (edits' reference images), checked against diffusers |
+| `QwenVisionTower.swift`, `Kernels/vision.metal` | Qwen3-VL's vision tower from the mmproj GGUF (edits' reference images): patches, 27 blocks, merger and deepstack outputs |
| `QwenTokenizer.swift` | Qwen's byte-level BPE tokenizer, read from the encoder GGUF's metadata |
| `QwenTextEncoder.swift`, `Kernels/text.metal` | the text encoder: Qwen3-VL-8B's text decoder on the GPU, diffusers' prompt template |
| `QwenPipeline.swift` | text to image: noise, Turbo's 8 flow-Euler steps (`planner::TURBO_SIGMAS`), VAE, PNG; models kept loaded for the server |
@@ -387,5 +388,22 @@ and the head as one graph at 1/16 of the image, then the posterior's mode normal
texture-fix VAE, M2 Pro, 2026-10-25): the latent within 6.5e-6. 1024²: 0.59 s an encode, peak 2.4 GB, 0.87 GB kept after
(its weights); in its own autorelease pool (without one, a loop on a thread without a pool kept 1.5 GB per encode).
+**Vision tower** (`QwenVisionTower.swift`; what an edit's reference image becomes in the text encoder's
+`<|image_pad|>` slots): Qwen3-VL's ViT from Kokon's mmproj GGUF (llama.cpp's layout: the Conv3d as one kernel per frame,
+fused q/k/v, `mm.0`/`mm.2` the merger, `v.deepstack.8/16/24`), fp32 on the GPU with the F16 weights unpacked before each
+product. The patches in the processor's 2 × 2 merge-block order, the two frames' kernels summed (both frames are the
+image), the 48 × 48 position table resampled bilinearly, axial 2D RoPE, the 72-dimension heads on the DiT's flash kernel
+zero-padded to 128. Its input is the processor's: alpha over white as PIL's paste, and below 65 536 pixels the processor's
+resize (torchvision's uint8 antialiased bicubic: PIL's kernel with int16 weights, horizontal pass first; bit for bit).
+`check-vision` against `tools/reference_vision.py` (transformers' `Qwen3VLVisionModel`, M2 Pro, 2026-10-25), 320 × 224:
+the pixels within 1.6e-7 (float rounding), the merged rows within 1.35e-5 of torch fp32 and **7.6e-6 of fp64**
+(`--dtype float64`; torch's own fp32 is 1.55e-5 from it), deepstack 3.8–9.4e-6; 256 × 192 (resized to 320 × 224) and
+an RGBA image the same. 1024²: 2.5 s an encode (4 096 patches; the padded heads cost ~1.8× the attention's work).
+
+```bash
+~/kokon-bench/venv-oracle/bin/python -I tools/reference_vision.py <mmproj.gguf> <processor dir> <text_encoder dir> ~/kokon-bench/ref/vision-320 [--dtype float64]
+.build/release/kokon-engine-mac check-vision <mmproj.gguf> ~/kokon-bench/ref/vision-320
+```
+
Credits: the Neural Engine program technique and the GPU + AMX + Neural Engine split come from
[Siliconed](https://github.com/gwenn-ha-dev/Siliconed) (Apache-2.0), written again here.
diff --git a/engine-mac/Sources/KokonEngine/Kernels/vision.metal b/engine-mac/Sources/KokonEngine/Kernels/vision.metal
new file mode 100644
index 0000000..9d6d7ea
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/Kernels/vision.metal
@@ -0,0 +1,98 @@
+#include <metal_stdlib>
+using namespace metal;
+
+// The Qwen3-VL vision tower's kernels, fp32, following transformers' modeling_qwen3_vl.py (QwenVisionTower.swift says
+// which module each one replaces). Rows are patches (or merged groups of 4 patches) of `columns` floats.
+
+struct AffineParams {
+ uint columns;
+ float eps;
+};
+
+// nn.LayerNorm(columns, eps) with weight and bias: norm1/norm2, the mergers' norms. One threadgroup of 256 per row;
+// the mean first, then the variance around it (two passes, as torch's CPU kernel's result).
+kernel void layernorm_affine(device const float *x [[buffer(0)]], device float *out [[buffer(1)]],
+ device const float *weight [[buffer(2)]], device const float *bias [[buffer(3)]],
+ constant AffineParams &p [[buffer(4)]],
+ uint row [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup float shared[8];
+ device const float *r = x + (ulong)row * p.columns;
+ device float *o = out + (ulong)row * p.columns;
+ float s = 0;
+ for (uint j = tid; j < p.columns; j += 256) s += r[j];
+ s = simd_sum(s);
+ if (tid % 32 == 0) shared[tid / 32] = s;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float mean = 0;
+ for (uint i = 0; i < 8; i++) mean += shared[i];
+ mean /= float(p.columns);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float v = 0;
+ for (uint j = tid; j < p.columns; j += 256) { float d = r[j] - mean; v += d * d; }
+ v = simd_sum(v);
+ if (tid % 32 == 0) shared[tid / 32] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float var = 0;
+ for (uint i = 0; i < 8; i++) var += shared[i];
+ float inv = rsqrt(var / float(p.columns) + p.eps);
+ for (uint j = tid; j < p.columns; j += 256) o[j] = (r[j] - mean) * inv * weight[j] + bias[j];
+}
+
+// x[r, c] += b[c]: the Linear layers' bias after the product (count = rows × columns).
+kernel void bias_rows(device float *x [[buffer(0)]], device const float *b [[buffer(1)]],
+ constant uint2 &p [[buffer(2)]], uint i [[thread_position_in_grid]]) {
+ if (i >= p.x) return;
+ x[i] += b[i % p.y];
+}
+
+// nn.GELU() (erf), in place, the mergers' activation: 0.5 x erfc(−x / √2), erfc as Numerical Recipes' erfcc (fractional
+// error below 1.2e-7 everywhere, so the tail for negative x keeps its relative precision).
+kernel void gelu_erf(device float *x [[buffer(0)]], constant uint &count [[buffer(1)]],
+ uint i [[thread_position_in_grid]]) {
+ if (i >= count) return;
+ float v = x[i];
+ float z = -v * 0.7071067811865476f;
+ float a = fabs(z), t = 1.0f / (1.0f + 0.5f * a);
+ float e = t * precise::exp(-a * a - 1.26551223f + t * (1.00002368f + t * (0.37409196f + t * (0.09678418f +
+ t * (-0.18628806f + t * (0.27886807f + t * (-1.13520398f + t * (1.48851587f +
+ t * (-0.82215223f + t * 0.17087277f)))))))));
+ float tail = z >= 0 ? e : 2.0f - e;
+ x[i] = 0.5f * v * tail;
+}
+
+// Qwen3VLVisionAttention's qkv split and apply_rotary_pos_emb_vision, into the flash kernel's layout: q, k, v as
+// [rows, 16 heads, 128] with the head's 72 dimensions first and zeros after (the scores and the output are the
+// 72-dimension ones). rotate_half pairs dimension i with i + 36; `rope` is [rows, 36, (cos, sin)].
+// One thread per (row, head, j < 64): j < 36 rotates the pair (j, j + 36), the others clear two padding columns.
+kernel void vision_rope_pad(device const float *qkv [[buffer(0)]], device float *q [[buffer(1)]],
+ device float *k [[buffer(2)]], device float *v [[buffer(3)]],
+ device const float *rope [[buffer(4)]], constant uint &rows [[buffer(5)]],
+ uint i [[thread_position_in_grid]]) {
+ uint j = i % 64, head = (i / 64) % 16, row = i / 1024;
+ if (row >= rows) return;
+ device const float *in = qkv + (ulong)row * 3456 + head * 72;
+ ulong o = (ulong)row * 2048 + head * 128;
+ if (j < 36) {
+ float c = rope[((ulong)row * 36 + j) * 2], s = rope[((ulong)row * 36 + j) * 2 + 1];
+ float q0 = in[j], q1 = in[j + 36], k0 = in[1152 + j], k1 = in[1152 + j + 36];
+ q[o + j] = q0 * c - q1 * s;
+ q[o + j + 36] = q1 * c + q0 * s;
+ k[o + j] = k0 * c - k1 * s;
+ k[o + j + 36] = k1 * c + k0 * s;
+ v[o + j] = in[2304 + j];
+ v[o + j + 36] = in[2304 + j + 36];
+ } else {
+ uint c = 72 + (j - 36) * 2;
+ q[o + c] = 0; q[o + c + 1] = 0;
+ k[o + c] = 0; k[o + c + 1] = 0;
+ v[o + c] = 0; v[o + c + 1] = 0;
+ }
+}
+
+// The attention's output back to [rows, 16 × 72] for attn.proj.
+kernel void vision_unpad(device const float *x [[buffer(0)]], device float *out [[buffer(1)]],
+ constant uint &rows [[buffer(2)]], uint i [[thread_position_in_grid]]) {
+ if (i >= rows * 1152) return;
+ uint row = i / 1152, c = i % 1152;
+ out[i] = x[(ulong)row * 2048 + (c / 72) * 128 + c % 72];
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenPipeline.swift b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
index 642cbc8..b5c0ab1 100644
--- a/engine-mac/Sources/KokonEngine/QwenPipeline.swift
+++ b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
@@ -232,6 +232,44 @@ public final class QwenPipeline {
try pngData(rgba, width: width, height: height).write(to: URL(fileURLWithPath: path))
}
+ /// An image file (PNG, JPEG…) as interleaved 8-bit RGBA, straight alpha. 8-bit RGB(A) files are read as stored, without
+ /// color management, as PIL reads them for diffusers; other layouts (16-bit, gray, palette) are drawn into an sRGB
+ /// context, premultiplied, and divided back.
+ public static func decodeImage(_ data: Data) throws -> (rgba: [UInt8], width: Int, height: Int) {
+ guard let source = CGImageSourceCreateWithData(data as CFData, nil),
+ let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else {
+ throw MetalContext.Failure.buffer("cannot decode the image")
+ }
+ let (w, h) = (image.width, image.height)
+ var out = [UInt8](repeating: 255, count: w * h * 4)
+ let alpha = image.alphaInfo, order = image.bitmapInfo.intersection(.byteOrderMask)
+ let direct = image.bitsPerComponent == 8 && (order == [] || order == .byteOrderDefault || order == .byteOrder32Big)
+ && image.colorSpace?.model == .rgb
+ && ((image.bitsPerPixel == 32 && [.last, .noneSkipLast].contains(alpha)) || (image.bitsPerPixel == 24 && alpha == .none))
+ if direct, let bytes = image.dataProvider?.data, let p = CFDataGetBytePtr(bytes) {
+ let step = image.bitsPerPixel / 8
+ for y in 0..<h {
+ let row = p + y * image.bytesPerRow
+ for x in 0..<w {
+ for c in 0..<3 { out[(y * w + x) * 4 + c] = row[x * step + c] }
+ if alpha == .last { out[(y * w + x) * 4 + 3] = row[x * step + 3] }
+ }
+ }
+ return (out, w, h)
+ }
+ guard let context = CGContext(data: &out, width: w, height: h, bitsPerComponent: 8, bytesPerRow: w * 4,
+ space: CGColorSpace(name: CGColorSpace.sRGB)!,
+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else {
+ throw MetalContext.Failure.buffer("cannot decode the image")
+ }
+ context.draw(image, in: CGRect(x: 0, y: 0, width: w, height: h))
+ for p in 0..<(w * h) where out[p * 4 + 3] != 255 {
+ let a = Int(out[p * 4 + 3])
+ for c in 0..<3 { out[p * 4 + c] = a == 0 ? 0 : UInt8(min(255, (Int(out[p * 4 + c]) * 255 + a / 2) / a)) }
+ }
+ return (out, w, h)
+ }
+
/// Interleaved 8-bit RGBA (straight alpha) encoded as a PNG.
public static func pngData(_ rgba: [UInt8], width: Int, height: Int) throws -> Data {
let data = Data(rgba) as CFData
diff --git a/engine-mac/Sources/KokonEngine/QwenVisionTower.swift b/engine-mac/Sources/KokonEngine/QwenVisionTower.swift
new file mode 100644
index 0000000..ff474cb
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/QwenVisionTower.swift
@@ -0,0 +1,462 @@
+import Foundation
+import Metal
+import MetalPerformanceShaders
+
+/// **Qwen3-VL-8B's vision tower** (`Qwen3VLVisionModel`, transformers 5) on the GPU, fp32, from Kokon's mmproj GGUF
+/// (llama.cpp's layout, F16 weights unpacked just before each product): what an edit's reference image becomes in the
+/// text encoder's `<|image_pad|>` slots.
+///
+/// image ─ 16 × 16 patches, in 2 × 2 merge blocks ─ patch_embed (Conv3d over the image twice) + bilinear position table
+/// 27 × { x += proj(SDPA(rope2d(q), rope2d(k), v)) on LN1(x) (16 heads of 72, every patch sees every patch)
+/// x += fc2(GELU_tanh(fc1(LN2(x)))) }
+/// merged = fc2(GELU(fc1(LN(x) as [N/4, 4608]))) [N/4, 4096], one row per text encoder slot
+/// deepstack j = the same after block 8 / 16 / 24 (LN after the 2 × 2 merge) added to the text encoder's layers 0–2
+///
+/// The heads (72 dimensions) run on the DiT's flash kernel zero-padded to 128 (`vision_rope_pad`): the scores and the
+/// output are the 72-dimension ones. The processor's input is the image with alpha composited over white (as the
+/// pipeline's PIL paste), scaled to [−1, 1], each frame of the Conv3d's two the same image.
+public final class QwenVisionTower {
+ public static let dim = 1152, heads = 16, headDim = 72, hidden = 4304, layers = 27, outDim = 4096
+ static let patch = 16, merge = 2, positionSide = 48, eps: Float = 1e-6, theta: Float = 10_000
+ /// Blocks whose output also goes through a deepstack merger (`deepstack_visual_indexes`).
+ public static let deepstackLayers = [8, 16, 24]
+ /// The processor's `smart_resize` bounds (`size.shortest_edge`, `longest_edge`): pixels of the image it encodes.
+ static let minPixels = 65_536, maxPixels = 16_777_216
+
+ public let file: GGUFFile
+ let context: MetalContext
+ let dequant: GPUDequant
+ private var products: [String: MPSMatrixMultiplication] = [:]
+ /// Seconds of the last encode.
+ public private(set) var seconds: Double = 0
+
+ public init(file: GGUFFile, context: MetalContext) throws {
+ guard file.metadata["clip.projector_type"]?.string == "qwen3vl_merger",
+ file.metadata["clip.vision.block_count"]?.int == QwenVisionTower.layers,
+ file.metadata["clip.vision.embedding_length"]?.int == QwenVisionTower.dim else {
+ throw GGUFFile.Failure.unsupported("\(file.path): not Qwen3-VL-8B's vision tower (mmproj)")
+ }
+ self.file = file
+ self.context = context
+ dequant = GPUDequant(context: context)
+ }
+
+ public func release() { context.release(file) }
+
+ /// What the tower makes of one image: `merged` `[slots, 4096]` and one deepstack `[slots, 4096]` per entry of
+ /// `deepstackLayers`, slots = (height / 32) · (width / 32) in merge-block order (row-major over 32 × 32 cells).
+ public struct Output {
+ public let merged: [Float]
+ public let deepstack: [[Float]]
+ public let height, width: Int
+ public var slots: Int { (height / 32) * (width / 32) }
+ }
+
+ // MARK: - the processor
+
+ /// Qwen2VLImageProcessor's `smart_resize` for sides already multiples of 32 (as the pipeline makes them): the size
+ /// the tower sees, which differs only below 65 536 or above 16 777 216 pixels.
+ public static func visionSize(width: Int, height: Int) -> (width: Int, height: Int) {
+ let factor = Double(patch * merge)
+ var h = Int((Double(height) / factor).rounded(.toNearestOrEven) * factor)
+ var w = Int((Double(width) / factor).rounded(.toNearestOrEven) * factor)
+ if h * w > maxPixels {
+ let beta = (Double(height * width) / Double(maxPixels)).squareRoot()
+ h = max(Int(factor), Int((Double(height) / beta / factor).rounded(.down) * factor))
+ w = max(Int(factor), Int((Double(width) / beta / factor).rounded(.down) * factor))
+ } else if h * w < minPixels {
+ let beta = (Double(minPixels) / Double(height * width)).squareRoot()
+ h = Int((Double(height) * beta / factor).rounded(.up) * factor)
+ w = Int((Double(width) * beta / factor).rounded(.up) * factor)
+ }
+ return (w, h)
+ }
+
+ /// Interleaved 8-bit RGBA → the tower's input: alpha composited over white as PIL's `paste` with a mask does
+ /// (the pipeline's `white.paste(img, mask=alpha)`), resized when `visionSize` says so (bicubic), then planar RGB
+ /// `[3, H, W]` scaled to [−1, 1] (`rescale 1/255`, `normalize 0.5, 0.5`).
+ public static func pixels(rgba: [UInt8], width: Int, height: Int) -> (rgb: [Float], width: Int, height: Int) {
+ var rgb = [UInt8](repeating: 0, count: width * height * 3)
+ for p in 0..<(width * height) {
+ let a = Int(rgba[p * 4 + 3])
+ for c in 0..<3 {
+ // ImagingPaste's BLEND: in·a + 255·(255 − a), divided by 255 with rounding (PIL's MULDIV255).
+ let t = Int(rgba[p * 4 + c]) * a + 255 * (255 - a) + 128
+ rgb[p * 3 + c] = UInt8((t + (t >> 8)) >> 8)
+ }
+ }
+ let (w, h) = visionSize(width: width, height: height)
+ if (w, h) != (width, height) { rgb = resizeBicubic(rgb, width: width, height: height, to: w, h) }
+ var out = [Float](repeating: 0, count: 3 * w * h)
+ for p in 0..<(w * h) {
+ for c in 0..<3 { out[c * w * h + p] = (Float(rgb[p * 3 + c]) * (1 / 255.0) - 0.5) / 0.5 }
+ }
+ return (out, w, h)
+ }
+
+ /// Bicubic resize of interleaved 8-bit RGB exactly as torchvision's antialiased resize of a uint8 image (the
+ /// processor's path; PIL's algorithm with torch's integer arithmetic): a = −0.5, the kernel stretched by the scale
+ /// when shrinking, each output's weights normalized then quantized to int16 with as many fraction bits as the
+ /// largest fits in, the horizontal pass rounded to 8 bits before the vertical one. Bit for bit against torchvision
+ /// 0.26 on a 256 × 192 → 320 × 224 image (2026-10-25). Only images outside the processor's pixel bounds go through it.
+ static func resizeBicubic(_ src: [UInt8], width: Int, height: Int, to w: Int, _ h: Int) -> [UInt8] {
+ func taps(_ out: Int, _ inSize: Int) -> (taps: [(Int, [Int])], bits: Int) {
+ let scale = Double(inSize) / Double(out), support = 2 * max(scale, 1), invScale = 1 / max(scale, 1)
+ func cubic(_ x: Double) -> Double {
+ let a = -0.5, x = abs(x)
+ if x <= 1 { return ((a + 2) * x - (a + 3)) * x * x + 1 }
+ if x < 2 { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a }
+ return 0
+ }
+ let real = (0..<out).map { i -> (Int, [Double]) in
+ let center = scale * (Double(i) + 0.5)
+ let lo = max(Int(center - support + 0.5), 0), hi = min(Int(center + support + 0.5), inSize)
+ let weights = (lo..<hi).map { cubic((Double($0) - center + 0.5) * invScale) }
+ let total = weights.reduce(0, +)
+ return (lo, weights.map { $0 / total })
+ }
+ let largest = real.flatMap { $0.1 }.map(abs).max() ?? 1
+ var bits = 0
+ while bits < 22, Int(0.5 + largest * Double(1 << (bits + 1))) < 1 << 15 { bits += 1 }
+ let scaleBits = Double(1 << bits)
+ return (real.map { ($0.0, $0.1.map { v in Int(v >= 0 ? (v * scaleBits + 0.5).rounded(.down) : (v * scaleBits - 0.5).rounded(.up)) }) }, bits)
+ }
+ func clamp(_ v: Int, _ bits: Int) -> UInt8 { UInt8(min(max((v + (1 << (bits - 1))) >> bits, 0), 255)) }
+ let (tx, bx) = taps(w, width), (ty, by) = taps(h, height)
+ var horizontal = [UInt8](repeating: 0, count: w * height * 3)
+ for y in 0..<height {
+ for x in 0..<w {
+ let (lo, weights) = tx[x]
+ for c in 0..<3 {
+ var s = 0
+ for (j, wt) in weights.enumerated() { s += wt * Int(src[(y * width + lo + j) * 3 + c]) }
+ horizontal[(y * w + x) * 3 + c] = clamp(s, bx)
+ }
+ }
+ }
+ var out = [UInt8](repeating: 0, count: w * h * 3)
+ for y in 0..<h {
+ let (lo, weights) = ty[y]
+ for x in 0..<w {
+ for c in 0..<3 {
+ var s = 0
+ for (j, wt) in weights.enumerated() { s += wt * Int(horizontal[((lo + j) * w + x) * 3 + c]) }
+ out[(y * w + x) * 3 + c] = clamp(s, by)
+ }
+ }
+ }
+ return out
+ }
+
+ // MARK: - forward
+
+ /// Called with a stage's name and its rows (CPU pointer, floats); the pass then waits on the GPU at each stage.
+ /// Stages: patches, block0, block8, block26 (the processor's order: 2 × 2 merge blocks).
+ public typealias Probe = (String, UnsafePointer<Float>, Int) -> Void
+
+ struct Buffers {
+ let arena: Arena
+ let x, normed, qkv, q, k, v, attended, a72, h, mergeIn, mergeHidden, smalls, rope: Arena.Slice
+ let outputs: [Arena.Slice]
+ let reserves: [Arena.Slice]
+ }
+ private var nextReserve = 0
+ /// Float offsets of the small F32 tensors (norms, biases) in `smalls`.
+ private var small: [String: Int] = [:]
+
+ /// The tower on planar RGB `[3, height, width]` in [−1, 1] (`pixels`), sides multiples of 32. In its own
+ /// autorelease pool.
+ public func encode(rgb: [Float], width: Int, height: Int, probe: Probe? = nil) throws -> Output {
+ try autoreleasepool { try forward(rgb: rgb, width: width, height: height, probe: probe) }
+ }
+
+ private func forward(rgb: [Float], width: Int, height: Int, probe: Probe?) throws -> Output {
+ guard width % 32 == 0, height % 32 == 0, rgb.count == 3 * width * height else {
+ throw QwenSequence.Failure.layout("vision tower: \(rgb.count) values for 3 × \(height) × \(width) (sides multiples of 32)")
+ }
+ let started = DispatchTime.now().uptimeNanoseconds
+ let d = QwenVisionTower.dim, (gh, gw) = (height / 16, width / 16), n = gh * gw, slots = n / 4
+ let merged = 4 * d // 4608
+ // Rows padded as the DiT's (64), plus 64 for the attention kernel's overrun; x is also read as [slots, 4608].
+ let rows = max(QwenDiT.padded(n) + 64, 4 * QwenDiT.padded(slots))
+ let mrows = QwenDiT.padded(slots)
+ let smallNames = smallTensors()
+ let smallCount = try smallNames.reduce(0) { $0 + (try file.tensor($1)).count } + n * d
+ let sizes = [rows * d, rows * d, rows * 3 * d, rows * 2048, rows * 2048, rows * 2048, rows * 2048, rows * d,
+ rows * QwenVisionTower.hidden, mrows * merged, mrows * merged, smallCount, n * 72,
+ slots * 4096, slots * 4096, slots * 4096, slots * 4096, merged * merged, merged * merged]
+ let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))
+ var s = try sizes.map { try arena.slice(floats: $0) }.makeIterator()
+ let b = Buffers(arena: arena, x: s.next()!, normed: s.next()!, qkv: s.next()!, q: s.next()!, k: s.next()!,
+ v: s.next()!, attended: s.next()!, a72: s.next()!, h: s.next()!, mergeIn: s.next()!,
+ mergeHidden: s.next()!, smalls: s.next()!, rope: s.next()!,
+ outputs: [s.next()!, s.next()!, s.next()!, s.next()!], reserves: [s.next()!, s.next()!])
+
+ // Norms and biases (F32), then the position table after them.
+ small = [:]
+ var at = 0
+ for name in smallNames {
+ let values = try Dequant.array(file, name)
+ arena.pointer(b.smalls, floatOffset: at).update(from: values, count: values.count)
+ small[name] = at
+ at += values.count
+ }
+ small["positions"] = at
+ let table = try positions(gridHeight: gh, gridWidth: gw)
+ arena.pointer(b.smalls, floatOffset: at).update(from: table, count: table.count)
+
+ // Patches [n, 768] in merge-block order (each row: channel, then 16 × 16 pixels): the Conv3d's two frames see
+ // the same image, so its two kernels are summed (`v.patch_embd.weight` + `.weight.1`) into one [768, 1152].
+ let patchRows = arena.pointer(b.normed)
+ var index = 0
+ for by in 0..<(gh / 2) { for bx in 0..<(gw / 2) { for iy in 0..<2 { for ix in 0..<2 {
+ let (py, px) = ((by * 2 + iy) * 16, (bx * 2 + ix) * 16)
+ for c in 0..<3 { for y in 0..<16 { for x in 0..<16 {
+ patchRows[index * 768 + c * 256 + y * 16 + x] = rgb[(c * height + py + y) * width + px + x]
+ } } }
+ index += 1
+ } } } }
+ let w0 = try Dequant.array(file, "v.patch_embd.weight"), w1 = try Dequant.array(file, "v.patch_embd.weight.1")
+ let kernel = arena.pointer(b.reserves[0])
+ for o in 0..<d { for i in 0..<768 { kernel[i * d + o] = w0[o * 768 + i] + w1[o * 768 + i] } }
+
+ var commands = MPSCommandBuffer(from: context.queue) as MTLCommandBuffer
+ try product(b, a: b.normed, w: b.reserves[0], c: b.x, m: n, k: 768, n: d, commands: commands)
+ try bias(b, b.x, "v.patch_embd.bias", rows: n, columns: d, commands: commands)
+ if let probe {
+ try MetalContext.finish(commands, "vision tower")
+ probe("patches", arena.pointer(b.x), n * d)
+ commands = MPSCommandBuffer(from: context.queue)
+ }
+ try add(b, b.x, b.smalls, offset: small["positions"]!, count: n * d, commands: commands)
+ try ropeTable(b, gridHeight: gh, gridWidth: gw)
+
+ for layer in 0..<QwenVisionTower.layers {
+ try encodeBlock(layer, rows: n, b, commands: commands)
+ if let j = QwenVisionTower.deepstackLayers.firstIndex(of: layer) {
+ // Deepstack merger: the block's output as [slots, 4608], then its LayerNorm over 4608.
+ try layerNorm(b, b.x, into: b.mergeIn, prefix: "v.deepstack.\(layer).norm", rows: slots, columns: merged,
+ commands: commands)
+ try mergerMLP(b, prefix: "v.deepstack.\(layer)", fc1: "fc1", fc2: "fc2", out: b.outputs[1 + j], slots: slots,
+ commands: commands)
+ }
+ if let probe, [0, 8, 26].contains(layer) {
+ try MetalContext.finish(commands, "vision tower")
+ probe("block\(layer)", arena.pointer(b.x), n * d)
+ } else {
+ commands.commit()
+ }
+ commands = MPSCommandBuffer(from: context.queue)
+ }
+ // Merger: post_ln on each patch, then the 2 × 2 groups as rows of 4608.
+ try layerNorm(b, b.x, into: b.mergeIn, prefix: "v.post_ln", rows: n, columns: d, commands: commands)
+ try mergerMLP(b, prefix: "mm", fc1: "0", fc2: "2", out: b.outputs[0], slots: slots, commands: commands)
+ try MetalContext.finish(commands, "vision tower")
+ seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
+ func read(_ slice: Arena.Slice) -> [Float] { Array(UnsafeBufferPointer(start: arena.pointer(slice), count: slots * 4096)) }
+ return Output(merged: read(b.outputs[0]), deepstack: (1...3).map { read(b.outputs[$0]) }, height: height, width: width)
+ }
+
+ func smallTensors() -> [String] {
+ var names = ["v.patch_embd.bias", "v.post_ln.weight", "v.post_ln.bias", "mm.0.bias", "mm.2.bias"]
+ for i in 0..<QwenVisionTower.layers {
+ for t in ["ln1.weight", "ln1.bias", "ln2.weight", "ln2.bias", "attn_qkv.bias", "attn_out.bias", "ffn_up.bias", "ffn_down.bias"] {
+ names.append("v.blk.\(i).\(t)")
+ }
+ }
+ for l in QwenVisionTower.deepstackLayers {
+ for t in ["norm.weight", "norm.bias", "fc1.bias", "fc2.bias"] { names.append("v.deepstack.\(l).\(t)") }
+ }
+ return names
+ }
+
+ /// `pos_embed` (48 × 48 learned rows) resampled bilinearly, corners aligned, to each patch (merge-block order), as
+ /// `get_vision_interpolation_indices_and_weights`: the four taps' rows weighted and summed in fp32.
+ func positions(gridHeight gh: Int, gridWidth gw: Int) throws -> [Float] {
+ let d = QwenVisionTower.dim, side = QwenVisionTower.positionSide
+ let table = try Dequant.array(file, "v.position_embd.weight") // [2304, 1152]
+ func taps(_ index: Int, _ size: Int) -> [(Int, Float)] {
+ let src = Float(index) * Float(side - 1) / Float(max(size - 1, 1))
+ let floor = src.rounded(.down)
+ return (0..<2).map { o in
+ let tap = min(max(Int(floor) + o, 0), side - 1)
+ return (tap, max(1 - abs(src - floor - Float(o)), 0))
+ }
+ }
+ var out = [Float](repeating: 0, count: gh * gw * d)
+ var index = 0
+ for by in 0..<(gh / 2) { for bx in 0..<(gw / 2) { for iy in 0..<2 { for ix in 0..<2 {
+ let (row, col) = (by * 2 + iy, bx * 2 + ix)
+ let hs = taps(row, gh), ws = taps(col, gw)
+ var acc = [Float](repeating: 0, count: d)
+ for (ht, hw) in hs { for (wt, ww) in ws {
+ let weight = hw * ww, base = (ht * side + wt) * d
+ for c in 0..<d { acc[c] += table[base + c] * weight }
+ } }
+ for c in 0..<d { out[index * d + c] = acc[c] }
+ index += 1
+ } } } }
+ return out
+ }
+
+ /// `[rows, 36, (cos, sin)]` of the axial rotary embedding: dimensions 0–17 rotate by the patch's row, 18–35 by its
+ /// column (`inv_freq = 1 / 10000^(2i / 36)`), and dimension i + 36 shares dimension i's angle.
+ func ropeTable(_ b: Buffers, gridHeight gh: Int, gridWidth gw: Int) throws {
+ let inv = (0..<18).map { i in 1 / powf(QwenVisionTower.theta, Float(2 * i) / 36) }
+ let p = b.arena.pointer(b.rope)
+ var index = 0
+ for by in 0..<(gh / 2) { for bx in 0..<(gw / 2) { for iy in 0..<2 { for ix in 0..<2 {
+ let (row, col) = (Float(by * 2 + iy), Float(bx * 2 + ix))
+ for i in 0..<36 {
+ let angle = i < 18 ? row * inv[i] : col * inv[i - 18]
+ p[(index * 36 + i) * 2] = cosf(angle)
+ p[(index * 36 + i) * 2 + 1] = sinf(angle)
+ }
+ index += 1
+ } } } }
+ }
+
+ func encodeBlock(_ layer: Int, rows n: Int, _ b: Buffers, commands: MTLCommandBuffer) throws {
+ let d = QwenVisionTower.dim, block = "v.blk.\(layer)."
+ try layerNorm(b, b.x, into: b.normed, prefix: block + "ln1", rows: n, columns: d, commands: commands)
+ try linear(block + "attn_qkv", b, a: b.normed, c: b.qkv, m: n, commands: commands)
+ try compute("vision_rope_pad", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.qkv.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 1)
+ e.setBuffer(b.arena.buffer, offset: b.k.offset, index: 2)
+ e.setBuffer(b.arena.buffer, offset: b.v.offset, index: 3)
+ e.setBuffer(b.arena.buffer, offset: b.rope.offset, index: 4)
+ var r = UInt32(n)
+ e.setBytes(&r, length: 4, index: 5)
+ e.dispatchThreads(MTLSize(width: n * 1024, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ try compute("flash_attention", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: b.k.offset, index: 1)
+ e.setBuffer(b.arena.buffer, offset: b.v.offset, index: 2)
+ e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 3)
+ var params = QwenDiT.FlashParams(queries: UInt32(n), keys: UInt32(n), causal: 0, stride: 2048,
+ scale: 1 / Float(QwenVisionTower.headDim).squareRoot() * Float(M_LOG2E), headBase: 0)
+ e.setBytes(&params, length: MemoryLayout<QwenDiT.FlashParams>.stride, index: 4)
+ e.dispatchThreadgroups(MTLSize(width: (n + 63) / 64, height: QwenVisionTower.heads, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ try compute("vision_unpad", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: b.a72.offset, index: 1)
+ var r = UInt32(n)
+ e.setBytes(&r, length: 4, index: 2)
+ e.dispatchThreads(MTLSize(width: n * d, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ try linear(block + "attn_out", b, a: b.a72, c: b.normed, m: n, commands: commands)
+ try add(b, b.x, b.normed, offset: 0, count: n * d, commands: commands)
+ try layerNorm(b, b.x, into: b.normed, prefix: block + "ln2", rows: n, columns: d, commands: commands)
+ try linear(block + "ffn_up", b, a: b.normed, c: b.h, m: n, commands: commands)
+ try compute("gelu_tanh", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.h.offset, index: 0)
+ var c = UInt32(n * QwenVisionTower.hidden)
+ e.setBytes(&c, length: 4, index: 1)
+ e.dispatchThreads(MTLSize(width: Int(c), height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ try linear(block + "ffn_down", b, a: b.h, c: b.normed, m: n, commands: commands)
+ try add(b, b.x, b.normed, offset: 0, count: n * d, commands: commands)
+ }
+
+ /// `linear_fc2(GELU(linear_fc1(mergeIn)))` into `out` `[slots, 4096]`.
+ func mergerMLP(_ b: Buffers, prefix: String, fc1: String, fc2: String, out: Arena.Slice, slots: Int,
+ commands: MTLCommandBuffer) throws {
+ try linear("\(prefix).\(fc1)", b, a: b.mergeIn, c: b.mergeHidden, m: slots, commands: commands)
+ try compute("gelu_erf", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.mergeHidden.offset, index: 0)
+ var c = UInt32(slots * 4 * QwenVisionTower.dim)
+ e.setBytes(&c, length: 4, index: 1)
+ e.dispatchThreads(MTLSize(width: Int(c), height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ // The product writes padded rows; `out` holds only the real ones, so it goes through mergeIn.
+ try linear("\(prefix).\(fc2)", b, a: b.mergeHidden, c: b.mergeIn, m: slots, commands: commands)
+ let blit = commands.makeBlitCommandEncoder()!
+ blit.copy(from: b.arena.buffer, sourceOffset: b.mergeIn.offset, to: b.arena.buffer, destinationOffset: out.offset,
+ size: slots * QwenVisionTower.outDim * 4)
+ blit.endEncoding()
+ }
+
+ // MARK: - encoders
+
+ func compute(_ name: String, commands: MTLCommandBuffer, _ body: (MTLComputeCommandEncoder) throws -> Void) throws {
+ let encoder = commands.makeComputeCommandEncoder()!
+ encoder.label = name
+ encoder.setComputePipelineState(try context.pipeline(name))
+ try body(encoder)
+ encoder.endEncoding()
+ }
+
+ /// `C[m, n] = A[m, k] · W + bias`, W the file's Linear `[n, k]` unpacked to `[k, n]` fp32 in a reserve first; m
+ /// padded to 64 as the text encoder's products.
+ func linear(_ name: String, _ b: Buffers, a: Arena.Slice, c: Arena.Slice, m: Int, commands: MTLCommandBuffer) throws {
+ let tensor = try file.tensor(name + ".weight")
+ let (kk, nn) = (tensor.columns, tensor.rows)
+ let reserve = b.reserves[nextReserve]
+ nextReserve = (nextReserve + 1) % b.reserves.count
+ try dequant.encode(tensor, of: file, into: b.arena.buffer, outOffset: reserve.offset, transposed: true, commands: commands)
+ try product(b, a: a, w: reserve, c: c, m: m, k: kk, n: nn, commands: commands)
+ try bias(b, c, name + ".bias", rows: m, columns: nn, commands: commands)
+ }
+
+ func product(_ b: Buffers, a: Arena.Slice, w: Arena.Slice, c: Arena.Slice, m: Int, k: Int, n: Int,
+ commands: MTLCommandBuffer) throws {
+ let m = QwenDiT.padded(m)
+ guard m * k * 4 <= a.bytes, m * n * 4 <= c.bytes, k * n * 4 <= w.bytes else {
+ throw MetalContext.Failure.buffer("vision product \(m)×\(k)×\(n): no room")
+ }
+ let key = "\(m)×\(k)×\(n)"
+ let product = products[key] ?? MPSMatrixMultiplication(device: context.device, transposeLeft: false, transposeRight: false,
+ resultRows: m, resultColumns: n, interiorColumns: k, alpha: 1, beta: 0)
+ products[key] = product
+ func matrix(_ offset: Int, _ rows: Int, _ columns: Int) -> MPSMatrix {
+ MPSMatrix(buffer: b.arena.buffer, offset: offset,
+ descriptor: MPSMatrixDescriptor(rows: rows, columns: columns, rowBytes: columns * 4, dataType: .float32))
+ }
+ product.encode(commandBuffer: commands, leftMatrix: matrix(a.offset, m, k), rightMatrix: matrix(w.offset, k, n),
+ resultMatrix: matrix(c.offset, m, n))
+ }
+
+ func bias(_ b: Buffers, _ x: Arena.Slice, _ name: String, rows: Int, columns: Int, commands: MTLCommandBuffer) throws {
+ guard let at = small[name] else { throw GGUFFile.Failure.unsupported("no tensor \(name)") }
+ try compute("bias_rows", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: x.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: b.smalls.offset + at * 4, index: 1)
+ var p = SIMD2<UInt32>(UInt32(rows * columns), UInt32(columns))
+ e.setBytes(&p, length: 8, index: 2)
+ e.dispatchThreads(MTLSize(width: rows * columns, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+
+ struct AffineParams { var columns: UInt32; var eps: Float }
+
+ func layerNorm(_ b: Buffers, _ input: Arena.Slice, into out: Arena.Slice, prefix: String, rows: Int, columns: Int,
+ commands: MTLCommandBuffer) throws {
+ guard let w = small[prefix + ".weight"], let bias = small[prefix + ".bias"] else {
+ throw GGUFFile.Failure.unsupported("no tensor \(prefix)")
+ }
+ try compute("layernorm_affine", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: input.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: out.offset, index: 1)
+ e.setBuffer(b.arena.buffer, offset: b.smalls.offset + w * 4, index: 2)
+ e.setBuffer(b.arena.buffer, offset: b.smalls.offset + bias * 4, index: 3)
+ var p = AffineParams(columns: UInt32(columns), eps: QwenVisionTower.eps)
+ e.setBytes(&p, length: MemoryLayout<AffineParams>.stride, index: 4)
+ e.dispatchThreadgroups(MTLSize(width: rows, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+
+ /// x += y (y at `offset` floats into its slice).
+ func add(_ b: Buffers, _ x: Arena.Slice, _ y: Arena.Slice, offset: Int, count: Int, commands: MTLCommandBuffer) throws {
+ try compute("add_inplace", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: x.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: y.offset + offset * 4, index: 1)
+ var c = UInt32(count)
+ e.setBytes(&c, length: 4, index: 2)
+ e.dispatchThreads(MTLSize(width: count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/TextChecks.swift b/engine-mac/Sources/KokonEngine/TextChecks.swift
index 95c7ce3..333b844 100644
--- a/engine-mac/Sources/KokonEngine/TextChecks.swift
+++ b/engine-mac/Sources/KokonEngine/TextChecks.swift
@@ -224,3 +224,60 @@ extension Checks {
}
}
}
+
+extension Checks {
+ /// The vision tower against `tools/reference_vision.py` (transformers fp32): its input pixels (the processor's), the
+ /// patches, blocks 0, 8 and 26, the merged rows and the three deepstack outputs; then the time of a 1024² encode.
+ public static func vision(path: String, reference directory: String) throws {
+ let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory)
+ .appendingPathComponent("meta.json"))) as! [String: Any]
+ let (w, h) = (meta["width"] as! Int, meta["height"] as! Int)
+ let png = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: directory).appendingPathComponent("image.png")))
+ let input = QwenVisionTower.pixels(rgba: png.rgba, width: png.width, height: png.height)
+ let grid = meta["grid"] as! [Int]
+ print("image \(png.width)×\(png.height) → \(input.width)×\(input.height)"
+ + (grid == [1, input.height / 16, input.width / 16] ? "" : " (reference grid \(grid))") + ", reference \(w)×\(h)")
+ // The processor's pixel rows: 2 × 2 merge blocks, each patch's 3 × 2 frames × 16 × 16 values.
+ let wantPixels = try reference(directory, "pixels")
+ var pixels = [Float](), (gh, gw) = (input.height / 16, input.width / 16)
+ for by in 0..<(gh / 2) { for bx in 0..<(gw / 2) { for iy in 0..<2 { for ix in 0..<2 {
+ for c in 0..<3 { for _ in 0..<2 { for y in 0..<16 { for x in 0..<16 {
+ pixels.append(input.rgb[(c * input.height + (by * 2 + iy) * 16 + y) * input.width + (bx * 2 + ix) * 16 + x])
+ } } } }
+ } } } }
+ if pixels.count == wantPixels.count {
+ let r = pixels.withUnsafeBufferPointer { compare($0.baseAddress!, wantPixels) }
+ print(String(format: " pixels: rel %.2e, max |Δ| %.3g", r.relative, r.maxAbs))
+ } else {
+ print(" pixels: \(pixels.count) values, reference \(wantPixels.count)")
+ }
+ let context = try MetalContext()
+ let tower = try QwenVisionTower(file: try GGUFFile(path: path), context: context)
+ var stages: [(String, [Float])] = []
+ let out = try tower.encode(rgb: input.rgb, width: input.width, height: input.height) { name, p, count in
+ stages.append((name, Array(UnsafeBufferPointer(start: p, count: count))))
+ }
+ var worst = 0.0
+ func report(_ name: String, _ got: [Float]) throws {
+ let want = try reference(directory, name)
+ guard want.count == got.count else { print(" \(name): \(got.count) values, reference \(want.count)"); worst = .infinity; return }
+ let r = got.withUnsafeBufferPointer { compare($0.baseAddress!, want) }
+ worst = max(worst, r.relative)
+ print(String(format: " %@: rel %.2e, max |Δ| %.3g%@", name, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))
+ }
+ for (name, values) in stages { try report(name, values) }
+ try report("merged", out.merged)
+ for (j, d) in out.deepstack.enumerated() { try report("deepstack\(j)", d) }
+ print(String(format: "worst %.2e; encode %.2f s with the probes", worst, tower.seconds))
+ let random = (0..<(3 * 1024 * 1024)).map { i -> Float in
+ let (y, x) = ((i / 1024) % 1024, i % 1024)
+ return sinf(Float(x) * 0.013 + Float(i / (1024 * 1024))) * cosf(Float(y) * 0.021) * 0.9
+ }
+ for run in 1...2 {
+ _ = try tower.encode(rgb: random, width: 1024, height: 1024)
+ Thread.sleep(forTimeInterval: 1.5)
+ let m = MemoryUsage.now()
+ print(String(format: "encode %d at 1024²: %.2f s, footprint after %.2f GB (peak %.2f GB)", run, tower.seconds, m.footprint, m.footprintPeak))
+ }
+ }
+}
diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swift
index 445bf47..e542430 100644
--- a/engine-mac/Sources/kokon-engine-mac/main.swift
+++ b/engine-mac/Sources/kokon-engine-mac/main.swift
@@ -13,6 +13,7 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)
time-vae <vae.safetensors> [grid] the VAE decoder's time and memory (64 = 1024²; KOKON_VAE_TILE)
check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.py
check-text <encoder.gguf> <reference dir> the text encoder against tools/reference_text_encoder.py
+ check-vision <mmproj.gguf> <reference dir> the vision tower against tools/reference_vision.py, then its time
check-generate <encoder.gguf> <dit.gguf> <vae.safetensors> <reference dir> [text reference dir]
text to image against tools/reference_generate.py, from its noise
(and from the text reference's conditioning when given)
@@ -94,6 +95,9 @@ do {
case "check-text":
guard arguments.count == 3 else { fail(usage) }
try Checks.textEncoder(path: arguments[1], reference: arguments[2])
+ case "check-vision":
+ guard arguments.count == 3 else { fail(usage) }
+ try Checks.vision(path: arguments[1], reference: arguments[2])
case "check-generate":
guard arguments.count == 5 || arguments.count == 6 else { fail(usage) }
try Checks.generate(encoder: arguments[1], dit: arguments[2], vae: arguments[3], reference: arguments[4],
diff --git a/engine-mac/Tests/KokonEngineTests/VisionTests.swift b/engine-mac/Tests/KokonEngineTests/VisionTests.swift
new file mode 100644
index 0000000..acf329a
--- /dev/null
+++ b/engine-mac/Tests/KokonEngineTests/VisionTests.swift
@@ -0,0 +1,40 @@
+import XCTest
+@testable import KokonEngine
+
+/// The vision tower's processor, without a model: the size it encodes at, the white composite, the resize.
+final class VisionTests: XCTestCase {
+ func testVisionSizeKeepsTheImageInsideTheProcessorsBounds() {
+ XCTAssert(QwenVisionTower.visionSize(width: 1024, height: 768) == (1024, 768))
+ XCTAssert(QwenVisionTower.visionSize(width: 320, height: 224) == (320, 224)) // 71 680 pixels: kept
+ XCTAssert(QwenVisionTower.visionSize(width: 256, height: 192) == (320, 224)) // under 65 536: the processor's
+ XCTAssert(QwenVisionTower.visionSize(width: 64, height: 64) == (256, 256))
+ }
+
+ func testAlphaIsCompositedOverWhiteAsPILPastes() {
+ // Opaque, transparent and half-transparent pixels; PIL: white.paste(img, mask=alpha) on 8-bit values.
+ let rgba: [UInt8] = [10, 200, 30, 255, 10, 200, 30, 0, 0, 100, 255, 128, 77, 77, 77, 1]
+ let input = QwenVisionTower.pixels(rgba: rgba + [UInt8](repeating: 255, count: 4 * (256 * 256 - 4)), width: 256, height: 256)
+ let plane = 256 * 256
+ func level(_ c: Int, _ p: Int) -> Int { Int(((input.rgb[c * plane + p] * 0.5 + 0.5) * 255).rounded()) }
+ XCTAssertEqual([level(0, 0), level(1, 0), level(2, 0)], [10, 200, 30])
+ XCTAssertEqual([level(0, 1), level(1, 1), level(2, 1)], [255, 255, 255])
+ // (v·128 + 255·127) / 255, rounded: 0 → 127, 100 → 177, 255 → 255.
+ XCTAssertEqual([level(0, 2), level(1, 2), level(2, 2)], [127, 177, 255])
+ XCTAssertEqual(level(0, 3), 254)
+ }
+
+ func testResizeKeepsFlatImagesFlatAndCoversTheOutput() {
+ let flat = [UInt8](repeating: 90, count: 3 * 192 * 256)
+ let up = QwenVisionTower.resizeBicubic(flat, width: 256, height: 192, to: 320, 224)
+ XCTAssertEqual(up.count, 3 * 320 * 224)
+ XCTAssert(up.allSatisfy { $0 == 90 })
+ // A horizontal ramp stays monotonic (no ringing on a smooth ramp) and keeps its ends.
+ var ramp = [UInt8](repeating: 0, count: 3 * 192 * 256)
+ for y in 0..<192 { for x in 0..<256 { for c in 0..<3 { ramp[(y * 256 + x) * 3 + c] = UInt8(x) } } }
+ let r = QwenVisionTower.resizeBicubic(ramp, width: 256, height: 192, to: 320, 224)
+ let row = (0..<320).map { Int(r[(100 * 320 + $0) * 3]) }
+ XCTAssertEqual(row.first, 0)
+ XCTAssertEqual(row.last, 255)
+ XCTAssert(zip(row, row.dropFirst()).allSatisfy { $0 <= $1 })
+ }
+}
diff --git a/engine-mac/tools/reference_vision.py b/engine-mac/tools/reference_vision.py
new file mode 100644
index 0000000..eb52649
--- /dev/null
+++ b/engine-mac/tools/reference_vision.py
@@ -0,0 +1,130 @@
+"""The Qwen-Image 2.1 vision tower reference for the Mac engine: transformers' own `Qwen3VLVisionModel` in fp32 on the
+CPU, on the weights of Kokon's mmproj GGUF (llama.cpp's layout) unpacked by gguf-py, on an image prepared by the
+official processor as diffusers' QwenImage21Pipeline hands it over (`_get_qwen_prompt_embeds`: RGBA composited over
+white, then `Qwen3VLProcessor`). The image's sides must be multiples of 32, as the pipeline makes them.
+
+The input is a PNG (`--image`) or, by default, a smooth random image (random low-resolution noise upsampled
+bilinearly, so it has edges and flat areas as photos do), saved as `image.png` for the engine to read.
+
+ python -I engine-mac/tools/reference_vision.py <mmproj.gguf> <processor dir> <text_encoder dir> <out dir>
+ [--image in.png] [--size 320x224] [--dtype float64]
+
+Writes `<out>/meta.json` (grid, sizes) and one raw little-endian fp32 file per tensor (`<name>.f32`): pixels (the
+processor's `pixel_values`), embeddings (patches + positions), block0/8/26 outputs, deepstack0..2, merged.
+"""
+import argparse
+import json
+import os
+
+import gguf
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+from transformers import AutoConfig, AutoProcessor
+from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLVisionModel
+
+BLOCK = {"ln1": "norm1", "ln2": "norm2", "attn_qkv": "attn.qkv", "attn_out": "attn.proj",
+ "ffn_up": "mlp.linear_fc1", "ffn_down": "mlp.linear_fc2"}
+MERGER = {"norm": "norm", "fc1": "linear_fc1", "fc2": "linear_fc2"}
+
+
+def tensor(reader, name):
+ t = reader[name]
+ values = gguf.quants.dequantize(t.data, t.tensor_type).astype(np.float32)
+ return torch.from_numpy(values.reshape([int(d) for d in reversed(t.shape)]).copy())
+
+
+def state_dict(path, config):
+ """mmproj names (llama.cpp's clip.cpp, projector qwen3vl_merger) -> Qwen3VLVisionModel names."""
+ reader = {t.name: t for t in gguf.GGUFReader(path).tensors}
+ get = lambda name: tensor(reader, name) # noqa: E731
+ # The Conv3d over 2 frames is stored as one 2D kernel per frame.
+ state = {"patch_embed.proj.weight": torch.stack([get("v.patch_embd.weight"), get("v.patch_embd.weight.1")], dim=2),
+ "patch_embed.proj.bias": get("v.patch_embd.bias"), "pos_embed.weight": get("v.position_embd.weight")}
+ for i in range(config.depth):
+ for g, h in BLOCK.items():
+ for kind in ("weight", "bias"):
+ state[f"blocks.{i}.{h}.{kind}"] = get(f"v.blk.{i}.{g}.{kind}")
+ for kind in ("weight", "bias"):
+ state[f"merger.norm.{kind}"] = get(f"v.post_ln.{kind}")
+ state[f"merger.linear_fc1.{kind}"] = get(f"mm.0.{kind}")
+ state[f"merger.linear_fc2.{kind}"] = get(f"mm.2.{kind}")
+ for j, layer in enumerate(config.deepstack_visual_indexes):
+ for g, h in MERGER.items():
+ state[f"deepstack_merger_list.{j}.{h}.{kind}"] = get(f"v.deepstack.{layer}.{g}.{kind}")
+ return state
+
+
+def smooth_image(width, height, seed=0):
+ g = torch.Generator().manual_seed(seed)
+ low = torch.rand(1, 3, height // 16, width // 16, generator=g)
+ img = F.interpolate(low, size=(height, width), mode="bilinear", align_corners=False)[0]
+ return Image.fromarray((img.permute(1, 2, 0).numpy() * 255).round().clip(0, 255).astype(np.uint8))
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("gguf")
+ parser.add_argument("processor")
+ parser.add_argument("text_encoder", help="text_encoder/ of the official repository (config.json)")
+ parser.add_argument("out")
+ parser.add_argument("--image")
+ parser.add_argument("--size", default="320x224")
+ parser.add_argument("--dtype", default="float32", help="float64: the model in fp64, to tell torch's fp32 drift apart")
+ args = parser.parse_args()
+ torch.set_grad_enabled(False)
+ os.makedirs(args.out, exist_ok=True)
+
+ if args.image:
+ image = Image.open(args.image)
+ else:
+ width, height = (int(v) for v in args.size.split("x"))
+ image = smooth_image(width, height)
+ assert image.width % 32 == 0 and image.height % 32 == 0, "sides must be multiples of 32"
+ image.save(os.path.join(args.out, "image.png"))
+ if image.mode == "RGBA": # as the pipeline: alpha composited over white for the vision encoder
+ white = Image.new("RGB", image.size, (255, 255, 255))
+ white.paste(image, mask=image.getchannel("A"))
+ image = white
+ image = image.convert("RGB")
+
+ processor = AutoProcessor.from_pretrained(args.processor)
+ inputs = processor.image_processor(images=[image], return_tensors="pt")
+ dtype = getattr(torch, args.dtype)
+ pixels, grid = inputs.pixel_values.to(dtype), inputs.image_grid_thw
+
+ config = AutoConfig.from_pretrained(args.text_encoder).vision_config
+ config._attn_implementation = "eager"
+ model = Qwen3VLVisionModel(config).eval()
+ model.load_state_dict(state_dict(args.gguf, config), strict=True)
+ model = model.to(dtype)
+
+ saved = {}
+
+ def save(name, t):
+ array = t.detach().to(torch.float32).contiguous().numpy()
+ array.tofile(os.path.join(args.out, name + ".f32"))
+ saved[name] = list(array.shape)
+
+ save("pixels", pixels)
+ hooks = [model.patch_embed.register_forward_hook(lambda m, a, out: save("patches", out))]
+ for i in (0, 8, 26):
+ hooks.append(model.blocks[i].register_forward_hook(lambda m, a, out, i=i: save(f"block{i}", out)))
+ out = model(pixels, grid_thw=grid, return_dict=True)
+ for h in hooks:
+ h.remove()
+ save("merged", out.pooler_output)
+ for j, feature in enumerate(out.deepstack_features):
+ save(f"deepstack{j}", feature)
+
+ meta = {"grid": grid[0].tolist(), "dtype": args.dtype, "width": image.width, "height": image.height, "tensors": saved,
+ "gguf": os.path.basename(args.gguf), "transformers": __import__("transformers").__version__,
+ "torch": torch.__version__}
+ with open(os.path.join(args.out, "meta.json"), "w") as f:
+ json.dump(meta, f, indent=1)
+ print(f"grid {grid[0].tolist()}, merged {saved['merged']}")
+
+
+if __name__ == "__main__":
+ main()