Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.
TypeScript ai-photo-editorcomfyui-alternativegenerative-fillimage-editingimage-generationinpaintinglinuxlocal-aimacosmcpmcp-servernano-banana-alternativeofflinephoto-editorprivacyqwen-imageqwen-image-2-1stable-diffusion-cppvulkanwindows
Mac engine: the vision tower, checked against transformers
Gabriele Lozupone76d0604c8bfa
diff --git a/engine-mac/README.md b/engine-mac/README.mdindex c82241d..4e58c89 100644--- a/engine-mac/README.md+++ b/engine-mac/README.md@@ -45,6 +45,7 @@ Windows and Linux.| `AttentionProbe.swift` | `probe-attention`: the target's attention on each unit alone || `QwenVAE.swift`, `Safetensors.swift` | the VAE decoder as one MPSGraph, from Kokon's VAE file (Wan 2.2 names) || `QwenVAEEncoder.swift` | the VAE encoder (edits' reference images), checked against diffusers |+| `QwenVisionTower.swift`, `Kernels/vision.metal` | Qwen3-VL's vision tower from the mmproj GGUF (edits' reference images): patches, 27 blocks, merger and deepstack outputs || `QwenTokenizer.swift` | Qwen's byte-level BPE tokenizer, read from the encoder GGUF's metadata || `QwenTextEncoder.swift`, `Kernels/text.metal` | the text encoder: Qwen3-VL-8B's text decoder on the GPU, diffusers' prompt template || `QwenPipeline.swift` | text to image: noise, Turbo's 8 flow-Euler steps (`planner::TURBO_SIGMAS`), VAE, PNG; models kept loaded for the server |@@ -387,5 +388,22 @@ and the head as one graph at 1/16 of the image, then the posterior's mode normaltexture-fix VAE, M2 Pro, 2026-10-25): the latent within 6.5e-6. 1024²: 0.59 s an encode, peak 2.4 GB, 0.87 GB kept after(its weights); in its own autorelease pool (without one, a loop on a thread without a pool kept 1.5 GB per encode).+**Vision tower** (`QwenVisionTower.swift`; what an edit's reference image becomes in the text encoder's+`<|image_pad|>` slots): Qwen3-VL's ViT from Kokon's mmproj GGUF (llama.cpp's layout: the Conv3d as one kernel per frame,+fused q/k/v, `mm.0`/`mm.2` the merger, `v.deepstack.8/16/24`), fp32 on the GPU with the F16 weights unpacked before each+product. The patches in the processor's 2 × 2 merge-block order, the two frames' kernels summed (both frames are the+image), the 48 × 48 position table resampled bilinearly, axial 2D RoPE, the 72-dimension heads on the DiT's flash kernel+zero-padded to 128. Its input is the processor's: alpha over white as PIL's paste, and below 65 536 pixels the processor's+resize (torchvision's uint8 antialiased bicubic: PIL's kernel with int16 weights, horizontal pass first; bit for bit).+`check-vision` against `tools/reference_vision.py` (transformers' `Qwen3VLVisionModel`, M2 Pro, 2026-10-25), 320 × 224:+the pixels within 1.6e-7 (float rounding), the merged rows within 1.35e-5 of torch fp32 and **7.6e-6 of fp64**+(`--dtype float64`; torch's own fp32 is 1.55e-5 from it), deepstack 3.8–9.4e-6; 256 × 192 (resized to 320 × 224) and+an RGBA image the same. 1024²: 2.5 s an encode (4 096 patches; the padded heads cost ~1.8× the attention's work).++```bash+~/kokon-bench/venv-oracle/bin/python -I tools/reference_vision.py <mmproj.gguf> <processor dir> <text_encoder dir> ~/kokon-bench/ref/vision-320 [--dtype float64]+.build/release/kokon-engine-mac check-vision <mmproj.gguf> ~/kokon-bench/ref/vision-320+```+Credits: the Neural Engine program technique and the GPU + AMX + Neural Engine split come from[Siliconed](https://github.com/gwenn-ha-dev/Siliconed) (Apache-2.0), written again here.diff --git a/engine-mac/Sources/KokonEngine/Kernels/vision.metal b/engine-mac/Sources/KokonEngine/Kernels/vision.metalnew file mode 100644index 0000000..9d6d7ea--- /dev/null+++ b/engine-mac/Sources/KokonEngine/Kernels/vision.metal@@ -0,0 +1,98 @@+#include <metal_stdlib>+using namespace metal;++// The Qwen3-VL vision tower's kernels, fp32, following transformers' modeling_qwen3_vl.py (QwenVisionTower.swift says+// which module each one replaces). Rows are patches (or merged groups of 4 patches) of `columns` floats.++struct AffineParams {+ uint columns;+ float eps;+};++// nn.LayerNorm(columns, eps) with weight and bias: norm1/norm2, the mergers' norms. One threadgroup of 256 per row;+// the mean first, then the variance around it (two passes, as torch's CPU kernel's result).+kernel void layernorm_affine(device const float *x [[buffer(0)]], device float *out [[buffer(1)]],+ device const float *weight [[buffer(2)]], device const float *bias [[buffer(3)]],+ constant AffineParams &p [[buffer(4)]],+ uint row [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ threadgroup float shared[8];+ device const float *r = x + (ulong)row * p.columns;+ device float *o = out + (ulong)row * p.columns;+ float s = 0;+ for (uint j = tid; j < p.columns; j += 256) s += r[j];+ s = simd_sum(s);+ if (tid % 32 == 0) shared[tid / 32] = s;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float mean = 0;+ for (uint i = 0; i < 8; i++) mean += shared[i];+ mean /= float(p.columns);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float v = 0;+ for (uint j = tid; j < p.columns; j += 256) { float d = r[j] - mean; v += d * d; }+ v = simd_sum(v);+ if (tid % 32 == 0) shared[tid / 32] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float var = 0;+ for (uint i = 0; i < 8; i++) var += shared[i];+ float inv = rsqrt(var / float(p.columns) + p.eps);+ for (uint j = tid; j < p.columns; j += 256) o[j] = (r[j] - mean) * inv * weight[j] + bias[j];+}++// x[r, c] += b[c]: the Linear layers' bias after the product (count = rows × columns).+kernel void bias_rows(device float *x [[buffer(0)]], device const float *b [[buffer(1)]],+ constant uint2 &p [[buffer(2)]], uint i [[thread_position_in_grid]]) {+ if (i >= p.x) return;+ x[i] += b[i % p.y];+}++// nn.GELU() (erf), in place, the mergers' activation: 0.5 x erfc(−x / √2), erfc as Numerical Recipes' erfcc (fractional+// error below 1.2e-7 everywhere, so the tail for negative x keeps its relative precision).+kernel void gelu_erf(device float *x [[buffer(0)]], constant uint &count [[buffer(1)]],+ uint i [[thread_position_in_grid]]) {+ if (i >= count) return;+ float v = x[i];+ float z = -v * 0.7071067811865476f;+ float a = fabs(z), t = 1.0f / (1.0f + 0.5f * a);+ float e = t * precise::exp(-a * a - 1.26551223f + t * (1.00002368f + t * (0.37409196f + t * (0.09678418f ++ t * (-0.18628806f + t * (0.27886807f + t * (-1.13520398f + t * (1.48851587f ++ t * (-0.82215223f + t * 0.17087277f)))))))));+ float tail = z >= 0 ? e : 2.0f - e;+ x[i] = 0.5f * v * tail;+}++// Qwen3VLVisionAttention's qkv split and apply_rotary_pos_emb_vision, into the flash kernel's layout: q, k, v as+// [rows, 16 heads, 128] with the head's 72 dimensions first and zeros after (the scores and the output are the+// 72-dimension ones). rotate_half pairs dimension i with i + 36; `rope` is [rows, 36, (cos, sin)].+// One thread per (row, head, j < 64): j < 36 rotates the pair (j, j + 36), the others clear two padding columns.+kernel void vision_rope_pad(device const float *qkv [[buffer(0)]], device float *q [[buffer(1)]],+ device float *k [[buffer(2)]], device float *v [[buffer(3)]],+ device const float *rope [[buffer(4)]], constant uint &rows [[buffer(5)]],+ uint i [[thread_position_in_grid]]) {+ uint j = i % 64, head = (i / 64) % 16, row = i / 1024;+ if (row >= rows) return;+ device const float *in = qkv + (ulong)row * 3456 + head * 72;+ ulong o = (ulong)row * 2048 + head * 128;+ if (j < 36) {+ float c = rope[((ulong)row * 36 + j) * 2], s = rope[((ulong)row * 36 + j) * 2 + 1];+ float q0 = in[j], q1 = in[j + 36], k0 = in[1152 + j], k1 = in[1152 + j + 36];+ q[o + j] = q0 * c - q1 * s;+ q[o + j + 36] = q1 * c + q0 * s;+ k[o + j] = k0 * c - k1 * s;+ k[o + j + 36] = k1 * c + k0 * s;+ v[o + j] = in[2304 + j];+ v[o + j + 36] = in[2304 + j + 36];+ } else {+ uint c = 72 + (j - 36) * 2;+ q[o + c] = 0; q[o + c + 1] = 0;+ k[o + c] = 0; k[o + c + 1] = 0;+ v[o + c] = 0; v[o + c + 1] = 0;+ }+}++// The attention's output back to [rows, 16 × 72] for attn.proj.+kernel void vision_unpad(device const float *x [[buffer(0)]], device float *out [[buffer(1)]],+ constant uint &rows [[buffer(2)]], uint i [[thread_position_in_grid]]) {+ if (i >= rows * 1152) return;+ uint row = i / 1152, c = i % 1152;+ out[i] = x[(ulong)row * 2048 + (c / 72) * 128 + c % 72];+}diff --git a/engine-mac/Sources/KokonEngine/QwenPipeline.swift b/engine-mac/Sources/KokonEngine/QwenPipeline.swiftindex 642cbc8..b5c0ab1 100644--- a/engine-mac/Sources/KokonEngine/QwenPipeline.swift+++ b/engine-mac/Sources/KokonEngine/QwenPipeline.swift@@ -232,6 +232,44 @@ public final class QwenPipeline {try pngData(rgba, width: width, height: height).write(to: URL(fileURLWithPath: path))}+ /// An image file (PNG, JPEG…) as interleaved 8-bit RGBA, straight alpha. 8-bit RGB(A) files are read as stored, without+ /// color management, as PIL reads them for diffusers; other layouts (16-bit, gray, palette) are drawn into an sRGB+ /// context, premultiplied, and divided back.+ public static func decodeImage(_ data: Data) throws -> (rgba: [UInt8], width: Int, height: Int) {+ guard let source = CGImageSourceCreateWithData(data as CFData, nil),+ let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else {+ throw MetalContext.Failure.buffer("cannot decode the image")+ }+ let (w, h) = (image.width, image.height)+ var out = [UInt8](repeating: 255, count: w * h * 4)+ let alpha = image.alphaInfo, order = image.bitmapInfo.intersection(.byteOrderMask)+ let direct = image.bitsPerComponent == 8 && (order == [] || order == .byteOrderDefault || order == .byteOrder32Big)+ && image.colorSpace?.model == .rgb+ && ((image.bitsPerPixel == 32 && [.last, .noneSkipLast].contains(alpha)) || (image.bitsPerPixel == 24 && alpha == .none))+ if direct, let bytes = image.dataProvider?.data, let p = CFDataGetBytePtr(bytes) {+ let step = image.bitsPerPixel / 8+ for y in 0..<h {+ let row = p + y * image.bytesPerRow+ for x in 0..<w {+ for c in 0..<3 { out[(y * w + x) * 4 + c] = row[x * step + c] }+ if alpha == .last { out[(y * w + x) * 4 + 3] = row[x * step + 3] }+ }+ }+ return (out, w, h)+ }+ guard let context = CGContext(data: &out, width: w, height: h, bitsPerComponent: 8, bytesPerRow: w * 4,+ space: CGColorSpace(name: CGColorSpace.sRGB)!,+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else {+ throw MetalContext.Failure.buffer("cannot decode the image")+ }+ context.draw(image, in: CGRect(x: 0, y: 0, width: w, height: h))+ for p in 0..<(w * h) where out[p * 4 + 3] != 255 {+ let a = Int(out[p * 4 + 3])+ for c in 0..<3 { out[p * 4 + c] = a == 0 ? 0 : UInt8(min(255, (Int(out[p * 4 + c]) * 255 + a / 2) / a)) }+ }+ return (out, w, h)+ }+/// Interleaved 8-bit RGBA (straight alpha) encoded as a PNG.public static func pngData(_ rgba: [UInt8], width: Int, height: Int) throws -> Data {let data = Data(rgba) as CFDatadiff --git a/engine-mac/Sources/KokonEngine/QwenVisionTower.swift b/engine-mac/Sources/KokonEngine/QwenVisionTower.swiftnew file mode 100644index 0000000..ff474cb--- /dev/null+++ b/engine-mac/Sources/KokonEngine/QwenVisionTower.swift@@ -0,0 +1,462 @@+import Foundation+import Metal+import MetalPerformanceShaders++/// **Qwen3-VL-8B's vision tower** (`Qwen3VLVisionModel`, transformers 5) on the GPU, fp32, from Kokon's mmproj GGUF+/// (llama.cpp's layout, F16 weights unpacked just before each product): what an edit's reference image becomes in the+/// text encoder's `<|image_pad|>` slots.+///+/// image ─ 16 × 16 patches, in 2 × 2 merge blocks ─ patch_embed (Conv3d over the image twice) + bilinear position table+/// 27 × { x += proj(SDPA(rope2d(q), rope2d(k), v)) on LN1(x) (16 heads of 72, every patch sees every patch)+/// x += fc2(GELU_tanh(fc1(LN2(x)))) }+/// merged = fc2(GELU(fc1(LN(x) as [N/4, 4608]))) [N/4, 4096], one row per text encoder slot+/// deepstack j = the same after block 8 / 16 / 24 (LN after the 2 × 2 merge) added to the text encoder's layers 0–2+///+/// The heads (72 dimensions) run on the DiT's flash kernel zero-padded to 128 (`vision_rope_pad`): the scores and the+/// output are the 72-dimension ones. The processor's input is the image with alpha composited over white (as the+/// pipeline's PIL paste), scaled to [−1, 1], each frame of the Conv3d's two the same image.+public final class QwenVisionTower {+ public static let dim = 1152, heads = 16, headDim = 72, hidden = 4304, layers = 27, outDim = 4096+ static let patch = 16, merge = 2, positionSide = 48, eps: Float = 1e-6, theta: Float = 10_000+ /// Blocks whose output also goes through a deepstack merger (`deepstack_visual_indexes`).+ public static let deepstackLayers = [8, 16, 24]+ /// The processor's `smart_resize` bounds (`size.shortest_edge`, `longest_edge`): pixels of the image it encodes.+ static let minPixels = 65_536, maxPixels = 16_777_216++ public let file: GGUFFile+ let context: MetalContext+ let dequant: GPUDequant+ private var products: [String: MPSMatrixMultiplication] = [:]+ /// Seconds of the last encode.+ public private(set) var seconds: Double = 0++ public init(file: GGUFFile, context: MetalContext) throws {+ guard file.metadata["clip.projector_type"]?.string == "qwen3vl_merger",+ file.metadata["clip.vision.block_count"]?.int == QwenVisionTower.layers,+ file.metadata["clip.vision.embedding_length"]?.int == QwenVisionTower.dim else {+ throw GGUFFile.Failure.unsupported("\(file.path): not Qwen3-VL-8B's vision tower (mmproj)")+ }+ self.file = file+ self.context = context+ dequant = GPUDequant(context: context)+ }++ public func release() { context.release(file) }++ /// What the tower makes of one image: `merged` `[slots, 4096]` and one deepstack `[slots, 4096]` per entry of+ /// `deepstackLayers`, slots = (height / 32) · (width / 32) in merge-block order (row-major over 32 × 32 cells).+ public struct Output {+ public let merged: [Float]+ public let deepstack: [[Float]]+ public let height, width: Int+ public var slots: Int { (height / 32) * (width / 32) }+ }++ // MARK: - the processor++ /// Qwen2VLImageProcessor's `smart_resize` for sides already multiples of 32 (as the pipeline makes them): the size+ /// the tower sees, which differs only below 65 536 or above 16 777 216 pixels.+ public static func visionSize(width: Int, height: Int) -> (width: Int, height: Int) {+ let factor = Double(patch * merge)+ var h = Int((Double(height) / factor).rounded(.toNearestOrEven) * factor)+ var w = Int((Double(width) / factor).rounded(.toNearestOrEven) * factor)+ if h * w > maxPixels {+ let beta = (Double(height * width) / Double(maxPixels)).squareRoot()+ h = max(Int(factor), Int((Double(height) / beta / factor).rounded(.down) * factor))+ w = max(Int(factor), Int((Double(width) / beta / factor).rounded(.down) * factor))+ } else if h * w < minPixels {+ let beta = (Double(minPixels) / Double(height * width)).squareRoot()+ h = Int((Double(height) * beta / factor).rounded(.up) * factor)+ w = Int((Double(width) * beta / factor).rounded(.up) * factor)+ }+ return (w, h)+ }++ /// Interleaved 8-bit RGBA → the tower's input: alpha composited over white as PIL's `paste` with a mask does+ /// (the pipeline's `white.paste(img, mask=alpha)`), resized when `visionSize` says so (bicubic), then planar RGB+ /// `[3, H, W]` scaled to [−1, 1] (`rescale 1/255`, `normalize 0.5, 0.5`).+ public static func pixels(rgba: [UInt8], width: Int, height: Int) -> (rgb: [Float], width: Int, height: Int) {+ var rgb = [UInt8](repeating: 0, count: width * height * 3)+ for p in 0..<(width * height) {+ let a = Int(rgba[p * 4 + 3])+ for c in 0..<3 {+ // ImagingPaste's BLEND: in·a + 255·(255 − a), divided by 255 with rounding (PIL's MULDIV255).+ let t = Int(rgba[p * 4 + c]) * a + 255 * (255 - a) + 128+ rgb[p * 3 + c] = UInt8((t + (t >> 8)) >> 8)+ }+ }+ let (w, h) = visionSize(width: width, height: height)+ if (w, h) != (width, height) { rgb = resizeBicubic(rgb, width: width, height: height, to: w, h) }+ var out = [Float](repeating: 0, count: 3 * w * h)+ for p in 0..<(w * h) {+ for c in 0..<3 { out[c * w * h + p] = (Float(rgb[p * 3 + c]) * (1 / 255.0) - 0.5) / 0.5 }+ }+ return (out, w, h)+ }++ /// Bicubic resize of interleaved 8-bit RGB exactly as torchvision's antialiased resize of a uint8 image (the+ /// processor's path; PIL's algorithm with torch's integer arithmetic): a = −0.5, the kernel stretched by the scale+ /// when shrinking, each output's weights normalized then quantized to int16 with as many fraction bits as the+ /// largest fits in, the horizontal pass rounded to 8 bits before the vertical one. Bit for bit against torchvision+ /// 0.26 on a 256 × 192 → 320 × 224 image (2026-10-25). Only images outside the processor's pixel bounds go through it.+ static func resizeBicubic(_ src: [UInt8], width: Int, height: Int, to w: Int, _ h: Int) -> [UInt8] {+ func taps(_ out: Int, _ inSize: Int) -> (taps: [(Int, [Int])], bits: Int) {+ let scale = Double(inSize) / Double(out), support = 2 * max(scale, 1), invScale = 1 / max(scale, 1)+ func cubic(_ x: Double) -> Double {+ let a = -0.5, x = abs(x)+ if x <= 1 { return ((a + 2) * x - (a + 3)) * x * x + 1 }+ if x < 2 { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a }+ return 0+ }+ let real = (0..<out).map { i -> (Int, [Double]) in+ let center = scale * (Double(i) + 0.5)+ let lo = max(Int(center - support + 0.5), 0), hi = min(Int(center + support + 0.5), inSize)+ let weights = (lo..<hi).map { cubic((Double($0) - center + 0.5) * invScale) }+ let total = weights.reduce(0, +)+ return (lo, weights.map { $0 / total })+ }+ let largest = real.flatMap { $0.1 }.map(abs).max() ?? 1+ var bits = 0+ while bits < 22, Int(0.5 + largest * Double(1 << (bits + 1))) < 1 << 15 { bits += 1 }+ let scaleBits = Double(1 << bits)+ return (real.map { ($0.0, $0.1.map { v in Int(v >= 0 ? (v * scaleBits + 0.5).rounded(.down) : (v * scaleBits - 0.5).rounded(.up)) }) }, bits)+ }+ func clamp(_ v: Int, _ bits: Int) -> UInt8 { UInt8(min(max((v + (1 << (bits - 1))) >> bits, 0), 255)) }+ let (tx, bx) = taps(w, width), (ty, by) = taps(h, height)+ var horizontal = [UInt8](repeating: 0, count: w * height * 3)+ for y in 0..<height {+ for x in 0..<w {+ let (lo, weights) = tx[x]+ for c in 0..<3 {+ var s = 0+ for (j, wt) in weights.enumerated() { s += wt * Int(src[(y * width + lo + j) * 3 + c]) }+ horizontal[(y * w + x) * 3 + c] = clamp(s, bx)+ }+ }+ }+ var out = [UInt8](repeating: 0, count: w * h * 3)+ for y in 0..<h {+ let (lo, weights) = ty[y]+ for x in 0..<w {+ for c in 0..<3 {+ var s = 0+ for (j, wt) in weights.enumerated() { s += wt * Int(horizontal[((lo + j) * w + x) * 3 + c]) }+ out[(y * w + x) * 3 + c] = clamp(s, by)+ }+ }+ }+ return out+ }++ // MARK: - forward++ /// Called with a stage's name and its rows (CPU pointer, floats); the pass then waits on the GPU at each stage.+ /// Stages: patches, block0, block8, block26 (the processor's order: 2 × 2 merge blocks).+ public typealias Probe = (String, UnsafePointer<Float>, Int) -> Void++ struct Buffers {+ let arena: Arena+ let x, normed, qkv, q, k, v, attended, a72, h, mergeIn, mergeHidden, smalls, rope: Arena.Slice+ let outputs: [Arena.Slice]+ let reserves: [Arena.Slice]+ }+ private var nextReserve = 0+ /// Float offsets of the small F32 tensors (norms, biases) in `smalls`.+ private var small: [String: Int] = [:]++ /// The tower on planar RGB `[3, height, width]` in [−1, 1] (`pixels`), sides multiples of 32. In its own+ /// autorelease pool.+ public func encode(rgb: [Float], width: Int, height: Int, probe: Probe? = nil) throws -> Output {+ try autoreleasepool { try forward(rgb: rgb, width: width, height: height, probe: probe) }+ }++ private func forward(rgb: [Float], width: Int, height: Int, probe: Probe?) throws -> Output {+ guard width % 32 == 0, height % 32 == 0, rgb.count == 3 * width * height else {+ throw QwenSequence.Failure.layout("vision tower: \(rgb.count) values for 3 × \(height) × \(width) (sides multiples of 32)")+ }+ let started = DispatchTime.now().uptimeNanoseconds+ let d = QwenVisionTower.dim, (gh, gw) = (height / 16, width / 16), n = gh * gw, slots = n / 4+ let merged = 4 * d // 4608+ // Rows padded as the DiT's (64), plus 64 for the attention kernel's overrun; x is also read as [slots, 4608].+ let rows = max(QwenDiT.padded(n) + 64, 4 * QwenDiT.padded(slots))+ let mrows = QwenDiT.padded(slots)+ let smallNames = smallTensors()+ let smallCount = try smallNames.reduce(0) { $0 + (try file.tensor($1)).count } + n * d+ let sizes = [rows * d, rows * d, rows * 3 * d, rows * 2048, rows * 2048, rows * 2048, rows * 2048, rows * d,+ rows * QwenVisionTower.hidden, mrows * merged, mrows * merged, smallCount, n * 72,+ slots * 4096, slots * 4096, slots * 4096, slots * 4096, merged * merged, merged * merged]+ let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))+ var s = try sizes.map { try arena.slice(floats: $0) }.makeIterator()+ let b = Buffers(arena: arena, x: s.next()!, normed: s.next()!, qkv: s.next()!, q: s.next()!, k: s.next()!,+ v: s.next()!, attended: s.next()!, a72: s.next()!, h: s.next()!, mergeIn: s.next()!,+ mergeHidden: s.next()!, smalls: s.next()!, rope: s.next()!,+ outputs: [s.next()!, s.next()!, s.next()!, s.next()!], reserves: [s.next()!, s.next()!])++ // Norms and biases (F32), then the position table after them.+ small = [:]+ var at = 0+ for name in smallNames {+ let values = try Dequant.array(file, name)+ arena.pointer(b.smalls, floatOffset: at).update(from: values, count: values.count)+ small[name] = at+ at += values.count+ }+ small["positions"] = at+ let table = try positions(gridHeight: gh, gridWidth: gw)+ arena.pointer(b.smalls, floatOffset: at).update(from: table, count: table.count)++ // Patches [n, 768] in merge-block order (each row: channel, then 16 × 16 pixels): the Conv3d's two frames see+ // the same image, so its two kernels are summed (`v.patch_embd.weight` + `.weight.1`) into one [768, 1152].+ let patchRows = arena.pointer(b.normed)+ var index = 0+ for by in 0..<(gh / 2) { for bx in 0..<(gw / 2) { for iy in 0..<2 { for ix in 0..<2 {+ let (py, px) = ((by * 2 + iy) * 16, (bx * 2 + ix) * 16)+ for c in 0..<3 { for y in 0..<16 { for x in 0..<16 {+ patchRows[index * 768 + c * 256 + y * 16 + x] = rgb[(c * height + py + y) * width + px + x]+ } } }+ index += 1+ } } } }+ let w0 = try Dequant.array(file, "v.patch_embd.weight"), w1 = try Dequant.array(file, "v.patch_embd.weight.1")+ let kernel = arena.pointer(b.reserves[0])+ for o in 0..<d { for i in 0..<768 { kernel[i * d + o] = w0[o * 768 + i] + w1[o * 768 + i] } }++ var commands = MPSCommandBuffer(from: context.queue) as MTLCommandBuffer+ try product(b, a: b.normed, w: b.reserves[0], c: b.x, m: n, k: 768, n: d, commands: commands)+ try bias(b, b.x, "v.patch_embd.bias", rows: n, columns: d, commands: commands)+ if let probe {+ try MetalContext.finish(commands, "vision tower")+ probe("patches", arena.pointer(b.x), n * d)+ commands = MPSCommandBuffer(from: context.queue)+ }+ try add(b, b.x, b.smalls, offset: small["positions"]!, count: n * d, commands: commands)+ try ropeTable(b, gridHeight: gh, gridWidth: gw)++ for layer in 0..<QwenVisionTower.layers {+ try encodeBlock(layer, rows: n, b, commands: commands)+ if let j = QwenVisionTower.deepstackLayers.firstIndex(of: layer) {+ // Deepstack merger: the block's output as [slots, 4608], then its LayerNorm over 4608.+ try layerNorm(b, b.x, into: b.mergeIn, prefix: "v.deepstack.\(layer).norm", rows: slots, columns: merged,+ commands: commands)+ try mergerMLP(b, prefix: "v.deepstack.\(layer)", fc1: "fc1", fc2: "fc2", out: b.outputs[1 + j], slots: slots,+ commands: commands)+ }+ if let probe, [0, 8, 26].contains(layer) {+ try MetalContext.finish(commands, "vision tower")+ probe("block\(layer)", arena.pointer(b.x), n * d)+ } else {+ commands.commit()+ }+ commands = MPSCommandBuffer(from: context.queue)+ }+ // Merger: post_ln on each patch, then the 2 × 2 groups as rows of 4608.+ try layerNorm(b, b.x, into: b.mergeIn, prefix: "v.post_ln", rows: n, columns: d, commands: commands)+ try mergerMLP(b, prefix: "mm", fc1: "0", fc2: "2", out: b.outputs[0], slots: slots, commands: commands)+ try MetalContext.finish(commands, "vision tower")+ seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9+ func read(_ slice: Arena.Slice) -> [Float] { Array(UnsafeBufferPointer(start: arena.pointer(slice), count: slots * 4096)) }+ return Output(merged: read(b.outputs[0]), deepstack: (1...3).map { read(b.outputs[$0]) }, height: height, width: width)+ }++ func smallTensors() -> [String] {+ var names = ["v.patch_embd.bias", "v.post_ln.weight", "v.post_ln.bias", "mm.0.bias", "mm.2.bias"]+ for i in 0..<QwenVisionTower.layers {+ for t in ["ln1.weight", "ln1.bias", "ln2.weight", "ln2.bias", "attn_qkv.bias", "attn_out.bias", "ffn_up.bias", "ffn_down.bias"] {+ names.append("v.blk.\(i).\(t)")+ }+ }+ for l in QwenVisionTower.deepstackLayers {+ for t in ["norm.weight", "norm.bias", "fc1.bias", "fc2.bias"] { names.append("v.deepstack.\(l).\(t)") }+ }+ return names+ }++ /// `pos_embed` (48 × 48 learned rows) resampled bilinearly, corners aligned, to each patch (merge-block order), as+ /// `get_vision_interpolation_indices_and_weights`: the four taps' rows weighted and summed in fp32.+ func positions(gridHeight gh: Int, gridWidth gw: Int) throws -> [Float] {+ let d = QwenVisionTower.dim, side = QwenVisionTower.positionSide+ let table = try Dequant.array(file, "v.position_embd.weight") // [2304, 1152]+ func taps(_ index: Int, _ size: Int) -> [(Int, Float)] {+ let src = Float(index) * Float(side - 1) / Float(max(size - 1, 1))+ let floor = src.rounded(.down)+ return (0..<2).map { o in+ let tap = min(max(Int(floor) + o, 0), side - 1)+ return (tap, max(1 - abs(src - floor - Float(o)), 0))+ }+ }+ var out = [Float](repeating: 0, count: gh * gw * d)+ var index = 0+ for by in 0..<(gh / 2) { for bx in 0..<(gw / 2) { for iy in 0..<2 { for ix in 0..<2 {+ let (row, col) = (by * 2 + iy, bx * 2 + ix)+ let hs = taps(row, gh), ws = taps(col, gw)+ var acc = [Float](repeating: 0, count: d)+ for (ht, hw) in hs { for (wt, ww) in ws {+ let weight = hw * ww, base = (ht * side + wt) * d+ for c in 0..<d { acc[c] += table[base + c] * weight }+ } }+ for c in 0..<d { out[index * d + c] = acc[c] }+ index += 1+ } } } }+ return out+ }++ /// `[rows, 36, (cos, sin)]` of the axial rotary embedding: dimensions 0–17 rotate by the patch's row, 18–35 by its+ /// column (`inv_freq = 1 / 10000^(2i / 36)`), and dimension i + 36 shares dimension i's angle.+ func ropeTable(_ b: Buffers, gridHeight gh: Int, gridWidth gw: Int) throws {+ let inv = (0..<18).map { i in 1 / powf(QwenVisionTower.theta, Float(2 * i) / 36) }+ let p = b.arena.pointer(b.rope)+ var index = 0+ for by in 0..<(gh / 2) { for bx in 0..<(gw / 2) { for iy in 0..<2 { for ix in 0..<2 {+ let (row, col) = (Float(by * 2 + iy), Float(bx * 2 + ix))+ for i in 0..<36 {+ let angle = i < 18 ? row * inv[i] : col * inv[i - 18]+ p[(index * 36 + i) * 2] = cosf(angle)+ p[(index * 36 + i) * 2 + 1] = sinf(angle)+ }+ index += 1+ } } } }+ }++ func encodeBlock(_ layer: Int, rows n: Int, _ b: Buffers, commands: MTLCommandBuffer) throws {+ let d = QwenVisionTower.dim, block = "v.blk.\(layer)."+ try layerNorm(b, b.x, into: b.normed, prefix: block + "ln1", rows: n, columns: d, commands: commands)+ try linear(block + "attn_qkv", b, a: b.normed, c: b.qkv, m: n, commands: commands)+ try compute("vision_rope_pad", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: b.qkv.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 1)+ e.setBuffer(b.arena.buffer, offset: b.k.offset, index: 2)+ e.setBuffer(b.arena.buffer, offset: b.v.offset, index: 3)+ e.setBuffer(b.arena.buffer, offset: b.rope.offset, index: 4)+ var r = UInt32(n)+ e.setBytes(&r, length: 4, index: 5)+ e.dispatchThreads(MTLSize(width: n * 1024, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ try compute("flash_attention", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: b.k.offset, index: 1)+ e.setBuffer(b.arena.buffer, offset: b.v.offset, index: 2)+ e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 3)+ var params = QwenDiT.FlashParams(queries: UInt32(n), keys: UInt32(n), causal: 0, stride: 2048,+ scale: 1 / Float(QwenVisionTower.headDim).squareRoot() * Float(M_LOG2E), headBase: 0)+ e.setBytes(¶ms, length: MemoryLayout<QwenDiT.FlashParams>.stride, index: 4)+ e.dispatchThreadgroups(MTLSize(width: (n + 63) / 64, height: QwenVisionTower.heads, depth: 1),+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ try compute("vision_unpad", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: b.a72.offset, index: 1)+ var r = UInt32(n)+ e.setBytes(&r, length: 4, index: 2)+ e.dispatchThreads(MTLSize(width: n * d, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ try linear(block + "attn_out", b, a: b.a72, c: b.normed, m: n, commands: commands)+ try add(b, b.x, b.normed, offset: 0, count: n * d, commands: commands)+ try layerNorm(b, b.x, into: b.normed, prefix: block + "ln2", rows: n, columns: d, commands: commands)+ try linear(block + "ffn_up", b, a: b.normed, c: b.h, m: n, commands: commands)+ try compute("gelu_tanh", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: b.h.offset, index: 0)+ var c = UInt32(n * QwenVisionTower.hidden)+ e.setBytes(&c, length: 4, index: 1)+ e.dispatchThreads(MTLSize(width: Int(c), height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ try linear(block + "ffn_down", b, a: b.h, c: b.normed, m: n, commands: commands)+ try add(b, b.x, b.normed, offset: 0, count: n * d, commands: commands)+ }++ /// `linear_fc2(GELU(linear_fc1(mergeIn)))` into `out` `[slots, 4096]`.+ func mergerMLP(_ b: Buffers, prefix: String, fc1: String, fc2: String, out: Arena.Slice, slots: Int,+ commands: MTLCommandBuffer) throws {+ try linear("\(prefix).\(fc1)", b, a: b.mergeIn, c: b.mergeHidden, m: slots, commands: commands)+ try compute("gelu_erf", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: b.mergeHidden.offset, index: 0)+ var c = UInt32(slots * 4 * QwenVisionTower.dim)+ e.setBytes(&c, length: 4, index: 1)+ e.dispatchThreads(MTLSize(width: Int(c), height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ // The product writes padded rows; `out` holds only the real ones, so it goes through mergeIn.+ try linear("\(prefix).\(fc2)", b, a: b.mergeHidden, c: b.mergeIn, m: slots, commands: commands)+ let blit = commands.makeBlitCommandEncoder()!+ blit.copy(from: b.arena.buffer, sourceOffset: b.mergeIn.offset, to: b.arena.buffer, destinationOffset: out.offset,+ size: slots * QwenVisionTower.outDim * 4)+ blit.endEncoding()+ }++ // MARK: - encoders++ func compute(_ name: String, commands: MTLCommandBuffer, _ body: (MTLComputeCommandEncoder) throws -> Void) throws {+ let encoder = commands.makeComputeCommandEncoder()!+ encoder.label = name+ encoder.setComputePipelineState(try context.pipeline(name))+ try body(encoder)+ encoder.endEncoding()+ }++ /// `C[m, n] = A[m, k] · W + bias`, W the file's Linear `[n, k]` unpacked to `[k, n]` fp32 in a reserve first; m+ /// padded to 64 as the text encoder's products.+ func linear(_ name: String, _ b: Buffers, a: Arena.Slice, c: Arena.Slice, m: Int, commands: MTLCommandBuffer) throws {+ let tensor = try file.tensor(name + ".weight")+ let (kk, nn) = (tensor.columns, tensor.rows)+ let reserve = b.reserves[nextReserve]+ nextReserve = (nextReserve + 1) % b.reserves.count+ try dequant.encode(tensor, of: file, into: b.arena.buffer, outOffset: reserve.offset, transposed: true, commands: commands)+ try product(b, a: a, w: reserve, c: c, m: m, k: kk, n: nn, commands: commands)+ try bias(b, c, name + ".bias", rows: m, columns: nn, commands: commands)+ }++ func product(_ b: Buffers, a: Arena.Slice, w: Arena.Slice, c: Arena.Slice, m: Int, k: Int, n: Int,+ commands: MTLCommandBuffer) throws {+ let m = QwenDiT.padded(m)+ guard m * k * 4 <= a.bytes, m * n * 4 <= c.bytes, k * n * 4 <= w.bytes else {+ throw MetalContext.Failure.buffer("vision product \(m)×\(k)×\(n): no room")+ }+ let key = "\(m)×\(k)×\(n)"+ let product = products[key] ?? MPSMatrixMultiplication(device: context.device, transposeLeft: false, transposeRight: false,+ resultRows: m, resultColumns: n, interiorColumns: k, alpha: 1, beta: 0)+ products[key] = product+ func matrix(_ offset: Int, _ rows: Int, _ columns: Int) -> MPSMatrix {+ MPSMatrix(buffer: b.arena.buffer, offset: offset,+ descriptor: MPSMatrixDescriptor(rows: rows, columns: columns, rowBytes: columns * 4, dataType: .float32))+ }+ product.encode(commandBuffer: commands, leftMatrix: matrix(a.offset, m, k), rightMatrix: matrix(w.offset, k, n),+ resultMatrix: matrix(c.offset, m, n))+ }++ func bias(_ b: Buffers, _ x: Arena.Slice, _ name: String, rows: Int, columns: Int, commands: MTLCommandBuffer) throws {+ guard let at = small[name] else { throw GGUFFile.Failure.unsupported("no tensor \(name)") }+ try compute("bias_rows", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: x.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: b.smalls.offset + at * 4, index: 1)+ var p = SIMD2<UInt32>(UInt32(rows * columns), UInt32(columns))+ e.setBytes(&p, length: 8, index: 2)+ e.dispatchThreads(MTLSize(width: rows * columns, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }++ struct AffineParams { var columns: UInt32; var eps: Float }++ func layerNorm(_ b: Buffers, _ input: Arena.Slice, into out: Arena.Slice, prefix: String, rows: Int, columns: Int,+ commands: MTLCommandBuffer) throws {+ guard let w = small[prefix + ".weight"], let bias = small[prefix + ".bias"] else {+ throw GGUFFile.Failure.unsupported("no tensor \(prefix)")+ }+ try compute("layernorm_affine", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: input.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: out.offset, index: 1)+ e.setBuffer(b.arena.buffer, offset: b.smalls.offset + w * 4, index: 2)+ e.setBuffer(b.arena.buffer, offset: b.smalls.offset + bias * 4, index: 3)+ var p = AffineParams(columns: UInt32(columns), eps: QwenVisionTower.eps)+ e.setBytes(&p, length: MemoryLayout<AffineParams>.stride, index: 4)+ e.dispatchThreadgroups(MTLSize(width: rows, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }++ /// x += y (y at `offset` floats into its slice).+ func add(_ b: Buffers, _ x: Arena.Slice, _ y: Arena.Slice, offset: Int, count: Int, commands: MTLCommandBuffer) throws {+ try compute("add_inplace", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: x.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: y.offset + offset * 4, index: 1)+ var c = UInt32(count)+ e.setBytes(&c, length: 4, index: 2)+ e.dispatchThreads(MTLSize(width: count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }+}diff --git a/engine-mac/Sources/KokonEngine/TextChecks.swift b/engine-mac/Sources/KokonEngine/TextChecks.swiftindex 95c7ce3..333b844 100644--- a/engine-mac/Sources/KokonEngine/TextChecks.swift+++ b/engine-mac/Sources/KokonEngine/TextChecks.swift@@ -224,3 +224,60 @@ extension Checks {}}}++extension Checks {+ /// The vision tower against `tools/reference_vision.py` (transformers fp32): its input pixels (the processor's), the+ /// patches, blocks 0, 8 and 26, the merged rows and the three deepstack outputs; then the time of a 1024² encode.+ public static func vision(path: String, reference directory: String) throws {+ let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory)+ .appendingPathComponent("meta.json"))) as! [String: Any]+ let (w, h) = (meta["width"] as! Int, meta["height"] as! Int)+ let png = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: directory).appendingPathComponent("image.png")))+ let input = QwenVisionTower.pixels(rgba: png.rgba, width: png.width, height: png.height)+ let grid = meta["grid"] as! [Int]+ print("image \(png.width)×\(png.height) → \(input.width)×\(input.height)"+ + (grid == [1, input.height / 16, input.width / 16] ? "" : " (reference grid \(grid))") + ", reference \(w)×\(h)")+ // The processor's pixel rows: 2 × 2 merge blocks, each patch's 3 × 2 frames × 16 × 16 values.+ let wantPixels = try reference(directory, "pixels")+ var pixels = [Float](), (gh, gw) = (input.height / 16, input.width / 16)+ for by in 0..<(gh / 2) { for bx in 0..<(gw / 2) { for iy in 0..<2 { for ix in 0..<2 {+ for c in 0..<3 { for _ in 0..<2 { for y in 0..<16 { for x in 0..<16 {+ pixels.append(input.rgb[(c * input.height + (by * 2 + iy) * 16 + y) * input.width + (bx * 2 + ix) * 16 + x])+ } } } }+ } } } }+ if pixels.count == wantPixels.count {+ let r = pixels.withUnsafeBufferPointer { compare($0.baseAddress!, wantPixels) }+ print(String(format: " pixels: rel %.2e, max |Δ| %.3g", r.relative, r.maxAbs))+ } else {+ print(" pixels: \(pixels.count) values, reference \(wantPixels.count)")+ }+ let context = try MetalContext()+ let tower = try QwenVisionTower(file: try GGUFFile(path: path), context: context)+ var stages: [(String, [Float])] = []+ let out = try tower.encode(rgb: input.rgb, width: input.width, height: input.height) { name, p, count in+ stages.append((name, Array(UnsafeBufferPointer(start: p, count: count))))+ }+ var worst = 0.0+ func report(_ name: String, _ got: [Float]) throws {+ let want = try reference(directory, name)+ guard want.count == got.count else { print(" \(name): \(got.count) values, reference \(want.count)"); worst = .infinity; return }+ let r = got.withUnsafeBufferPointer { compare($0.baseAddress!, want) }+ worst = max(worst, r.relative)+ print(String(format: " %@: rel %.2e, max |Δ| %.3g%@", name, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))+ }+ for (name, values) in stages { try report(name, values) }+ try report("merged", out.merged)+ for (j, d) in out.deepstack.enumerated() { try report("deepstack\(j)", d) }+ print(String(format: "worst %.2e; encode %.2f s with the probes", worst, tower.seconds))+ let random = (0..<(3 * 1024 * 1024)).map { i -> Float in+ let (y, x) = ((i / 1024) % 1024, i % 1024)+ return sinf(Float(x) * 0.013 + Float(i / (1024 * 1024))) * cosf(Float(y) * 0.021) * 0.9+ }+ for run in 1...2 {+ _ = try tower.encode(rgb: random, width: 1024, height: 1024)+ Thread.sleep(forTimeInterval: 1.5)+ let m = MemoryUsage.now()+ print(String(format: "encode %d at 1024²: %.2f s, footprint after %.2f GB (peak %.2f GB)", run, tower.seconds, m.footprint, m.footprintPeak))+ }+ }+}diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swiftindex 445bf47..e542430 100644--- a/engine-mac/Sources/kokon-engine-mac/main.swift+++ b/engine-mac/Sources/kokon-engine-mac/main.swift@@ -13,6 +13,7 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)time-vae <vae.safetensors> [grid] the VAE decoder's time and memory (64 = 1024²; KOKON_VAE_TILE)check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.pycheck-text <encoder.gguf> <reference dir> the text encoder against tools/reference_text_encoder.py+ check-vision <mmproj.gguf> <reference dir> the vision tower against tools/reference_vision.py, then its timecheck-generate <encoder.gguf> <dit.gguf> <vae.safetensors> <reference dir> [text reference dir]text to image against tools/reference_generate.py, from its noise(and from the text reference's conditioning when given)@@ -94,6 +95,9 @@ do {case "check-text":guard arguments.count == 3 else { fail(usage) }try Checks.textEncoder(path: arguments[1], reference: arguments[2])+ case "check-vision":+ guard arguments.count == 3 else { fail(usage) }+ try Checks.vision(path: arguments[1], reference: arguments[2])case "check-generate":guard arguments.count == 5 || arguments.count == 6 else { fail(usage) }try Checks.generate(encoder: arguments[1], dit: arguments[2], vae: arguments[3], reference: arguments[4],diff --git a/engine-mac/Tests/KokonEngineTests/VisionTests.swift b/engine-mac/Tests/KokonEngineTests/VisionTests.swiftnew file mode 100644index 0000000..acf329a--- /dev/null+++ b/engine-mac/Tests/KokonEngineTests/VisionTests.swift@@ -0,0 +1,40 @@+import XCTest+@testable import KokonEngine++/// The vision tower's processor, without a model: the size it encodes at, the white composite, the resize.+final class VisionTests: XCTestCase {+ func testVisionSizeKeepsTheImageInsideTheProcessorsBounds() {+ XCTAssert(QwenVisionTower.visionSize(width: 1024, height: 768) == (1024, 768))+ XCTAssert(QwenVisionTower.visionSize(width: 320, height: 224) == (320, 224)) // 71 680 pixels: kept+ XCTAssert(QwenVisionTower.visionSize(width: 256, height: 192) == (320, 224)) // under 65 536: the processor's+ XCTAssert(QwenVisionTower.visionSize(width: 64, height: 64) == (256, 256))+ }++ func testAlphaIsCompositedOverWhiteAsPILPastes() {+ // Opaque, transparent and half-transparent pixels; PIL: white.paste(img, mask=alpha) on 8-bit values.+ let rgba: [UInt8] = [10, 200, 30, 255, 10, 200, 30, 0, 0, 100, 255, 128, 77, 77, 77, 1]+ let input = QwenVisionTower.pixels(rgba: rgba + [UInt8](repeating: 255, count: 4 * (256 * 256 - 4)), width: 256, height: 256)+ let plane = 256 * 256+ func level(_ c: Int, _ p: Int) -> Int { Int(((input.rgb[c * plane + p] * 0.5 + 0.5) * 255).rounded()) }+ XCTAssertEqual([level(0, 0), level(1, 0), level(2, 0)], [10, 200, 30])+ XCTAssertEqual([level(0, 1), level(1, 1), level(2, 1)], [255, 255, 255])+ // (v·128 + 255·127) / 255, rounded: 0 → 127, 100 → 177, 255 → 255.+ XCTAssertEqual([level(0, 2), level(1, 2), level(2, 2)], [127, 177, 255])+ XCTAssertEqual(level(0, 3), 254)+ }++ func testResizeKeepsFlatImagesFlatAndCoversTheOutput() {+ let flat = [UInt8](repeating: 90, count: 3 * 192 * 256)+ let up = QwenVisionTower.resizeBicubic(flat, width: 256, height: 192, to: 320, 224)+ XCTAssertEqual(up.count, 3 * 320 * 224)+ XCTAssert(up.allSatisfy { $0 == 90 })+ // A horizontal ramp stays monotonic (no ringing on a smooth ramp) and keeps its ends.+ var ramp = [UInt8](repeating: 0, count: 3 * 192 * 256)+ for y in 0..<192 { for x in 0..<256 { for c in 0..<3 { ramp[(y * 256 + x) * 3 + c] = UInt8(x) } } }+ let r = QwenVisionTower.resizeBicubic(ramp, width: 256, height: 192, to: 320, 224)+ let row = (0..<320).map { Int(r[(100 * 320 + $0) * 3]) }+ XCTAssertEqual(row.first, 0)+ XCTAssertEqual(row.last, 255)+ XCTAssert(zip(row, row.dropFirst()).allSatisfy { $0 <= $1 })+ }+}diff --git a/engine-mac/tools/reference_vision.py b/engine-mac/tools/reference_vision.pynew file mode 100644index 0000000..eb52649--- /dev/null+++ b/engine-mac/tools/reference_vision.py@@ -0,0 +1,130 @@+"""The Qwen-Image 2.1 vision tower reference for the Mac engine: transformers' own `Qwen3VLVisionModel` in fp32 on the+CPU, on the weights of Kokon's mmproj GGUF (llama.cpp's layout) unpacked by gguf-py, on an image prepared by the+official processor as diffusers' QwenImage21Pipeline hands it over (`_get_qwen_prompt_embeds`: RGBA composited over+white, then `Qwen3VLProcessor`). The image's sides must be multiples of 32, as the pipeline makes them.++The input is a PNG (`--image`) or, by default, a smooth random image (random low-resolution noise upsampled+bilinearly, so it has edges and flat areas as photos do), saved as `image.png` for the engine to read.++ python -I engine-mac/tools/reference_vision.py <mmproj.gguf> <processor dir> <text_encoder dir> <out dir>+ [--image in.png] [--size 320x224] [--dtype float64]++Writes `<out>/meta.json` (grid, sizes) and one raw little-endian fp32 file per tensor (`<name>.f32`): pixels (the+processor's `pixel_values`), embeddings (patches + positions), block0/8/26 outputs, deepstack0..2, merged.+"""+import argparse+import json+import os++import gguf+import numpy as np+import torch+import torch.nn.functional as F+from PIL import Image+from transformers import AutoConfig, AutoProcessor+from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLVisionModel++BLOCK = {"ln1": "norm1", "ln2": "norm2", "attn_qkv": "attn.qkv", "attn_out": "attn.proj",+ "ffn_up": "mlp.linear_fc1", "ffn_down": "mlp.linear_fc2"}+MERGER = {"norm": "norm", "fc1": "linear_fc1", "fc2": "linear_fc2"}+++def tensor(reader, name):+ t = reader[name]+ values = gguf.quants.dequantize(t.data, t.tensor_type).astype(np.float32)+ return torch.from_numpy(values.reshape([int(d) for d in reversed(t.shape)]).copy())+++def state_dict(path, config):+ """mmproj names (llama.cpp's clip.cpp, projector qwen3vl_merger) -> Qwen3VLVisionModel names."""+ reader = {t.name: t for t in gguf.GGUFReader(path).tensors}+ get = lambda name: tensor(reader, name) # noqa: E731+ # The Conv3d over 2 frames is stored as one 2D kernel per frame.+ state = {"patch_embed.proj.weight": torch.stack([get("v.patch_embd.weight"), get("v.patch_embd.weight.1")], dim=2),+ "patch_embed.proj.bias": get("v.patch_embd.bias"), "pos_embed.weight": get("v.position_embd.weight")}+ for i in range(config.depth):+ for g, h in BLOCK.items():+ for kind in ("weight", "bias"):+ state[f"blocks.{i}.{h}.{kind}"] = get(f"v.blk.{i}.{g}.{kind}")+ for kind in ("weight", "bias"):+ state[f"merger.norm.{kind}"] = get(f"v.post_ln.{kind}")+ state[f"merger.linear_fc1.{kind}"] = get(f"mm.0.{kind}")+ state[f"merger.linear_fc2.{kind}"] = get(f"mm.2.{kind}")+ for j, layer in enumerate(config.deepstack_visual_indexes):+ for g, h in MERGER.items():+ state[f"deepstack_merger_list.{j}.{h}.{kind}"] = get(f"v.deepstack.{layer}.{g}.{kind}")+ return state+++def smooth_image(width, height, seed=0):+ g = torch.Generator().manual_seed(seed)+ low = torch.rand(1, 3, height // 16, width // 16, generator=g)+ img = F.interpolate(low, size=(height, width), mode="bilinear", align_corners=False)[0]+ return Image.fromarray((img.permute(1, 2, 0).numpy() * 255).round().clip(0, 255).astype(np.uint8))+++def main():+ parser = argparse.ArgumentParser()+ parser.add_argument("gguf")+ parser.add_argument("processor")+ parser.add_argument("text_encoder", help="text_encoder/ of the official repository (config.json)")+ parser.add_argument("out")+ parser.add_argument("--image")+ parser.add_argument("--size", default="320x224")+ parser.add_argument("--dtype", default="float32", help="float64: the model in fp64, to tell torch's fp32 drift apart")+ args = parser.parse_args()+ torch.set_grad_enabled(False)+ os.makedirs(args.out, exist_ok=True)++ if args.image:+ image = Image.open(args.image)+ else:+ width, height = (int(v) for v in args.size.split("x"))+ image = smooth_image(width, height)+ assert image.width % 32 == 0 and image.height % 32 == 0, "sides must be multiples of 32"+ image.save(os.path.join(args.out, "image.png"))+ if image.mode == "RGBA": # as the pipeline: alpha composited over white for the vision encoder+ white = Image.new("RGB", image.size, (255, 255, 255))+ white.paste(image, mask=image.getchannel("A"))+ image = white+ image = image.convert("RGB")++ processor = AutoProcessor.from_pretrained(args.processor)+ inputs = processor.image_processor(images=[image], return_tensors="pt")+ dtype = getattr(torch, args.dtype)+ pixels, grid = inputs.pixel_values.to(dtype), inputs.image_grid_thw++ config = AutoConfig.from_pretrained(args.text_encoder).vision_config+ config._attn_implementation = "eager"+ model = Qwen3VLVisionModel(config).eval()+ model.load_state_dict(state_dict(args.gguf, config), strict=True)+ model = model.to(dtype)++ saved = {}++ def save(name, t):+ array = t.detach().to(torch.float32).contiguous().numpy()+ array.tofile(os.path.join(args.out, name + ".f32"))+ saved[name] = list(array.shape)++ save("pixels", pixels)+ hooks = [model.patch_embed.register_forward_hook(lambda m, a, out: save("patches", out))]+ for i in (0, 8, 26):+ hooks.append(model.blocks[i].register_forward_hook(lambda m, a, out, i=i: save(f"block{i}", out)))+ out = model(pixels, grid_thw=grid, return_dict=True)+ for h in hooks:+ h.remove()+ save("merged", out.pooler_output)+ for j, feature in enumerate(out.deepstack_features):+ save(f"deepstack{j}", feature)++ meta = {"grid": grid[0].tolist(), "dtype": args.dtype, "width": image.width, "height": image.height, "tensors": saved,+ "gguf": os.path.basename(args.gguf), "transformers": __import__("transformers").__version__,+ "torch": torch.__version__}+ with open(os.path.join(args.out, "meta.json"), "w") as f:+ json.dump(meta, f, indent=1)+ print(f"grid {grid[0].tolist()}, merged {saved['merged']}")+++if __name__ == "__main__":+ main()