Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.
TypeScript ai-photo-editorcomfyui-alternativegenerative-fillimage-editingimage-generationinpaintinglinuxlocal-aimacosmcpmcp-servernano-banana-alternativeofflinephoto-editorprivacyqwen-imageqwen-image-2-1stable-diffusion-cppvulkanwindows
Mac engine: the vision tower's heads on a 72-wide attention kernel, 1 MP encode 2.6 → 1.7 s
Gabriele Lozupone71fb5eb2c6cc
diff --git a/engine-mac/README.md b/engine-mac/README.mdindex 9a0206d..700ff1a 100644--- a/engine-mac/README.md+++ b/engine-mac/README.md@@ -356,7 +356,7 @@ M2 Pro 16 GB, edits with the reference at the output's size, alternated:Page-ins vary with what else the Mac runs: during these runs free memory sat near zero (12.5 of 16 GB used, 4.6 GB of swap),and every configuration still reads ~2 GB of the DiT's pages a step. A 1 MP-edit: vision tower 2.5 s, text encoder 3.7 s (1 037 tokens), VAE encoder 0.8 s, step 1 ~20 s, then ~10.4 s, VAE 4.6–5 s; peak footprint+edit: vision tower 2.5 s (1.6–1.8 s since 2026-10-26, below), text encoder 3.7 s (1 037 tokens), VAE encoder 0.8 s, step 1 ~20 s, then ~10.4 s, VAE 4.6–5 s; peak footprint5.0 GB. Through the server (`tools/api_check.py --ref … --enhance`, 768 × 448): Prompt+ looking at the reference 5.5–6.8 s(vision 0.6 s, 742 prompt tokens), the edit 36.4–37.3 s, peak footprint 3.6 GB.@@ -495,13 +495,15 @@ texture-fix VAE, M2 Pro, 2026-10-25): the latent within 6.5e-6. 1024²: 0.59 s a`<|image_pad|>` slots): Qwen3-VL's ViT from Kokon's mmproj GGUF (llama.cpp's layout: the Conv3d as one kernel per frame,fused q/k/v, `mm.0`/`mm.2` the merger, `v.deepstack.8/16/24`), fp32 on the GPU with the F16 weights unpacked before eachproduct. The patches in the processor's 2 × 2 merge-block order, the two frames' kernels summed (both frames are the-image), the 48 × 48 position table resampled bilinearly, axial 2D RoPE, the 72-dimension heads on the DiT's flash kernel-zero-padded to 128. Its input is the processor's: alpha over white as PIL's paste, and below 65 536 pixels the processor's+image), the 48 × 48 position table resampled bilinearly, axial 2D RoPE, the 72-dimension heads on the DiT's fp32 flash+kernel built for 72 (`flash_attention_72`). Its input is the processor's: alpha over white as PIL's paste, and below 65 536 pixels the processor'sresize (torchvision's uint8 antialiased bicubic: PIL's kernel with int16 weights, horizontal pass first; bit for bit).`check-vision` against `tools/reference_vision.py` (transformers' `Qwen3VLVisionModel`, M2 Pro, 2026-10-25), 320 × 224:the pixels within 1.6e-7 (float rounding), the merged rows within 1.35e-5 of torch fp32 and **7.6e-6 of fp64**(`--dtype float64`; torch's own fp32 is 1.55e-5 from it), deepstack 3.8–9.4e-6; 256 × 192 (resized to 320 × 224) and-an RGBA image the same. 1024²: 2.5 s an encode (4 096 patches; the padded heads cost ~1.8× the attention's work).+an RGBA image the same. 1024²: 1.6–1.8 s an encode (4 096 patches), 2.6–2.7 s while the heads were zero-padded to 128+(~1.8× the attention's work; the same values to the bit, `check-vision` and `check-text` with the cat unchanged to every+digit; M2 Pro, 2026-10-26, other apps open).```bash~/kokon-bench/venv-oracle/bin/python -I tools/reference_vision.py <mmproj.gguf> <processor dir> <text_encoder dir> ~/kokon-bench/ref/vision-320 [--dtype float64]diff --git a/engine-mac/Sources/KokonEngine/Kernels/attention.metal b/engine-mac/Sources/KokonEngine/Kernels/attention.metalindex 90070ef..0268f60 100644--- a/engine-mac/Sources/KokonEngine/Kernels/attention.metal+++ b/engine-mac/Sources/KokonEngine/Kernels/attention.metal@@ -43,24 +43,25 @@ static inline thread float2 &elements(thread simdgroup_float8x8 &m) {return *(thread float2 *)&m.thread_elements();}-kernel void flash_attention(device const float *q [[buffer(0)]], device const float *k [[buffer(1)]],+template <uint HD>+kernel void flash_attention_d(device const float *q [[buffer(0)]], device const float *k [[buffer(1)]],device const float *v [[buffer(2)]], device float *out [[buffer(3)]],constant FlashParams &p [[buffer(4)]],uint2 tg [[threadgroup_position_in_grid]],uint tid [[thread_index_in_threadgroup]],uint sg [[simdgroup_index_in_threadgroup]],uint lane [[thread_index_in_simdgroup]]) {- threadgroup float tile[BK * D]; // 32 KB: the key tile, then the value tile+ threadgroup float tile[BK * HD]; // 32 KB at 128: the key tile, then the value tileconst uint head = p.headBase + tg.y;const uint q0 = tg.x * BQ + sg * 8;const uint stride = p.stride;- device const float *kh = k + head * D;- device const float *vh = v + head * D;+ device const float *kh = k + head * HD;+ device const float *vh = v + head * HD;- simdgroup_float8x8 Q[16], O[16];- for (uint d = 0; d < 16; d++) {- simdgroup_load(Q[d], q + (ulong)q0 * stride + head * D + 8 * d, stride);+ simdgroup_float8x8 Q[HD / 8], O[HD / 8];+ for (uint d = 0; d < HD / 8; d++) {+ simdgroup_load(Q[d], q + (ulong)q0 * stride + head * HD + 8 * d, stride);O[d] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);}@@ -76,18 +77,18 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const flfor (uint j0 = 0; j0 < kEnd; j0 += BK) {// Key tile: 64 rows × 128, 256 threads × 8 float4.- for (uint i = tid; i < BK * D / 4; i += 256) {- uint r = i / (D / 4), c = i % (D / 4);+ for (uint i = tid; i < BK * HD / 4; i += 256) {+ uint r = i / (HD / 4), c = i % (HD / 4);((threadgroup float4 *)tile)[i] = *((device const float4 *)(kh + (ulong)(j0 + r) * stride) + c);}threadgroup_barrier(mem_flags::mem_threadgroup);simdgroup_float8x8 S[8];for (uint c = 0; c < 8; c++) S[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- for (uint d = 0; d < 16; d++) {+ for (uint d = 0; d < HD / 8; d++) {for (uint c = 0; c < 8; c++) {simdgroup_float8x8 Kt;- simdgroup_load(Kt, tile + 8 * c * D + 8 * d, D, ulong2(0, 0), true);+ simdgroup_load(Kt, tile + 8 * c * HD + 8 * d, HD, ulong2(0, 0), true);simdgroup_multiply_accumulate(S[c], Q[d], Kt, S[c]);}}@@ -117,20 +118,20 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fll = l * alpha + sum;m = mNew;if (!simd_all(alpha == 1.0f)) {- for (uint d = 0; d < 16; d++) elements(O[d]) *= alpha;+ for (uint d = 0; d < HD / 8; d++) elements(O[d]) *= alpha;}// Value tile into the same memory, once every SIMD group is done with the keys.threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint i = tid; i < BK * D / 4; i += 256) {- uint r = i / (D / 4), c = i % (D / 4);+ for (uint i = tid; i < BK * HD / 4; i += 256) {+ uint r = i / (HD / 4), c = i % (HD / 4);((threadgroup float4 *)tile)[i] = *((device const float4 *)(vh + (ulong)(j0 + r) * stride) + c);}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint d = 0; d < 16; d++) {+ for (uint d = 0; d < HD / 8; d++) {for (uint c = 0; c < 8; c++) {simdgroup_float8x8 V;- simdgroup_load(V, tile + 8 * c * D + 8 * d, D);+ simdgroup_load(V, tile + 8 * c * HD + 8 * d, HD);simdgroup_multiply_accumulate(O[d], S[c], V, O[d]);}}@@ -140,8 +141,8 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fl// O / l, the valid rows to memory.if (qi < p.queries) {const float inv = 1.0f / l;- device float *o = out + (ulong)qi * stride + head * D + col;- for (uint d = 0; d < 16; d++) {+ device float *o = out + (ulong)qi * stride + head * HD + col;+ for (uint d = 0; d < HD / 8; d++) {float2 e = elements(O[d]) * inv;o[8 * d] = e.x;o[8 * d + 1] = e.y;@@ -149,6 +150,13 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fl}}+// The DiT's heads (128) and the vision tower's (72: Qwen3-VL's 1152 / 16), with no zero padding. At 72 the result is+// the padded kernel's to the bit: the 7 padding chunks only added exact zeros to every sum.+template [[host_name("flash_attention")]] kernel void flash_attention_d<128>(device const float *, device const float *,+ device const float *, device float *, constant FlashParams &, uint2, uint, uint, uint);+template [[host_name("flash_attention_72")]] kernel void flash_attention_d<72>(device const float *, device const float *,+ device const float *, device float *, constant FlashParams &, uint2, uint, uint, uint);+// The same with q, k, v and the probabilities in fp16 (scores, softmax and O accumulated in fp32; K and V tiles of 64// keys side by side in 32 KB), the engine's default (`KOKON_ATTENTION=fp32` for the one above): 94 ms against 120 at// 1024² (M2 Pro), head 0 within 6.3e-4 of exact, the velocity within 6.0e-4 of fp64 at 256² (fp32 kernel 1.0e-5).diff --git a/engine-mac/Sources/KokonEngine/Kernels/vision.metal b/engine-mac/Sources/KokonEngine/Kernels/vision.metalindex 9d6d7ea..fd441ff 100644--- a/engine-mac/Sources/KokonEngine/Kernels/vision.metal+++ b/engine-mac/Sources/KokonEngine/Kernels/vision.metal@@ -60,39 +60,23 @@ kernel void gelu_erf(device float *x [[buffer(0)]], constant uint &count [[buffex[i] = 0.5f * v * tail;}-// Qwen3VLVisionAttention's qkv split and apply_rotary_pos_emb_vision, into the flash kernel's layout: q, k, v as-// [rows, 16 heads, 128] with the head's 72 dimensions first and zeros after (the scores and the output are the-// 72-dimension ones). rotate_half pairs dimension i with i + 36; `rope` is [rows, 36, (cos, sin)].-// One thread per (row, head, j < 64): j < 36 rotates the pair (j, j + 36), the others clear two padding columns.-kernel void vision_rope_pad(device const float *qkv [[buffer(0)]], device float *q [[buffer(1)]],- device float *k [[buffer(2)]], device float *v [[buffer(3)]],- device const float *rope [[buffer(4)]], constant uint &rows [[buffer(5)]],- uint i [[thread_position_in_grid]]) {- uint j = i % 64, head = (i / 64) % 16, row = i / 1024;+// Qwen3VLVisionAttention's qkv split and apply_rotary_pos_emb_vision: q, k, v as [rows, 16 heads × 72], the layout+// `flash_attention_72` reads and attn.proj's input has. rotate_half pairs dimension i with i + 36; `rope` is+// [rows, 36, (cos, sin)]. One thread per (row, head, j < 36), rotating the pair (j, j + 36).+kernel void vision_rope(device const float *qkv [[buffer(0)]], device float *q [[buffer(1)]],+ device float *k [[buffer(2)]], device float *v [[buffer(3)]],+ device const float *rope [[buffer(4)]], constant uint &rows [[buffer(5)]],+ uint i [[thread_position_in_grid]]) {+ uint j = i % 36, head = (i / 36) % 16, row = i / 576;if (row >= rows) return;device const float *in = qkv + (ulong)row * 3456 + head * 72;- ulong o = (ulong)row * 2048 + head * 128;- if (j < 36) {- float c = rope[((ulong)row * 36 + j) * 2], s = rope[((ulong)row * 36 + j) * 2 + 1];- float q0 = in[j], q1 = in[j + 36], k0 = in[1152 + j], k1 = in[1152 + j + 36];- q[o + j] = q0 * c - q1 * s;- q[o + j + 36] = q1 * c + q0 * s;- k[o + j] = k0 * c - k1 * s;- k[o + j + 36] = k1 * c + k0 * s;- v[o + j] = in[2304 + j];- v[o + j + 36] = in[2304 + j + 36];- } else {- uint c = 72 + (j - 36) * 2;- q[o + c] = 0; q[o + c + 1] = 0;- k[o + c] = 0; k[o + c + 1] = 0;- v[o + c] = 0; v[o + c + 1] = 0;- }-}--// The attention's output back to [rows, 16 × 72] for attn.proj.-kernel void vision_unpad(device const float *x [[buffer(0)]], device float *out [[buffer(1)]],- constant uint &rows [[buffer(2)]], uint i [[thread_position_in_grid]]) {- if (i >= rows * 1152) return;- uint row = i / 1152, c = i % 1152;- out[i] = x[(ulong)row * 2048 + (c / 72) * 128 + c % 72];+ ulong o = (ulong)row * 1152 + head * 72;+ float c = rope[((ulong)row * 36 + j) * 2], s = rope[((ulong)row * 36 + j) * 2 + 1];+ float q0 = in[j], q1 = in[j + 36], k0 = in[1152 + j], k1 = in[1152 + j + 36];+ q[o + j] = q0 * c - q1 * s;+ q[o + j + 36] = q1 * c + q0 * s;+ k[o + j] = k0 * c - k1 * s;+ k[o + j + 36] = k1 * c + k0 * s;+ v[o + j] = in[2304 + j];+ v[o + j + 36] = in[2304 + j + 36];}diff --git a/engine-mac/Sources/KokonEngine/QwenVisionTower.swift b/engine-mac/Sources/KokonEngine/QwenVisionTower.swiftindex ff474cb..0b2ef3c 100644--- a/engine-mac/Sources/KokonEngine/QwenVisionTower.swift+++ b/engine-mac/Sources/KokonEngine/QwenVisionTower.swift@@ -12,8 +12,8 @@ import MetalPerformanceShaders/// merged = fc2(GELU(fc1(LN(x) as [N/4, 4608]))) [N/4, 4096], one row per text encoder slot/// deepstack j = the same after block 8 / 16 / 24 (LN after the 2 × 2 merge) added to the text encoder's layers 0–2///-/// The heads (72 dimensions) run on the DiT's flash kernel zero-padded to 128 (`vision_rope_pad`): the scores and the-/// output are the 72-dimension ones. The processor's input is the image with alpha composited over white (as the+/// The heads (72 dimensions) run on the DiT's fp32 flash kernel built for 72 (`flash_attention_72`): until 2026-10-26+/// they were zero-padded to 128, the same values to the bit for ~1.8× the work. The processor's input is the image with alpha composited over white (as the/// pipeline's PIL paste), scaled to [−1, 1], each frame of the Conv3d's two the same image.public final class QwenVisionTower {public static let dim = 1152, heads = 16, headDim = 72, hidden = 4304, layers = 27, outDim = 4096@@ -156,7 +156,7 @@ public final class QwenVisionTower {struct Buffers {let arena: Arena- let x, normed, qkv, q, k, v, attended, a72, h, mergeIn, mergeHidden, smalls, rope: Arena.Slice+ let x, normed, qkv, q, k, v, attended, h, mergeIn, mergeHidden, smalls, rope: Arena.Slicelet outputs: [Arena.Slice]let reserves: [Arena.Slice]}@@ -182,13 +182,13 @@ public final class QwenVisionTower {let mrows = QwenDiT.padded(slots)let smallNames = smallTensors()let smallCount = try smallNames.reduce(0) { $0 + (try file.tensor($1)).count } + n * d- let sizes = [rows * d, rows * d, rows * 3 * d, rows * 2048, rows * 2048, rows * 2048, rows * 2048, rows * d,+ let sizes = [rows * d, rows * d, rows * 3 * d, rows * d, rows * d, rows * d, rows * d,rows * QwenVisionTower.hidden, mrows * merged, mrows * merged, smallCount, n * 72,slots * 4096, slots * 4096, slots * 4096, slots * 4096, merged * merged, merged * merged]let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))var s = try sizes.map { try arena.slice(floats: $0) }.makeIterator()let b = Buffers(arena: arena, x: s.next()!, normed: s.next()!, qkv: s.next()!, q: s.next()!, k: s.next()!,- v: s.next()!, attended: s.next()!, a72: s.next()!, h: s.next()!, mergeIn: s.next()!,+ v: s.next()!, attended: s.next()!, h: s.next()!, mergeIn: s.next()!,mergeHidden: s.next()!, smalls: s.next()!, rope: s.next()!,outputs: [s.next()!, s.next()!, s.next()!, s.next()!], reserves: [s.next()!, s.next()!])@@ -320,7 +320,7 @@ public final class QwenVisionTower {let d = QwenVisionTower.dim, block = "v.blk.\(layer)."try layerNorm(b, b.x, into: b.normed, prefix: block + "ln1", rows: n, columns: d, commands: commands)try linear(block + "attn_qkv", b, a: b.normed, c: b.qkv, m: n, commands: commands)- try compute("vision_rope_pad", commands: commands) { e in+ try compute("vision_rope", commands: commands) { e ine.setBuffer(b.arena.buffer, offset: b.qkv.offset, index: 0)e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 1)e.setBuffer(b.arena.buffer, offset: b.k.offset, index: 2)@@ -328,27 +328,21 @@ public final class QwenVisionTower {e.setBuffer(b.arena.buffer, offset: b.rope.offset, index: 4)var r = UInt32(n)e.setBytes(&r, length: 4, index: 5)- e.dispatchThreads(MTLSize(width: n * 1024, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ e.dispatchThreads(MTLSize(width: n * QwenVisionTower.heads * 36, height: 1, depth: 1),+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))}- try compute("flash_attention", commands: commands) { e in+ try compute("flash_attention_72", commands: commands) { e ine.setBuffer(b.arena.buffer, offset: b.q.offset, index: 0)e.setBuffer(b.arena.buffer, offset: b.k.offset, index: 1)e.setBuffer(b.arena.buffer, offset: b.v.offset, index: 2)e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 3)- var params = QwenDiT.FlashParams(queries: UInt32(n), keys: UInt32(n), causal: 0, stride: 2048,+ var params = QwenDiT.FlashParams(queries: UInt32(n), keys: UInt32(n), causal: 0, stride: UInt32(d),scale: 1 / Float(QwenVisionTower.headDim).squareRoot() * Float(M_LOG2E), headBase: 0)e.setBytes(¶ms, length: MemoryLayout<QwenDiT.FlashParams>.stride, index: 4)e.dispatchThreadgroups(MTLSize(width: (n + 63) / 64, height: QwenVisionTower.heads, depth: 1),threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))}- try compute("vision_unpad", commands: commands) { e in- e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 0)- e.setBuffer(b.arena.buffer, offset: b.a72.offset, index: 1)- var r = UInt32(n)- e.setBytes(&r, length: 4, index: 2)- e.dispatchThreads(MTLSize(width: n * d, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))- }- try linear(block + "attn_out", b, a: b.a72, c: b.normed, m: n, commands: commands)+ try linear(block + "attn_out", b, a: b.attended, c: b.normed, m: n, commands: commands)try add(b, b.x, b.normed, offset: 0, count: n * d, commands: commands)try layerNorm(b, b.x, into: b.normed, prefix: block + "ln2", rows: n, columns: d, commands: commands)try linear(block + "ffn_up", b, a: b.normed, c: b.h, m: n, commands: commands)