Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: the vision tower's heads on a 72-wide attention kernel, 1 MP encode 2.6 → 1.7 s

Gabriele Lozupone71fb5eb2c6cc
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 9a0206d..700ff1a 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -356,7 +356,7 @@ M2 Pro 16 GB, edits with the reference at the output's size, alternated:
Page-ins vary with what else the Mac runs: during these runs free memory sat near zero (12.5 of 16 GB used, 4.6 GB of swap),
and every configuration still reads ~2 GB of the DiT's pages a step. A 1 MP
-edit: vision tower 2.5 s, text encoder 3.7 s (1 037 tokens), VAE encoder 0.8 s, step 1 ~20 s, then ~10.4 s, VAE 4.6–5 s; peak footprint
+edit: vision tower 2.5 s (1.6–1.8 s since 2026-10-26, below), text encoder 3.7 s (1 037 tokens), VAE encoder 0.8 s, step 1 ~20 s, then ~10.4 s, VAE 4.6–5 s; peak footprint
5.0 GB. Through the server (`tools/api_check.py --ref … --enhance`, 768 × 448): Prompt+ looking at the reference 5.5–6.8 s
(vision 0.6 s, 742 prompt tokens), the edit 36.4–37.3 s, peak footprint 3.6 GB.
@@ -495,13 +495,15 @@ texture-fix VAE, M2 Pro, 2026-10-25): the latent within 6.5e-6. 1024²: 0.59 s a
`<|image_pad|>` slots): Qwen3-VL's ViT from Kokon's mmproj GGUF (llama.cpp's layout: the Conv3d as one kernel per frame,
fused q/k/v, `mm.0`/`mm.2` the merger, `v.deepstack.8/16/24`), fp32 on the GPU with the F16 weights unpacked before each
product. The patches in the processor's 2 × 2 merge-block order, the two frames' kernels summed (both frames are the
-image), the 48 × 48 position table resampled bilinearly, axial 2D RoPE, the 72-dimension heads on the DiT's flash kernel
-zero-padded to 128. Its input is the processor's: alpha over white as PIL's paste, and below 65 536 pixels the processor's
+image), the 48 × 48 position table resampled bilinearly, axial 2D RoPE, the 72-dimension heads on the DiT's fp32 flash
+kernel built for 72 (`flash_attention_72`). Its input is the processor's: alpha over white as PIL's paste, and below 65 536 pixels the processor's
resize (torchvision's uint8 antialiased bicubic: PIL's kernel with int16 weights, horizontal pass first; bit for bit).
`check-vision` against `tools/reference_vision.py` (transformers' `Qwen3VLVisionModel`, M2 Pro, 2026-10-25), 320 × 224:
the pixels within 1.6e-7 (float rounding), the merged rows within 1.35e-5 of torch fp32 and **7.6e-6 of fp64**
(`--dtype float64`; torch's own fp32 is 1.55e-5 from it), deepstack 3.8–9.4e-6; 256 × 192 (resized to 320 × 224) and
-an RGBA image the same. 1024²: 2.5 s an encode (4 096 patches; the padded heads cost ~1.8× the attention's work).
+an RGBA image the same. 1024²: 1.6–1.8 s an encode (4 096 patches), 2.6–2.7 s while the heads were zero-padded to 128
+(~1.8× the attention's work; the same values to the bit, `check-vision` and `check-text` with the cat unchanged to every
+digit; M2 Pro, 2026-10-26, other apps open).
```bash
~/kokon-bench/venv-oracle/bin/python -I tools/reference_vision.py <mmproj.gguf> <processor dir> <text_encoder dir> ~/kokon-bench/ref/vision-320 [--dtype float64]
diff --git a/engine-mac/Sources/KokonEngine/Kernels/attention.metal b/engine-mac/Sources/KokonEngine/Kernels/attention.metal
index 90070ef..0268f60 100644
--- a/engine-mac/Sources/KokonEngine/Kernels/attention.metal
+++ b/engine-mac/Sources/KokonEngine/Kernels/attention.metal
@@ -43,24 +43,25 @@ static inline thread float2 &elements(thread simdgroup_float8x8 &m) {
return *(thread float2 *)&m.thread_elements();
}
-kernel void flash_attention(device const float *q [[buffer(0)]], device const float *k [[buffer(1)]],
+template <uint HD>
+kernel void flash_attention_d(device const float *q [[buffer(0)]], device const float *k [[buffer(1)]],
device const float *v [[buffer(2)]], device float *out [[buffer(3)]],
constant FlashParams &p [[buffer(4)]],
uint2 tg [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]],
uint sg [[simdgroup_index_in_threadgroup]],
uint lane [[thread_index_in_simdgroup]]) {
- threadgroup float tile[BK * D]; // 32 KB: the key tile, then the value tile
+ threadgroup float tile[BK * HD]; // 32 KB at 128: the key tile, then the value tile
const uint head = p.headBase + tg.y;
const uint q0 = tg.x * BQ + sg * 8;
const uint stride = p.stride;
- device const float *kh = k + head * D;
- device const float *vh = v + head * D;
+ device const float *kh = k + head * HD;
+ device const float *vh = v + head * HD;
- simdgroup_float8x8 Q[16], O[16];
- for (uint d = 0; d < 16; d++) {
- simdgroup_load(Q[d], q + (ulong)q0 * stride + head * D + 8 * d, stride);
+ simdgroup_float8x8 Q[HD / 8], O[HD / 8];
+ for (uint d = 0; d < HD / 8; d++) {
+ simdgroup_load(Q[d], q + (ulong)q0 * stride + head * HD + 8 * d, stride);
O[d] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
}
@@ -76,18 +77,18 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fl
for (uint j0 = 0; j0 < kEnd; j0 += BK) {
// Key tile: 64 rows × 128, 256 threads × 8 float4.
- for (uint i = tid; i < BK * D / 4; i += 256) {
- uint r = i / (D / 4), c = i % (D / 4);
+ for (uint i = tid; i < BK * HD / 4; i += 256) {
+ uint r = i / (HD / 4), c = i % (HD / 4);
((threadgroup float4 *)tile)[i] = *((device const float4 *)(kh + (ulong)(j0 + r) * stride) + c);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
simdgroup_float8x8 S[8];
for (uint c = 0; c < 8; c++) S[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- for (uint d = 0; d < 16; d++) {
+ for (uint d = 0; d < HD / 8; d++) {
for (uint c = 0; c < 8; c++) {
simdgroup_float8x8 Kt;
- simdgroup_load(Kt, tile + 8 * c * D + 8 * d, D, ulong2(0, 0), true);
+ simdgroup_load(Kt, tile + 8 * c * HD + 8 * d, HD, ulong2(0, 0), true);
simdgroup_multiply_accumulate(S[c], Q[d], Kt, S[c]);
}
}
@@ -117,20 +118,20 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fl
l = l * alpha + sum;
m = mNew;
if (!simd_all(alpha == 1.0f)) {
- for (uint d = 0; d < 16; d++) elements(O[d]) *= alpha;
+ for (uint d = 0; d < HD / 8; d++) elements(O[d]) *= alpha;
}
// Value tile into the same memory, once every SIMD group is done with the keys.
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint i = tid; i < BK * D / 4; i += 256) {
- uint r = i / (D / 4), c = i % (D / 4);
+ for (uint i = tid; i < BK * HD / 4; i += 256) {
+ uint r = i / (HD / 4), c = i % (HD / 4);
((threadgroup float4 *)tile)[i] = *((device const float4 *)(vh + (ulong)(j0 + r) * stride) + c);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint d = 0; d < 16; d++) {
+ for (uint d = 0; d < HD / 8; d++) {
for (uint c = 0; c < 8; c++) {
simdgroup_float8x8 V;
- simdgroup_load(V, tile + 8 * c * D + 8 * d, D);
+ simdgroup_load(V, tile + 8 * c * HD + 8 * d, HD);
simdgroup_multiply_accumulate(O[d], S[c], V, O[d]);
}
}
@@ -140,8 +141,8 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fl
// O / l, the valid rows to memory.
if (qi < p.queries) {
const float inv = 1.0f / l;
- device float *o = out + (ulong)qi * stride + head * D + col;
- for (uint d = 0; d < 16; d++) {
+ device float *o = out + (ulong)qi * stride + head * HD + col;
+ for (uint d = 0; d < HD / 8; d++) {
float2 e = elements(O[d]) * inv;
o[8 * d] = e.x;
o[8 * d + 1] = e.y;
@@ -149,6 +150,13 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fl
}
}
+// The DiT's heads (128) and the vision tower's (72: Qwen3-VL's 1152 / 16), with no zero padding. At 72 the result is
+// the padded kernel's to the bit: the 7 padding chunks only added exact zeros to every sum.
+template [[host_name("flash_attention")]] kernel void flash_attention_d<128>(device const float *, device const float *,
+ device const float *, device float *, constant FlashParams &, uint2, uint, uint, uint);
+template [[host_name("flash_attention_72")]] kernel void flash_attention_d<72>(device const float *, device const float *,
+ device const float *, device float *, constant FlashParams &, uint2, uint, uint, uint);
+
// The same with q, k, v and the probabilities in fp16 (scores, softmax and O accumulated in fp32; K and V tiles of 64
// keys side by side in 32 KB), the engine's default (`KOKON_ATTENTION=fp32` for the one above): 94 ms against 120 at
// 1024² (M2 Pro), head 0 within 6.3e-4 of exact, the velocity within 6.0e-4 of fp64 at 256² (fp32 kernel 1.0e-5).
diff --git a/engine-mac/Sources/KokonEngine/Kernels/vision.metal b/engine-mac/Sources/KokonEngine/Kernels/vision.metal
index 9d6d7ea..fd441ff 100644
--- a/engine-mac/Sources/KokonEngine/Kernels/vision.metal
+++ b/engine-mac/Sources/KokonEngine/Kernels/vision.metal
@@ -60,39 +60,23 @@ kernel void gelu_erf(device float *x [[buffer(0)]], constant uint &count [[buffe
x[i] = 0.5f * v * tail;
}
-// Qwen3VLVisionAttention's qkv split and apply_rotary_pos_emb_vision, into the flash kernel's layout: q, k, v as
-// [rows, 16 heads, 128] with the head's 72 dimensions first and zeros after (the scores and the output are the
-// 72-dimension ones). rotate_half pairs dimension i with i + 36; `rope` is [rows, 36, (cos, sin)].
-// One thread per (row, head, j < 64): j < 36 rotates the pair (j, j + 36), the others clear two padding columns.
-kernel void vision_rope_pad(device const float *qkv [[buffer(0)]], device float *q [[buffer(1)]],
- device float *k [[buffer(2)]], device float *v [[buffer(3)]],
- device const float *rope [[buffer(4)]], constant uint &rows [[buffer(5)]],
- uint i [[thread_position_in_grid]]) {
- uint j = i % 64, head = (i / 64) % 16, row = i / 1024;
+// Qwen3VLVisionAttention's qkv split and apply_rotary_pos_emb_vision: q, k, v as [rows, 16 heads × 72], the layout
+// `flash_attention_72` reads and attn.proj's input has. rotate_half pairs dimension i with i + 36; `rope` is
+// [rows, 36, (cos, sin)]. One thread per (row, head, j < 36), rotating the pair (j, j + 36).
+kernel void vision_rope(device const float *qkv [[buffer(0)]], device float *q [[buffer(1)]],
+ device float *k [[buffer(2)]], device float *v [[buffer(3)]],
+ device const float *rope [[buffer(4)]], constant uint &rows [[buffer(5)]],
+ uint i [[thread_position_in_grid]]) {
+ uint j = i % 36, head = (i / 36) % 16, row = i / 576;
if (row >= rows) return;
device const float *in = qkv + (ulong)row * 3456 + head * 72;
- ulong o = (ulong)row * 2048 + head * 128;
- if (j < 36) {
- float c = rope[((ulong)row * 36 + j) * 2], s = rope[((ulong)row * 36 + j) * 2 + 1];
- float q0 = in[j], q1 = in[j + 36], k0 = in[1152 + j], k1 = in[1152 + j + 36];
- q[o + j] = q0 * c - q1 * s;
- q[o + j + 36] = q1 * c + q0 * s;
- k[o + j] = k0 * c - k1 * s;
- k[o + j + 36] = k1 * c + k0 * s;
- v[o + j] = in[2304 + j];
- v[o + j + 36] = in[2304 + j + 36];
- } else {
- uint c = 72 + (j - 36) * 2;
- q[o + c] = 0; q[o + c + 1] = 0;
- k[o + c] = 0; k[o + c + 1] = 0;
- v[o + c] = 0; v[o + c + 1] = 0;
- }
-}
-
-// The attention's output back to [rows, 16 × 72] for attn.proj.
-kernel void vision_unpad(device const float *x [[buffer(0)]], device float *out [[buffer(1)]],
- constant uint &rows [[buffer(2)]], uint i [[thread_position_in_grid]]) {
- if (i >= rows * 1152) return;
- uint row = i / 1152, c = i % 1152;
- out[i] = x[(ulong)row * 2048 + (c / 72) * 128 + c % 72];
+ ulong o = (ulong)row * 1152 + head * 72;
+ float c = rope[((ulong)row * 36 + j) * 2], s = rope[((ulong)row * 36 + j) * 2 + 1];
+ float q0 = in[j], q1 = in[j + 36], k0 = in[1152 + j], k1 = in[1152 + j + 36];
+ q[o + j] = q0 * c - q1 * s;
+ q[o + j + 36] = q1 * c + q0 * s;
+ k[o + j] = k0 * c - k1 * s;
+ k[o + j + 36] = k1 * c + k0 * s;
+ v[o + j] = in[2304 + j];
+ v[o + j + 36] = in[2304 + j + 36];
}
diff --git a/engine-mac/Sources/KokonEngine/QwenVisionTower.swift b/engine-mac/Sources/KokonEngine/QwenVisionTower.swift
index ff474cb..0b2ef3c 100644
--- a/engine-mac/Sources/KokonEngine/QwenVisionTower.swift
+++ b/engine-mac/Sources/KokonEngine/QwenVisionTower.swift
@@ -12,8 +12,8 @@ import MetalPerformanceShaders
/// merged = fc2(GELU(fc1(LN(x) as [N/4, 4608]))) [N/4, 4096], one row per text encoder slot
/// deepstack j = the same after block 8 / 16 / 24 (LN after the 2 × 2 merge) added to the text encoder's layers 0–2
///
-/// The heads (72 dimensions) run on the DiT's flash kernel zero-padded to 128 (`vision_rope_pad`): the scores and the
-/// output are the 72-dimension ones. The processor's input is the image with alpha composited over white (as the
+/// The heads (72 dimensions) run on the DiT's fp32 flash kernel built for 72 (`flash_attention_72`): until 2026-10-26
+/// they were zero-padded to 128, the same values to the bit for ~1.8× the work. The processor's input is the image with alpha composited over white (as the
/// pipeline's PIL paste), scaled to [−1, 1], each frame of the Conv3d's two the same image.
public final class QwenVisionTower {
public static let dim = 1152, heads = 16, headDim = 72, hidden = 4304, layers = 27, outDim = 4096
@@ -156,7 +156,7 @@ public final class QwenVisionTower {
struct Buffers {
let arena: Arena
- let x, normed, qkv, q, k, v, attended, a72, h, mergeIn, mergeHidden, smalls, rope: Arena.Slice
+ let x, normed, qkv, q, k, v, attended, h, mergeIn, mergeHidden, smalls, rope: Arena.Slice
let outputs: [Arena.Slice]
let reserves: [Arena.Slice]
}
@@ -182,13 +182,13 @@ public final class QwenVisionTower {
let mrows = QwenDiT.padded(slots)
let smallNames = smallTensors()
let smallCount = try smallNames.reduce(0) { $0 + (try file.tensor($1)).count } + n * d
- let sizes = [rows * d, rows * d, rows * 3 * d, rows * 2048, rows * 2048, rows * 2048, rows * 2048, rows * d,
+ let sizes = [rows * d, rows * d, rows * 3 * d, rows * d, rows * d, rows * d, rows * d,
rows * QwenVisionTower.hidden, mrows * merged, mrows * merged, smallCount, n * 72,
slots * 4096, slots * 4096, slots * 4096, slots * 4096, merged * merged, merged * merged]
let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))
var s = try sizes.map { try arena.slice(floats: $0) }.makeIterator()
let b = Buffers(arena: arena, x: s.next()!, normed: s.next()!, qkv: s.next()!, q: s.next()!, k: s.next()!,
- v: s.next()!, attended: s.next()!, a72: s.next()!, h: s.next()!, mergeIn: s.next()!,
+ v: s.next()!, attended: s.next()!, h: s.next()!, mergeIn: s.next()!,
mergeHidden: s.next()!, smalls: s.next()!, rope: s.next()!,
outputs: [s.next()!, s.next()!, s.next()!, s.next()!], reserves: [s.next()!, s.next()!])
@@ -320,7 +320,7 @@ public final class QwenVisionTower {
let d = QwenVisionTower.dim, block = "v.blk.\(layer)."
try layerNorm(b, b.x, into: b.normed, prefix: block + "ln1", rows: n, columns: d, commands: commands)
try linear(block + "attn_qkv", b, a: b.normed, c: b.qkv, m: n, commands: commands)
- try compute("vision_rope_pad", commands: commands) { e in
+ try compute("vision_rope", commands: commands) { e in
e.setBuffer(b.arena.buffer, offset: b.qkv.offset, index: 0)
e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 1)
e.setBuffer(b.arena.buffer, offset: b.k.offset, index: 2)
@@ -328,27 +328,21 @@ public final class QwenVisionTower {
e.setBuffer(b.arena.buffer, offset: b.rope.offset, index: 4)
var r = UInt32(n)
e.setBytes(&r, length: 4, index: 5)
- e.dispatchThreads(MTLSize(width: n * 1024, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ e.dispatchThreads(MTLSize(width: n * QwenVisionTower.heads * 36, height: 1, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
}
- try compute("flash_attention", commands: commands) { e in
+ try compute("flash_attention_72", commands: commands) { e in
e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 0)
e.setBuffer(b.arena.buffer, offset: b.k.offset, index: 1)
e.setBuffer(b.arena.buffer, offset: b.v.offset, index: 2)
e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 3)
- var params = QwenDiT.FlashParams(queries: UInt32(n), keys: UInt32(n), causal: 0, stride: 2048,
+ var params = QwenDiT.FlashParams(queries: UInt32(n), keys: UInt32(n), causal: 0, stride: UInt32(d),
scale: 1 / Float(QwenVisionTower.headDim).squareRoot() * Float(M_LOG2E), headBase: 0)
e.setBytes(&params, length: MemoryLayout<QwenDiT.FlashParams>.stride, index: 4)
e.dispatchThreadgroups(MTLSize(width: (n + 63) / 64, height: QwenVisionTower.heads, depth: 1),
threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
}
- try compute("vision_unpad", commands: commands) { e in
- e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 0)
- e.setBuffer(b.arena.buffer, offset: b.a72.offset, index: 1)
- var r = UInt32(n)
- e.setBytes(&r, length: 4, index: 2)
- e.dispatchThreads(MTLSize(width: n * d, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
- }
- try linear(block + "attn_out", b, a: b.a72, c: b.normed, m: n, commands: commands)
+ try linear(block + "attn_out", b, a: b.attended, c: b.normed, m: n, commands: commands)
try add(b, b.x, b.normed, offset: 0, count: n * d, commands: commands)
try layerNorm(b, b.x, into: b.normed, prefix: block + "ln2", rows: n, columns: d, commands: commands)
try linear(block + "ffn_up", b, a: b.normed, c: b.h, m: n, commands: commands)