Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: next weight prepared during the current product; numbers of the pass

Gabriele Lozupone32e32a17ef4b
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 74433ee..7f0d5d8 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -156,10 +156,18 @@ is not linear in rows), the Neural Engine's rows (steps of 256, a program set pe
evaluations; every product's units then finish within ~0.3 ms of each other (before: the GPU 3–15 ms after the
AMX). `KOKON_BALANCE=0` keeps the starting shares; `KOKON_HEADS=g,a,c` fixes the heads.
-**One evaluation at 1024², all three units, unsloth's DiT: 8.8 s** (start of this pass 11.6 s): products ~5.2 s
-(≈11 TFLOP/s across the units, the probe's ceiling), attention 32 × 78 ms, the GPU alone between them ~1 s. The
-Neural Engine's per-column weight scales are kept per weight (computed once), and the modulation of all of the
-pipeline's timesteps is computed once (it was 0.2 s of CPU per evaluation).
+**One evaluation at 1024², all three units, unsloth's DiT: 8.7–8.9 s** (start of this pass 11.6 s): products ~5.3 s
+(≈11 TFLOP/s across the units, the probe's ceiling), attention 32 × 78 ms, the GPU alone between operations 0.6 s
+(`time-dit` prints it by transition). The Neural Engine's per-column weight scales are computed once per weight, the
+next product's weight is unpacked and packed for the Neural Engine while the current one runs (between operations
+1.12 → 0.60 s, the evaluation ~0.1 s faster: the units were already full), and the modulation of all of the
+pipeline's timesteps is computed once (it was 0.2 s of CPU per evaluation). The balanced shares are remembered per
+Mac and shape (`~/Library/Caches/kokon-engine-mac/balance.json`), so the first step starts balanced.
+
+**1024² image, M2 Pro, unsloth's DiT + texture-fix VAE (`generate`), 2026-10-25: 79.9 s** (text encoder 2.0 s, step 1
+10.2 s, steps 2–8 8.8–9.1 s, VAE 5.2 s); at the start of this pass 100.2 s; Kokon's sd.cpp ~4.5 min. With
+`KOKON_ATTENTION=half` (fp16 operands in the flash kernel): 76.0 s, the velocity within 6.0e-4 of fp64 at 256²
+(fp32 kernel 1.0e-5); the two 1024² images differ by 0.25 levels on average, 0.18 % of values by more than 8.
**VAE decoder** (`check-vae`, against `tools/reference_vae.py`, diffusers fp32): within 1.1e-5; 1024² in 2.75 s. Its
graph builds and compiles in 0.3 s, but its first run at a size costs ~3.7 s more, GPU time (~1 s of CPU): it cannot
diff --git a/engine-mac/Sources/KokonEngine/Coexecution.swift b/engine-mac/Sources/KokonEngine/Coexecution.swift
index 4724329..461ef4d 100644
--- a/engine-mac/Sources/KokonEngine/Coexecution.swift
+++ b/engine-mac/Sources/KokonEngine/Coexecution.swift
@@ -29,9 +29,14 @@ final class Coexecutor {
private(set) var aneRows: Int
var amxRows: [String: Int] = [:]
typealias ProductSlot = (product: NeuralEngineProduct, x: MTLBuffer, w: MTLBuffer, y: MTLBuffer)
- private var slots: [ProductSlot]
+ /// Two sets of the three tile programs, alternating product by product: the next product's weight is packed into
+ /// one set while the Neural Engine reads the other (`prepareWeight`). The sets share their row surfaces.
+ private var slots: [[ProductSlot]]
/// Programs of other row counts, kept while the balancing loop may come back to them.
- private var spareSlots: [Int: [ProductSlot]] = [:]
+ private var spareSlots: [Int: [[ProductSlot]]] = [:]
+ /// The set of the last product encoded; the weight packed ahead and its set.
+ private var currentSet = 0
+ private var packedAhead: (weight: String, set: Int)?
var balancer: Balancer
/// The timeline of the last evaluation, once the balancing loop has read it.
var lastTimeline: [Timeline.Entry] = []
@@ -76,27 +81,34 @@ final class Coexecutor {
}
(ready, amxDone, aneDone, gpuDone) = (r, a, e, g)
scales = try context.scratch(rows * 4, label: "ane row scales")
- slots = []
+ slots = [[], []]
slots = try productSlots(aneRows)
}
- /// Three programs of `rows` rows (the tiles of a product, at most 3), compiled and warmed.
- private func productSlots(_ rows: Int) throws -> [ProductSlot] {
- guard rows > 0 else { return [] }
+ /// Two sets of three programs of `rows` rows (the tiles of a product, at most 3), compiled and warmed.
+ private func productSlots(_ rows: Int) throws -> [[ProductSlot]] {
+ guard rows > 0 else { return [[], []] }
let device = context.device
- return try (0..<3).map { _ in
+ let first: [ProductSlot] = try (0..<3).map { _ in
let p = try NeuralEngineProduct(rows: rows, k: Coexecutor.tile, n: Coexecutor.tile)
try p.run()
return (p, try p.gpuBuffer(p.x, device: device), try p.gpuBuffer(p.w, device: device), try p.gpuBuffer(p.y, device: device))
}
+ let second: [ProductSlot] = try first.map { f in
+ let p = try NeuralEngineProduct(rows: rows, k: Coexecutor.tile, n: Coexecutor.tile, sharing: f.product)
+ try p.run()
+ return (p, f.x, try p.gpuBuffer(p.w, device: device), f.y)
+ }
+ return [first, second]
}
/// Moves the Neural Engine to `rows` rows per product; between evaluations only (the GPU must be idle).
func setAneRows(_ rows: Int) throws {
guard rows != aneRows else { return }
let next = try spareSlots.removeValue(forKey: rows) ?? productSlots(rows)
- if aneRows > 0 { spareSlots = [aneRows: slots] } // the last one only: 192 MB of surfaces at 2 048 rows
+ if aneRows > 0 { spareSlots = [aneRows: slots] } // the last one only: 288 MB of surfaces at 2 048 rows
(slots, aneRows) = (next, rows)
+ packedAhead = nil
}
/// Whether a product takes the split: the Neural Engine's tiles need K and N in multiples of 4 096 with at
@@ -118,13 +130,14 @@ final class Coexecutor {
func encode(weight: String, a: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
w: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
c: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
- m: Int, k: Int, n: Int, commands: MTLCommandBuffer, gpuRows: (Int) throws -> Void) throws {
+ m: Int, k: Int, n: Int, commands: MTLCommandBuffer,
+ ahead: ((MTLCommandBuffer) throws -> Void)? = nil, gpuRows: (Int) throws -> Void) throws {
let s = split(m: m, k: k, n: n)
counter += 1
let p = counter
open(p, kind: "\(k)×\(n)", shares: (s.gpu, s.amx, s.ane))
let (kParts, nParts) = (k / Coexecutor.tile, n / Coexecutor.tile)
- let (lambda, norm, state, sigma) = (0, n * 4, 2 * n * 4, 0)
+ let (lambda, state, sigma) = (0, 2 * n * 4, 0)
func shape(kp: Int = 0, np: Int = 0) -> Shape {
Shape(k: UInt32(k), n: UInt32(n), k0: UInt32(kp * Coexecutor.tile), kPart: UInt32(Coexecutor.tile),
n0: UInt32(np * Coexecutor.tile), nPart: UInt32(Coexecutor.tile), row0: UInt32(s.gpu + s.amx),
@@ -138,24 +151,19 @@ final class Coexecutor {
e.endEncoding()
}
// (Read only when the Neural Engine takes rows.)
- let columns = s.ane > 0 ? try columnScales[weight] ?? context.scratch((2 * n + 16) * 4, label: "λ \(weight)") : scales
+ let columns = s.ane > 0 ? try columnScales(weight, w: w, k: k, n: n, commands: commands) : scales
+ let set: Int
+ if s.ane > 0, let ahead = packedAhead, ahead.weight == weight {
+ set = ahead.set
+ } else {
+ set = 1 - currentSet
+ if s.ane > 0 { try packWeight(w: w, k: k, n: n, columns: columns, set: set, commands: commands) }
+ }
+ packedAhead = nil
+ currentSet = set
+ let slots = self.slots[set]
if s.ane > 0 {
var sh = shape()
- if columnScales[weight] == nil {
- columnScales[weight] = columns
- try kernel("ane_columns") { e in
- e.setBuffer(w.buffer, offset: w.offset, index: 0)
- e.setBuffer(columns, offset: lambda, index: 1); e.setBuffer(columns, offset: norm, index: 2)
- e.setBytes(&sh, length: MemoryLayout<Shape>.stride, index: 3)
- e.dispatchThreadgroups(MTLSize(width: (n + 63) / 64, height: 1, depth: 1),
- threadsPerThreadgroup: MTLSize(width: 64, height: 16, depth: 1))
- }
- try kernel("ane_norm") { e in
- e.setBuffer(columns, offset: norm, index: 0); e.setBuffer(columns, offset: state, index: 1)
- e.setBytes(&sh, length: MemoryLayout<Shape>.stride, index: 2)
- e.dispatchThreadgroups(MTLSize(width: 1, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 1024, height: 1, depth: 1))
- }
- }
try kernel("ane_row_scales") { e in
e.setBuffer(a.buffer, offset: a.offset, index: 0); e.setBuffer(columns, offset: state, index: 1)
e.setBuffer(scales, offset: sigma, index: 2)
@@ -166,13 +174,6 @@ final class Coexecutor {
for np in 0..<nParts {
let slot = slots[kp * nParts + np]
var t = shape(kp: kp, np: np)
- try kernel("ane_pack_weight") { e in
- e.setBuffer(w.buffer, offset: w.offset, index: 0); e.setBuffer(columns, offset: lambda, index: 1)
- e.setBuffer(slot.w, offset: 0, index: 2)
- e.setBytes(&t, length: MemoryLayout<Shape>.stride, index: 3)
- e.dispatchThreads(MTLSize(width: Coexecutor.tile, height: Coexecutor.tile, depth: 1),
- threadsPerThreadgroup: MTLSize(width: 64, height: 4, depth: 1))
- }
try kernel("ane_pack_rows") { e in
e.setBuffer(a.buffer, offset: a.offset, index: 0); e.setBuffer(scales, offset: sigma, index: 1)
e.setBuffer(slot.x, offset: 0, index: 2)
@@ -184,6 +185,8 @@ final class Coexecutor {
}
}
if s.amx > 0 || s.ane > 0 { commands.encodeSignalEvent(ready, value: p) }
+ // The next weight's own work while the AMX and the Neural Engine run this product.
+ try ahead?(commands)
if s.amx > 0 {
nonisolated(unsafe) let (pa, pw, pc) = (a.pointer + s.gpu * k, w.pointer, c.pointer + s.gpu * n)
let rows = s.amx
@@ -236,6 +239,66 @@ final class Coexecutor {
return false
}
+ func kernel(_ name: String, _ commands: MTLCommandBuffer, _ body: (MTLComputeCommandEncoder) throws -> Void) throws {
+ let e = commands.makeComputeCommandEncoder()!
+ e.label = name
+ e.setComputePipelineState(try context.pipeline(name))
+ try body(e)
+ e.endEncoding()
+ }
+
+ /// The weight's column scales `[λ (n) | norms (n) | L]`, encoded the first time it is used, then kept.
+ func columnScales(_ weight: String, w: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
+ k: Int, n: Int, commands: MTLCommandBuffer) throws -> MTLBuffer {
+ if let c = columnScales[weight] { return c }
+ let columns = try context.scratch((2 * n + 16) * 4, label: "λ \(weight)")
+ columnScales[weight] = columns
+ var sh = Shape(k: UInt32(k), n: UInt32(n), k0: 0, kPart: 0, n0: 0, nPart: 0, row0: 0, rows: 0, kParts: 0)
+ try kernel("ane_columns", commands) { e in
+ e.setBuffer(w.buffer, offset: w.offset, index: 0)
+ e.setBuffer(columns, offset: 0, index: 1); e.setBuffer(columns, offset: n * 4, index: 2)
+ e.setBytes(&sh, length: MemoryLayout<Shape>.stride, index: 3)
+ e.dispatchThreadgroups(MTLSize(width: (n + 63) / 64, height: 1, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 64, height: 16, depth: 1))
+ }
+ try kernel("ane_norm", commands) { e in
+ e.setBuffer(columns, offset: n * 4, index: 0); e.setBuffer(columns, offset: 2 * n * 4, index: 1)
+ e.setBytes(&sh, length: MemoryLayout<Shape>.stride, index: 2)
+ e.dispatchThreadgroups(MTLSize(width: 1, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 1024, height: 1, depth: 1))
+ }
+ return columns
+ }
+
+ /// The weight's tiles, lifted, into the weight surfaces of `set`.
+ private func packWeight(w: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>), k: Int, n: Int,
+ columns: MTLBuffer, set: Int, commands: MTLCommandBuffer) throws {
+ let (kParts, nParts) = (k / Coexecutor.tile, n / Coexecutor.tile)
+ for kp in 0..<kParts {
+ for np in 0..<nParts {
+ var t = Shape(k: UInt32(k), n: UInt32(n), k0: UInt32(kp * Coexecutor.tile), kPart: UInt32(Coexecutor.tile),
+ n0: UInt32(np * Coexecutor.tile), nPart: UInt32(Coexecutor.tile), row0: 0, rows: 0, kParts: UInt32(kParts))
+ try kernel("ane_pack_weight", commands) { e in
+ e.setBuffer(w.buffer, offset: w.offset, index: 0); e.setBuffer(columns, offset: 0, index: 1)
+ e.setBuffer(slots[set][kp * nParts + np].w, offset: 0, index: 2)
+ e.setBytes(&t, length: MemoryLayout<Shape>.stride, index: 3)
+ e.dispatchThreads(MTLSize(width: Coexecutor.tile, height: Coexecutor.tile, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 64, height: 4, depth: 1))
+ }
+ }
+ }
+ }
+
+ /// Packs the next product's weight (already unpacked to fp32 [k, n] at `w`) into the set the current product does
+ /// not use. Encoded inside the current product, after its release: the set was last read by the product before.
+ func prepareWeight(_ weight: String, w: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
+ k: Int, n: Int, commands: MTLCommandBuffer) throws {
+ guard aneRows > 0, applies(m: rows, k: k, n: n) else { return }
+ let set = 1 - currentSet
+ try packWeight(w: w, k: k, n: n, columns: try columnScales(weight, w: w, k: k, n: n, commands: commands),
+ set: set, commands: commands)
+ packedAhead = (weight, set)
+ }
+
func resetTimings() { lock.withLock { amxSeconds = 0; aneSeconds = 0 } }
// MARK: - the timeline
diff --git a/engine-mac/Sources/KokonEngine/NeuralEngine.swift b/engine-mac/Sources/KokonEngine/NeuralEngine.swift
index 7402970..0d19f68 100644
--- a/engine-mac/Sources/KokonEngine/NeuralEngine.swift
+++ b/engine-mac/Sources/KokonEngine/NeuralEngine.swift
@@ -37,14 +37,15 @@ public final class NeuralEngineProduct: @unchecked Sendable {
}
}
- public init(rows: Int, k: Int, n: Int) throws {
+ /// `sharing`: a program of the same shape whose `x` and `y` surfaces this one uses too (its own weight only).
+ public init(rows: Int, k: Int, n: Int, sharing: NeuralEngineProduct? = nil) throws {
self.rows = rows; self.k = k; self.n = n
(model, directory) = try NeuralEngineProduct.compile(
"\(rows)x\(k)x\(n)", program: NeuralEngineProduct.program(rows: rows, k: k, n: n),
inputs: [("x", [rows, k]), ("w", [k, n])], outputs: [("y", [rows, n])])
- x = try NeuralEngineProduct.surface(rows: rows, columns: k)
+ x = try sharing?.x ?? NeuralEngineProduct.surface(rows: rows, columns: k)
w = try NeuralEngineProduct.surface(rows: k, columns: n)
- y = try NeuralEngineProduct.surface(rows: rows, columns: n)
+ y = try sharing?.y ?? NeuralEngineProduct.surface(rows: rows, columns: n)
func shape(_ dims: Int...) -> [NSNumber] { dims.map { NSNumber(value: $0) } }
options = MLPredictionOptions()
options.outputBackings = ["y": MLMultiArray(pixelBuffer: y, shape: shape(rows, n))]
diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swift
index 24cd782..b70abff 100644
--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift
+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift
@@ -245,10 +245,10 @@ public final class QwenDiT {
let block = "transformer_blocks.\(layer)."
let band = 5 * d // the target's modulation row follows the prefix's
try layerNorm(x, into: normed, rows: n, prefixMod: 0, targetMod: band, commands: commands)
- try linear(block + "attn.to_q.weight", a: normed, c: q, m: n, commands: commands)
- try linear(block + "attn.to_k.weight", a: normed, c: k, m: n, commands: commands)
+ try linear(block + "attn.to_q.weight", a: normed, c: q, m: n, next: block + "attn.to_k.weight", commands: commands)
+ try linear(block + "attn.to_k.weight", a: normed, c: k, m: n, next: block + "attn.to_v.weight", commands: commands)
try mark("norm1", normed)
- try linear(block + "attn.to_v.weight", a: normed, c: v, m: n, commands: commands)
+ try linear(block + "attn.to_v.weight", a: normed, c: v, m: n, next: block + "attn.to_out.0.weight", commands: commands)
try headNormRope(q, weightOffset: (layer * 2) * QwenDiT.headDim, rows: n, commands: commands)
try headNormRope(k, weightOffset: (layer * 2 + 1) * QwenDiT.headDim, rows: n, commands: commands)
try mark("q", q); try mark("k", k); try mark("v", v)
@@ -257,16 +257,17 @@ public final class QwenDiT {
causal: segment.isText, commands: commands)
}
try attend(queries: p, count: n - p, keys: n, causal: false, commands: commands)
- try linear(block + "attn.to_out.0.weight", a: attended, c: o, m: n, commands: commands)
+ try linear(block + "attn.to_out.0.weight", a: attended, c: o, m: n, next: block + "img_mlp.gate_layer.weight", commands: commands)
try mark("attn_out", o)
try residual(gate: d, rows: n, commands: commands)
try mark("res1", x)
try layerNorm(x, into: normed, rows: n, prefixMod: 2 * d, targetMod: band + 2 * d, commands: commands)
try mark("norm2", normed)
- try linear(block + "img_mlp.gate_layer.weight", a: normed, c: h1, m: n, commands: commands)
- try linear(block + "img_mlp.proj.weight", a: normed, c: h2, m: n, commands: commands)
+ try linear(block + "img_mlp.gate_layer.weight", a: normed, c: h1, m: n, next: block + "img_mlp.proj.weight", commands: commands)
+ try linear(block + "img_mlp.proj.weight", a: normed, c: h2, m: n, next: block + "img_mlp.out.weight", commands: commands)
try elementwise("swiglu", [(h1, 0), (h2, 0)], count: n * QwenDiT.hidden, commands: commands)
- try linear(block + "img_mlp.out.weight", a: h1, c: o, m: n, commands: commands)
+ try linear(block + "img_mlp.out.weight", a: h1, c: o, m: n,
+ next: layer + 1 < QwenDiT.layers ? "transformer_blocks.\(layer + 1).attn.to_q.weight" : nil, commands: commands)
try mark("mlp", o)
try residual(gate: 3 * d, rows: n, commands: commands)
}
@@ -316,21 +317,33 @@ public final class QwenDiT {
return try file.tensor(QwenDiT.prefix + name)
}
+ /// The weight unpacked (and packed for the Neural Engine) during the product before, and its reserve.
+ private var unpackedAhead: (name: String, reserve: Int)?
+
/// `C[m, n] = A[m, k] · W` with W the file's Linear `[n, k]`, unpacked to `[k, n]` fp32 in a reserve first.
- /// `aRow`, `cRow`: first row of A and C in their slices.
+ /// `aRow`, `cRow`: first row of A and C in their slices. `next`: the weight of the product that follows, unpacked
+ /// into the other reserve and packed for the Neural Engine while the AMX and the Neural Engine run this one (the
+ /// GPU's ~1.8 ms per 4096 × 4096 weight otherwise ran alone between products).
func linear(_ name: String, a: Arena.Slice, aRow: Int = 0, c: Arena.Slice, m: Int, cRow: Int = 0,
- commands: MTLCommandBuffer) throws {
+ next: String? = nil, commands: MTLCommandBuffer) throws {
let tensor = try weight(name)
let (kk, nn) = (tensor.columns, tensor.rows)
let m = QwenDiT.padded(m)
guard (aRow + m) * kk * 4 <= a.bytes, (cRow + m) * nn * 4 <= c.bytes else {
throw MetalContext.Failure.buffer("\(name): no room for \(m) padded rows")
}
- let reserve = reserves[nextReserve]
- nextReserve = (nextReserve + 1) % reserves.count
- try gpu("dequant", commands) {
- try dequant.encode(tensor, of: file, into: arena.buffer, outOffset: reserve.offset, transposed: true, commands: $0)
+ let index: Int
+ if let ahead = unpackedAhead, ahead.name == name {
+ index = ahead.reserve
+ } else {
+ index = nextReserve
+ try gpu("dequant", commands) {
+ try dequant.encode(tensor, of: file, into: arena.buffer, outOffset: reserves[index].offset, transposed: true, commands: $0)
+ }
}
+ unpackedAhead = nil
+ nextReserve = (index + 1) % reserves.count
+ let reserve = reserves[index]
// The GPU's rows: all of them, or its share of a split product.
func gpuRows(_ rows: Int) throws {
let key = "\(rows)×\(kk)×\(nn)"
@@ -354,8 +367,17 @@ public final class QwenDiT {
func operand(_ slice: Arena.Slice, _ row: Int, _ width: Int) -> (MTLBuffer, Int, UnsafeMutablePointer<Float>) {
(arena.buffer, slice.offset + row * width * 4, arena.pointer(slice, floatOffset: row * width))
}
+ var ahead: ((MTLCommandBuffer) throws -> Void)?
+ if let next {
+ let t = try weight(next), other = nextReserve
+ ahead = { [self] commands in
+ try dequant.encode(t, of: file, into: arena.buffer, outOffset: reserves[other].offset, transposed: true, commands: commands)
+ try coexecutor.prepareWeight(next, w: operand(reserves[other], 0, t.rows), k: t.columns, n: t.rows, commands: commands)
+ unpackedAhead = (next, other)
+ }
+ }
try coexecutor.encode(weight: name, a: operand(a, aRow, kk), w: operand(reserve, 0, nn), c: operand(c, cRow, nn),
- m: m, k: kk, n: nn, commands: commands, gpuRows: gpuRows)
+ m: m, k: kk, n: nn, commands: commands, ahead: ahead, gpuRows: gpuRows)
} else {
try gpuRows(m)
}