Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: layernorm and swiglu write the products' fp16 input (no pack_half pass)

Gabriele Lozupone1818a2d4cbb1
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 9500dbb..7b5c11c 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -202,6 +202,12 @@ rows of 4 128, and the products drop from 4 160 padded rows to 4 096); for an ed
as long as the target, about half of each step. A pass of the prefix alone, before the first evaluation, took
1.86 s (products of 64 rows run far below the GPU's rate), so the first evaluation carries the prefix instead.
+**The row kernels write the products' fp16 input** (`KOKON_FUSED_CAST=0` for the comparison): `layernorm_modulate`
+(before q/k/v and gate/proj) and `swiglu` (before the MLP's out) write the GPU's and the AMX's rows straight to fp16
+and fp32 only for the Neural Engine's rows, so the separate `pack_half` pass, which read the fp32 rows back, is gone.
+The same values to the bit (check-dit at 896²: velocity 6.74e-4 either way). 1024², 3 alternated pairs of 4
+evaluations (M2 Pro, 2026-10-25): 7.67 → 7.58 s per evaluation, the GPU alone between operations 0.71 → 0.64 s.
+
**1024² image after this pass (M2 Pro, `generate`, 2026-10-25): 67.3–68.0 s** (text encoder 2.0 s, step 1 8.35 s,
steps 2–8 7.3–7.7 s, VAE 5.0–5.3 s with its first run at the size; 79.9 s at the start of the fp16 work, 76.0 with
fp16 attention). `MTLBinaryArchive` was not added: the engine's kernels compile from source in 0.1 s at start (macOS
diff --git a/engine-mac/Sources/KokonEngine/Checks.swift b/engine-mac/Sources/KokonEngine/Checks.swift
index c411864..520105d 100644
--- a/engine-mac/Sources/KokonEngine/Checks.swift
+++ b/engine-mac/Sources/KokonEngine/Checks.swift
@@ -80,7 +80,7 @@ extension Checks {
func report(_ name: String, _ got: UnsafePointer<Float>, _ want: [Float]) {
let r = compare(got, want)
- print(String(format: " %-12@ rel %.2e max |Δ| %.3g%@", name as NSString, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))
+ print(String(format: " %-22@ rel %.2e max |Δ| %.3g%@", name as NSString, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))
}
let rope = sequence.ropeTable()
rope.withUnsafeBufferPointer { report("rope", $0.baseAddress!, try! reference(directory, "rope")) }
@@ -94,11 +94,14 @@ extension Checks {
precondition(want.count == count, "\(name): \(count) floats, reference \(want.count)")
report(name, pointer, want)
}
- // Timing without the stage waits.
- let started = DispatchTime.now().uptimeNanoseconds
- _ = try dit.forward(encoder: encoder, target: latent, sigma: sigma)
- print(String(format: "evaluation: %.3f s (modulation on the CPU %.3f s)",
- Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9, dit.timings["modulation"] ?? 0))
+ // Without the stage waits (and with the prefix cache, when on): timed, and its velocity checked too.
+ for run in 1...2 {
+ let started = DispatchTime.now().uptimeNanoseconds
+ let out = try dit.forward(encoder: encoder, target: latent, sigma: sigma)
+ let seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
+ out.withUnsafeBufferPointer { report("model_out, unprobed \(run)", $0.baseAddress!, try! reference(directory, "model_out")) }
+ print(String(format: "evaluation: %.3f s (modulation on the CPU %.3f s)", seconds, dit.timings["modulation"] ?? 0))
+ }
}
}
@@ -133,6 +136,7 @@ extension Checks {
c.resetTimings()
}
}
+ guard ProcessInfo.processInfo.environment["KOKON_PROFILE"] != "0" else { return }
dit.profiling = true
_ = try dit.forward(encoder: encoder, target: latent, sigma: 0.5)
print("profile (each operation alone, GPU seconds):")
diff --git a/engine-mac/Sources/KokonEngine/Coexecution.swift b/engine-mac/Sources/KokonEngine/Coexecution.swift
index e4ccb5d..5504866 100644
--- a/engine-mac/Sources/KokonEngine/Coexecution.swift
+++ b/engine-mac/Sources/KokonEngine/Coexecution.swift
@@ -62,7 +62,10 @@ final class Coexecutor {
/// for `cblas_sgemm`. The velocity at 896² against diffusers: 7.2e-4 (fp32 A 5.7e-4, fp32 weights 5.1e-4; the
/// Neural Engine's rows are fp16 in every case).
let halfActivations = ProcessInfo.processInfo.environment["KOKON_ACTIVATIONS"] != "fp32"
- private lazy var aHalf: MTLBuffer? = try? context.scratch(rows * QwenDiT.hidden * 2, label: "A fp16")
+ /// The longest split product's rows: the first evaluation of a conditioning carries the prefix's rows too, and
+ /// without the Neural Engine they are all cast.
+ let maxRows: Int
+ private lazy var aHalf: MTLBuffer? = try? context.scratch(maxRows * QwenDiT.hidden * 2, label: "A fp16")
/// Seconds each worker spent computing (not waiting), and failures (a failure leaves rows unwritten).
var amxSeconds = 0.0, aneSeconds = 0.0
var failures: [String] = []
@@ -75,11 +78,12 @@ final class Coexecutor {
/// Starting shares measured on the M2 Pro 16 GB, 1024² + 32 text rows (`time-dit`, 2026-10-25): 2 048 rows on the
/// Neural Engine and 1/16 on the AMX (AMX 512 rows 12.8 s per evaluation, 256 rows 11.3 s, none 11.8 s; GPU alone
/// 16.2 s). The balancing loop moves them from the first evaluation on.
- init(context: MetalContext, rows: Int, ane: Bool = true, amx: Bool = true, aneAttention: Bool = false,
+ init(context: MetalContext, rows: Int, maxRows: Int? = nil, ane: Bool = true, amx: Bool = true, aneAttention: Bool = false,
halfWeights: Bool = false) throws {
self.context = context
self.halfWeights = halfWeights
self.rows = rows
+ self.maxRows = max(rows, maxRows ?? rows)
self.aneEnabled = ane
self.amxEnabled = amx
aneRows = ane ? min(2048, max(0, (rows - 512) / 256 * 256)) : 0
@@ -138,6 +142,15 @@ final class Coexecutor {
struct Shape { var k, n, k0, kPart, n0, nPart, row0, rows, kParts: UInt32 }
+ /// Where the kernel that writes a split product's input writes its fp16 copy, and how many rows of it: the
+ /// GPU's and the AMX's (all but the Neural Engine's last ones, whatever the AMX's share). The product then takes
+ /// it as cast (`castReady`), so no `pack_half` runs between the two. nil: the product casts for itself, or
+ /// reads fp32. `m` padded.
+ func halfInput(m: Int, k: Int, n: Int) -> (buffer: MTLBuffer, rows: Int)? {
+ guard halfActivations, applies(m: m, k: k, n: n), m <= maxRows, let aHalf else { return nil }
+ return (aHalf, m - split(m: m, k: k, n: n).ane)
+ }
+
/// Encodes the split product of the weight named `weight`. `gpuRows` encodes the GPU's own rows `[0, g)` into the
/// command buffer.
func encode(weight: String, a: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
diff --git a/engine-mac/Sources/KokonEngine/Kernels/dit.metal b/engine-mac/Sources/KokonEngine/Kernels/dit.metal
index f68db79..3930547 100644
--- a/engine-mac/Sources/KokonEngine/Kernels/dit.metal
+++ b/engine-mac/Sources/KokonEngine/Kernels/dit.metal
@@ -26,13 +26,35 @@ static inline float row_sum(float v, threadgroup float *shared, uint tid) {
return total;
}
+// The fp16 copy of a row kernel's output that the next split product reads (`Coexecution.swift`): rows below
+// `halfRows` go to `outHalf` (the GPU's and the AMX's rows, as `pack_half` would cast them), fp32 is written from
+// `fullFrom` on (the Neural Engine's rows, which its packing scales from fp32; 0 when every row is read in fp32).
+// Rows from `rows` (the real ones) to `halfRows` are the product's padding: zeros in the copy.
+struct HalfOut {
+ uint rows;
+ uint halfRows;
+ uint fullFrom;
+};
+
+struct NormParams {
+ uint columns;
+ uint targetStart;
+ float eps;
+ HalfOut half_out;
+};
+
// out = LayerNorm(x) (no affine) * onePlusScale[band]: img_norm1/2 + _modulate, norm_out.
-// One threadgroup per row.
+// One threadgroup per row (max(rows, halfRows) of them).
kernel void layernorm_modulate(device const float *x [[buffer(0)]], device float *out [[buffer(1)]],
device const float *prefixMod [[buffer(2)]], device const float *targetMod [[buffer(3)]],
- constant RowParams &p [[buffer(4)]],
+ constant NormParams &p [[buffer(4)]], device half *outHalf [[buffer(5)]],
uint row [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
threadgroup float shared[ROW_THREADS / 32];
+ device half *h = outHalf + (ulong)row * p.columns;
+ if (row >= p.half_out.rows) { // the whole threadgroup: no barrier is skipped by part of it
+ for (uint j = tid; j < p.columns; j += ROW_THREADS) h[j] = 0;
+ return;
+ }
device const float *r = x + (ulong)row * p.columns;
float s = 0;
for (uint j = tid; j < p.columns; j += ROW_THREADS) s += r[j];
@@ -42,7 +64,12 @@ kernel void layernorm_modulate(device const float *x [[buffer(0)]], device float
float inv = rsqrt(row_sum(q, shared, tid) / float(p.columns) + p.eps);
device const float *m = row < p.targetStart ? prefixMod : targetMod;
device float *o = out + (ulong)row * p.columns;
- for (uint j = tid; j < p.columns; j += ROW_THREADS) o[j] = ((r[j] - mean) * inv) * m[j];
+ bool toHalf = row < p.half_out.halfRows, toFull = row >= p.half_out.fullFrom;
+ for (uint j = tid; j < p.columns; j += ROW_THREADS) {
+ float v = ((r[j] - mean) * inv) * m[j];
+ if (toFull) o[j] = v;
+ if (toHalf) h[j] = half(v);
+ }
}
// QwenImage21ZeroCenterRMSNorm: x * rsqrt(mean(x²) + eps) * (w + 1), `onePlusWeight` precomputed.
@@ -104,12 +131,25 @@ kernel void gated_residual(device float *x [[buffer(0)]], device const float *o
x[i] = x[i] + g * o[i];
}
-// QwenImage21SwiGLUFeedForward: gate = SiLU(gate) * proj, in place in `gate`.
+// QwenImage21SwiGLUFeedForward: gate = SiLU(gate) * proj, in place in `gate`; with a fp16 copy as in
+// layernorm_modulate (`columns` floats a row, one thread per value of max(rows, halfRows) rows).
+struct SwigluParams {
+ uint columns;
+ HalfOut half_out;
+};
+
kernel void swiglu(device float *gate [[buffer(0)]], device const float *proj [[buffer(1)]],
- constant uint &count [[buffer(2)]], uint i [[thread_position_in_grid]]) {
- if (i >= count) return;
+ constant SwigluParams &p [[buffer(2)]], device half *outHalf [[buffer(3)]],
+ uint i [[thread_position_in_grid]]) {
+ uint row = i / p.columns;
+ if (row >= p.half_out.rows) {
+ if (row < p.half_out.halfRows) outHalf[i] = 0;
+ return;
+ }
float g = gate[i];
- gate[i] = (g / (1.0f + exp(-g))) * proj[i];
+ float v = (g / (1.0f + exp(-g))) * proj[i];
+ if (row >= p.half_out.fullFrom) gate[i] = v;
+ if (row < p.half_out.halfRows) outHalf[i] = half(v);
}
// nn.GELU(approximate="tanh"), in place.
diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swift
index 0125e37..5517208 100644
--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift
+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift
@@ -100,7 +100,7 @@ public final class QwenDiT {
// With the cache the split products are the target's rows; without it, the whole sequence's.
let splitRows = QwenDiT.padded(prefixCache != nil ? sequence.target : sequence.count)
if environment["KOKON_COEXEC"] != "0", splitRows >= 1024 {
- coexecutor = try Coexecutor(context: context, rows: splitRows,
+ coexecutor = try Coexecutor(context: context, rows: splitRows, maxRows: QwenDiT.padded(sequence.count),
ane: environment["KOKON_ANE"] != "0", amx: environment["KOKON_AMX"] != "0",
aneAttention: environment["KOKON_ANE_ATTENTION"] == "1", halfWeights: halfWeights)
} else {
@@ -211,6 +211,7 @@ public final class QwenDiT {
// The cached path when there is a cache and no probe (the probes compare every row of every stage).
let cached = prefixCache != nil && probe == nil
+ probing = probe != nil
var commands = try begin()
// A new conditioning: the whole sequence once, the prefix's K and V saved on the way. A pass of the prefix
// alone took 1.86 s at 1024² (products of 64 rows run far below the GPU's rate); riding along the first
@@ -304,7 +305,8 @@ public final class QwenDiT {
let (first, count) = (rows.lowerBound, rows.count)
let block = "transformer_blocks.\(layer)."
let band = 5 * d // the target's modulation row follows the prefix's
- try layerNorm(x, into: normed, rows: rows, prefixMod: 0, targetMod: band, commands: commands)
+ try layerNorm(x, into: normed, rows: rows, prefixMod: 0, targetMod: band,
+ half: halfOutput(count: count, k: d, n: d), commands: commands)
try linear(block + "attn.to_q.weight", a: normed, aRow: first, c: q, m: count, cRow: first, next: block + "attn.to_k.weight", commands: commands)
try linear(block + "attn.to_k.weight", a: normed, aRow: first, c: k, m: count, cRow: first, next: block + "attn.to_v.weight", commands: commands)
try mark("norm1", normed)
@@ -326,11 +328,12 @@ public final class QwenDiT {
try mark("attn_out", o)
try residual(gate: d, rows: rows, commands: commands)
try mark("res1", x)
- try layerNorm(x, into: normed, rows: rows, prefixMod: 2 * d, targetMod: band + 2 * d, commands: commands)
+ try layerNorm(x, into: normed, rows: rows, prefixMod: 2 * d, targetMod: band + 2 * d,
+ half: halfOutput(count: count, k: d, n: QwenDiT.hidden), commands: commands)
try mark("norm2", normed)
try linear(block + "img_mlp.gate_layer.weight", a: normed, aRow: first, c: h1, m: count, cRow: first, next: block + "img_mlp.proj.weight", commands: commands)
try linear(block + "img_mlp.proj.weight", a: normed, aRow: first, c: h2, m: count, cRow: first, next: block + "img_mlp.out.weight", commands: commands)
- try elementwise("swiglu", [(h1, first * QwenDiT.hidden), (h2, first * QwenDiT.hidden)], count: count * QwenDiT.hidden, commands: commands)
+ try swiglu(first: first, count: count, half: halfOutput(count: count, k: QwenDiT.hidden, n: d), commands: commands)
try linear(block + "img_mlp.out.weight", a: h1, aRow: first, c: o, m: count, cRow: first,
next: layer + 1 < QwenDiT.layers ? "transformer_blocks.\(layer + 1).attn.to_q.weight" : nil, commands: commands)
try mark("mlp", o)
@@ -468,22 +471,64 @@ public final class QwenDiT {
}
struct RowParams { var columns: UInt32; var targetStart: UInt32; var eps: Float }
+ struct HalfOut { var rows, halfRows, fullFrom: UInt32 }
+ struct NormParams { var columns, targetStart: UInt32; var eps: Float; var half: HalfOut }
+ struct SwigluParams { var columns: UInt32; var half: HalfOut }
+
+ /// **The row kernels write the next split product's fp16 input** (`layernorm_modulate` before q/k/v and gate/proj,
+ /// `swiglu` before the MLP's out): the GPU's and the AMX's rows straight to fp16, fp32 only for the Neural Engine's
+ /// rows, instead of fp32 everywhere and a `pack_half` pass that reads it back. The same values to the bit
+ /// (`half(v)` of the same fp32 v). `KOKON_FUSED_CAST=0`: the separate cast, for comparison.
+ let fusedCast = ProcessInfo.processInfo.environment["KOKON_FUSED_CAST"] != "0"
+ /// Set while a probe reads the stages: every row is written in fp32 too.
+ private var probing = false
+
+ /// Where a row kernel of `count` real rows writes the fp16 input of the split product `k × n` that follows.
+ func halfOutput(count: Int, k: Int, n: Int) -> (buffer: MTLBuffer, out: HalfOut)? {
+ guard fusedCast, !profiling, let coexecutor, let h = coexecutor.halfInput(m: QwenDiT.padded(count), k: k, n: n) else { return nil }
+ return (h.buffer, HalfOut(rows: UInt32(count), halfRows: UInt32(h.rows), fullFrom: probing ? 0 : UInt32(h.rows)))
+ }
+
+ /// No fp16 copy: every row in fp32.
+ static func fullOnly(_ rows: Int) -> HalfOut { HalfOut(rows: UInt32(rows), halfRows: 0, fullFrom: 0) }
struct HeadParams { var heads: UInt32; var rows: UInt32; var eps: Float }
struct ResidualParams { var columns: UInt32; var targetStart: UInt32; var count: UInt32 }
- /// `out = LN(x) · mod[band]`, `prefixMod` / `targetMod` floats into the modulation slice.
+ /// `out = LN(x) · mod[band]`, `prefixMod` / `targetMod` floats into the modulation slice; `half`: the next
+ /// product's fp16 input too (`halfOutput`).
func layerNorm(_ input: Arena.Slice, into out: Arena.Slice, rows: Range<Int>, prefixMod: Int, targetMod: Int,
- commands: MTLCommandBuffer) throws {
+ half: (buffer: MTLBuffer, out: HalfOut)? = nil, commands: MTLCommandBuffer) throws {
let at = rows.lowerBound * QwenDiT.dim * 4
+ let h = half?.out ?? QwenDiT.fullOnly(rows.count)
try compute("layernorm_modulate", commands: commands) { e in
e.setBuffer(arena.buffer, offset: input.offset + at, index: 0)
e.setBuffer(arena.buffer, offset: out.offset + at, index: 1)
e.setBuffer(arena.buffer, offset: modulation.offset + prefixMod * 4, index: 2)
e.setBuffer(arena.buffer, offset: modulation.offset + targetMod * 4, index: 3)
- var params = RowParams(columns: UInt32(QwenDiT.dim), targetStart: UInt32(max(0, sequence.prefix - rows.lowerBound)), eps: QwenDiT.eps)
- e.setBytes(&params, length: MemoryLayout<RowParams>.stride, index: 4)
- e.dispatchThreadgroups(MTLSize(width: rows.count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ var params = NormParams(columns: UInt32(QwenDiT.dim), targetStart: UInt32(max(0, sequence.prefix - rows.lowerBound)),
+ eps: QwenDiT.eps, half: h)
+ e.setBytes(&params, length: MemoryLayout<NormParams>.stride, index: 4)
+ e.setBuffer(half?.buffer ?? arena.buffer, offset: half == nil ? out.offset + at : 0, index: 5)
+ e.dispatchThreadgroups(MTLSize(width: max(rows.count, Int(h.halfRows)), height: 1, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ if half != nil { castInput = (out.offset, rows.lowerBound, QwenDiT.dim) }
+ }
+
+ /// `h1 = SiLU(h1) ⊙ h2` on rows `[first, first + count)`; `half`: the MLP out's fp16 input too.
+ func swiglu(first: Int, count: Int, half: (buffer: MTLBuffer, out: HalfOut)?, commands: MTLCommandBuffer) throws {
+ let w = QwenDiT.hidden
+ let h = half?.out ?? QwenDiT.fullOnly(count)
+ try compute("swiglu", commands: commands) { e in
+ e.setBuffer(arena.buffer, offset: h1.offset + first * w * 4, index: 0)
+ e.setBuffer(arena.buffer, offset: h2.offset + first * w * 4, index: 1)
+ var params = SwigluParams(columns: UInt32(w), half: h)
+ e.setBytes(&params, length: MemoryLayout<SwigluParams>.stride, index: 2)
+ e.setBuffer(half?.buffer ?? arena.buffer, offset: half == nil ? h1.offset : 0, index: 3)
+ e.dispatchThreads(MTLSize(width: max(count, Int(h.halfRows)) * w, height: 1, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
}
+ if half != nil { castInput = (h1.offset, first, w) }
}
func rmsNorm(_ input: Arena.Slice, into out: Arena.Slice, rows: Int, weight: Arena.Slice, commands: MTLCommandBuffer) throws {