Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: fp16-operand attention as an option; VAE compiled ahead, kept between images

Gabriele Lozupone589e7773c696
diff --git a/engine-mac/Sources/KokonEngine/AttentionProbe.swift b/engine-mac/Sources/KokonEngine/AttentionProbe.swift
index 2a58623..0db6637 100644
--- a/engine-mac/Sources/KokonEngine/AttentionProbe.swift
+++ b/engine-mac/Sources/KokonEngine/AttentionProbe.swift
@@ -51,10 +51,11 @@ extension Probe {
// The GPU's flash kernel, all heads.
struct FlashParams { var queries, keys, causal, stride: UInt32; var scale: Float; var headBase: UInt32 }
+ for kernel in ["flash_attention", "flash_attention_half"] {
let gpu = try best(3) {
let commands = context.queue.makeCommandBuffer()!
let e = commands.makeComputeCommandEncoder()!
- e.setComputePipelineState(try context.pipeline("flash_attention"))
+ e.setComputePipelineState(try context.pipeline(kernel))
e.setBuffer(qb, offset: offset * stride * 4, index: 0); e.setBuffer(kb, offset: 0, index: 1)
e.setBuffer(vb, offset: 0, index: 2); e.setBuffer(ob, offset: offset * stride * 4, index: 3)
var p = FlashParams(queries: UInt32(queries), keys: UInt32(keys), causal: 0, stride: UInt32(stride),
@@ -66,8 +67,9 @@ extension Probe {
return try MetalContext.finish(commands, "flash")
}
let gpuError = error { i, c in f(ob)[(offset + i) * stride + c] }
- print(String(format: "GPU flash kernel, 32 heads: %.1f ms, %.2f ms/head, %.2f TFLOP/s; head 0 rel %.1e",
+ print(String(format: "GPU %@, 32 heads: %.1f ms, %.2f ms/head, %.2f TFLOP/s; head 0 rel %.1e", kernel,
gpu * 1e3, gpu * 1e3 / Double(heads), 4 * Double(queries) * Double(keys) * Double(d) * Double(heads) / gpu / 1e12, gpuError))
+ }
if ProcessInfo.processInfo.environment["KOKON_PROBE_GPU_ONLY"] == "1" { return }
// The Neural Engine.
diff --git a/engine-mac/Sources/KokonEngine/Balancer.swift b/engine-mac/Sources/KokonEngine/Balancer.swift
index 362be3f..9d311fc 100644
--- a/engine-mac/Sources/KokonEngine/Balancer.swift
+++ b/engine-mac/Sources/KokonEngine/Balancer.swift
@@ -101,10 +101,11 @@ extension Coexecutor {
}
if lastTimeline.contains(where: { $0.kind == "attention" }), HeadSplit.fromEnvironment() == nil {
- let maxAne = aneEnabled && balancer.aneAttention ? QwenDiT.heads : 0
+ // The GPU keeps at least one head (its text segments run there anyway).
+ let maxAne = aneEnabled && balancer.aneAttention ? QwenDiT.heads - Coexecutor.aneHeadGroup : 0
var best = heads
for ane in stride(from: 0, through: maxAne, by: Coexecutor.aneHeadGroup) {
- for cpu in 0...(amxEnabled ? QwenDiT.heads - 1 - ane : 0) where ane + cpu < QwenDiT.heads {
+ for cpu in 0...(amxEnabled ? QwenDiT.heads - 1 - ane : 0) {
let h = HeadSplit(gpu: QwenDiT.heads - ane - cpu, ane: ane, cpu: cpu)
if balancer.attentionCost(h) < balancer.attentionCost(best) { best = h }
}
diff --git a/engine-mac/Sources/KokonEngine/Coexecution.swift b/engine-mac/Sources/KokonEngine/Coexecution.swift
index fa3610d..4b09c9f 100644
--- a/engine-mac/Sources/KokonEngine/Coexecution.swift
+++ b/engine-mac/Sources/KokonEngine/Coexecution.swift
@@ -305,6 +305,19 @@ final class Timeline: @unchecked Sendable {
if groups[e.kind] == nil { kinds.append(e.kind) }
groups[e.kind, default: []].append(e)
}
+ // Between operations: from the last unit's end of one to the GPU's release of the next (the GPU alone:
+ // unpacking, the elementwise kernels, the next weight's unpacking and packing, the waits' latency).
+ var gaps: [String: (count: Int, seconds: Double)] = [:]
+ for (a, b) in zip(entries, entries.dropFirst()) {
+ let end = max(a.gpu, a.shares.amx > 0 ? a.amx.end : 0, a.shares.ane > 0 ? a.ane.end : 0)
+ let key = "\(a.kind) → \(b.kind)"
+ gaps[key] = ((gaps[key]?.count ?? 0) + 1, (gaps[key]?.seconds ?? 0) + max(0, b.released - end))
+ }
+ let between = gaps.sorted { $0.value.seconds > $1.value.seconds }.map {
+ String(format: " %-26@ ×%3d %6.2f ms each, %5.2f s", $0.key as NSString, $0.value.count,
+ $0.value.seconds / Double($0.value.count) * 1e3, $0.value.seconds)
+ }
+ let total = gaps.values.reduce(0) { $0 + $1.seconds }
return kinds.map { kind in
let g = groups[kind]!, n = Double(g.count), s = g[0].shares
func mean(_ f: (Entry) -> Double) -> Double { g.map(f).reduce(0, +) / n * 1e3 }
@@ -313,7 +326,7 @@ final class Timeline: @unchecked Sendable {
mean { $0.amx.seconds }, mean { $0.shares.amx > 0 ? $0.amx.start - $0.released : 0 },
mean { $0.ane.seconds }, mean { $0.shares.ane > 0 ? $0.ane.start - $0.released : 0 },
mean { max($0.walls.gpu, $0.walls.amx, $0.walls.ane) })
- }.joined(separator: "\n")
+ }.joined(separator: "\n") + String(format: "\n between operations %.2f s:\n", total) + between.joined(separator: "\n")
}
}
diff --git a/engine-mac/Sources/KokonEngine/Kernels/attention.metal b/engine-mac/Sources/KokonEngine/Kernels/attention.metal
index 263faf7..077de4e 100644
--- a/engine-mac/Sources/KokonEngine/Kernels/attention.metal
+++ b/engine-mac/Sources/KokonEngine/Kernels/attention.metal
@@ -148,3 +148,114 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fl
}
}
}
+
+// The same with q, k, v and the probabilities in fp16 (scores, softmax and O accumulated in fp32; K and V tiles of 64
+// keys side by side in 32 KB), `KOKON_ATTENTION=half`: 94 ms against 120 at 1024² (M2 Pro), head 0 within 6.3e-4 of
+// exact, the velocity within 6.0e-4 of fp64 at 256² (fp32 kernel 1.0e-5). Only V and the probabilities in fp16
+// (q·k fp32): 118 ms, 2.2e-4, so not kept. Off until decided.
+static inline thread half2 &elementsHalf(thread simdgroup_half8x8 &m) {
+ return *(thread half2 *)&m.thread_elements();
+}
+
+kernel void flash_attention_half(device const float *q [[buffer(0)]], device const float *k [[buffer(1)]],
+ device const float *v [[buffer(2)]], device float *out [[buffer(3)]],
+ constant FlashParams &p [[buffer(4)]],
+ uint2 tg [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]],
+ uint sg [[simdgroup_index_in_threadgroup]],
+ uint lane [[thread_index_in_simdgroup]]) {
+ threadgroup half ktile[BK * D]; // 16 KB; first each SIMD group's 8 query rows, once
+ threadgroup half vtile[BK * D]; // 16 KB
+ threadgroup half *qstage = ktile + sg * 8 * D;
+
+ const uint head = p.headBase + tg.y;
+ const uint q0 = tg.x * BQ + sg * 8;
+ const uint stride = p.stride;
+ device const float *kh = k + head * D;
+ device const float *vh = v + head * D;
+
+ for (uint i = lane; i < 8 * D; i += 32) qstage[i] = half(q[(ulong)(q0 + i / D) * stride + head * D + i % D]);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ simdgroup_half8x8 Q[16];
+ simdgroup_float8x8 O[16];
+ for (uint d = 0; d < 16; d++) {
+ simdgroup_load(Q[d], qstage + 8 * d, D);
+ O[d] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const uint row = (lane / 16) * 4 + (lane / 2) % 4;
+ const uint col = ((lane / 8) % 2) * 4 + (lane % 2) * 2;
+ const uint qi = q0 + row;
+ const uint offset = p.keys - p.queries;
+ const int limit = p.causal ? int(offset + qi) : int(p.keys) - 1;
+ float m = -INFINITY, l = 0.0f;
+
+ uint kEnd = p.keys;
+ if (p.causal) kEnd = min(p.keys, offset + min(tg.x * BQ + BQ - 1, p.queries - 1) + 1);
+
+ for (uint j0 = 0; j0 < kEnd; j0 += BK) {
+ for (uint i = tid; i < BK * D / 4; i += 256) {
+ uint r = i / (D / 4), c = i % (D / 4);
+ ((threadgroup half4 *)ktile)[i] = half4(*((device const float4 *)(kh + (ulong)(j0 + r) * stride) + c));
+ ((threadgroup half4 *)vtile)[i] = half4(*((device const float4 *)(vh + (ulong)(j0 + r) * stride) + c));
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ simdgroup_float8x8 S[8];
+ for (uint c = 0; c < 8; c++) S[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ for (uint d = 0; d < 16; d++) {
+ for (uint c = 0; c < 8; c++) {
+ simdgroup_half8x8 Kt;
+ simdgroup_load(Kt, ktile + 8 * c * D + 8 * d, D, ulong2(0, 0), true);
+ simdgroup_multiply_accumulate(S[c], Q[d], Kt, S[c]);
+ }
+ }
+
+ float2 s[8];
+ float blockMax = -INFINITY;
+ for (uint c = 0; c < 8; c++) {
+ s[c] = elements(S[c]) * p.scale;
+ int j = int(j0 + 8 * c + col);
+ if (j > limit) s[c].x = -INFINITY;
+ if (j + 1 > limit) s[c].y = -INFINITY;
+ blockMax = max(blockMax, max(s[c].x, s[c].y));
+ }
+ blockMax = max(blockMax, simd_shuffle_xor(blockMax, 1));
+ blockMax = max(blockMax, simd_shuffle_xor(blockMax, 8));
+ float mNew = max(m, blockMax);
+ float alpha = (mNew == -INFINITY) ? 1.0f : exp2(m - mNew);
+ float sum = 0.0f;
+ simdgroup_half8x8 P[8];
+ for (uint c = 0; c < 8; c++) {
+ float2 e = (mNew == -INFINITY) ? float2(0.0f) : exp2(s[c] - mNew);
+ elementsHalf(P[c]) = half2(e);
+ sum += e.x + e.y;
+ }
+ sum += simd_shuffle_xor(sum, 1);
+ sum += simd_shuffle_xor(sum, 8);
+ l = l * alpha + sum;
+ m = mNew;
+ if (!simd_all(alpha == 1.0f)) {
+ for (uint d = 0; d < 16; d++) elements(O[d]) *= alpha;
+ }
+ for (uint d = 0; d < 16; d++) {
+ for (uint c = 0; c < 8; c++) {
+ simdgroup_half8x8 V;
+ simdgroup_load(V, vtile + 8 * c * D + 8 * d, D);
+ simdgroup_multiply_accumulate(O[d], P[c], V, O[d]);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ if (qi < p.queries) {
+ const float inv = 1.0f / l;
+ device float *o = out + (ulong)qi * stride + head * D + col;
+ for (uint d = 0; d < 16; d++) {
+ float2 e = elements(O[d]) * inv;
+ o[8 * d] = e.x;
+ o[8 * d + 1] = e.y;
+ }
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swift
index d50039c..20f6f53 100644
--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift
+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift
@@ -435,8 +435,10 @@ public final class QwenDiT {
}
}
- /// Attention through `Kernels/attention.metal` (default) or MPSGraph (`KOKON_ATTENTION=graph`, the check).
+ /// Attention through `Kernels/attention.metal` (default) or MPSGraph (`KOKON_ATTENTION=graph`, the check);
+ /// `KOKON_ATTENTION=half`: the kernel with fp16 operands (faster, 6e-4 of fp64; not the default until decided).
let graphAttention = ProcessInfo.processInfo.environment["KOKON_ATTENTION"] == "graph"
+ let flashKernel = ProcessInfo.processInfo.environment["KOKON_ATTENTION"] == "half" ? "flash_attention_half" : "flash_attention"
struct FlashParams { var queries, keys, causal, stride: UInt32; var scale: Float; var headBase: UInt32 }
/// Queries `[queries, queries + count)` against keys `[0, keys)`, into the same rows of `attended`. The target's
@@ -475,7 +477,7 @@ public final class QwenDiT {
func flash(queries: Int, count: Int, keys: Int, causal: Bool, heads: Int, commands: MTLCommandBuffer) throws {
guard heads > 0 else { return }
let row = QwenDiT.dim * 4
- try compute("flash_attention", commands: commands) { e in
+ try compute(flashKernel, commands: commands) { e in
e.setBuffer(arena.buffer, offset: q.offset + queries * row, index: 0)
e.setBuffer(arena.buffer, offset: k.offset, index: 1)
e.setBuffer(arena.buffer, offset: v.offset, index: 2)
diff --git a/engine-mac/Sources/KokonEngine/QwenPipeline.swift b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
index d7db681..843a016 100644
--- a/engine-mac/Sources/KokonEngine/QwenPipeline.swift
+++ b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
@@ -21,8 +21,9 @@ public final class QwenPipeline {
public var log: (String) -> Void = { print($0) }
/// Seconds per phase of the last run: encode, steps (each), dit load, vae, total.
public private(set) var timings: [String: Double] = [:]
- /// Kept between images: its first decode of a size costs ~3.7 s more than the next ones at 1024² (M2 Pro), GPU
- /// time (~1 s of CPU), so it cannot hide under the DiT; a long-lived engine pays it once per size.
+ /// Kept between images: its first decode of a size costs ~4 s more than the next ones at 1024² (M2 Pro). Compiled
+ /// beside the DiT (from the second step) it slowed the steps by ~9 s to save 1.9: a long-lived engine pays it
+ /// once per size, or ahead while idle (`QwenVAE.prepare`).
private var vae: QwenVAE?
public init(encoder: String, dit: String, vae: String) throws {
diff --git a/engine-mac/Sources/KokonEngine/QwenVAE.swift b/engine-mac/Sources/KokonEngine/QwenVAE.swift
index b242a57..d5e97d9 100644
--- a/engine-mac/Sources/KokonEngine/QwenVAE.swift
+++ b/engine-mac/Sources/KokonEngine/QwenVAE.swift
@@ -35,10 +35,12 @@ public final class QwenVAE {
let context: MetalContext
let file: SafetensorsFile
- /// One compiled graph per latent size. Building and compiling the 1024² one takes 0.3 s; its first run then costs
- /// ~3.7 s more than the next ones (M2 Pro), GPU time with ~1 s of CPU, so the pipeline keeps its VAE between
- /// images rather than building it beside the DiT. fp16 convolutions (norms and attention in fp32) were wrong
- /// (2.7e-1: an overflow) and 5.7× slower, MPSGraph trying to move them to the Neural Engine: the VAE stays fp32.
+ /// One compiled graph per latent size, compiled to the end (`waitForCompilationCompletion`): 2.7 s at 1024² (M2
+ /// Pro), mostly the shader compiler, which otherwise ran inside the first decode (6.4 s against 2.75 s; with it
+ /// up front the first decode is 4.15 s). Beside the DiT it slows the steps far more than it saves: the pipeline
+ /// keeps its VAE between images, and an idle engine can prepare ahead. fp16 convolutions (norms and attention in
+ /// fp32) were wrong (2.7e-1: an overflow) and 5.7× slower, MPSGraph trying to move them to the Neural Engine: the
+ /// VAE stays fp32.
private var executables: [String: MPSGraphExecutable] = [:]
private let lock = NSLock()
@@ -73,8 +75,10 @@ public final class QwenVAE {
if let e = executables[key] { return e }
let built = try build(height: h, width: w)
let shape = MPSGraphShapedType(shape: [1, 64, NSNumber(value: h), NSNumber(value: w)], dataType: .float32)
+ let descriptor = MPSGraphCompilationDescriptor()
+ descriptor.waitForCompilationCompletion = true
let e = built.graph.compile(with: MPSGraphDevice(mtlDevice: context.device), feeds: [built.input: shape],
- targetTensors: [built.output], targetOperations: nil, compilationDescriptor: nil)
+ targetTensors: [built.output], targetOperations: nil, compilationDescriptor: descriptor)
executables[key] = e
return e
}