Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: the AMX and the Neural Engine beside the GPU on the DiT's products

Gabriele Lozuponede8ca8a70307
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 685d8d8..991c55b 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -63,5 +63,11 @@ m = 4 096 rows (a 1024² image), TFLOP/s:
(MPSGraph) 4.1 s, unpacking 0.2 s, the rest 0.3 s. Every product goes to MPS on a multiple of 64 rows: one product
of 32 rows (the text projection) slowed all later ones to 18.5 s.
+**With the AMX and the Neural Engine on the products** (`Coexecution.swift`; rows GPU 1856 / AMX 256 / Neural
+Engine 2048 at 1024²): **11.3 s per evaluation at 1024²**, 8.8 s at 896². The GPU packs the Neural Engine's fp16
+operands straight into its IOSurfaces and unpacks its output, so the CPU only runs the AMX. Accuracy at 896²
+against diffusers fp32: the velocity within 3.6e-4 relative (GPU alone 4.4e-6); running the model in bf16, as
+most tools do, deviates by roughly 1e-2. The attention (4.1 s, GPU only, the other units idle) is next.
+
Credits: the Neural Engine program technique and the GPU + AMX + Neural Engine split come from
[Siliconed](https://github.com/gwenn-ha-dev/Siliconed) (Apache-2.0), written again here.
diff --git a/engine-mac/Sources/KokonEngine/Checks.swift b/engine-mac/Sources/KokonEngine/Checks.swift
index c8352d0..62fca2c 100644
--- a/engine-mac/Sources/KokonEngine/Checks.swift
+++ b/engine-mac/Sources/KokonEngine/Checks.swift
@@ -117,12 +117,18 @@ extension Checks {
}
}
let encoder = random(text * QwenDiT.dim, 8), latent = random(sequence.target * QwenDiT.channels, 2)
- print("sequence: \(sequence.count) rows (\(sequence.prefix) text, \(sequence.target) target)")
+ print("sequence: \(sequence.count) rows (\(sequence.prefix) text, \(sequence.target) target)"
+ + (dit.coexecutor.map { ", products split \($0.split(m: QwenDiT.padded(sequence.count)))" } ?? ", GPU only"))
for run in 0..<runs {
let started = DispatchTime.now().uptimeNanoseconds
let out = try dit.forward(encoder: encoder, target: latent, sigma: 0.5)
let seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
print(String(format: "evaluation %d: %.2f s%@", run + 1, seconds, out.allSatisfy(\.isFinite) ? "" : " NON-FINITE OUTPUT"))
+ if let c = dit.coexecutor {
+ print(String(format: " AMX busy %.2f s, Neural Engine busy %.2f s%@", c.amxSeconds, c.aneSeconds,
+ c.failures.isEmpty ? "" : "; FAILURES: \(c.failures.prefix(3))"))
+ c.resetTimings()
+ }
}
dit.profiling = true
_ = try dit.forward(encoder: encoder, target: latent, sigma: 0.5)
@@ -132,3 +138,44 @@ extension Checks {
}
}
}
+
+extension Checks {
+ /// The GPU writes the Neural Engine's operands straight into its surfaces; the product is compared with the CPU.
+ public static func aneSurfaces() throws {
+ let context = try MetalContext()
+ let (rows, k, n) = (512, 512, 256)
+ let product = try NeuralEngineProduct(rows: rows, k: k, n: n)
+ let xb = try product.gpuBuffer(product.x, device: context.device)
+ let wb = try product.gpuBuffer(product.w, device: context.device)
+ var a = [Float](repeating: 0, count: rows * k), w = [Float](repeating: 0, count: k * n)
+ for i in a.indices { a[i] = Float((i * 7919) % 97) / 97 - 0.5 }
+ for i in w.indices { w[i] = Float((i * 104729) % 89) / 89 - 0.5 }
+ let src = try context.scratch((a.count + w.count) * 4, label: "src")
+ src.contents().copyMemory(from: a, byteCount: a.count * 4)
+ (src.contents() + a.count * 4).copyMemory(from: w, byteCount: w.count * 4)
+ let commands = context.queue.makeCommandBuffer()!
+ let encoder = commands.makeComputeCommandEncoder()!
+ encoder.setComputePipelineState(try context.pipeline("pack_half"))
+ for (offset, count, target) in [(0, a.count, xb), (a.count * 4, w.count, wb)] {
+ var c = UInt32(count)
+ encoder.setBuffer(src, offset: offset, index: 0)
+ encoder.setBuffer(target, offset: 0, index: 1)
+ encoder.setBytes(&c, length: 4, index: 2)
+ encoder.dispatchThreads(MTLSize(width: count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ encoder.endEncoding()
+ try MetalContext.finish(commands, "pack")
+ try product.run()
+ var worst: Float = 0, scale: Float = 0
+ product.read { y in
+ for i in 0..<rows {
+ for j in 0..<n {
+ var s: Float = 0
+ for l in 0..<k { s += Float(Float16(a[i * k + l])) * Float(Float16(w[l * n + j])) }
+ worst = max(worst, abs(Float(y[i * n + j]) - s)); scale = max(scale, abs(s))
+ }
+ }
+ }
+ print(String(format: "Neural Engine on GPU-written surfaces: max |Δ| %.4g of max |y| %.4g (fp16 output)", worst, scale))
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/Coexecution.swift b/engine-mac/Sources/KokonEngine/Coexecution.swift
new file mode 100644
index 0000000..270faa6
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/Coexecution.swift
@@ -0,0 +1,214 @@
+import Accelerate
+import Foundation
+import Metal
+
+/// **One product on the GPU, the AMX and the Neural Engine at the same time**, cut by rows:
+///
+/// C[m, n] = A[m, k] · W[k, n] W already unpacked to fp32 [k, n] (the reserve) by the GPU
+/// rows [0, g) GPU, MPS fp32
+/// rows [g, g + a) AMX, cblas_sgemm fp32 on the same reserve (unified memory)
+/// rows [g + a, m) Neural Engine, fp16 in tiles of 4 096 × 4 096 (`ane.metal` packs and unpacks)
+///
+/// GPU timeline per product p: unpack W → pack the Neural Engine's tiles → signal `ready` = p → its own rows →
+/// wait `amxDone` ≥ p and `aneDone` ≥ p → unpack the Neural Engine's rows. The two CPU workers wait on `ready`,
+/// compute, signal. Each output row is computed by exactly one unit, so the split never reorders a sum.
+///
+/// Nothing waits on the CPU while the GPU runs: the GPU encodes ahead, the workers follow the events. A reserve
+/// or a surface is rewritten only after the GPU has waited for the product that read it.
+final class Coexecutor {
+ struct Split: CustomStringConvertible { var gpu: Int, amx: Int, ane: Int
+ var description: String { "GPU \(gpu) / AMX \(amx) / ANE \(ane) rows" }
+ }
+
+ static let tile = 4096
+ let context: MetalContext
+ let aneRows: Int
+ private let slots: [(product: NeuralEngineProduct, x: MTLBuffer, w: MTLBuffer, y: MTLBuffer)]
+ private let ready: MTLSharedEvent, amxDone: MTLSharedEvent, aneDone: MTLSharedEvent
+ private var counter: UInt64 = 0
+ private let amxWorker = Worker(name: "kokon.amx"), aneWorker = Worker(name: "kokon.ane")
+ /// λ (12 288), lifted norms (12 288), L, σ (aneRows): the Neural Engine's scales.
+ private let scales: MTLBuffer
+ /// The share of rows each unit takes; `KOKON_SPLIT=gpu,amx,ane` (fractions) overrides.
+ var amxShare: Double, aneEnabled: Bool, amxEnabled: Bool
+ /// Seconds each worker spent computing (not waiting), and failures (a failure leaves rows unwritten).
+ private(set) var amxSeconds = 0.0, aneSeconds = 0.0
+ private(set) var failures: [String] = []
+ private let lock = NSLock()
+
+ /// Defaults measured on the M2 Pro 16 GB, 1024² + 32 text rows (`time-dit`, 2026-10-25): AMX 512 rows 12.8 s per
+ /// evaluation (the AMX the slowest, ~35 % under its solo rate beside the others), 256 rows 11.3 s, none 11.8 s;
+ /// GPU alone 16.2 s. A balancing loop per shape replaces the fixed share later.
+ init(context: MetalContext, aneRows: Int = 2048, amxShare: Double = 0.0625, ane: Bool = true, amx: Bool = true) throws {
+ self.context = context
+ self.aneRows = aneRows
+ self.amxShare = amxShare
+ self.aneEnabled = ane
+ self.amxEnabled = amx
+ let device = context.device
+ guard let r = device.makeSharedEvent(), let a = device.makeSharedEvent(), let e = device.makeSharedEvent() else {
+ throw MetalContext.Failure.buffer("shared events")
+ }
+ (ready, amxDone, aneDone) = (r, a, e)
+ scales = try context.scratch((2 * 12288 + 16 + aneRows) * 4, label: "ane scales")
+ slots = ane ? try (0..<3).map { _ in
+ let p = try NeuralEngineProduct(rows: aneRows, k: Coexecutor.tile, n: Coexecutor.tile)
+ try p.run() // compile and warm
+ return (p, try p.gpuBuffer(p.x, device: device), try p.gpuBuffer(p.w, device: device), try p.gpuBuffer(p.y, device: device))
+ } : []
+ }
+
+ /// Whether a product takes the split: the Neural Engine's tiles need K and N in multiples of 4 096 with at
+ /// most 3 tiles; below 1 024 rows the split costs more than it brings.
+ func applies(m: Int, k: Int, n: Int) -> Bool {
+ m >= 1024 && k % Coexecutor.tile == 0 && n % Coexecutor.tile == 0 && (k / Coexecutor.tile) * (n / Coexecutor.tile) <= 3
+ }
+
+ func split(m: Int) -> Split {
+ let ane = aneEnabled && m >= aneRows + 512 ? aneRows : 0
+ let amx = amxEnabled ? Int((Double(m) * amxShare / 256).rounded()) * 256 : 0
+ return Split(gpu: m - ane - amx, amx: amx, ane: ane)
+ }
+
+ struct Shape { var k, n, k0, kPart, n0, nPart, row0, rows, kParts: UInt32 }
+
+ /// Encodes the split product. `gpuRows` encodes the GPU's own rows `[0, g)` into the command buffer.
+ func encode(a: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
+ w: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
+ c: (buffer: MTLBuffer, offset: Int, pointer: UnsafeMutablePointer<Float>),
+ m: Int, k: Int, n: Int, commands: MTLCommandBuffer, gpuRows: (Int) throws -> Void) throws {
+ let s = split(m: m)
+ counter += 1
+ let p = counter
+ let (kParts, nParts) = (k / Coexecutor.tile, n / Coexecutor.tile)
+ let lambda = 0, norm = 12288 * 4, state = 2 * 12288 * 4, sigma = 2 * 12288 * 4 + 64
+ func shape(kp: Int = 0, np: Int = 0) -> Shape {
+ Shape(k: UInt32(k), n: UInt32(n), k0: UInt32(kp * Coexecutor.tile), kPart: UInt32(Coexecutor.tile),
+ n0: UInt32(np * Coexecutor.tile), nPart: UInt32(Coexecutor.tile), row0: UInt32(s.gpu + s.amx),
+ rows: UInt32(s.ane), kParts: UInt32(kParts))
+ }
+ func kernel(_ name: String, _ body: (MTLComputeCommandEncoder) throws -> Void) throws {
+ let e = commands.makeComputeCommandEncoder()!
+ e.label = name
+ e.setComputePipelineState(try context.pipeline(name))
+ try body(e)
+ e.endEncoding()
+ }
+ if s.ane > 0 {
+ var sh = shape()
+ try kernel("ane_columns") { e in
+ e.setBuffer(w.buffer, offset: w.offset, index: 0)
+ e.setBuffer(scales, offset: lambda, index: 1); e.setBuffer(scales, offset: norm, index: 2)
+ e.setBytes(&sh, length: MemoryLayout<Shape>.stride, index: 3)
+ e.dispatchThreads(MTLSize(width: n, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 64, height: 1, depth: 1))
+ }
+ try kernel("ane_norm") { e in
+ e.setBuffer(scales, offset: norm, index: 0); e.setBuffer(scales, offset: state, index: 1)
+ e.setBytes(&sh, length: MemoryLayout<Shape>.stride, index: 2)
+ e.dispatchThreadgroups(MTLSize(width: 1, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 1024, height: 1, depth: 1))
+ }
+ try kernel("ane_row_scales") { e in
+ e.setBuffer(a.buffer, offset: a.offset, index: 0); e.setBuffer(scales, offset: state, index: 1)
+ e.setBuffer(scales, offset: sigma, index: 2)
+ e.setBytes(&sh, length: MemoryLayout<Shape>.stride, index: 3)
+ e.dispatchThreadgroups(MTLSize(width: s.ane, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ for kp in 0..<kParts {
+ for np in 0..<nParts {
+ let slot = slots[kp * nParts + np]
+ var t = shape(kp: kp, np: np)
+ try kernel("ane_pack_weight") { e in
+ e.setBuffer(w.buffer, offset: w.offset, index: 0); e.setBuffer(scales, offset: lambda, index: 1)
+ e.setBuffer(slot.w, offset: 0, index: 2)
+ e.setBytes(&t, length: MemoryLayout<Shape>.stride, index: 3)
+ e.dispatchThreads(MTLSize(width: Coexecutor.tile, height: Coexecutor.tile, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 64, height: 4, depth: 1))
+ }
+ try kernel("ane_pack_rows") { e in
+ e.setBuffer(a.buffer, offset: a.offset, index: 0); e.setBuffer(scales, offset: sigma, index: 1)
+ e.setBuffer(slot.x, offset: 0, index: 2)
+ e.setBytes(&t, length: MemoryLayout<Shape>.stride, index: 3)
+ e.dispatchThreads(MTLSize(width: Coexecutor.tile, height: s.ane, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 64, height: 4, depth: 1))
+ }
+ }
+ }
+ }
+ if s.amx > 0 || s.ane > 0 { commands.encodeSignalEvent(ready, value: p) }
+ if s.amx > 0 {
+ nonisolated(unsafe) let (pa, pw, pc) = (a.pointer + s.gpu * k, w.pointer, c.pointer + s.gpu * n)
+ let rows = s.amx
+ amxWorker.submit { [self] in
+ defer { amxDone.signaledValue = p }
+ guard wait(ready, p, "AMX") else { return }
+ let t = DispatchTime.now().uptimeNanoseconds
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, Int32(rows), Int32(n), Int32(k), 1,
+ pa, Int32(k), pw, Int32(n), 0, pc, Int32(n))
+ lock.withLock { amxSeconds += Double(DispatchTime.now().uptimeNanoseconds - t) * 1e-9 }
+ }
+ }
+ if s.ane > 0 {
+ let tiles = Array(slots.prefix(kParts * nParts).map(\.product))
+ aneWorker.submit { [self] in
+ defer { aneDone.signaledValue = p }
+ guard wait(ready, p, "ANE") else { return }
+ let t = DispatchTime.now().uptimeNanoseconds
+ do { for tile in tiles { try tile.run() } } catch { fail("ANE: \(error)") }
+ lock.withLock { aneSeconds += Double(DispatchTime.now().uptimeNanoseconds - t) * 1e-9 }
+ }
+ }
+ try gpuRows(s.gpu)
+ if s.amx > 0 { commands.encodeWaitForEvent(amxDone, value: p) }
+ if s.ane > 0 {
+ commands.encodeWaitForEvent(aneDone, value: p)
+ for np in 0..<nParts {
+ var t = shape(np: np)
+ let ys = (0..<3).map { kp in slots[min(kp, kParts - 1) * nParts + np].y }
+ try kernel("ane_unpack") { e in
+ for (i, y) in ys.enumerated() { e.setBuffer(y, offset: 0, index: i) }
+ e.setBuffer(scales, offset: lambda, index: 3); e.setBuffer(scales, offset: sigma, index: 4)
+ e.setBuffer(c.buffer, offset: c.offset, index: 5)
+ e.setBytes(&t, length: MemoryLayout<Shape>.stride, index: 6)
+ e.dispatchThreads(MTLSize(width: Coexecutor.tile, height: s.ane, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 64, height: 4, depth: 1))
+ }
+ }
+ }
+ }
+
+ private func fail(_ message: String) { lock.withLock { failures.append(message) } }
+
+ /// Waits for the GPU to reach `value`; false (and a failure) after a minute, so a broken command buffer never
+ /// hangs a worker for good.
+ private func wait(_ event: MTLSharedEvent, _ value: UInt64, _ who: String) -> Bool {
+ if event.wait(untilSignaledValue: value, timeoutMS: 60_000) { return true }
+ fail("\(who): the GPU never reached product \(value)")
+ return false
+ }
+
+ func resetTimings() { lock.withLock { amxSeconds = 0; aneSeconds = 0 } }
+}
+
+/// A thread running submitted jobs in order.
+final class Worker: @unchecked Sendable {
+ private var jobs: [() -> Void] = []
+ private let lock = NSLock(), available = DispatchSemaphore(value: 0)
+
+ init(name: String) {
+ let thread = Thread { [unowned self] in
+ while true {
+ available.wait()
+ let job = lock.withLock { jobs.removeFirst() }
+ job()
+ }
+ }
+ thread.name = name
+ thread.qualityOfService = .userInteractive
+ thread.start()
+ }
+
+ func submit(_ job: @escaping () -> Void) {
+ lock.withLock { jobs.append(job) }
+ available.signal()
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/Kernels/ane.metal b/engine-mac/Sources/KokonEngine/Kernels/ane.metal
new file mode 100644
index 0000000..3328586
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/Kernels/ane.metal
@@ -0,0 +1,113 @@
+#include <metal_stdlib>
+using namespace metal;
+
+// The Neural Engine's side of a product C[m, n] = A[m, k] · W[k, n], packed and unpacked on the GPU so the CPU
+// stays free for the AMX. The Neural Engine computes in fp16 and flushes values under fp16's smallest normal,
+// so (scheme from Siliconed):
+// each weight column j is lifted by λ_j = 2^-floor(log2 max_k |W_kj|), its absmax into [1, 2): exact;
+// each activation row i by σ_i = 2^floor(log2(2^15 / (max_k |A_ik| · L))), L = max_j Σ_k |W_kj| λ_j,
+// so every output |Y_ij| <= max|A_i| σ_i L <= 2^15 stays under fp16's 65 504;
+// C_ij = Y_ij / (σ_i λ_j) in fp32.
+// The product runs in tiles of at most 4 096 × 4 096 (one Core ML program each, the Neural Engine's fast range);
+// tiles along K are summed here in fp32.
+
+// fp32 -> fp16 (check-ane).
+kernel void pack_half(device const float *src [[buffer(0)]], device half *dst [[buffer(1)]],
+ constant uint &count [[buffer(2)]], uint i [[thread_position_in_grid]]) {
+ if (i < count) dst[i] = half(src[i]);
+}
+
+struct AneShape {
+ uint k; // full K of the product
+ uint n; // full N
+ uint k0, kPart; // this tile's K range
+ uint n0, nPart; // this tile's N range
+ uint row0; // first row of A / C the Neural Engine takes
+ uint rows; // its rows
+ uint kParts; // tiles along K (unpack)
+};
+
+// Per column of W [k, n]: λ_j and the lifted column's L1 norm. One thread per column (coalesced rows).
+kernel void ane_columns(device const float *w [[buffer(0)]], device float *lambda [[buffer(1)]],
+ device float *norm [[buffer(2)]], constant AneShape &s [[buffer(3)]],
+ uint j [[thread_position_in_grid]]) {
+ if (j >= s.n) return;
+ float peak = 0, sum = 0;
+ for (uint r = 0; r < s.k; r++) {
+ float v = fabs(w[(ulong)r * s.n + j]);
+ peak = max(peak, v);
+ sum += v;
+ }
+ float l = (peak > 0 && isfinite(peak)) ? exp2(-floor(log2(peak))) : 1.0f;
+ lambda[j] = l;
+ norm[j] = sum * l;
+}
+
+// L = max_j norm[j], into state[0]. One threadgroup of 1 024.
+kernel void ane_norm(device const float *norm [[buffer(0)]], device float *state [[buffer(1)]],
+ constant AneShape &s [[buffer(2)]], uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup float shared[32];
+ float m = 0;
+ for (uint j = tid; j < s.n; j += 1024) m = max(m, norm[j]);
+ m = simd_max(m);
+ if (tid % 32 == 0) shared[tid / 32] = m;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0) {
+ float total = 0;
+ for (uint i = 0; i < 32; i++) total = max(total, shared[i]);
+ state[0] = total;
+ }
+}
+
+// One weight tile, lifted, into its surface [kPart, nPart] fp16.
+kernel void ane_pack_weight(device const float *w [[buffer(0)]], device const float *lambda [[buffer(1)]],
+ device half *tile [[buffer(2)]], constant AneShape &s [[buffer(3)]],
+ uint2 g [[thread_position_in_grid]]) {
+ uint jj = g.x, kk = g.y;
+ if (jj >= s.nPart || kk >= s.kPart) return;
+ uint j = s.n0 + jj;
+ tile[(ulong)kk * s.nPart + jj] = half(w[(ulong)(s.k0 + kk) * s.n + j] * lambda[j]);
+}
+
+// σ_i of each Neural Engine row. One threadgroup of 256 per row.
+kernel void ane_row_scales(device const float *a [[buffer(0)]], device const float *state [[buffer(1)]],
+ device float *sigma [[buffer(2)]], constant AneShape &s [[buffer(3)]],
+ uint i [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup float shared[8];
+ device const float *row = a + (ulong)(s.row0 + i) * s.k;
+ float m = 0;
+ for (uint c = tid; c < s.k; c += 256) m = max(m, fabs(row[c]));
+ m = simd_max(m);
+ if (tid % 32 == 0) shared[tid / 32] = m;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0) {
+ float peak = 0;
+ for (uint q = 0; q < 8; q++) peak = max(peak, shared[q]);
+ float bound = peak * state[0];
+ sigma[i] = (bound > 0 && isfinite(bound)) ? exp2(floor(log2(32768.0f / bound))) : 1.0f;
+ }
+}
+
+// The Neural Engine's rows of A, scaled, into a tile's input surface [rows, kPart] fp16.
+kernel void ane_pack_rows(device const float *a [[buffer(0)]], device const float *sigma [[buffer(1)]],
+ device half *x [[buffer(2)]], constant AneShape &s [[buffer(3)]],
+ uint2 g [[thread_position_in_grid]]) {
+ uint kk = g.x, i = g.y;
+ if (kk >= s.kPart || i >= s.rows) return;
+ x[(ulong)i * s.kPart + kk] = half(a[(ulong)(s.row0 + i) * s.k + s.k0 + kk] * sigma[i]);
+}
+
+// C rows from the outputs of the K tiles of one N range (up to 3), summed in fp32, unscaled.
+kernel void ane_unpack(device const half *y0 [[buffer(0)]], device const half *y1 [[buffer(1)]],
+ device const half *y2 [[buffer(2)]], device const float *lambda [[buffer(3)]],
+ device const float *sigma [[buffer(4)]], device float *c [[buffer(5)]],
+ constant AneShape &s [[buffer(6)]], uint2 g [[thread_position_in_grid]]) {
+ uint jj = g.x, i = g.y;
+ if (jj >= s.nPart || i >= s.rows) return;
+ ulong at = (ulong)i * s.nPart + jj;
+ float sum = float(y0[at]);
+ if (s.kParts > 1) sum += float(y1[at]);
+ if (s.kParts > 2) sum += float(y2[at]);
+ uint j = s.n0 + jj;
+ c[(ulong)(s.row0 + i) * s.n + j] = (sum / lambda[j]) / sigma[i];
+}
diff --git a/engine-mac/Sources/KokonEngine/NeuralEngine.swift b/engine-mac/Sources/KokonEngine/NeuralEngine.swift
index 1de9c17..ae20099 100644
--- a/engine-mac/Sources/KokonEngine/NeuralEngine.swift
+++ b/engine-mac/Sources/KokonEngine/NeuralEngine.swift
@@ -1,4 +1,5 @@
import CoreML
+import Metal
import CoreVideo
import Foundation
@@ -144,3 +145,21 @@ public final class NeuralEngineProduct: @unchecked Sendable {
return Data(header + payload + trailer)
}
}
+
+extension NeuralEngineProduct {
+ /// A GPU buffer over a surface's memory, no copy: the GPU writes the Neural Engine's operands in place.
+ /// The surface stays locked for the product's life (unified memory: the lock only pins the address).
+ public func gpuBuffer(_ surface: CVPixelBuffer, device: MTLDevice) throws -> MTLBuffer {
+ CVPixelBufferLockBaseAddress(surface, [])
+ guard let io = CVPixelBufferGetIOSurface(surface)?.takeUnretainedValue(),
+ let base = CVPixelBufferGetBaseAddress(surface) else { throw Failure.surface("no IOSurface") }
+ // The surface reports its exact size (data + 64 bytes); its memory is whole pages.
+ let page = Int(getpagesize())
+ let length = (IOSurfaceGetAllocSize(io) + page - 1) / page * page
+ guard Int(bitPattern: base) % page == 0,
+ let buffer = device.makeBuffer(bytesNoCopy: base, length: length, options: .storageModeShared, deallocator: nil) else {
+ throw Failure.surface("surface memory at \(base) is not page-aligned (\(length) bytes)")
+ }
+ return buffer
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swift
index 34832f0..aee47f5 100644
--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift
+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift
@@ -44,6 +44,9 @@ public final class QwenDiT {
let rope, modulation, qkNorms, textNorm, textIn, textOut, latent, output: Arena.Slice
private var nextReserve = 0
private var products: [String: MPSMatrixMultiplication] = [:]
+ /// The AMX and the Neural Engine beside the GPU on the large products (`Coexecution.swift`); nil: GPU alone.
+ /// `KOKON_COEXEC=0` turns it off, `KOKON_ANE=0` / `KOKON_AMX=0` one unit, `KOKON_AMX_SHARE` the AMX's rows.
+ let coexecutor: Coexecutor?
private var attentions: [String: GraphAttention] = [:]
/// Seconds per phase of the last evaluation.
public private(set) var timings: [String: Double] = [:]
@@ -69,6 +72,14 @@ public final class QwenDiT {
rope = slices.next()!; modulation = slices.next()!; qkNorms = slices.next()!; textNorm = slices.next()!
textIn = slices.next()!; textOut = slices.next()!; latent = slices.next()!; output = slices.next()!
+ let environment = ProcessInfo.processInfo.environment
+ if environment["KOKON_COEXEC"] != "0", sequence.count >= 1024 {
+ coexecutor = try Coexecutor(context: context,
+ amxShare: Double(environment["KOKON_AMX_SHARE"] ?? "") ?? 0.0625,
+ ane: environment["KOKON_ANE"] != "0", amx: environment["KOKON_AMX"] != "0")
+ } else {
+ coexecutor = nil
+ }
let table = sequence.ropeTable()
arena.pointer(rope).update(from: table, count: table.count)
// Small weights unpacked once on the CPU: the QK norms of every layer, txt_in's norm as (w + 1).
@@ -281,21 +292,33 @@ public final class QwenDiT {
try gpu("dequant", commands) {
try dequant.encode(tensor, of: file, into: arena.buffer, outOffset: reserve.offset, transposed: true, commands: $0)
}
- let key = "\(m)×\(kk)×\(nn)"
- let product: MPSMatrixMultiplication
- if let existing = products[key] { product = existing } else {
- product = MPSMatrixMultiplication(device: context.device, transposeLeft: false, transposeRight: false,
- resultRows: m, resultColumns: nn, interiorColumns: kk, alpha: 1, beta: 0)
- products[key] = product
+ // The GPU's rows: all of them, or its share of a split product.
+ func gpuRows(_ rows: Int) throws {
+ let key = "\(rows)×\(kk)×\(nn)"
+ let product: MPSMatrixMultiplication
+ if let existing = products[key] { product = existing } else {
+ product = MPSMatrixMultiplication(device: context.device, transposeLeft: false, transposeRight: false,
+ resultRows: rows, resultColumns: nn, interiorColumns: kk, alpha: 1, beta: 0)
+ products[key] = product
+ }
+ let left = MPSMatrix(buffer: arena.buffer, offset: a.offset + aRow * kk * 4,
+ descriptor: MPSMatrixDescriptor(rows: rows, columns: kk, rowBytes: kk * 4, dataType: .float32))
+ let right = MPSMatrix(buffer: arena.buffer, offset: reserve.offset,
+ descriptor: MPSMatrixDescriptor(rows: kk, columns: nn, rowBytes: nn * 4, dataType: .float32))
+ let result = MPSMatrix(buffer: arena.buffer, offset: c.offset + cRow * nn * 4,
+ descriptor: MPSMatrixDescriptor(rows: rows, columns: nn, rowBytes: nn * 4, dataType: .float32))
+ try gpu(m < 512 ? "products (few rows)" : "products", commands) {
+ product.encode(commandBuffer: $0, leftMatrix: left, rightMatrix: right, resultMatrix: result)
+ }
}
- let left = MPSMatrix(buffer: arena.buffer, offset: a.offset + aRow * kk * 4,
- descriptor: MPSMatrixDescriptor(rows: m, columns: kk, rowBytes: kk * 4, dataType: .float32))
- let right = MPSMatrix(buffer: arena.buffer, offset: reserve.offset,
- descriptor: MPSMatrixDescriptor(rows: kk, columns: nn, rowBytes: nn * 4, dataType: .float32))
- let result = MPSMatrix(buffer: arena.buffer, offset: c.offset + cRow * nn * 4,
- descriptor: MPSMatrixDescriptor(rows: m, columns: nn, rowBytes: nn * 4, dataType: .float32))
- try gpu(m < 512 ? "products (few rows)" : "products", commands) {
- product.encode(commandBuffer: $0, leftMatrix: left, rightMatrix: right, resultMatrix: result)
+ if let coexecutor, !profiling, coexecutor.applies(m: m, k: kk, n: nn) {
+ func operand(_ slice: Arena.Slice, _ row: Int, _ width: Int) -> (MTLBuffer, Int, UnsafeMutablePointer<Float>) {
+ (arena.buffer, slice.offset + row * width * 4, arena.pointer(slice, floatOffset: row * width))
+ }
+ try coexecutor.encode(a: operand(a, aRow, kk), w: operand(reserve, 0, nn), c: operand(c, cRow, nn),
+ m: m, k: kk, n: nn, commands: commands, gpuRows: gpuRows)
+ } else {
+ try gpuRows(m)
}
}
diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swift
index b905c20..68a5651 100644
--- a/engine-mac/Sources/kokon-engine-mac/main.swift
+++ b/engine-mac/Sources/kokon-engine-mac/main.swift
@@ -46,6 +46,8 @@ do {
let numbers = arguments.dropFirst(2).compactMap { Int($0) }
try Checks.timeDiT(path: arguments[1], grid: numbers.first ?? 64, text: numbers.dropFirst().first ?? 32,
runs: numbers.dropFirst(2).first ?? 3)
+ case "check-ane":
+ try Checks.aneSurfaces()
case "probe":
try Probe.run()
default: