Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.
TypeScript ai-photo-editorcomfyui-alternativegenerative-fillimage-editingimage-generationinpaintinglinuxlocal-aimacosmcpmcp-servernano-banana-alternativeofflinephoto-editorprivacyqwen-imageqwen-image-2-1stable-diffusion-cppvulkanwindows
Mac engine: fp16-operand attention as an option; VAE compiled ahead, kept between images
Gabriele Lozupone589e7773c696
diff --git a/engine-mac/Sources/KokonEngine/AttentionProbe.swift b/engine-mac/Sources/KokonEngine/AttentionProbe.swiftindex 2a58623..0db6637 100644--- a/engine-mac/Sources/KokonEngine/AttentionProbe.swift+++ b/engine-mac/Sources/KokonEngine/AttentionProbe.swift@@ -51,10 +51,11 @@ extension Probe {// The GPU's flash kernel, all heads.struct FlashParams { var queries, keys, causal, stride: UInt32; var scale: Float; var headBase: UInt32 }+ for kernel in ["flash_attention", "flash_attention_half"] {let gpu = try best(3) {let commands = context.queue.makeCommandBuffer()!let e = commands.makeComputeCommandEncoder()!- e.setComputePipelineState(try context.pipeline("flash_attention"))+ e.setComputePipelineState(try context.pipeline(kernel))e.setBuffer(qb, offset: offset * stride * 4, index: 0); e.setBuffer(kb, offset: 0, index: 1)e.setBuffer(vb, offset: 0, index: 2); e.setBuffer(ob, offset: offset * stride * 4, index: 3)var p = FlashParams(queries: UInt32(queries), keys: UInt32(keys), causal: 0, stride: UInt32(stride),@@ -66,8 +67,9 @@ extension Probe {return try MetalContext.finish(commands, "flash")}let gpuError = error { i, c in f(ob)[(offset + i) * stride + c] }- print(String(format: "GPU flash kernel, 32 heads: %.1f ms, %.2f ms/head, %.2f TFLOP/s; head 0 rel %.1e",+ print(String(format: "GPU %@, 32 heads: %.1f ms, %.2f ms/head, %.2f TFLOP/s; head 0 rel %.1e", kernel,gpu * 1e3, gpu * 1e3 / Double(heads), 4 * Double(queries) * Double(keys) * Double(d) * Double(heads) / gpu / 1e12, gpuError))+ }if ProcessInfo.processInfo.environment["KOKON_PROBE_GPU_ONLY"] == "1" { return }// The Neural Engine.diff --git a/engine-mac/Sources/KokonEngine/Balancer.swift b/engine-mac/Sources/KokonEngine/Balancer.swiftindex 362be3f..9d311fc 100644--- a/engine-mac/Sources/KokonEngine/Balancer.swift+++ b/engine-mac/Sources/KokonEngine/Balancer.swift@@ -101,10 +101,11 @@ extension Coexecutor {}if lastTimeline.contains(where: { $0.kind == "attention" }), HeadSplit.fromEnvironment() == nil {- let maxAne = aneEnabled && balancer.aneAttention ? QwenDiT.heads : 0+ // The GPU keeps at least one head (its text segments run there anyway).+ let maxAne = aneEnabled && balancer.aneAttention ? QwenDiT.heads - Coexecutor.aneHeadGroup : 0var best = headsfor ane in stride(from: 0, through: maxAne, by: Coexecutor.aneHeadGroup) {- for cpu in 0...(amxEnabled ? QwenDiT.heads - 1 - ane : 0) where ane + cpu < QwenDiT.heads {+ for cpu in 0...(amxEnabled ? QwenDiT.heads - 1 - ane : 0) {let h = HeadSplit(gpu: QwenDiT.heads - ane - cpu, ane: ane, cpu: cpu)if balancer.attentionCost(h) < balancer.attentionCost(best) { best = h }}diff --git a/engine-mac/Sources/KokonEngine/Coexecution.swift b/engine-mac/Sources/KokonEngine/Coexecution.swiftindex fa3610d..4b09c9f 100644--- a/engine-mac/Sources/KokonEngine/Coexecution.swift+++ b/engine-mac/Sources/KokonEngine/Coexecution.swift@@ -305,6 +305,19 @@ final class Timeline: @unchecked Sendable {if groups[e.kind] == nil { kinds.append(e.kind) }groups[e.kind, default: []].append(e)}+ // Between operations: from the last unit's end of one to the GPU's release of the next (the GPU alone:+ // unpacking, the elementwise kernels, the next weight's unpacking and packing, the waits' latency).+ var gaps: [String: (count: Int, seconds: Double)] = [:]+ for (a, b) in zip(entries, entries.dropFirst()) {+ let end = max(a.gpu, a.shares.amx > 0 ? a.amx.end : 0, a.shares.ane > 0 ? a.ane.end : 0)+ let key = "\(a.kind) → \(b.kind)"+ gaps[key] = ((gaps[key]?.count ?? 0) + 1, (gaps[key]?.seconds ?? 0) + max(0, b.released - end))+ }+ let between = gaps.sorted { $0.value.seconds > $1.value.seconds }.map {+ String(format: " %-26@ ×%3d %6.2f ms each, %5.2f s", $0.key as NSString, $0.value.count,+ $0.value.seconds / Double($0.value.count) * 1e3, $0.value.seconds)+ }+ let total = gaps.values.reduce(0) { $0 + $1.seconds }return kinds.map { kind inlet g = groups[kind]!, n = Double(g.count), s = g[0].sharesfunc mean(_ f: (Entry) -> Double) -> Double { g.map(f).reduce(0, +) / n * 1e3 }@@ -313,7 +326,7 @@ final class Timeline: @unchecked Sendable {mean { $0.amx.seconds }, mean { $0.shares.amx > 0 ? $0.amx.start - $0.released : 0 },mean { $0.ane.seconds }, mean { $0.shares.ane > 0 ? $0.ane.start - $0.released : 0 },mean { max($0.walls.gpu, $0.walls.amx, $0.walls.ane) })- }.joined(separator: "\n")+ }.joined(separator: "\n") + String(format: "\n between operations %.2f s:\n", total) + between.joined(separator: "\n")}}diff --git a/engine-mac/Sources/KokonEngine/Kernels/attention.metal b/engine-mac/Sources/KokonEngine/Kernels/attention.metalindex 263faf7..077de4e 100644--- a/engine-mac/Sources/KokonEngine/Kernels/attention.metal+++ b/engine-mac/Sources/KokonEngine/Kernels/attention.metal@@ -148,3 +148,114 @@ kernel void flash_attention(device const float *q [[buffer(0)]], device const fl}}}++// The same with q, k, v and the probabilities in fp16 (scores, softmax and O accumulated in fp32; K and V tiles of 64+// keys side by side in 32 KB), `KOKON_ATTENTION=half`: 94 ms against 120 at 1024² (M2 Pro), head 0 within 6.3e-4 of+// exact, the velocity within 6.0e-4 of fp64 at 256² (fp32 kernel 1.0e-5). Only V and the probabilities in fp16+// (q·k fp32): 118 ms, 2.2e-4, so not kept. Off until decided.+static inline thread half2 &elementsHalf(thread simdgroup_half8x8 &m) {+ return *(thread half2 *)&m.thread_elements();+}++kernel void flash_attention_half(device const float *q [[buffer(0)]], device const float *k [[buffer(1)]],+ device const float *v [[buffer(2)]], device float *out [[buffer(3)]],+ constant FlashParams &p [[buffer(4)]],+ uint2 tg [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]],+ uint sg [[simdgroup_index_in_threadgroup]],+ uint lane [[thread_index_in_simdgroup]]) {+ threadgroup half ktile[BK * D]; // 16 KB; first each SIMD group's 8 query rows, once+ threadgroup half vtile[BK * D]; // 16 KB+ threadgroup half *qstage = ktile + sg * 8 * D;++ const uint head = p.headBase + tg.y;+ const uint q0 = tg.x * BQ + sg * 8;+ const uint stride = p.stride;+ device const float *kh = k + head * D;+ device const float *vh = v + head * D;++ for (uint i = lane; i < 8 * D; i += 32) qstage[i] = half(q[(ulong)(q0 + i / D) * stride + head * D + i % D]);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ simdgroup_half8x8 Q[16];+ simdgroup_float8x8 O[16];+ for (uint d = 0; d < 16; d++) {+ simdgroup_load(Q[d], qstage + 8 * d, D);+ O[d] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const uint row = (lane / 16) * 4 + (lane / 2) % 4;+ const uint col = ((lane / 8) % 2) * 4 + (lane % 2) * 2;+ const uint qi = q0 + row;+ const uint offset = p.keys - p.queries;+ const int limit = p.causal ? int(offset + qi) : int(p.keys) - 1;+ float m = -INFINITY, l = 0.0f;++ uint kEnd = p.keys;+ if (p.causal) kEnd = min(p.keys, offset + min(tg.x * BQ + BQ - 1, p.queries - 1) + 1);++ for (uint j0 = 0; j0 < kEnd; j0 += BK) {+ for (uint i = tid; i < BK * D / 4; i += 256) {+ uint r = i / (D / 4), c = i % (D / 4);+ ((threadgroup half4 *)ktile)[i] = half4(*((device const float4 *)(kh + (ulong)(j0 + r) * stride) + c));+ ((threadgroup half4 *)vtile)[i] = half4(*((device const float4 *)(vh + (ulong)(j0 + r) * stride) + c));+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ simdgroup_float8x8 S[8];+ for (uint c = 0; c < 8; c++) S[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ for (uint d = 0; d < 16; d++) {+ for (uint c = 0; c < 8; c++) {+ simdgroup_half8x8 Kt;+ simdgroup_load(Kt, ktile + 8 * c * D + 8 * d, D, ulong2(0, 0), true);+ simdgroup_multiply_accumulate(S[c], Q[d], Kt, S[c]);+ }+ }++ float2 s[8];+ float blockMax = -INFINITY;+ for (uint c = 0; c < 8; c++) {+ s[c] = elements(S[c]) * p.scale;+ int j = int(j0 + 8 * c + col);+ if (j > limit) s[c].x = -INFINITY;+ if (j + 1 > limit) s[c].y = -INFINITY;+ blockMax = max(blockMax, max(s[c].x, s[c].y));+ }+ blockMax = max(blockMax, simd_shuffle_xor(blockMax, 1));+ blockMax = max(blockMax, simd_shuffle_xor(blockMax, 8));+ float mNew = max(m, blockMax);+ float alpha = (mNew == -INFINITY) ? 1.0f : exp2(m - mNew);+ float sum = 0.0f;+ simdgroup_half8x8 P[8];+ for (uint c = 0; c < 8; c++) {+ float2 e = (mNew == -INFINITY) ? float2(0.0f) : exp2(s[c] - mNew);+ elementsHalf(P[c]) = half2(e);+ sum += e.x + e.y;+ }+ sum += simd_shuffle_xor(sum, 1);+ sum += simd_shuffle_xor(sum, 8);+ l = l * alpha + sum;+ m = mNew;+ if (!simd_all(alpha == 1.0f)) {+ for (uint d = 0; d < 16; d++) elements(O[d]) *= alpha;+ }+ for (uint d = 0; d < 16; d++) {+ for (uint c = 0; c < 8; c++) {+ simdgroup_half8x8 V;+ simdgroup_load(V, vtile + 8 * c * D + 8 * d, D);+ simdgroup_multiply_accumulate(O[d], P[c], V, O[d]);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ if (qi < p.queries) {+ const float inv = 1.0f / l;+ device float *o = out + (ulong)qi * stride + head * D + col;+ for (uint d = 0; d < 16; d++) {+ float2 e = elements(O[d]) * inv;+ o[8 * d] = e.x;+ o[8 * d + 1] = e.y;+ }+ }+}diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swiftindex d50039c..20f6f53 100644--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift@@ -435,8 +435,10 @@ public final class QwenDiT {}}- /// Attention through `Kernels/attention.metal` (default) or MPSGraph (`KOKON_ATTENTION=graph`, the check).+ /// Attention through `Kernels/attention.metal` (default) or MPSGraph (`KOKON_ATTENTION=graph`, the check);+ /// `KOKON_ATTENTION=half`: the kernel with fp16 operands (faster, 6e-4 of fp64; not the default until decided).let graphAttention = ProcessInfo.processInfo.environment["KOKON_ATTENTION"] == "graph"+ let flashKernel = ProcessInfo.processInfo.environment["KOKON_ATTENTION"] == "half" ? "flash_attention_half" : "flash_attention"struct FlashParams { var queries, keys, causal, stride: UInt32; var scale: Float; var headBase: UInt32 }/// Queries `[queries, queries + count)` against keys `[0, keys)`, into the same rows of `attended`. The target's@@ -475,7 +477,7 @@ public final class QwenDiT {func flash(queries: Int, count: Int, keys: Int, causal: Bool, heads: Int, commands: MTLCommandBuffer) throws {guard heads > 0 else { return }let row = QwenDiT.dim * 4- try compute("flash_attention", commands: commands) { e in+ try compute(flashKernel, commands: commands) { e ine.setBuffer(arena.buffer, offset: q.offset + queries * row, index: 0)e.setBuffer(arena.buffer, offset: k.offset, index: 1)e.setBuffer(arena.buffer, offset: v.offset, index: 2)diff --git a/engine-mac/Sources/KokonEngine/QwenPipeline.swift b/engine-mac/Sources/KokonEngine/QwenPipeline.swiftindex d7db681..843a016 100644--- a/engine-mac/Sources/KokonEngine/QwenPipeline.swift+++ b/engine-mac/Sources/KokonEngine/QwenPipeline.swift@@ -21,8 +21,9 @@ public final class QwenPipeline {public var log: (String) -> Void = { print($0) }/// Seconds per phase of the last run: encode, steps (each), dit load, vae, total.public private(set) var timings: [String: Double] = [:]- /// Kept between images: its first decode of a size costs ~3.7 s more than the next ones at 1024² (M2 Pro), GPU- /// time (~1 s of CPU), so it cannot hide under the DiT; a long-lived engine pays it once per size.+ /// Kept between images: its first decode of a size costs ~4 s more than the next ones at 1024² (M2 Pro). Compiled+ /// beside the DiT (from the second step) it slowed the steps by ~9 s to save 1.9: a long-lived engine pays it+ /// once per size, or ahead while idle (`QwenVAE.prepare`).private var vae: QwenVAE?public init(encoder: String, dit: String, vae: String) throws {diff --git a/engine-mac/Sources/KokonEngine/QwenVAE.swift b/engine-mac/Sources/KokonEngine/QwenVAE.swiftindex b242a57..d5e97d9 100644--- a/engine-mac/Sources/KokonEngine/QwenVAE.swift+++ b/engine-mac/Sources/KokonEngine/QwenVAE.swift@@ -35,10 +35,12 @@ public final class QwenVAE {let context: MetalContextlet file: SafetensorsFile- /// One compiled graph per latent size. Building and compiling the 1024² one takes 0.3 s; its first run then costs- /// ~3.7 s more than the next ones (M2 Pro), GPU time with ~1 s of CPU, so the pipeline keeps its VAE between- /// images rather than building it beside the DiT. fp16 convolutions (norms and attention in fp32) were wrong- /// (2.7e-1: an overflow) and 5.7× slower, MPSGraph trying to move them to the Neural Engine: the VAE stays fp32.+ /// One compiled graph per latent size, compiled to the end (`waitForCompilationCompletion`): 2.7 s at 1024² (M2+ /// Pro), mostly the shader compiler, which otherwise ran inside the first decode (6.4 s against 2.75 s; with it+ /// up front the first decode is 4.15 s). Beside the DiT it slows the steps far more than it saves: the pipeline+ /// keeps its VAE between images, and an idle engine can prepare ahead. fp16 convolutions (norms and attention in+ /// fp32) were wrong (2.7e-1: an overflow) and 5.7× slower, MPSGraph trying to move them to the Neural Engine: the+ /// VAE stays fp32.private var executables: [String: MPSGraphExecutable] = [:]private let lock = NSLock()@@ -73,8 +75,10 @@ public final class QwenVAE {if let e = executables[key] { return e }let built = try build(height: h, width: w)let shape = MPSGraphShapedType(shape: [1, 64, NSNumber(value: h), NSNumber(value: w)], dataType: .float32)+ let descriptor = MPSGraphCompilationDescriptor()+ descriptor.waitForCompilationCompletion = truelet e = built.graph.compile(with: MPSGraphDevice(mtlDevice: context.device), feeds: [built.input: shape],- targetTensors: [built.output], targetOperations: nil, compilationDescriptor: nil)+ targetTensors: [built.output], targetOperations: nil, compilationDescriptor: descriptor)executables[key] = ereturn e}