Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

13.2 KB
Raw Blame
#include <metal_stdlib>
using namespace metal;

// The Neural Engine's side of a product C[m, n] = A[m, k] · W[k, n], packed and unpacked on the GPU so the CPU
// stays free for the AMX. The Neural Engine computes in fp16 and flushes values under fp16's smallest normal,
// so (scheme from Siliconed):
//   each weight column j is lifted by λ_j = 2^-floor(log2 max_k |W_kj|), its absmax into [1, 2): exact;
//   each activation row i by σ_i = 2^floor(log2(2^15 / (max_k |A_ik| · L))), L = max_j Σ_k |W_kj| λ_j,
//   so every output |Y_ij| <= max|A_i| σ_i L <= 2^15 stays under fp16's 65 504;
//   C_ij = Y_ij / (σ_i λ_j) in fp32.
// The product runs in tiles of at most 4 096 × 4 096 (one Core ML program each, the Neural Engine's fast range);
// tiles along K are summed here in fp32.

// A few milliseconds of arithmetic per thread and nothing else: the GPU work around the probe's signal
// (`Coexecutor.signalArrivesLate`).
kernel void spin(device float *o [[buffer(0)]], constant uint &n [[buffer(1)]], uint i [[thread_position_in_grid]]) {
    float x = float(i);
    for (uint k = 0; k < n; k++) x = fma(x, 1.0000001f, 0.5f);
    o[i] = x;
}

// fp32 -> fp16 (check-ane).
kernel void pack_half(device const float *src [[buffer(0)]], device half *dst [[buffer(1)]],
                      constant uint &count [[buffer(2)]], uint i [[thread_position_in_grid]]) {
    if (i < count) dst[i] = half(src[i]);
}

struct AneShape {
    uint k;        // full K of the product
    uint n;        // full N
    uint k0, kPart;  // this tile's K range
    uint n0, nPart;  // this tile's N range
    uint row0;     // first row of A / C the Neural Engine takes
    uint rows;     // its rows
    uint kParts;   // tiles along K (unpack)
};

// Per column of W [k, n]: λ_j and the lifted column's L1 norm. A threadgroup takes 64 columns, 16 row slices of
// each (coalesced rows), combined in threadgroup memory: one thread per column walking every row took ~8 ms per
// 4096 × 12288 weight (M2 Pro).
template <typename T>
kernel void ane_columns(device const T *w [[buffer(0)]], device float *lambda [[buffer(1)]],
                        device float *norm [[buffer(2)]], constant AneShape &s [[buffer(3)]],
                        uint2 tg [[threadgroup_position_in_grid]], uint2 t [[thread_position_in_threadgroup]]) {
    threadgroup float peaks[16][64], sums[16][64];
    uint j = tg.x * 64 + t.x;
    float peak = 0, sum = 0;
    if (j < s.n) {
        for (uint r = t.y; r < s.k; r += 16) {
            float v = fabs(float(w[(ulong)r * s.n + j]));
            peak = max(peak, v);
            sum += v;
        }
    }
    peaks[t.y][t.x] = peak;
    sums[t.y][t.x] = sum;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (t.y == 0 && j < s.n) {
        for (uint i = 1; i < 16; i++) { peak = max(peak, peaks[i][t.x]); sum += sums[i][t.x]; }
        float l = (peak > 0 && isfinite(peak)) ? exp2(-floor(log2(peak))) : 1.0f;
        lambda[j] = l;
        norm[j] = sum * l;
    }
}

template [[host_name("ane_columns")]] kernel void ane_columns<float>(device const float *, device float *, device float *,
    constant AneShape &, uint2, uint2);
template [[host_name("ane_columns_h")]] kernel void ane_columns<half>(device const half *, device float *, device float *,
    constant AneShape &, uint2, uint2);

// L = max_j norm[j], into state[0]. One threadgroup of 1 024.
kernel void ane_norm(device const float *norm [[buffer(0)]], device float *state [[buffer(1)]],
                     constant AneShape &s [[buffer(2)]], uint tid [[thread_index_in_threadgroup]]) {
    threadgroup float shared[32];
    float m = 0;
    for (uint j = tid; j < s.n; j += 1024) m = max(m, norm[j]);
    m = simd_max(m);
    if (tid % 32 == 0) shared[tid / 32] = m;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (tid == 0) {
        float total = 0;
        for (uint i = 0; i < 32; i++) total = max(total, shared[i]);
        state[0] = total;
    }
}

// One weight tile, lifted, into its surface [kPart, nPart] fp16. W fp32 or fp16 (`_h`).
template <typename T>
kernel void ane_pack_weight(device const T *w [[buffer(0)]], device const float *lambda [[buffer(1)]],
                            device half *tile [[buffer(2)]], constant AneShape &s [[buffer(3)]],
                            uint2 g [[thread_position_in_grid]]) {
    uint jj = g.x, kk = g.y;
    if (jj >= s.nPart || kk >= s.kPart) return;
    uint j = s.n0 + jj;
    tile[(ulong)kk * s.nPart + jj] = half(float(w[(ulong)(s.k0 + kk) * s.n + j]) * lambda[j]);
}
template [[host_name("ane_pack_weight")]] kernel void ane_pack_weight<float>(device const float *, device const float *,
    device half *, constant AneShape &, uint2);
template [[host_name("ane_pack_weight_h")]] kernel void ane_pack_weight<half>(device const half *, device const float *,
    device half *, constant AneShape &, uint2);

// σ_i of each Neural Engine row, for up to three weights reading the same rows (their L in `state`, `state1`,
// `state2`; the same buffer again for fewer): the largest L bounds them all. One threadgroup of 256 per row.
kernel void ane_row_scales(device const float *a [[buffer(0)]], device const float *state [[buffer(1)]],
                           device float *sigma [[buffer(2)]], constant AneShape &s [[buffer(3)]],
                           device const float *state1 [[buffer(4)]], device const float *state2 [[buffer(5)]],
                           uint i [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
    threadgroup float shared[8];
    device const float *row = a + (ulong)(s.row0 + i) * s.k;
    float m = 0;
    for (uint c = tid; c < s.k; c += 256) m = max(m, fabs(row[c]));
    m = simd_max(m);
    if (tid % 32 == 0) shared[tid / 32] = m;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (tid == 0) {
        float peak = 0;
        for (uint q = 0; q < 8; q++) peak = max(peak, shared[q]);
        float bound = peak * max(state[0], max(state1[0], state2[0]));
        sigma[i] = (bound > 0 && isfinite(bound)) ? exp2(floor(log2(32768.0f / bound))) : 1.0f;
    }
}

// The Neural Engine's rows of A, scaled, into a tile's input surface [rows, kPart] fp16.
kernel void ane_pack_rows(device const float *a [[buffer(0)]], device const float *sigma [[buffer(1)]],
                          device half *x [[buffer(2)]], constant AneShape &s [[buffer(3)]],
                          uint2 g [[thread_position_in_grid]]) {
    uint kk = g.x, i = g.y;
    if (kk >= s.kPart || i >= s.rows) return;
    x[(ulong)i * s.kPart + kk] = half(a[(ulong)(s.row0 + i) * s.k + s.k0 + kk] * sigma[i]);
}

// C rows from the outputs of the K tiles of one N range (up to 3), summed in fp32, unscaled.
kernel void ane_unpack(device const half *y0 [[buffer(0)]], device const half *y1 [[buffer(1)]],
                       device const half *y2 [[buffer(2)]], device const float *lambda [[buffer(3)]],
                       device const float *sigma [[buffer(4)]], device float *c [[buffer(5)]],
                       constant AneShape &s [[buffer(6)]], uint2 g [[thread_position_in_grid]]) {
    uint jj = g.x, i = g.y;
    if (jj >= s.nPart || i >= s.rows) return;
    ulong at = (ulong)i * s.nPart + jj;
    float sum = float(y0[at]);
    if (s.kParts > 1) sum += float(y1[at]);
    if (s.kParts > 2) sum += float(y2[at]);
    uint j = s.n0 + jj;
    c[(ulong)(s.row0 + i) * s.n + j] = (sum / lambda[j]) / sigma[i];
}

// Attention heads to and from the Neural Engine's surfaces [heads, rows, 128] fp16 (heads first), from the rows of
// `stride` floats the products write (head h at columns column0 + 128 h). Surface rows past `valid` are zeros.
struct HeadsShape {
    uint stride;
    uint column0;
    uint rows;      // rows per head in the surface
    uint valid;     // of which exist in the fp32 tensor, from row0
    uint row0;
    uint heads;
    float factor;   // q: 1/sqrt(128)
};

kernel void ane_heads_pack(device const float *src [[buffer(0)]], device half *dst [[buffer(1)]],
                           constant HeadsShape &s [[buffer(2)]], uint3 g [[thread_position_in_grid]]) {
    uint c = g.x, r = g.y, h = g.z;
    if (c >= 128 || r >= s.rows || h >= s.heads) return;
    float x = r < s.valid ? src[(ulong)(s.row0 + r) * s.stride + s.column0 + h * 128 + c] * s.factor : 0.0f;
    dst[((ulong)h * s.rows + r) * 128 + c] = half(x);
}

kernel void ane_heads_unpack(device const half *src [[buffer(0)]], device float *dst [[buffer(1)]],
                             constant HeadsShape &s [[buffer(2)]], uint3 g [[thread_position_in_grid]]) {
    uint c = g.x, r = g.y, h = g.z;
    if (c >= 128 || r >= s.valid || h >= s.heads) return;
    dst[(ulong)(s.row0 + r) * s.stride + s.column0 + h * 128 + c] = float(src[((ulong)h * s.rows + r) * 128 + c]);
}

// The int8 products (`NeuralEngineProduct.int8Program`, for the Neural Engines with an int8 path): W quantized per
// column, s_j = max_k |W_kj| / 127; A per row, t_i = max_k |A_ik| / 127; the program returns y = (A_q · W_q) / K per
// 4 096 × 4 096 tile; C_ij = K · t_i · s_j · Σ_tiles y_ij in fp32. Following libnnc's row-wise int8 GEMM (ccv, BSD-3).

static inline char quantize8(float v, float scale) { return char(clamp(rint(v / scale), -127.0f, 127.0f)); }

// s_j per column of W [k, n] (as ane_columns: 64 columns, 16 row slices each).
template <typename T>
kernel void ane8_columns(device const T *w [[buffer(0)]], device float *scale [[buffer(1)]],
                         constant AneShape &s [[buffer(2)]],
                         uint2 tg [[threadgroup_position_in_grid]], uint2 t [[thread_position_in_threadgroup]]) {
    threadgroup float peaks[16][64];
    uint j = tg.x * 64 + t.x;
    float peak = 0;
    if (j < s.n) {
        for (uint r = t.y; r < s.k; r += 16) peak = max(peak, fabs(float(w[(ulong)r * s.n + j])));
    }
    peaks[t.y][t.x] = peak;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (t.y == 0 && j < s.n) {
        for (uint i = 1; i < 16; i++) peak = max(peak, peaks[i][t.x]);
        scale[j] = (peak > 0 && isfinite(peak)) ? peak / 127.0f : 1.0f;
    }
}
template [[host_name("ane8_columns")]] kernel void ane8_columns<float>(device const float *, device float *, constant AneShape &, uint2, uint2);
template [[host_name("ane8_columns_h")]] kernel void ane8_columns<half>(device const half *, device float *, constant AneShape &, uint2, uint2);

// One weight tile, quantized, into its surface [kPart, nPart] int8.
template <typename T>
kernel void ane8_pack_weight(device const T *w [[buffer(0)]], device const float *scale [[buffer(1)]],
                             device char *tile [[buffer(2)]], constant AneShape &s [[buffer(3)]],
                             uint2 g [[thread_position_in_grid]]) {
    uint jj = g.x, kk = g.y;
    if (jj >= s.nPart || kk >= s.kPart) return;
    uint j = s.n0 + jj;
    tile[(ulong)kk * s.nPart + jj] = quantize8(float(w[(ulong)(s.k0 + kk) * s.n + j]), scale[j]);
}
template [[host_name("ane8_pack_weight")]] kernel void ane8_pack_weight<float>(device const float *, device const float *, device char *, constant AneShape &, uint2);
template [[host_name("ane8_pack_weight_h")]] kernel void ane8_pack_weight<half>(device const half *, device const float *, device char *, constant AneShape &, uint2);

// t_i of each Neural Engine row. One threadgroup of 256 per row.
kernel void ane8_row_scales(device const float *a [[buffer(0)]], device float *scale [[buffer(1)]],
                            constant AneShape &s [[buffer(2)]],
                            uint i [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
    threadgroup float shared[8];
    device const float *row = a + (ulong)(s.row0 + i) * s.k;
    float m = 0;
    for (uint c = tid; c < s.k; c += 256) m = max(m, fabs(row[c]));
    m = simd_max(m);
    if (tid % 32 == 0) shared[tid / 32] = m;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (tid == 0) {
        float peak = 0;
        for (uint q = 0; q < 8; q++) peak = max(peak, shared[q]);
        scale[i] = (peak > 0 && isfinite(peak)) ? peak / 127.0f : 1.0f;
    }
}

// The Neural Engine's rows of A, quantized, into a tile's input surface [rows, kPart] int8.
kernel void ane8_pack_rows(device const float *a [[buffer(0)]], device const float *scale [[buffer(1)]],
                           device char *x [[buffer(2)]], constant AneShape &s [[buffer(3)]],
                           uint2 g [[thread_position_in_grid]]) {
    uint kk = g.x, i = g.y;
    if (kk >= s.kPart || i >= s.rows) return;
    x[(ulong)i * s.kPart + kk] = quantize8(a[(ulong)(s.row0 + i) * s.k + s.k0 + kk], scale[i]);
}

// C rows from the int8 tiles' outputs of one N range (up to 3 along K), scaled back.
kernel void ane8_unpack(device const half *y0 [[buffer(0)]], device const half *y1 [[buffer(1)]],
                        device const half *y2 [[buffer(2)]], device const float *columnScale [[buffer(3)]],
                        device const float *rowScale [[buffer(4)]], device float *c [[buffer(5)]],
                        constant AneShape &s [[buffer(6)]], uint2 g [[thread_position_in_grid]]) {
    uint jj = g.x, i = g.y;
    if (jj >= s.nPart || i >= s.rows) return;
    ulong at = (ulong)i * s.nPart + jj;
    float sum = float(y0[at]);
    if (s.kParts > 1) sum += float(y1[at]);
    if (s.kParts > 2) sum += float(y2[at]);
    uint j = s.n0 + jj;
    c[(ulong)(s.row0 + i) * s.n + j] = sum * float(s.kPart) * rowScale[i] * columnScale[j];
}