Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
4.3 KB
Raw Blame
import MLX

/// Language-level Swift port of mlx-lm's vector-gated delta Metal kernel used
/// by the official Edge0 8B KDA prefill path. The Metal body, grid and
/// template parameters intentionally retain the upstream structure.
public enum OfficialGatedDelta {
    private static let vectorKernel = MLXFast.metalKernel(
        name: "edge0_gated_delta_step_vec",
        inputNames: ["q", "k", "v", "g", "beta", "state_in", "T"],
        outputNames: ["y", "state_out"],
        source: """
            auto n = thread_position_in_grid.z;
            auto b_idx = n / Hv;
            auto hv_idx = n % Hv;
            auto hk_idx = hv_idx / (Hv / Hk);
            constexpr int n_per_t = Dk / 32;

            // q, k: [B, T, Hk, Dk]
            auto q_ = q + b_idx * T * Hk * Dk + hk_idx * Dk;
            auto k_ = k + b_idx * T * Hk * Dk + hk_idx * Dk;

            // v, y: [B, T, Hv, Dv]
            auto v_ = v + b_idx * T * Hv * Dv + hv_idx * Dv;
            y += b_idx * T * Hv * Dv + hv_idx * Dv;

            auto dk_idx = thread_position_in_threadgroup.x;
            auto dv_idx = thread_position_in_grid.y;

            // state_in, state_out: [B, Hv, Dv, Dk]
            auto i_state = state_in + (n * Dv + dv_idx) * Dk;
            auto o_state = state_out + (n * Dv + dv_idx) * Dk;

            float state[n_per_t];
            for (int i = 0; i < n_per_t; ++i) {
              auto s_idx = n_per_t * dk_idx + i;
              state[i] = static_cast<float>(i_state[s_idx]);
            }

            // g: [B, T, Hv, Dk]
            auto g_ = g + (b_idx * T * Hv + hv_idx) * Dk;
            auto beta_ = beta + b_idx * T * Hv;

            for (int t = 0; t < T; ++t) {
              float kv_mem = 0.0f;
              for (int i = 0; i < n_per_t; ++i) {
                auto s_idx = n_per_t * dk_idx + i;
                state[i] = state[i] * g_[s_idx];
                kv_mem += state[i] * k_[s_idx];
              }
              kv_mem = simd_sum(kv_mem);

              auto delta = (v_[dv_idx] - kv_mem) * beta_[hv_idx];

              float out = 0.0f;
              for (int i = 0; i < n_per_t; ++i) {
                auto s_idx = n_per_t * dk_idx + i;
                state[i] = state[i] + k_[s_idx] * delta;
                out += state[i] * q_[s_idx];
              }
              out = simd_sum(out);
              if (thread_index_in_simdgroup == 0) {
                y[dv_idx] = static_cast<InT>(out);
              }

              q_ += Hk * Dk;
              k_ += Hk * Dk;
              v_ += Hv * Dv;
              y += Hv * Dv;
              g_ += Hv * Dk;
              beta_ += Hv;
            }
            for (int i = 0; i < n_per_t; ++i) {
              auto s_idx = n_per_t * dk_idx + i;
              o_state[s_idx] = static_cast<InT>(state[i]);
            }
        """)

    public static func callAsFunction(q: MLXArray, k: MLXArray, v: MLXArray,
                                      decay: MLXArray, beta: MLXArray,
                                      state: MLXArray) throws
        -> (output: MLXArray, state: MLXArray) {
        guard q.ndim == 4, k.shape == q.shape, v.ndim == 4,
              decay.shape == q.shape,
              beta.shape == [q.dim(0), q.dim(1), v.dim(2)],
              q.dim(2) > 0, v.dim(2) % q.dim(2) == 0,
              q.dim(3) > 0, q.dim(3) % 32 == 0,
              state.shape == [q.dim(0), v.dim(2), v.dim(3), q.dim(3)],
              q.dtype == k.dtype, q.dtype == v.dtype,
              q.dtype == decay.dtype, q.dtype == state.dtype else {
            throw M1Error.invalid("Invalid official gated-delta tensor shapes")
        }
        let batch = q.dim(0)
        let tokens = q.dim(1)
        let keyHeads = q.dim(2)
        let keyDimension = q.dim(3)
        let valueHeads = v.dim(2)
        let valueDimension = v.dim(3)
        let outputs = vectorKernel(
            [q, k, v, decay, beta, state, tokens],
            template: [
                ("InT", q.dtype), ("Dk", keyDimension),
                ("Dv", valueDimension), ("Hk", keyHeads),
                ("Hv", valueHeads),
            ],
            grid: (32, valueDimension, batch * valueHeads),
            threadGroup: (32, 4, 1),
            outputShapes: [
                [batch, tokens, valueHeads, valueDimension],
                state.shape,
            ],
            outputDTypes: [q.dtype, q.dtype])
        return (outputs[0], outputs[1])
    }
}