Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
12.8 KB
Raw Blame
import Edge0Core
import Foundation
import MLX

public struct DecoderBlockOutput {
    public let hidden: MLXArray
    public let expertIndices: [Int]
    public let expertWeights: [Float]
}

public final class MLAKVCache {
    private var keys: MLXArray?
    private var values: MLXArray?
    public private(set) var count = 0

    public init() {}

    func reset() { keys = nil; values = nil; count = 0 }

    fileprivate func append(keys newKeys: MLXArray, values newValues: MLXArray) -> (MLXArray, MLXArray) {
        let allKeys: MLXArray
        let allValues: MLXArray
        if let keys, let values {
            allKeys = concatenated([keys, newKeys], axis: 2)
            allValues = concatenated([values, newValues], axis: 2)
        } else {
            allKeys = newKeys
            allValues = newValues
        }
        keys = allKeys
        values = allValues
        count += newKeys.dim(2)
        return (allKeys, allValues)
    }
}

/// Correctness-first implementation of one Edge0 MLA decoder layer for a
/// single decode token. The current token attends to itself; `position`
/// exercises the checkpoint's interleaved RoPE convention. KV persistence is
/// intentionally the next milestone.
public final class StreamingMLADecoderBlock {
    private let configuration: Edge0Configuration8B
    private let layer: Int
    private let inputNorm: MLXArray
    private let postAttentionNorm: MLXArray
    private let qANorm: MLXArray
    private let kvANorm: MLXArray
    private let qA: ParallelLinear
    private let qB: ParallelLinear
    private let kvA: ParallelLinear
    private let kvB: ParallelLinear
    private let gate: ParallelLinear
    private let dense: ParallelLinear
    private let moe: StreamingMoE
    public private(set) var prerouterInput: MLXArray?
    public private(set) var prerouterIndices: MLXArray?

    public init(configuration c: Edge0Configuration8B, store: ExpertTensorStore, layer: Int,
                loraStore: ExpertTensorStore? = nil,
                bundleCache: ExpertBundleCache? = nil,
                progress: (String) -> Void = { _ in }) throws {
        let queryHeadDim = c.qkNopeHeadDim + c.qkRopeHeadDim
        guard c.isMLALayer(layer), c.ropeInterleave,
              c.qkRopeHeadDim > 0, c.qkRopeHeadDim % 2 == 0,
              c.attentionHeads == c.keyValueHeads else {
            throw M1Error.invalid("M2 requires an interleaved-RoPE MLA layer with matching Q/KV heads")
        }
        configuration = c
        self.layer = layer
        let base = "model.layers.\(layer)"
        inputNorm = try store.mlxArrayCopying(named: base + ".input_layernorm.weight")
        postAttentionNorm = try store.mlxArrayCopying(named: base + ".post_attention_layernorm.weight")
        qANorm = try store.mlxArrayCopying(named: base + ".attention.q_a_layernorm.weight")
        kvANorm = try store.mlxArrayCopying(named: base + ".attention.kv_a_layernorm.weight")
        progress("norm tensors loaded")
        qA = try Self.linear(store, base + ".attention.q_a_proj", nil)
        progress("q_a loaded")
        qB = try Self.linear(store, base + ".attention.q_b_proj", loraStore)
        progress("q_b loaded")
        kvA = try Self.linear(store, base + ".attention.kv_a_proj_with_mqa", nil)
        progress("kv_a loaded")
        kvB = try Self.linear(store, base + ".attention.kv_b_proj", loraStore)
        progress("kv_b loaded")
        gate = try Self.linear(store, base + ".attention.g_proj", loraStore)
        progress("gate loaded")
        dense = try Self.linear(store, base + ".attention.dense", loraStore)
        progress("dense loaded")
        moe = try StreamingMoE(configuration: c, store: store, layer: layer,
                               bundleCache: bundleCache)
        progress("moe loaded")
        guard qA.inputSize == c.hiddenSize, qA.outputSize == c.qLoraRank,
              qB.inputSize == c.qLoraRank, qB.outputSize == c.attentionHeads * queryHeadDim,
              kvA.inputSize == c.hiddenSize, kvA.outputSize == c.kvLoraRank + c.qkRopeHeadDim,
              kvB.inputSize == c.kvLoraRank,
              kvB.outputSize == c.attentionHeads * (c.qkNopeHeadDim + c.vHeadDim),
              gate.outputSize == c.attentionHeads,
              dense.inputSize == c.attentionHeads * c.vHeadDim,
              dense.outputSize == c.hiddenSize else {
            throw M1Error.invalid("M2 attention tensor dimensions disagree with config")
        }
        eval(inputNorm, postAttentionNorm, qANorm, kvANorm)
        progress("norm tensors evaluated")
    }

    private static func linear(_ store: ExpertTensorStore, _ prefix: String,
                               _ loraStore: ExpertTensorStore?) throws -> ParallelLinear {
        let base = try QuantizedExpertLinear(
            weight: store.mlxArrayCopying(named: prefix + ".weight"),
            scales: store.mlxArrayCopying(named: prefix + ".scales"),
            biases: store.mlxArrayCopying(named: prefix + ".biases")
        )
        return try ParallelLinear(base: base, loraStore: loraStore, prefix: prefix)
    }

    public func callAsFunction(_ input: MLXArray, position: Int,
                               cache: MLAKVCache? = nil,
                               predictedRoute: PredictedMoERoute? = nil) throws -> DecoderBlockOutput {
        let c = configuration
        guard input.shape == [1, c.hiddenSize], position >= 0,
              cache == nil || cache?.count == position else {
            throw M1Error.invalid("M2 accepts one token [1,\(c.hiddenSize)] and a nonnegative position")
        }
        let normalized = MLXFast.rmsNorm(input, weight: inputNorm, eps: c.rmsNormEps)
        let attention = try selfAttention(normalized, position: position, cache: cache)
        let residual = input + attention
        let moeInput = MLXFast.rmsNorm(residual, weight: postAttentionNorm, eps: c.rmsNormEps)
        let moeOutput = if let predictedRoute {
            try moe.predicted(moeInput, route: predictedRoute)
        } else {
            try moe(moeInput)
        }
        prerouterInput = moeInput
        prerouterIndices = moeOutput.routeIndices
        let hidden = residual + moeOutput.hidden
        return DecoderBlockOutput(hidden: hidden, expertIndices: moeOutput.expertIndices,
                                  expertWeights: moeOutput.expertWeights)
    }

    /// Official multi-token Edge0 8B MLA path used by chunked prefill.
    public func prefill(_ input: MLXArray, position: Int,
                        cache: MLAKVCache) throws -> DecoderBlockOutput {
        let c = configuration
        guard input.ndim == 2, input.dim(0) > 1, input.dim(1) == c.hiddenSize,
              position >= 0, cache.count == position else {
            throw M1Error.invalid("MLA prefill accepts [T,\(c.hiddenSize)], T > 1")
        }
        let x = input
        let normalized = MLXFast.rmsNorm(x, weight: inputNorm, eps: c.rmsNormEps)
        let attention = try selfAttentionPrefill(
            normalized, position: position, cache: cache)
        let residual = x + attention
        let moeInput = MLXFast.rmsNorm(
            residual, weight: postAttentionNorm, eps: c.rmsNormEps)
        let moeOutput = try moe.fullLayerPrefill(moeInput)
        prerouterInput = moeInput
        prerouterIndices = moeOutput.routeIndices
        let hidden = residual + moeOutput.hidden
        return DecoderBlockOutput(hidden: hidden,
                                  expertIndices: moeOutput.expertIndices,
                                  expertWeights: moeOutput.expertWeights)
    }

    public func loadFullExpertLayer() throws { try moe.loadFullLayer() }
    public func clearFullExpertLayer() { moe.clearFullLayer() }
    public func stage(experts: [Int]) throws { try moe.stage(experts: experts) }
    public func resetStaging() { moe.resetStaging() }
    public var moeStats: StreamingMoEStats { moe.stats }

    private func selfAttention(_ x: MLXArray, position: Int, cache: MLAKVCache?) throws -> MLXArray {
        let c = configuration
        let queryHeadDim = c.qkNopeHeadDim + c.qkRopeHeadDim
        let qLatent = MLXFast.rmsNorm(try qA(x), weight: qANorm, eps: c.rmsNormEps)
        let q = try qB(qLatent).reshaped([1, 1, c.attentionHeads, queryHeadDim])
            .transposed(0, 2, 1, 3)
        let qParts = split(q, indices: [c.qkNopeHeadDim], axis: -1)

        let compressedParts = split(try kvA(x), indices: [c.kvLoraRank], axis: -1)
        let kvLatent = MLXFast.rmsNorm(compressedParts[0], weight: kvANorm, eps: c.rmsNormEps)
        let kv = try kvB(kvLatent)
            .reshaped([1, 1, c.attentionHeads, c.qkNopeHeadDim + c.vHeadDim])
            .transposed(0, 2, 1, 3)
        let kvParts = split(kv, indices: [c.qkNopeHeadDim], axis: -1)
        var kRope = compressedParts[1].reshaped([1, 1, 1, c.qkRopeHeadDim])

        let qRope = interleavedRope(qParts[1], position: position)
        kRope = interleavedRope(kRope, position: position)
        kRope = broadcast(kRope, to: [1, c.attentionHeads, 1, c.qkRopeHeadDim])
        let queries = concatenated([qParts[0], qRope], axis: -1)
        var keys = concatenated([kvParts[0], kRope], axis: -1)
        var values = kvParts[1]
        if let cache {
            (keys, values) = cache.append(keys: keys, values: values)
        }
        var output = MLXFast.scaledDotProductAttention(
            queries: queries, keys: keys, values: values,
            scale: 1.0 / Float(queryHeadDim).squareRoot(), mask: .none
        )
        output = output.transposed(0, 2, 1, 3).reshaped([1, c.attentionHeads, c.vHeadDim])
        let gates = sigmoid(try gate(x)).reshaped([1, c.attentionHeads, 1])
        output = (output * gates).reshaped([1, c.attentionHeads * c.vHeadDim])
        return try dense(output)
    }

    private func interleavedRope(_ x: MLXArray, position: Int) -> MLXArray {
        interleavedRope(x, startPosition: position, tokenCount: 1)
    }

    private func selfAttentionPrefill(_ x: MLXArray, position: Int,
                                      cache: MLAKVCache) throws -> MLXArray {
        let c = configuration
        let tokens = x.dim(0)
        let queryHeadDim = c.qkNopeHeadDim + c.qkRopeHeadDim
        let qLatent = MLXFast.rmsNorm(
            try qA(x), weight: qANorm, eps: c.rmsNormEps)
        let q = try qB(qLatent)
            .reshaped([1, tokens, c.attentionHeads, queryHeadDim])
            .transposed(0, 2, 1, 3)
        let qParts = split(q, indices: [c.qkNopeHeadDim], axis: -1)

        let compressedParts = split(
            try kvA(x), indices: [c.kvLoraRank], axis: -1)
        let kvLatent = MLXFast.rmsNorm(
            compressedParts[0], weight: kvANorm, eps: c.rmsNormEps)
        let kv = try kvB(kvLatent)
            .reshaped([1, tokens, c.attentionHeads,
                       c.qkNopeHeadDim + c.vHeadDim])
            .transposed(0, 2, 1, 3)
        let kvParts = split(kv, indices: [c.qkNopeHeadDim], axis: -1)
        var kRope = compressedParts[1]
            .reshaped([1, 1, tokens, c.qkRopeHeadDim])

        let qRope = interleavedRope(
            qParts[1], startPosition: position, tokenCount: tokens)
        kRope = interleavedRope(
            kRope, startPosition: position, tokenCount: tokens)
        kRope = broadcast(
            kRope,
            to: [1, c.attentionHeads, tokens, c.qkRopeHeadDim])
        let queries = concatenated([qParts[0], qRope], axis: -1)
        let newKeys = concatenated([kvParts[0], kRope], axis: -1)
        let (keys, values) = cache.append(keys: newKeys, values: kvParts[1])
        var output = MLXFast.scaledDotProductAttention(
            queries: queries, keys: keys, values: values,
            scale: 1.0 / Float(queryHeadDim).squareRoot(), mask: .causal)
        output = output.transposed(0, 2, 1, 3)
            .reshaped([tokens, c.attentionHeads, c.vHeadDim])
        let gates = sigmoid(try gate(x))
            .reshaped([tokens, c.attentionHeads, 1])
        output = (output * gates)
            .reshaped([tokens, c.attentionHeads * c.vHeadDim])
        return try dense(output)
    }

    private func interleavedRope(_ x: MLXArray, startPosition: Int,
                                 tokenCount: Int) -> MLXArray {
        let pairs = configuration.qkRopeHeadDim / 2
        let angles = (0..<tokenCount).flatMap { token in
            (0..<pairs).map { i in
                Float(startPosition + token)
                    / pow(configuration.ropeTheta,
                          Float(2 * i) / Float(configuration.qkRopeHeadDim))
            }
        }
        let cosines = MLXArray(
            angles.map { cos($0) }, [1, 1, tokenCount, pairs, 1])
        let sines = MLXArray(
            angles.map { sin($0) }, [1, 1, tokenCount, pairs, 1])
        let pairParts = split(
            x.reshaped([1, x.dim(1), tokenCount, pairs, 2]),
            indices: [1], axis: -1)
        let even = pairParts[0]
        let odd = pairParts[1]
        return concatenated(
            [even * cosines - odd * sines, even * sines + odd * cosines],
            axis: -1)
            .reshaped(x.shape)
    }
}