Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
13.9 KB
Raw Blame
import Edge0Core
import MLX

public final class KDAStateCache {
    fileprivate var qConv: MLXArray?
    fileprivate var kConv: MLXArray?
    fileprivate var vConv: MLXArray?
    fileprivate var recurrent: MLXArray?
    public fileprivate(set) var count = 0

    public init() {}

    func reset() {
        qConv = nil; kConv = nil; vConv = nil; recurrent = nil; count = 0
    }
}

/// Correctness-first one-token Kimi Delta Attention decoder layer.
/// The recurrent update is expressed with MLX array operations; a fused Metal
/// kernel can replace it after numerical parity is established.
public final class StreamingKDADecoderBlock {
    private let c: Edge0Configuration8B
    private let layer: Int
    private let inputNorm: MLXArray
    private let postAttentionNorm: MLXArray
    private let qConvWeight: MLXArray
    private let kConvWeight: MLXArray
    private let vConvWeight: MLXArray
    private let aLog: MLXArray
    private let dtBias: MLXArray
    private let outputNorm: MLXArray
    private let qProj: ParallelLinear
    private let kProj: ParallelLinear
    private let vProj: ParallelLinear
    private let fProj: ParallelLinear
    private let gProj: ParallelLinear
    private let bProj: ParallelLinear
    private let outputProj: ParallelLinear
    private let moe: StreamingMoE?
    private let denseMLP: ParallelExpert?
    public private(set) var prerouterInput: MLXArray?
    public private(set) var prerouterIndices: MLXArray?

    public init(configuration c: Edge0Configuration8B, store: ExpertTensorStore, layer: Int,
                loraStore: ExpertTensorStore? = nil,
                bundleCache: ExpertBundleCache? = nil,
                progress: (String) -> Void = { _ in }) throws {
        guard !c.isMLALayer(layer), layer >= 0, layer < c.hiddenLayers,
              c.kdaSafeGate, c.shortConvKernelSize == 4,
              c.headDim * c.attentionHeads == 2048 else {
            throw M1Error.invalid("KDA decoder requires a KDA layer with the preview safe-gate configuration")
        }
        self.c = c
        self.layer = layer
        let base = "model.layers.\(layer)"
        let attention = base + ".attention"
        inputNorm = try store.mlxArrayCopying(named: base + ".input_layernorm.weight")
        postAttentionNorm = try store.mlxArrayCopying(named: base + ".post_attention_layernorm.weight")
        qConvWeight = try store.mlxArrayCopying(named: attention + ".q_conv1d.weight")
        kConvWeight = try store.mlxArrayCopying(named: attention + ".k_conv1d.weight")
        vConvWeight = try store.mlxArrayCopying(named: attention + ".v_conv1d.weight")
        aLog = try store.mlxArrayCopying(named: attention + ".A_log")
        dtBias = try store.mlxArrayCopying(named: attention + ".dt_bias")
        outputNorm = try store.mlxArrayCopying(named: attention + ".o_norm.weight")
        progress("KDA state tensors loaded")
        qProj = try Self.linear(store, attention + ".q_proj", loraStore)
        kProj = try Self.linear(store, attention + ".k_proj", loraStore)
        vProj = try Self.linear(store, attention + ".v_proj", loraStore)
        fProj = try Self.linear(store, attention + ".f_proj", loraStore)
        gProj = try Self.linear(store, attention + ".g_proj", loraStore)
        bProj = try Self.linear(store, attention + ".b_proj", loraStore)
        outputProj = try Self.linear(store, attention + ".o_proj", loraStore)
        if layer < c.firstKDenseReplace {
            denseMLP = try ParallelExpert(
                base: store.loadDenseMLP(layer: layer), loraStore: loraStore,
                prefix: base + ".mlp")
            moe = nil
            progress("dense MLP loaded")
        } else {
            moe = try StreamingMoE(configuration: c, store: store, layer: layer,
                                   bundleCache: bundleCache)
            denseMLP = nil
            progress("sparse MoE loaded")
        }
        let projection = c.attentionHeads * c.headDim
        guard [qProj, kProj, vProj, fProj, gProj].allSatisfy({
            $0.inputSize == c.hiddenSize && $0.outputSize == projection
        }), bProj.inputSize == c.hiddenSize, bProj.outputSize == c.attentionHeads,
              outputProj.inputSize == projection, outputProj.outputSize == c.hiddenSize,
              qConvWeight.shape == [projection, 1, c.shortConvKernelSize],
              kConvWeight.shape == qConvWeight.shape, vConvWeight.shape == qConvWeight.shape,
              aLog.shape == [c.attentionHeads], dtBias.shape == [projection],
              outputNorm.shape == [c.headDim] else {
            throw M1Error.invalid("M4 KDA tensor dimensions disagree with config")
        }
        eval(inputNorm, postAttentionNorm, qConvWeight, kConvWeight, vConvWeight,
             aLog, dtBias, outputNorm)
        progress("KDA block initialized")
    }

    private static func linear(_ store: ExpertTensorStore, _ prefix: String,
                               _ loraStore: ExpertTensorStore?) throws -> ParallelLinear {
        let base = try QuantizedExpertLinear(weight: store.mlxArrayCopying(named: prefix + ".weight"),
            scales: store.mlxArrayCopying(named: prefix + ".scales"),
            biases: store.mlxArrayCopying(named: prefix + ".biases"))
        return try ParallelLinear(base: base, loraStore: loraStore, prefix: prefix)
    }

    public func callAsFunction(_ input: MLXArray, cache: KDAStateCache,
                               predictedRoute: PredictedMoERoute? = nil) throws -> DecoderBlockOutput {
        guard input.shape == [1, c.hiddenSize] else {
            throw M1Error.invalid("M4 accepts one token [1,\(c.hiddenSize)]")
        }
        let x = input
        let normalized = MLXFast.rmsNorm(x, weight: inputNorm, eps: c.rmsNormEps)
        let attention = try kda(normalized, cache: cache)
        let residual = x + attention
        let moeInput = MLXFast.rmsNorm(residual, weight: postAttentionNorm, eps: c.rmsNormEps)
        let feedForward: MLXArray
        let expertIndices: [Int]
        let expertWeights: [Float]
        if let moe {
            let output = if let predictedRoute {
                try moe.predicted(moeInput, route: predictedRoute)
            } else {
                try moe(moeInput)
            }
            feedForward = output.hidden
            expertIndices = output.expertIndices
            expertWeights = output.expertWeights
            prerouterInput = moeInput
            prerouterIndices = output.routeIndices
        } else if let denseMLP {
            feedForward = try denseMLP(moeInput)
            expertIndices = []
            expertWeights = []
            prerouterInput = nil
            prerouterIndices = nil
        } else {
            throw M1Error.invalid("KDA decoder has no feed-forward block")
        }
        let hidden = residual + feedForward
        cache.count += 1
        return DecoderBlockOutput(hidden: hidden, expertIndices: expertIndices,
                                  expertWeights: expertWeights)
    }

    /// Official multi-token Edge0 8B KDA path used by chunked prefill.
    public func prefill(_ input: MLXArray, cache: KDAStateCache) throws -> DecoderBlockOutput {
        guard input.ndim == 2, input.dim(0) > 1, input.dim(1) == c.hiddenSize else {
            throw M1Error.invalid("KDA prefill accepts [T,\(c.hiddenSize)], T > 1")
        }
        let x = input
        let normalized = MLXFast.rmsNorm(x, weight: inputNorm, eps: c.rmsNormEps)
        let attention = try kdaPrefill(normalized, cache: cache)
        let residual = x + attention
        let moeInput = MLXFast.rmsNorm(residual, weight: postAttentionNorm, eps: c.rmsNormEps)
        let feedForward: MLXArray
        let expertIndices: [Int]
        let expertWeights: [Float]
        if let moe {
            let output = try moe.fullLayerPrefill(moeInput)
            feedForward = output.hidden
            expertIndices = output.expertIndices
            expertWeights = output.expertWeights
            prerouterInput = moeInput
            prerouterIndices = output.routeIndices
        } else if let denseMLP {
            feedForward = try denseMLP(moeInput)
            expertIndices = []
            expertWeights = []
        } else {
            throw M1Error.invalid("KDA decoder has no feed-forward block")
        }
        let hidden = residual + feedForward
        cache.count += input.dim(0)
        return DecoderBlockOutput(hidden: hidden, expertIndices: expertIndices,
                                  expertWeights: expertWeights)
    }

    public func loadFullExpertLayer() throws { try moe?.loadFullLayer() }
    public func clearFullExpertLayer() { moe?.clearFullLayer() }
    public func stage(experts: [Int]) throws { try moe?.stage(experts: experts) }
    public func resetStaging() { moe?.resetStaging() }
    public var moeStats: StreamingMoEStats? { moe?.stats }

    private func convStep(_ x: MLXArray, weight: MLXArray,
                          state: inout MLXArray?) -> MLXArray {
        let channels = c.attentionHeads * c.headDim
        let history = state ?? MLXArray.zeros(
            [c.shortConvKernelSize - 1, channels], dtype: x.dtype)
        let window = concatenated([history, x], axis: 0)
        let kernel = weight.reshaped([channels, c.shortConvKernelSize]).T
        let output = (window * kernel).sum(axis: 0, keepDims: true)
        state = split(window, indices: [1], axis: 0)[1]
        return output * sigmoid(output)
    }

    private func kda(_ x: MLXArray, cache: KDAStateCache) throws -> MLXArray {
        let heads = c.attentionHeads
        let dimension = c.headDim
        let qConv = convStep(try qProj(x), weight: qConvWeight, state: &cache.qConv)
        let kConv = convStep(try kProj(x), weight: kConvWeight, state: &cache.kConv)
        let vConv = convStep(try vProj(x), weight: vConvWeight, state: &cache.vConv)
        var q = qConv.reshaped([1, heads, dimension]).asType(.float32)
        var k = kConv.reshaped([1, heads, dimension]).asType(.float32)
        let v = vConv.reshaped([1, heads, dimension]).asType(.float32)
        let qNorm = sqrt((q * q).sum(axis: -1, keepDims: true)) + 1e-6
        let kNorm = sqrt((k * k).sum(axis: -1, keepDims: true)) + 1e-6
        q = q * (1.0 / Float(dimension).squareRoot()) / qNorm
        k = k / kNorm

        let f = try fProj(x).reshaped([1, heads, dimension]).asType(.float32)
        let gateInput = exp(aLog.asType(.float32)).reshaped([1, heads, 1])
            * (f + dtBias.asType(.float32).reshaped([1, heads, dimension]))
        let decay = exp(Float(c.kdaLowerBound) * sigmoid(gateInput)).expandedDimensions(axis: 2)
        let beta = sigmoid(try bProj(x).asType(.float32)).expandedDimensions(axis: -1)
        var state = cache.recurrent ?? MLXArray.zeros([1, heads, dimension, dimension])
        state = state * decay
        let kRows = k.expandedDimensions(axis: 2)
        let memory = (state * kRows).sum(axis: -1)
        let delta = (v - memory) * beta
        state = state + kRows * delta.expandedDimensions(axis: -1)
        var output = (state * q.expandedDimensions(axis: 2)).sum(axis: -1)
        cache.recurrent = state

        output = MLXFast.rmsNorm(
            output.asType(x.dtype), weight: outputNorm, eps: c.rmsNormEps)
        let outputGate = sigmoid(try gProj(x).reshaped([1, heads, dimension]))
        return try outputProj((output * outputGate).reshaped([1, heads * dimension]))
    }

    private func convPrefill(_ x: MLXArray, weight: MLXArray,
                             state: inout MLXArray?) -> MLXArray {
        let channels = c.attentionHeads * c.headDim
        let history = state ?? MLXArray.zeros(
            [c.shortConvKernelSize - 1, channels], dtype: x.dtype)
        let convInput = concatenated([history, x], axis: 0)
        let kernel = weight.transposed(0, 2, 1)
        let output = conv1d(
            convInput.expandedDimensions(axis: 0), kernel,
            groups: channels).squeezed(axis: 0)
        let stateStart = convInput.dim(0) - (c.shortConvKernelSize - 1)
        state = split(convInput, indices: [stateStart], axis: 0)[1]
        return output * sigmoid(output)
    }

    private func kdaPrefill(_ x: MLXArray, cache: KDAStateCache) throws -> MLXArray {
        let tokens = x.dim(0)
        let heads = c.attentionHeads
        let dimension = c.headDim
        let qConv = convPrefill(try qProj(x), weight: qConvWeight, state: &cache.qConv)
        let kConv = convPrefill(try kProj(x), weight: kConvWeight, state: &cache.kConv)
        let vConv = convPrefill(try vProj(x), weight: vConvWeight, state: &cache.vConv)
        var q = qConv.reshaped([1, tokens, heads, dimension]).asType(.float32)
        var k = kConv.reshaped([1, tokens, heads, dimension]).asType(.float32)
        let v = vConv.reshaped([1, tokens, heads, dimension]).asType(.float32)
        let qNorm = sqrt((q * q).sum(axis: -1, keepDims: true)) + 1e-6
        let kNorm = sqrt((k * k).sum(axis: -1, keepDims: true)) + 1e-6
        q = q * (1.0 / Float(dimension).squareRoot()) / qNorm
        k = k / kNorm

        let f = try fProj(x).reshaped([1, tokens, heads, dimension])
            .asType(.float32)
        let gateInput = exp(aLog.asType(.float32)).reshaped([1, 1, heads, 1])
            * (f + dtBias.asType(.float32)
                .reshaped([1, 1, heads, dimension]))
        let decay = exp(Float(c.kdaLowerBound) * sigmoid(gateInput))
        let beta = sigmoid(try bProj(x).asType(.float32))
            .reshaped([1, tokens, heads])
        let initialState = cache.recurrent
            ?? MLXArray.zeros([1, heads, dimension, dimension])
        let update = try OfficialGatedDelta.callAsFunction(
            q: q, k: k, v: v, decay: decay, beta: beta,
            state: initialState)
        cache.recurrent = update.state
        var output = MLXFast.rmsNorm(
            update.output.asType(x.dtype), weight: outputNorm,
            eps: c.rmsNormEps)
        let outputGate = sigmoid(
            try gProj(x).reshaped([1, tokens, heads, dimension]))
        output = output * outputGate
        return try outputProj(output.reshaped([tokens, heads * dimension]))
    }
}