Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
11.8 KB
Raw Blame
import Edge0Core
import Foundation
import MLX

public struct StreamingModelOutput {
    public let logits: MLXArray
    public let activeMemory: Int
    public let peakMemory: Int
    public let cacheMemory: Int
}

private enum StreamingLayer {
    case kda(StreamingKDADecoderBlock, KDAStateCache)
    case mla(StreamingMLADecoderBlock, MLAKVCache)

    func callAsFunction(_ hidden: MLXArray,
                        predictedRoute: PredictedMoERoute? = nil) throws -> MLXArray {
        switch self {
        case .kda(let block, let cache):
            return try block(hidden, cache: cache, predictedRoute: predictedRoute).hidden
        case .mla(let block, let cache):
            return try block(hidden, position: cache.count, cache: cache,
                             predictedRoute: predictedRoute).hidden
        }
    }

    var prerouterFeature: LingPrerouterFeature? {
        switch self {
        case .kda(let block, _):
            guard let input = block.prerouterInput,
                  let indices = block.prerouterIndices else { return nil }
            return LingPrerouterFeature(input: input, routeIndices: indices)
        case .mla(let block, _):
            guard let input = block.prerouterInput,
                  let indices = block.prerouterIndices else { return nil }
            return LingPrerouterFeature(input: input, routeIndices: indices)
        }
    }

    func stage(experts: [Int]) throws {
        switch self {
        case .kda(let block, _): try block.stage(experts: experts)
        case .mla(let block, _): try block.stage(experts: experts)
        }
    }

    func reset() {
        switch self {
        case .kda(let block, let cache):
            cache.reset(); block.resetStaging()
        case .mla(let block, let cache):
            cache.reset(); block.resetStaging()
        }
    }

    func prefill(_ hidden: MLXArray) throws -> MLXArray {
        switch self {
        case .kda(let block, let cache):
            return try block.prefill(hidden, cache: cache).hidden
        case .mla(let block, let cache):
            return try block.prefill(
                hidden, position: cache.count, cache: cache).hidden
        }
    }

    func loadFullExpertLayer() throws {
        switch self {
        case .kda(let block, _): try block.loadFullExpertLayer()
        case .mla(let block, _): try block.loadFullExpertLayer()
        }
    }

    func clearFullExpertLayer() {
        switch self {
        case .kda(let block, _): block.clearFullExpertLayer()
        case .mla(let block, _): block.clearFullExpertLayer()
        }
    }

    var moeStats: StreamingMoEStats? {
        switch self {
        case .kda(let block, _): block.moeStats
        case .mla(let block, _): block.moeStats
        }
    }
}

public struct StreamingRuntimeStats: Sendable {
    public let layers: [StreamingMoEStats]
    public let cacheHits: Int
    public let cacheMisses: Int
    public let cachedExperts: Int
}

/// One-token exact-streaming Edge0 model.
///
/// Resident tensors are limited to the non-routed backbone, routers, and shared
/// experts. Routed experts remain in the checkpoint mmap and are copied only for
/// the selected top-8, matching the official repository's memory contract. MLX's
/// reusable free-buffer cache is capped at 256 MiB, also matching the official
/// engine default.
public final class StreamingEdge0Model8B {
    private let configuration: Edge0Configuration8B
    private let store: ExpertTensorStore
    private let layers: [StreamingLayer]
    private let finalNorm: MLXArray
    private let lmHead: QuantizedExpertLinear
    private let prerouter: LingPrerouter?
    private let bundleCache: ExpertBundleCache
    private var predictedRoutes: [Int: PredictedMoERoute] = [:]

    public init(configuration c: Edge0Configuration8B, store: ExpertTensorStore,
                cacheLimitBytes: Int = 256 * 1024 * 1024,
                loraWeightsURL: URL? = nil,
                prerouterWeightsURL: URL? = nil,
                progress: (String) -> Void = { _ in }) throws {
        guard c.quantization.bits == 4, c.quantization.groupSize == 64,
              c.quantization.mode == "affine" else {
            throw M1Error.invalid("Full model requires affine INT4/group-64 weights")
        }
        Memory.cacheLimit = cacheLimitBytes
        configuration = c
        self.store = store
        finalNorm = try store.mlxArrayCopying(named: "model.norm.weight")
        lmHead = try QuantizedExpertLinear(
            weight: store.mlxArrayCopying(named: "lm_head.weight"),
            scales: store.mlxArrayCopying(named: "lm_head.scales"),
            biases: store.mlxArrayCopying(named: "lm_head.biases"))
        guard finalNorm.shape == [c.hiddenSize], lmHead.inputSize == c.hiddenSize,
              lmHead.outputSize == c.vocabularySize else {
            throw M1Error.invalid("Final norm or lm-head dimensions disagree with config")
        }
        progress("final norm and lm-head loaded")

        let bundleCache = ExpertBundleCache(capacity: 64)
        self.bundleCache = bundleCache
        let loraStore = try loraWeightsURL.map {
            try ExpertTensorStore(modelURL: $0, expertCount: c.numExperts)
        }
        var built: [StreamingLayer] = []
        built.reserveCapacity(c.hiddenLayers)
        for layer in 0..<c.hiddenLayers {
            if c.isMLALayer(layer) {
                let block = try StreamingMLADecoderBlock(
                    configuration: c, store: store, layer: layer,
                    loraStore: loraStore,
                    bundleCache: bundleCache)
                built.append(.mla(block, MLAKVCache()))
            } else {
                let block = try StreamingKDADecoderBlock(
                    configuration: c, store: store, layer: layer,
                    loraStore: loraStore,
                    bundleCache: bundleCache)
                built.append(.kda(block, KDAStateCache()))
            }
            progress("layer \(layer) resident backbone loaded")
        }
        layers = built
        if let prerouterWeightsURL {
            prerouter = try LingPrerouter(configuration: c,
                                         weightsURL: prerouterWeightsURL)
            progress("Ling prerouter loaded")
        } else {
            prerouter = nil
        }
        eval(finalNorm)
        progress("24-layer model initialized")
    }

    public func callAsFunction(tokenID: Int,
                               afterLayer: (Int, MLXArray) -> Void = { _, _ in }) throws -> StreamingModelOutput {
        let c = configuration
        guard tokenID >= 0, tokenID < c.vocabularySize else {
            throw M1Error.invalid("Token ID out of vocabulary range")
        }
        var hidden = try embedding(tokenID: tokenID)
        var features: [Int: LingPrerouterFeature] = [:]
        for (index, layer) in layers.enumerated() {
            hidden = try layer(hidden, predictedRoute: predictedRoutes[index])
            if (7...22).contains(index), let feature = layer.prerouterFeature {
                features[index] = feature
            }
            afterLayer(index, hidden)
        }
        hidden = MLXFast.rmsNorm(hidden, weight: finalNorm, eps: c.rmsNormEps)
        let logits = try lmHead(hidden)
        eval(logits)
        try updateStagedRoutes(features)
        let memory = Memory.snapshot()
        return StreamingModelOutput(logits: logits,
                                    activeMemory: memory.activeMemory,
                                    peakMemory: memory.peakMemory,
                                    cacheMemory: memory.cacheMemory)
    }

    /// Official chunked prefill path: each sparse layer's complete expert
    /// stack is loaded immediately before that layer and dropped when the
    /// next layer begins. KDA and MLA process the full token chunk at once.
    public func prefill(tokenIDs: [Int],
                        afterLayer: (Int, MLXArray) -> Void = { _, _ in }) throws
        -> StreamingModelOutput {
        let c = configuration
        guard tokenIDs.count > 1,
              tokenIDs.allSatisfy({ (0..<c.vocabularySize).contains($0) }) else {
            throw M1Error.invalid("Chunked prefill requires at least two valid token IDs")
        }
        defer { layers.forEach { $0.clearFullExpertLayer() } }

        var hidden = try embedding(tokenIDs: tokenIDs)
        var features: [Int: LingPrerouterFeature] = [:]
        for (index, layer) in layers.enumerated() {
            if index > 0 { layers[index - 1].clearFullExpertLayer() }
            try layer.loadFullExpertLayer()
            hidden = try layer.prefill(hidden)
            asyncEval(hidden)
            if (7...22).contains(index), let feature = layer.prerouterFeature {
                features[index] = feature
            }
            afterLayer(index, hidden)
        }
        hidden = MLXFast.rmsNorm(hidden, weight: finalNorm, eps: c.rmsNormEps)
        let last = split(hidden, indices: [hidden.dim(0) - 1], axis: 0)[1]
        let logits = try lmHead(last)
        eval(logits)
        try updateStagedRoutes(features)
        let memory = Memory.snapshot()
        return StreamingModelOutput(logits: logits,
                                    activeMemory: memory.activeMemory,
                                    peakMemory: memory.peakMemory,
                                    cacheMemory: memory.cacheMemory)
    }

    private func embedding(tokenID: Int) throws -> MLXArray {
        let prefix = "model.word_embeddings"
        let weight = try store.mlxAxisZeroSliceCopying(named: prefix + ".weight", index: tokenID)
            .expandedDimensions(axis: 0)
        let scales = try store.mlxAxisZeroSliceCopying(named: prefix + ".scales", index: tokenID)
            .expandedDimensions(axis: 0)
        let biases = try store.mlxAxisZeroSliceCopying(named: prefix + ".biases", index: tokenID)
            .expandedDimensions(axis: 0)
        let hidden = dequantized(weight, scales: scales, biases: biases,
                                 groupSize: 64, bits: 4, mode: .affine,
                                 dtype: .bfloat16)
        guard hidden.shape == [1, configuration.hiddenSize] else {
            throw M1Error.invalid("Embedding row has an unexpected shape")
        }
        return hidden
    }

    /// Official step boundary: owner layers 7...22 predict consumers 8...23,
    /// then their bundles are staged before the next token begins.
    private func updateStagedRoutes(_ features: [Int: LingPrerouterFeature]) throws {
        guard let prerouter else { return }
        let next = try prerouter.predict(features: features)
        for (consumer, route) in next {
            try layers[consumer].stage(experts: route.expertIndices)
        }
        predictedRoutes = next
    }

    private func embedding(tokenIDs: [Int]) throws -> MLXArray {
        try concatenated(tokenIDs.map { try embedding(tokenID: $0) }, axis: 0)
    }

    public func reset() {
        layers.forEach { $0.reset() }
        predictedRoutes.removeAll(keepingCapacity: true)
        prerouter?.reset()
    }

    public var stats: StreamingRuntimeStats {
        let cache = bundleCache.stats
        return StreamingRuntimeStats(
            layers: layers.compactMap(\.moeStats), cacheHits: cache.hits,
            cacheMisses: cache.misses, cachedExperts: cache.size)
    }

    /// Official Ling startup warm-up: request sequential pages, execute a
    /// tiny prompt plus one decode step, then clear only sequence state.
    ///
    /// The four-token warm-up uses the top-8 decode path. Full-layer prefill
    /// of four tokens loads every expert and is the opposite of a warm-up.
    public func prewarm(tokenID: Int = 0) throws {
        store.mappedFile.adviseWillNeed()
        store.mappedFile.adviseSequential()
        store.mappedFile.sequentialRead()
        reset()
        for _ in 0..<4 {
            _ = try callAsFunction(tokenID: tokenID)
        }
        _ = try callAsFunction(tokenID: tokenID)
        reset()
    }

}