Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
20.0 KB
Raw Blame
import Edge0Core
import Foundation
import MLX

public struct MoEOutput {
    public let hidden: MLXArray
    public let expertIndices: [Int]
    public let expertWeights: [Float]
    public let routeIndices: MLXArray
}

public struct PredictedMoERoute: Sendable {
    public let expertIndices: [Int]
    public let expertWeights: [Float]

    public init(expertIndices: [Int], expertWeights: [Float]) {
        self.expertIndices = expertIndices
        self.expertWeights = expertWeights
    }
}

/// Official-style whole-model expert-bundle LRU. The production profile uses
/// 64 slots globally, not per layer.
public final class ExpertBundleCache: @unchecked Sendable {
    private let capacity: Int
    private var values: [String: QuantizedExpert] = [:]
    private var order: [String] = []
    private let lock = NSLock()
    private var hitCount = 0
    private var missCount = 0

    public init(capacity: Int = 64) { self.capacity = max(0, capacity) }

    func get(layer: Int, expert: Int) -> QuantizedExpert? {
        lock.lock(); defer { lock.unlock() }
        let key = "\(layer):\(expert)"
        guard let value = values[key] else {
            missCount += 1
            return nil
        }
        hitCount += 1
        order.removeAll { $0 == key }
        order.append(key)
        return value
    }

    func put(_ value: QuantizedExpert, layer: Int, expert: Int) {
        lock.lock(); defer { lock.unlock() }
        guard capacity > 0 else { return }
        let key = "\(layer):\(expert)"
        values[key] = value
        order.removeAll { $0 == key }
        order.append(key)
        while order.count > capacity {
            values.removeValue(forKey: order.removeFirst())
        }
    }

    public var stats: (hits: Int, misses: Int, size: Int) {
        lock.lock(); defer { lock.unlock() }
        return (hitCount, missCount, values.count)
    }
}

public struct StreamingMoEStats: Sendable {
    public let calls: Int
    public let loads: Int
    public let hits: Int
    public let stagedUsed: Int
    public let stagedFallback: Int
    public let stageBuilds: Int
    public let stageCacheHits: Int
    public let stageWallSeconds: TimeInterval
}

private final class ExpertLoadSlots: @unchecked Sendable {
    private let lock = NSLock()
    private var values: [QuantizedExpert?]
    private var firstError: Error?

    init(count: Int) {
        values = Array(repeating: nil, count: count)
    }

    func store(_ result: Result<QuantizedExpert, Error>, at index: Int) {
        lock.lock(); defer { lock.unlock() }
        switch result {
        case .success(let expert): values[index] = expert
        case .failure(let error): if firstError == nil { firstError = error }
        }
    }

    func finish() throws -> [QuantizedExpert] {
        lock.lock(); defer { lock.unlock() }
        if let firstError { throw firstError }
        guard values.allSatisfy({ $0 != nil }) else {
            throw M1Error.invalid("Parallel expert load did not fill every slot")
        }
        return values.map { $0! }
    }
}

/// One decode token's feed-forward block, including the unconditional shared expert.
/// It includes the official gathered exact path, global cache and prerouter staging paths.
public final class StreamingMoE: @unchecked Sendable {
    private let configuration: Edge0Configuration8B
    private let store: ExpertTensorStore
    private let layer: Int
    private let routerWeight: MLXArray
    private let expertBias: MLXArray
    private let shared: QuantizedExpert
    private let bundleCache: ExpertBundleCache?
    private var fullLayer: QuantizedExpertStack?
    private var stagedBundles: [Int: QuantizedExpert] = [:]
    private var stagedStack: QuantizedExpertStack?
    private var stagedOrder: [Int] = []
    private let stagingLock = NSLock()
    private var pendingOrder: [Int] = []
    private var pendingBundles: [QuantizedExpert]?
    private var pendingError: Error?
    private var calls = 0
    private var loads = 0
    private var hits = 0
    private var stagedUsed = 0
    private var stagedFallback = 0
    private var stageBuilds = 0
    private var stageCacheHits = 0
    private var stageWall: TimeInterval = 0
    private let compiledStagedMath: @Sendable ([MLXArray]) -> [MLXArray]

    public init(configuration c: Edge0Configuration8B, store: ExpertTensorStore,
                layer: Int, bundleCache: ExpertBundleCache? = nil) throws {
        guard c.quantization.bits == 4, c.quantization.groupSize == 64,
              c.quantization.mode == "affine", c.numSharedExperts == 1,
              layer >= c.firstKDenseReplace, layer < c.hiddenLayers,
              c.numExperts == store.expertCount, c.nGroup > 0,
              c.numExperts % c.nGroup == 0, c.numExperts / c.nGroup >= 2,
              c.topkGroup > 0, c.topkGroup <= c.nGroup,
              c.expertsPerToken > 0,
              c.expertsPerToken <= c.topkGroup * (c.numExperts / c.nGroup) else {
            throw M1Error.invalid("M1 requires an affine 4-bit/group-64 MoE layer with one shared expert and valid grouped routing")
        }
        configuration = c; self.store = store; self.layer = layer
        self.bundleCache = bundleCache
        compiledStagedMath = compile { arrays in
            let x = arrays[0], local = arrays[1]
            let up = gatherQuantizedMM(
                x, arrays[2], scales: arrays[3], biases: arrays[4],
                rhsIndices: local, transpose: true,
                groupSize: 64, bits: 4, mode: .affine,
                sortedIndices: false)
            let gate = gatherQuantizedMM(
                x, arrays[5], scales: arrays[6], biases: arrays[7],
                rhsIndices: local, transpose: true,
                groupSize: 64, bits: 4, mode: .affine,
                sortedIndices: false)
            let activated = (gate * sigmoid(gate)) * up
            return [gatherQuantizedMM(
                activated, arrays[8], scales: arrays[9], biases: arrays[10],
                rhsIndices: local, transpose: true,
                groupSize: 64, bits: 4, mode: .affine,
                sortedIndices: false)]
        }
        routerWeight = try store.mlxArrayCopying(named: "model.layers.\(layer).mlp.gate.weight").asType(.float32)
        expertBias = c.routerHasExpertBias
            ? try store.mlxArrayCopying(named: "model.layers.\(layer).mlp.gate.expert_bias").asType(.float32)
            : MLXArray.zeros([c.numExperts])
        shared = try store.loadSharedExpert(layer: layer)
        guard routerWeight.shape == [c.numExperts, c.hiddenSize], expertBias.shape == [c.numExperts],
              shared.up.inputSize == c.hiddenSize,
              shared.up.outputSize == c.sharedExpertIntermediateSize else {
            throw M1Error.invalid("Router/shared expert dimensions disagree with config")
        }
        eval(routerWeight, expertBias)
    }

    public func callAsFunction(_ input: MLXArray) throws -> MoEOutput {
        stagingLock.lock(); calls += 1; stagingLock.unlock()
        let c = configuration
        guard input.shape == [1, c.hiddenSize],
              [.float32, .float16, .bfloat16].contains(input.dtype) else {
            throw M1Error.invalid("M1 MoE accepts exactly one floating token [1,\(c.hiddenSize)]")
        }
        let x = input
        let route = teacherRoute(x.asType(.float32))
        eval(route.indices, route.weights)
        let indices = route.indices.asArray(Int32.self).map(Int.init)
        let weights = route.weights.asArray(Float.self)
        let result = try executeGathered(x, indices: indices, weights: weights)
        return MoEOutput(hidden: result, expertIndices: indices,
                         expertWeights: weights, routeIndices: route.indices)
    }

    /// Consumer-layer path from the official Ling prerouter. Current
    /// production configuration feeds the executed top-k to the next head.
    public func predicted(_ input: MLXArray, route: PredictedMoERoute) throws -> MoEOutput {
        stagingLock.lock(); calls += 1; stagingLock.unlock()
        try activatePendingStage(for: route.expertIndices)
        let x = input
        let hidden: MLXArray
        if Set(route.expertIndices) == Set(stagedOrder), let stagedStack {
            stagingLock.lock(); stagedUsed += route.expertIndices.count; stagingLock.unlock()
            hidden = try executeStaged(
                x, stack: stagedStack, indices: route.expertIndices,
                weights: route.expertWeights)
        } else {
            stagingLock.lock(); stagedFallback += 1; stagingLock.unlock()
            hidden = try executeGathered(
                x, indices: route.expertIndices, weights: route.expertWeights)
        }
        // Current upstream Edge0-8B uses feature_topk="executed": the
        // prerouter's selected set feeds the next head directly. Do not
        // recompute the original gate on consumer layers.
        let executed = MLXArray(
            route.expertIndices.map(Int32.init),
            [1, route.expertIndices.count])
        return MoEOutput(hidden: hidden, expertIndices: route.expertIndices,
                         expertWeights: route.expertWeights,
                         routeIndices: executed)
    }

    /// Official `prod_k8` next-state submission (`staged_sync=True`): fill the
    /// NEXT set synchronously and promote it only at the next consume boundary.
    public func stage(experts: [Int]) throws {
        let ordered = Array(Set(experts)).sorted()
        let started = Date()
        var cacheHits = 0
        var bundles = [QuantizedExpert?](repeating: nil, count: ordered.count)
        var missingIDs: [Int] = []
        var missingPositions: [Int] = []
        store.adviseExpertsWillNeed(layer: layer, experts: ordered)
        for (position, id) in ordered.enumerated() {
            if let existing = stagedBundles[id] {
                cacheHits += 1
                bundles[position] = existing
            } else if let cached = bundleCache?.get(layer: layer, expert: id) {
                cacheHits += 1
                bundles[position] = cached
            } else {
                missingIDs.append(id)
                missingPositions.append(position)
            }
        }
        let loaded = try loadExpertsFromStore(missingIDs)
        for ((id, position), expert) in zip(
            zip(missingIDs, missingPositions), loaded) {
            bundles[position] = expert
            bundleCache?.put(expert, layer: layer, expert: id)
        }
        let resolved = bundles.map { $0! }
        let loadCount = missingIDs.count
        stagingLock.lock()
        pendingOrder = ordered
        pendingBundles = resolved
        pendingError = nil
        stageBuilds += loadCount
        stageCacheHits += cacheHits
        loads += loadCount
        hits += cacheHits
        stageWall += Date().timeIntervalSince(started)
        stagingLock.unlock()
    }

    private func activatePendingStage(for expected: [Int]) throws {
        stagingLock.lock()
        defer { stagingLock.unlock() }
        if let pendingError { throw pendingError }
        guard pendingOrder == Array(Set(expected)).sorted(),
              let bundles = pendingBundles else { return }
        if pendingOrder == stagedOrder, stagedStack != nil {
            // Upstream asm-cache behavior: an unchanged staged set reuses
            // the existing nine stacked graph nodes; only route weights vary.
            pendingBundles = nil
            pendingOrder = []
            return
        }
        stagedBundles = Dictionary(uniqueKeysWithValues: zip(pendingOrder, bundles))
        stagedStack = try QuantizedExpertStack(experts: bundles)
        stagedOrder = pendingOrder
        pendingBundles = nil
        pendingOrder = []
    }

    public func resetStaging() {
        stagingLock.lock()
        pendingBundles = nil
        pendingError = nil
        pendingOrder = []
        stagedBundles.removeAll(keepingCapacity: true)
        stagedStack = nil
        stagedOrder = []
        stagingLock.unlock()
        clearFullLayer()
    }

    private func teacherRoute(_ x: MLXArray) -> (indices: MLXArray, weights: MLXArray) {
        let logits = matmul(x, routerWeight.T)
        return Edge0GroupedRouter8B.select(
            logits: logits, expertBias: expertBias,
            topK: configuration.expertsPerToken, nGroup: configuration.nGroup,
            topkGroup: configuration.topkGroup,
            normalize: configuration.normTopkProb,
            routedScale: configuration.routedScalingFactor)
    }

    private func resolveExpert(_ id: Int) throws -> QuantizedExpert {
        if let staged = stagedBundles[id] {
            stagingLock.lock(); hits += 1; stagingLock.unlock()
            return staged
        }
        if let cached = bundleCache?.get(layer: layer, expert: id) {
            stagingLock.lock(); hits += 1; stagingLock.unlock()
            return cached
        }
        let expert = try store.loadExpert(layer: layer, expert: id)
        bundleCache?.put(expert, layer: layer, expert: id)
        stagingLock.lock(); loads += 1; stagingLock.unlock()
        return expert
    }

    /// Match the upstream loader pool: resolve independent expert bundles in
    /// parallel after issuing page-cache readahead for their tensor ranges.
    private func resolveExperts(_ ids: [Int]) throws -> [QuantizedExpert] {
        guard !ids.isEmpty else { return [] }
        store.adviseExpertsWillNeed(layer: layer, experts: ids)
        let slots = ExpertLoadSlots(count: ids.count)
        DispatchQueue.concurrentPerform(iterations: ids.count) { index in
            slots.store(Result { try self.resolveExpert(ids[index]) }, at: index)
        }
        return try slots.finish()
    }

    private func loadExpertsFromStore(_ ids: [Int]) throws -> [QuantizedExpert] {
        guard !ids.isEmpty else { return [] }
        let slots = ExpertLoadSlots(count: ids.count)
        DispatchQueue.concurrentPerform(iterations: ids.count) { index in
            slots.store(
                Result { try self.store.loadExpert(layer: self.layer,
                                                   expert: ids[index]) },
                at: index)
        }
        return try slots.finish()
    }

    public var stats: StreamingMoEStats {
        stagingLock.lock(); defer { stagingLock.unlock() }
        return StreamingMoEStats(
            calls: calls, loads: loads, hits: hits, stagedUsed: stagedUsed,
            stagedFallback: stagedFallback, stageBuilds: stageBuilds,
            stageCacheHits: stageCacheHits, stageWallSeconds: stageWall)
    }

    /// Upstream exact path: stack the unique routed bundles and execute the
    /// same compiled gathered-QMM graph used by fixed-slot decode.
    private func executeGathered(_ x: MLXArray, indices: [Int],
                                 weights: [Float]) throws -> MLXArray {
        let c = configuration
        guard indices.count == weights.count else {
            throw M1Error.invalid("Expert IDs and weights disagree")
        }
        let unique = Array(Set(indices)).sorted()
        let experts = try resolveExperts(unique).map { expert in
            guard expert.up.inputSize == c.hiddenSize,
                  expert.up.outputSize == c.moeIntermediateSize else {
                throw M1Error.invalid("Routed expert dimensions disagree with config")
            }
            return expert
        }
        let localByID = Dictionary(uniqueKeysWithValues:
            unique.enumerated().map { ($0.element, $0.offset) })
        return try executeStacked(
            x, stack: QuantizedExpertStack(experts: experts),
            localIndices: indices.map { localByID[$0]! }, weights: weights)
    }

    /// Direct Swift translation of the official fixed staged-slot math. The
    /// selected K experts map into canonical staged slots and run through three
    /// gathered INT4 matrix multiplications instead of K independent graphs.
    private func executeStaged(_ x: MLXArray, stack: QuantizedExpertStack,
                               indices: [Int],
                               weights: [Float]) throws -> MLXArray {
        let k = indices.count
        guard k == weights.count, stagedOrder.count == stack.up.expertCount else {
            throw M1Error.invalid("Staged expert stack and route disagree")
        }
        let slotByID = Dictionary(uniqueKeysWithValues:
            stagedOrder.enumerated().map { ($0.element, $0.offset) })
        return try executeStacked(
            x, stack: stack, localIndices: indices.map { slotByID[$0]! },
            weights: weights)
    }

    private func executeStacked(_ x: MLXArray, stack: QuantizedExpertStack,
                                localIndices: [Int], weights: [Float]) throws
        -> MLXArray {
        let k = localIndices.count
        guard k == weights.count, !localIndices.isEmpty,
              localIndices.allSatisfy({
                  (0..<stack.up.expertCount).contains($0)
              }) else {
            throw M1Error.invalid("Stacked expert indices and route disagree")
        }
        let local = MLXArray(localIndices.map(Int32.init))
        let routedInput = broadcast(x, to: [k, configuration.hiddenSize])
            .expandedDimensions(axis: 1)
        let outputs = compiledStagedMath([
            routedInput, local,
            stack.up.weight, stack.up.scales, stack.up.biases,
            stack.gate.weight, stack.gate.scales, stack.gate.biases,
            stack.down.weight, stack.down.scales, stack.down.biases,
        ])[0].squeezed(axis: 1)
        let routedWeights = MLXArray(weights, [k, 1]).asType(outputs.dtype)
        let routed = (outputs * routedWeights).sum(axis: 0, keepDims: true)
        return try routed + shared(x)
    }

    /// Official whole-layer prefill lifecycle (`load_full_layer`).
    public func loadFullLayer() throws {
        if fullLayer == nil {
            fullLayer = try store.loadFullExpertLayer(layer: layer)
        }
    }

    /// Official whole-layer prefill lifecycle (`clear_full_layer`).
    public func clearFullLayer() {
        fullLayer = nil
    }

    public var hasFullLayer: Bool { fullLayer != nil }

    /// Official full-layer `StreamingSwitchGLU.__call__` branch for a flat
    /// token batch `[T, H]`. The sorted gather/unsort sequence is kept in the
    /// same order as mlx-lm's `_gather_sort` and `_scatter_unsort`.
    public func fullLayerPrefill(_ input: MLXArray) throws -> MoEOutput {
        let c = configuration
        guard input.ndim == 2, input.dim(0) > 1, input.dim(1) == c.hiddenSize,
              let fullLayer else {
            throw M1Error.invalid("Full-layer prefill requires loaded weights and [T,H], T > 1")
        }
        let x = input
        let logits = matmul(x.asType(.float32), routerWeight.T)
        let route = Edge0GroupedRouter8B.select(
            logits: logits, expertBias: expertBias,
            topK: c.expertsPerToken, nGroup: c.nGroup,
            topkGroup: c.topkGroup, normalize: c.normTopkProb,
            routedScale: c.routedScalingFactor)

        let tokenCount = input.dim(0)
        let topK = c.expertsPerToken
        let flatIndices = route.indices.flattened()
        let order = argSort(flatIndices)
        let inverseOrder = argSort(order)
        let tokenRows = order.floorDivide(topK)
        let sortedInput = x.take(tokenRows, axis: 0).expandedDimensions(axis: 1)
        let sortedExperts = flatIndices.take(order, axis: 0)

        let up = try fullLayer.up(sortedInput, expertIndices: sortedExperts)
        let gate = try fullLayer.gate(sortedInput, expertIndices: sortedExperts)
        let activated = (gate * sigmoid(gate)) * up
        let sortedOutput = try fullLayer.down(activated, expertIndices: sortedExperts)
        let expertOutput = sortedOutput.take(inverseOrder, axis: 0)
            .reshaped([tokenCount, topK, c.hiddenSize])
        let routedWeights = route.weights.asType(expertOutput.dtype)
            .expandedDimensions(axis: -1)
        let routed = (expertOutput * routedWeights)
            .sum(axis: 1)
        let hidden = try routed + shared(x)
        eval(hidden, route.indices, route.weights)
        return MoEOutput(
            hidden: hidden,
            expertIndices: route.indices.asArray(Int32.self).map(Int.init),
            expertWeights: route.weights.asArray(Float.self),
            routeIndices: route.indices)
    }
}