Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
10.3 KB
Raw Blame
import Edge0Core
import Foundation
import MLX

public enum M1Error: Error, LocalizedError {
    case invalid(String)
    public var errorDescription: String? {
        switch self { case .invalid(let message): return message }
    }
}

/// Affine INT4: W[o,i] = nibble[o,i] * scale[o,i/64] + bias[o,i/64].
/// Production affine INT4 linear. Coefficients retain their checkpoint dtype
/// and the output follows the activation dtype, matching upstream MLX.
public struct QuantizedExpertLinear {
    public let weight: MLXArray
    public let scales: MLXArray
    public let biases: MLXArray
    public let inputSize: Int
    public let outputSize: Int

    public init(weight: MLXArray, scales: MLXArray, biases: MLXArray) throws {
        guard weight.ndim == 2, weight.dtype == .uint32,
              weight.dim(0) > 0, weight.dim(1) > 0, weight.dim(1) % 8 == 0,
              scales.shape == [weight.dim(0), weight.dim(1) / 8],
              biases.shape == scales.shape,
              [.float32, .float16, .bfloat16].contains(scales.dtype),
              [.float32, .float16, .bfloat16].contains(biases.dtype) else {
            throw M1Error.invalid("Expected affine INT4 [out,in/8] U32 and floating [out,in/64] coefficients")
        }
        self.weight = weight
        self.scales = scales
        self.biases = biases
        inputSize = weight.dim(1) * 8
        outputSize = weight.dim(0)
    }

    public func callAsFunction(_ input: MLXArray) throws -> MLXArray {
        guard input.ndim == 2, input.dim(0) > 0, input.dim(1) == inputSize,
              [.float32, .float16, .bfloat16].contains(input.dtype) else {
            throw M1Error.invalid("Linear input must be floating [tokens,\(inputSize)]")
        }
        return quantizedMM(input, weight, scales: scales,
                           biases: biases, transpose: true, groupSize: 64, bits: 4, mode: .affine)
    }
}

public struct QuantizedExpert {
    public let up: QuantizedExpertLinear
    public let gate: QuantizedExpertLinear
    public let down: QuantizedExpertLinear

    public init(up: QuantizedExpertLinear, gate: QuantizedExpertLinear,
                down: QuantizedExpertLinear) throws {
        guard up.inputSize == gate.inputSize, up.outputSize == gate.outputSize,
              down.inputSize == up.outputSize, down.outputSize == up.inputSize else {
            throw M1Error.invalid("Inconsistent SwiGLU projection dimensions")
        }
        self.up = up; self.gate = gate; self.down = down
    }

    public func callAsFunction(_ input: MLXArray) throws -> MLXArray {
        let g = try gate(input)
        return try down((g * sigmoid(g)) * up(input))
    }
}

/// Official Edge0 whole-layer prefill representation: every routed expert for
/// one projection remains stacked on axis zero and is selected by
/// `gatherQuantizedMM`. Unlike the one-expert correctness path, checkpoint
/// BF16 scales/biases stay BF16, matching `StreamingSwitchGLU.load_full_layer`.
public struct QuantizedExpertStackedLinear {
    public let weight: MLXArray
    public let scales: MLXArray
    public let biases: MLXArray
    public let expertCount: Int
    public let inputSize: Int
    public let outputSize: Int

    public init(weight: MLXArray, scales: MLXArray, biases: MLXArray) throws {
        guard weight.ndim == 3, weight.dtype == .uint32,
              weight.dim(0) > 0, weight.dim(1) > 0,
              weight.dim(2) > 0, weight.dim(2) % 8 == 0,
              scales.shape == [weight.dim(0), weight.dim(1), weight.dim(2) / 8],
              biases.shape == scales.shape,
              [.float32, .float16, .bfloat16].contains(scales.dtype),
              [.float32, .float16, .bfloat16].contains(biases.dtype) else {
            throw M1Error.invalid("Expected stacked affine INT4 expert tensors")
        }
        self.weight = weight
        self.scales = scales
        self.biases = biases
        expertCount = weight.dim(0)
        inputSize = weight.dim(2) * 8
        outputSize = weight.dim(1)
    }

    public func callAsFunction(_ input: MLXArray, expertIndices: MLXArray,
                               sortedIndices: Bool = true) throws -> MLXArray {
        guard input.ndim >= 2, input.dim(-1) == inputSize,
              [.uint8, .uint16, .uint32, .uint64,
               .int8, .int16, .int32, .int64].contains(expertIndices.dtype),
              [.float32, .float16, .bfloat16].contains(input.dtype) else {
            throw M1Error.invalid("Invalid stacked expert input or indices")
        }
        return gatherQuantizedMM(
            input, weight, scales: scales, biases: biases,
            rhsIndices: expertIndices, transpose: true,
            groupSize: 64, bits: 4, mode: .affine,
            sortedIndices: sortedIndices)
    }
}

public struct QuantizedExpertStack {
    public let up: QuantizedExpertStackedLinear
    public let gate: QuantizedExpertStackedLinear
    public let down: QuantizedExpertStackedLinear

    public init(up: QuantizedExpertStackedLinear, gate: QuantizedExpertStackedLinear,
                down: QuantizedExpertStackedLinear) throws {
        guard up.expertCount == gate.expertCount,
              up.expertCount == down.expertCount,
              up.inputSize == gate.inputSize,
              up.outputSize == gate.outputSize,
              down.inputSize == up.outputSize,
              down.outputSize == up.inputSize else {
            throw M1Error.invalid("Inconsistent stacked SwiGLU dimensions")
        }
        self.up = up
        self.gate = gate
        self.down = down
    }

    /// Official staged-slot assembly: selected expert bundles are stacked in
    /// prediction order, then consumed by gathered quantized matmuls.
    public init(experts: [QuantizedExpert]) throws {
        guard !experts.isEmpty else {
            throw M1Error.invalid("Cannot stack an empty expert selection")
        }
        func projection(_ keyPath: KeyPath<QuantizedExpert, QuantizedExpertLinear>) throws
            -> QuantizedExpertStackedLinear {
            let values = experts.map { $0[keyPath: keyPath] }
            return try QuantizedExpertStackedLinear(
                weight: stacked(values.map(\.weight), axis: 0),
                scales: stacked(values.map(\.scales), axis: 0),
                biases: stacked(values.map(\.biases), axis: 0))
        }
        try self.init(up: projection(\.up), gate: projection(\.gate),
                      down: projection(\.down))
    }
}

public extension ExpertTensorStore {
    /// Copies one named tensor, used for the small router and shared expert.
    func mlxArrayCopying(named name: String) throws -> MLXArray {
        let descriptor = try index.tensor(named: name)
        let pointer = try mappedFile.pointer(to: descriptor.byteRange)
        return try copyTensor(descriptor: descriptor, pointer: pointer)
    }

    func loadExpert(layer: Int, expert: Int) throws -> QuantizedExpert {
        guard (0..<expertCount).contains(expert) else { throw M1Error.invalid("Expert out of range") }
        func projection(_ p: ExpertProjection) throws -> QuantizedExpertLinear {
            try QuantizedExpertLinear(
                weight: mlxArrayCopying(layer: layer, expert: expert, projection: p, part: .weight),
                scales: mlxArrayCopying(layer: layer, expert: expert, projection: p, part: .scales),
                biases: mlxArrayCopying(layer: layer, expert: expert, projection: p, part: .biases))
        }
        return try QuantizedExpert(up: projection(.up), gate: projection(.gate), down: projection(.down))
    }

    func loadSharedExpert(layer: Int) throws -> QuantizedExpert {
        func projection(_ p: ExpertProjection) throws -> QuantizedExpertLinear {
            let prefix = "model.layers.\(layer).mlp.shared_experts.\(p.rawValue)"
            return try QuantizedExpertLinear(weight: mlxArrayCopying(named: prefix + ".weight"),
                scales: mlxArrayCopying(named: prefix + ".scales"),
                biases: mlxArrayCopying(named: prefix + ".biases"))
        }
        return try QuantizedExpert(up: projection(.up), gate: projection(.gate), down: projection(.down))
    }

    func loadDenseMLP(layer: Int) throws -> QuantizedExpert {
        func projection(_ p: ExpertProjection) throws -> QuantizedExpertLinear {
            let prefix = "model.layers.\(layer).mlp.\(p.rawValue)"
            return try QuantizedExpertLinear(weight: mlxArrayCopying(named: prefix + ".weight"),
                scales: mlxArrayCopying(named: prefix + ".scales"),
                biases: mlxArrayCopying(named: prefix + ".biases"))
        }
        return try QuantizedExpert(up: projection(.up), gate: projection(.gate), down: projection(.down))
    }

    /// Direct translation of official `StreamingSwitchGLU.load_full_layer` for
    /// Edge0-8B's separate gate/up/down layout: nine whole stacked tensors.
    func loadFullExpertLayer(layer: Int) throws -> QuantizedExpertStack {
        func projection(_ p: ExpertProjection) throws -> QuantizedExpertStackedLinear {
            let prefix = "model.layers.\(layer).mlp.experts.\(p.rawValue)."
            return try QuantizedExpertStackedLinear(
                weight: mlxArrayCopying(named: prefix + "weight"),
                scales: mlxArrayCopying(named: prefix + "scales"),
                biases: mlxArrayCopying(named: prefix + "biases"))
        }
        return try QuantizedExpertStack(up: projection(.up), gate: projection(.gate),
                                        down: projection(.down))
    }
}

func copyTensor(descriptor: TensorDescriptor, pointer: UnsafeMutableRawPointer) throws -> MLXArray {
    let dtype: DType
    switch descriptor.dtype {
    case .u32: dtype = .uint32
    case .bf16: dtype = .bfloat16
    case .f16: dtype = .float16
    case .f32: dtype = .float32
    default: throw Edge0MLXError.unsupportedDType(descriptor.dtype)
    }
    var elements = 1
    for dimension in descriptor.shape {
        let product = elements.multipliedReportingOverflow(by: dimension)
        guard dimension > 0, !product.overflow else { throw M1Error.invalid("Invalid tensor shape") }
        elements = product.partialValue
    }
    let size = elements.multipliedReportingOverflow(by: descriptor.dtype.byteWidth)
    guard !size.overflow, size.partialValue == descriptor.byteCount else {
        throw M1Error.invalid("Tensor shape/byte count mismatch: \(descriptor.name)")
    }
    return MLXArray(Data(bytes: pointer, count: descriptor.byteCount), descriptor.shape, dtype: dtype)
}