Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
2.2 KB
Raw Blame
import Edge0Core
import MLX

/// Parallel, unmerged LoRA used by the official Edge0 deployment.
/// Base INT4 weights stay untouched; fp16 rank-16 adapters are added at run time.
struct ParallelLinear {
    let base: QuantizedExpertLinear
    private let loraA: MLXArray?
    private let loraB: MLXArray?
    private let scale: Float

    init(base: QuantizedExpertLinear, loraStore: ExpertTensorStore? = nil,
         prefix: String? = nil, rank: Int = 16, alpha: Float = 32) throws {
        self.base = base
        scale = alpha / Float(rank)
        if let loraStore, let prefix {
            let a = try loraStore.mlxArrayCopying(named: prefix + ".lora_A")
            let b = try loraStore.mlxArrayCopying(named: prefix + ".lora_B")
            guard a.shape == [rank, base.inputSize],
                  b.shape == [base.outputSize, rank] else {
                throw M1Error.invalid("LoRA dimensions disagree for \(prefix)")
            }
            loraA = a
            loraB = b
            eval(a, b)
        } else {
            loraA = nil
            loraB = nil
        }
    }

    var inputSize: Int { base.inputSize }
    var outputSize: Int { base.outputSize }

    func callAsFunction(_ input: MLXArray) throws -> MLXArray {
        let output = try base(input)
        guard let loraA, let loraB else { return output }
        let x = input.asType(loraA.dtype)
        let delta = matmul(matmul(x, loraA.T), loraB.T) * scale
        return output + delta.asType(output.dtype)
    }
}

struct ParallelExpert {
    let up: ParallelLinear
    let gate: ParallelLinear
    let down: ParallelLinear

    init(base: QuantizedExpert, loraStore: ExpertTensorStore?, prefix: String) throws {
        up = try ParallelLinear(base: base.up, loraStore: loraStore,
                                prefix: prefix + ".up_proj")
        gate = try ParallelLinear(base: base.gate, loraStore: loraStore,
                                  prefix: prefix + ".gate_proj")
        down = try ParallelLinear(base: base.down, loraStore: loraStore,
                                  prefix: prefix + ".down_proj")
    }

    func callAsFunction(_ input: MLXArray) throws -> MLXArray {
        let g = try gate(input)
        return try down((g * sigmoid(g)) * up(input))
    }
}