Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
8.6 KB
Raw Blame
import Edge0Core
@testable import Edge0MLX
import Foundation
import MLX
import Testing

// CPU arithmetic by default; MLX still requires an Xcode-built Metal bundle.
// EDGE0_TEST_GPU=1 is used by the documented xcodebuild verification.
private func onDevice(_ body: () throws -> Void) rethrows {
    try Device.withDefaultDevice(ProcessInfo.processInfo.environment["EDGE0_TEST_GPU"] == "1" ? .gpu : .cpu, body)
}

@Test func packedAffineLinearMatchesScalarOracle() throws {
    try onDevice {
        // Two groups, all nibble positions, nonzero offsets, non-square matrix, two tokens.
        let width = 128, rows = 64
        var packed = [UInt32](repeating: 0, count: rows * width / 8)
        var scales = [Float](), biases = [Float]()
        for o in 0..<rows {
            scales += [Float(o % 3 + 1) / 32, Float(o % 5 + 1) / 64]
            biases += [-0.25, 0.125]
            for i in 0..<width { packed[o * width / 8 + i / 8] |= UInt32((o * 3 + i) % 16) << (4 * (i % 8)) }
        }
        let x = (0..<(width * 2)).map { Float($0 % 19 - 9) / 16 }
        let linear = try QuantizedExpertLinear(weight: MLXArray(packed, [rows, width/8]),
            scales: MLXArray(scales, [rows, 2]).asType(.bfloat16),
            biases: MLXArray(biases, [rows, 2]).asType(.bfloat16))
        let y = try linear(MLXArray(x, [2, width])).asArray(Float.self)
        for t in 0..<2 {
            for o in 0..<rows {
                var expected: Float = 0
                for i in 0..<width {
                    let q = Float((o * 3 + i) % 16)
                    expected += x[t * width + i] * (q * scales[o*2+i/64] + biases[o*2+i/64])
                }
                #expect(abs(y[t*rows+o] - expected) < 1e-5)
            }
        }
        #expect(throws: M1Error.self) { try linear(MLXArray.zeros([1, 64])) }
    }
}

@Test func malformedQuantizedShapesRejected() {
    onDevice {
        #expect(throws: M1Error.self) {
            try QuantizedExpertLinear(weight: MLXArray.zeros([64, 8]),
                scales: MLXArray.zeros([64, 1]), biases: MLXArray.zeros([64, 1]))
        }
        #expect(throws: M1Error.self) {
            try QuantizedExpertLinear(weight: MLXArray.zeros([64, 8], dtype: .uint32),
                scales: MLXArray.zeros([64, 2]), biases: MLXArray.zeros([64, 2]))
        }
    }
}

@Test func groupedMLXMatchesCPUWithSelectionBias() {
    onDevice {
        let logits: [Float] = [-4, -3, 5, 4, 1, 0, 3, 2]
        let bias: [Float] = [3, 2, 0, 0, 0.2, 0, -1, -1]
        for keep in [2, 4] {
            for k in [1, 2] {
                for normalize in [false, true] {
                    let cpu = GroupedExpertRouter.select(logits: logits, expertBias: bias, topK: k,
                        nGroup: 4, topkGroup: keep, normalize: normalize)
                    let gpu = Edge0GroupedRouter8B.select(logits: MLXArray(logits, [1, 8]),
                        expertBias: MLXArray(bias), topK: k, nGroup: 4, topkGroup: keep, normalize: normalize)
                    let ids = gpu.indices.asArray(Int32.self).map(Int.init)
                    let weights = gpu.weights.asArray(Float.self)
                    #expect(Set(ids) == Set(cpu.indices))
                    for (id, w) in zip(ids, weights) {
                        #expect(abs(w - cpu.weights[cpu.indices.firstIndex(of: id)!]) < 1e-6)
                    }
                }
            }
        }
    }
}

@Test func swigluUsesGateAndDownProjection() throws {
    try onDevice {
        // Identity INT4 projections make the expected result exactly x * sigmoid(x) * x.
        var packed = [UInt32](repeating: 0, count: 64 * 8)
        for i in 0..<64 { packed[i*8+i/8] = 1 << (4*(i%8)) }
        let linear = try QuantizedExpertLinear(weight: MLXArray(packed, [64, 8]),
            scales: MLXArray.ones([64, 1]), biases: MLXArray.zeros([64, 1]))
        let expert = try QuantizedExpert(up: linear, gate: linear, down: linear)
        let x = (0..<64).map { Float($0-32)/8 }
        let actual = try expert(MLXArray(x, [1, 64])).asArray(Float.self)
        for (v, y) in zip(x, actual) { #expect(abs(y - v*v/(1+exp(-v))) < 2e-6) }
    }
}

@Test func streamedMoEIncludesSharedAndOnlySelectedExperts() throws {
    try onDevice {
        let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
        try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
        defer { try? FileManager.default.removeItem(at: directory) }
        var config = try JSONSerialization.jsonObject(with: Data(contentsOf:
            Bundle.module.url(forResource: "config", withExtension: "json", subdirectory: "Fixtures")!)) as! [String: Any]
        for key in ["hidden_size", "moe_intermediate_size", "moe_shared_expert_intermediate_size"] { config[key] = 64 }
        config["num_experts"] = 8; config["n_group"] = 4
        config["topk_group"] = 2; config["num_experts_per_tok"] = 2
        let c = try JSONDecoder().decode(Edge0Configuration8B.self, from: JSONSerialization.data(withJSONObject: config))
        var header = [String: Any](), payload = Data()
        func append<T>(_ name: String, _ dtype: String, _ shape: [Int], _ values: [T]) {
            let bytes = values.withUnsafeBytes { Data($0) }
            header[name] = ["dtype": dtype, "shape": shape, "data_offsets": [payload.count, payload.count + bytes.count]]
            payload.append(bytes)
        }
        var identity = [UInt32](repeating: 0, count: 64*8)
        for i in 0..<64 { identity[i*8+i/8] = 1 << (4*(i%8)) }
        let prefix = "model.layers.1.mlp"
        let logits: [Float] = [-4, -3, -2, -1, 0, 1, 2, 3]
        let bias: [Float] = [3, 2, 0, 0, 0, 0, -1, -1]
        var router = [Float](repeating: 0, count: 8*64)
        for i in 0..<8 { router[i*64] = logits[i] }
        append(prefix + ".gate.weight", "F32", [8, 64], router)
        append(prefix + ".gate.expert_bias", "F32", [8], bias)
        let route = GroupedExpertRouter.select(logits: logits, expertBias: bias, topK: 2, nGroup: 4, topkGroup: 2)
        for projection in ExpertProjection.allCases {
            let name = prefix + ".experts." + projection.rawValue
            append(name + ".weight", "U32", [8, 64, 8], Array(repeating: identity, count: 8).flatMap { $0 })
            // Unselected experts deliberately contain NaN scales: their execution contaminates output.
            let scales: [Float] = (0..<8).flatMap { id in
                Array(repeating: route.indices.contains(id) ? (projection == .up ? Float(id+1)/8 : 1) : Float.nan, count: 64)
            }
            append(name + ".scales", "F32", [8, 64, 1], scales)
            append(name + ".biases", "F32", [8, 64, 1], [Float](repeating: 0, count: 8*64))
            let shared = prefix + ".shared_experts." + projection.rawValue
            append(shared + ".weight", "U32", [64, 8], identity)
            append(shared + ".scales", "F32", [64, 1], [Float](repeating: projection == .up ? 2 : 1, count: 64))
            append(shared + ".biases", "F32", [64, 1], [Float](repeating: 0, count: 64))
        }
        var json = try JSONSerialization.data(withJSONObject: header, options: .sortedKeys)
        while json.count % 8 != 0 { json.append(32) }
        var length = UInt64(json.count).littleEndian
        var file = withUnsafeBytes(of: &length) { Data($0) }
        file.append(json); file.append(payload)
        let url = directory.appendingPathComponent("model.safetensors")
        try file.write(to: url)
        let store = try ExpertTensorStore(modelURL: url, expertCount: 8)
        let block = try StreamingMoE(configuration: c, store: store, layer: 1)
        var input = (0..<64).map { Float($0-32)/16 }; input[0] = 1
        let output = try block(MLXArray(input, [1, 64]))
        #expect(Set(output.expertIndices) == Set(route.indices))
        let multiplier = zip(route.indices, route.weights).reduce(Float(2)) { $0 + Float($1.0+1)/8 * $1.1 }
        for (x, y) in zip(input, output.hidden.asArray(Float.self)) {
            #expect(y.isFinite && abs(y - multiplier*x*x/(1+exp(-x))) < 3e-6)
        }
        #expect(throws: M1Error.self) { try block(MLXArray.zeros([2, 64])) }
        #expect(throws: M1Error.self) { try store.loadExpert(layer: 1, expert: 8) }
        #expect(throws: M1Error.self) { try StreamingMoE(configuration: c, store: store, layer: 0) }
        // A copied expert is safe after the originating mapping closes.
        let detached: QuantizedExpert = try {
            let temporary = try ExpertTensorStore(modelURL: url, expertCount: 8)
            return try temporary.loadExpert(layer: 1, expert: route.indices[0])
        }()
        let detachedY = try detached(MLXArray(input, [1,64])).asArray(Float.self)
        #expect(detachedY.allSatisfy { $0.isFinite })
    }
}