Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
47.9 KB
Raw Blame
import Foundation
import MLX
import MLXFast

/// Streaming Edge0 35B model implementation used by the iOS chat engine.
///
/// This implementation is kept independent of the Python reference so parity checks can
/// compare separate code paths. It is not main-actor isolated because model steps perform
/// synchronous GPU and file I/O. The chat engine serializes access to one model instance.
final class Edge0Model35B: @unchecked Sendable {

    /// How many tokens may go through one forward pass.
    ///
    /// Sets the expert buffer ceiling: a batch of T tokens can in the worst case route to
    /// T*K distinct experts. The loader grows staging storage only to the union actually
    /// observed, so this ceiling no longer reserves the worst case up front.
    ///
    /// Storage is sized for the observed expert union rather than the worst-case T*K.
    /// Top-4 routing makes the batch size a tradeoff between repeated base-weight reads
    /// and the memory required for the active graph and expert union.
    /// Tokens per prefill forward. Decode ignores this.
    ///
    /// A full-layer read is about 8 s of flash for 40 layers, independent of
    /// how many tokens share it. 512 is the first size at which that fixed
    /// cost can land near 60 tok/s before GPU time; the Gated DeltaNet scan
    /// is what will jetsam first if 512 is too large.
    static let maximumBatch = 512
    /// Below this, prefill keeps the decode-style expert union. A full-layer
    /// read is 432 MiB × 40 and only pays for itself on a long chunk.
    static let fullLayerPrefillMinimum = 64

    struct Configuration {
        var layers = 40
        var hidden = 2048
        var moeIntermediate = 512
        var expertsPerLayer = 256
        // The public Edge0-35B-A3B-preview adapter and prerouter were trained with K=4.
        // This must stay aligned with the shipped artifacts: changing K changes both
        // the MoE output and the routing history consumed by the next-step heads.
        var activeExperts = 4
        /// Round-6 pregate + LoRA, trained at K=2. The two files are a pair.
        var round6 = false
        var groupSize = 64
        var bits = 4
        var fullAttentionInterval = 4
        var vocabulary = 248_320

        /// When true, keep every non-expert resident tensor after first materialisation.
        /// The default retains only selected tensors to limit active memory use.
        var retainAllResidentWeights = false

        /// Source used to prefetch expert reads for the next decode step.
        var routingGuess: RoutingGuess = .trainedHeads

        /// Number of recent routing steps used by `.recentRouting`.
        var routingLookback = 1

        enum RoutingGuess {
            /// Do not prefetch expert reads.
            case none
            /// Prefetch the experts selected by each layer on the previous step.
            case recentRouting
            /// The trained pregate heads, when their file is present. Predicts the
            /// *next* layer a *token* ahead, so it is the only source that can cover a
            /// layer before that layer has run — and it replaces the router to do it.
            case trainedHeads
        }

        /// Identifies this configuration in caches whose contents depend on it.
        var fingerprint: String { "full" }

        /// `(index + 1) % interval == 0` are full attention; the rest are linear.
        func isLinear(_ layer: Int) -> Bool {
            (layer + 1) % fullAttentionInterval != 0
        }
    }

    enum Failure: LocalizedError {
        case weightsMissing(URL)
        case tensorMissing(String)
        case layerFailed(Int)

        var errorDescription: String? {
            switch self {
            case .weightsMissing(let url):
                return "no model at \(url.lastPathComponent)"
            case .tensorMissing(let name):
                return "missing tensor \(name)"
            case .layerFailed(let layer):
                return "layer \(layer) could not run"
            }
        }
    }

    /// State carried between decode steps.
    ///
    /// Two kinds, and **they are not interchangeable**. Linear layers hold a fixed-size
    /// `[1, heads, dim, dim]` float32 matrix updated in place: it does not grow with
    /// context and cannot be pruned. Full-attention layers hold a KV cache that does
    /// grow. Any context policy that drops old turns shortens the second and leaves the
    /// first untouched — a model that appears to forget while still carrying everything.
    final class State {
        var convolution: [Int: MLXArray] = [:]
        var recurrent: [Int: MLXArray] = [:]
        var keys: [Int: MLXArray] = [:]
        var values: [Int: MLXArray] = [:]
        var offset = 0

        /// Routing carried between decode steps. Per-conversation, not per-model: two
        /// conversations share one set of heads but must never share the routing
        /// history that feeds them.
        let pregate = PregateHeads.State()

        /// Layer → the experts it ran, most recent step first, capped at the lookback.
        /// Previous-step router output, used by `.recentRouting`.
        var recentRouting: [Int: [[Int]]] = [:]

        /// Layer → what was asked for on its behalf. Exists so the hit rate is a
        /// measurement of what was *issued* rather than a restatement of the guess:
        /// `recentRouting` is overwritten as each layer runs, so comparing against it
        /// would compare a layer's routing with its own.
        var prefetched: [Int: [Int]] = [:]

        var byteCount: Int {
            [convolution, recurrent, keys, values].reduce(0) { total, table in
                total + table.values.reduce(0) { $0 + $1.size * $1.itemSize }
            }
        }

        func reset() {
            convolution.removeAll(); recurrent.removeAll()
            keys.removeAll(); values.removeAll()
            offset = 0
            pregate.reset()
            recentRouting.removeAll(); prefetched.removeAll()
        }
    }

    let configuration: Configuration
    private let weights: SafetensorsBundle
    private let experts: ExpertLoader
    private let lora: SafetensorsShard?
    private let directory: URL
    private let coreMLHead: CoreMLVocabularyHead?
    private var coreMLHeadEnabled: Bool
    private var coreMLHeadValidated = false

    init(directory: URL, configuration: Configuration = Configuration()) throws {
        let index = directory.appendingPathComponent("model.safetensors.index.json")
        guard FileManager.default.fileExists(atPath: index.path) else {
            throw Failure.weightsMissing(directory)
        }
        self.directory = directory
        self.configuration = configuration
        self.weights = try SafetensorsBundle(directory: directory)
        self.experts = try ExpertLoader(
            directory: directory, layers: configuration.layers,
            slots: Edge0Model35B.maximumBatch * configuration.activeExperts,
            hidden: configuration.hidden,
            moeIntermediate: configuration.moeIntermediate,
            groupSize: configuration.groupSize)
        self.experts.profile = self.profile
        let round6LoRA = directory.appendingPathComponent("lora-round6.safetensors")
        let officialLoRA = directory.appendingPathComponent("lora_edge0_35b.safetensors")
        let legacyLoRA = directory.appendingPathComponent("lora.safetensors")
        let loraURL: URL
        if configuration.round6 {
            loraURL = round6LoRA
        } else if FileManager.default.fileExists(atPath: officialLoRA.path) {
            loraURL = officialLoRA
        } else {
            loraURL = legacyLoRA
        }
        self.lora = try? SafetensorsShard(url: loraURL)
        if configuration.round6 {
            print("[round6] K=\(configuration.activeExperts) lora=\(self.lora == nil ? "missing" : "loaded")")
        }
        let compiledHead = directory.appendingPathComponent(
            CoreMLVocabularyHead.directoryName, isDirectory: true)
        if CommandLine.arguments.contains("--ane-35b"),
           FileManager.default.fileExists(atPath: compiledHead.path) {
            do {
                self.coreMLHead = try CoreMLVocabularyHead(directory: directory)
                self.coreMLHeadEnabled = true
                print("[coreml-head] loaded with computeUnits=cpuAndNeuralEngine")
            } catch {
                self.coreMLHead = nil
                self.coreMLHeadEnabled = false
                print("[coreml-head] unavailable: \(error)")
            }
        } else {
            self.coreMLHead = nil
            self.coreMLHeadEnabled = false
            print("[coreml-head] disabled or absent")
        }
        let free = (try? directory.resourceValues(
            forKeys: [.volumeAvailableCapacityForImportantUsageKey]))?
            .volumeAvailableCapacityForImportantUsage
        print("[weights] free=\(free.map { String($0 / 1_048_576) } ?? "?")MiB")

        // Pregate weights are optional. Report an invalid present file so a fallback to
        // the normal router is visible during diagnostics.
        let pregateURL = PregateHeads.fileURL(in: directory, round6: configuration.round6)
        if configuration.routingGuess == .trainedHeads,
           FileManager.default.fileExists(atPath: pregateURL.path) {
            do {
                let heads = try PregateHeads(
                    url: pregateURL, hidden: configuration.hidden,
                    experts: RepackedExperts.expertsPerLayer)
                self.pregate = heads
                print("[pregate] heads loaded: owners \(heads.owners.first ?? -1)..\(heads.owners.last ?? -1) n=\(heads.owners.count)")
                if let report = heads.selfCheck(
                    fixture: directory.appendingPathComponent("pregate-fixture.safetensors")) {
                    print(report)
                }
            } catch {
                print("[pregate] present but unusable: \(error)")
                self.pregate = nil
            }
        } else {
            self.pregate = nil
        }
    }

    /// The trained routing heads, when the file is present. See `PregateHeads`.
    private let pregate: PregateHeads?

    /// See `StepProfile` for why only the synchronous phases are timed.
    let profile = StepProfile()

    var loraLoaded: Bool { lora != nil }
    var availableLayers: Int { experts.availableLayers.count }

    // MARK: - Weights

    /// Selected tensors retained after first use to reduce repeated materialization.
    private var retainedWeights: [String: MLXArray] = [:]
    private var retainAttentionWeights = false

    /// First-turn prefill is latency-sensitive and sees every attention weight only
    /// once, so retaining them there adds pressure without reuse. Decode immediately
    /// reuses them; enable the cache at that boundary and keep it warm for later turns.
    func beginDecodeCaching() {
        retainAttentionWeights = true
    }

    /// Prepare selected attention weights and issue a discarded token before generation
    /// so initial Metal setup is outside the first reply's latency measurement.
    func prepareForFirstTurn() throws {
        beginDecodeCaching()
        let started = Date()
        var count = 0
        for name in weights.index.keys where
            name.hasPrefix("language_model.lm_head.")
            || name.contains(".linear_attn.")
            || name.contains(".self_attn.") {
            if weight(name) != nil { count += 1 }
        }
        print(String(
            format: "[weights] first-turn cache tensors=%d %.2fs",
            count, Date().timeIntervalSince(started)))

        let gpu = Date()
        let scratch = State()
        _ = try step(tokens: MLXArray([Int32(0)], [1, 1]), state: scratch)
        profile.reset()
        print(String(format: "[weights] gpu warm %.2fs", Date().timeIntervalSince(gpu)))
    }

    /// Materialize a tensor from the mapped checkpoint. Retain selected tensors according
    /// to the configured memory policy; other tensors are released after use.
    private func weight(_ name: String) -> MLXArray? {
        if let retained = retainedWeights[name] { return retained }
        guard let entry = weights.entry(name), let pointer = weights.pointer(to: name)
        else { return nil }
        let raw = UnsafeRawBufferPointer(start: pointer, count: entry.byteCount)
        profile.recordWeight(bytes: entry.byteCount)
        let made = profile.measure(.weightCopy) { () -> MLXArray? in
            switch entry.dtype {
            case "U32", "I32": return MLXArray(raw, entry.shape, type: UInt32.self)
            case "BF16":
                return MLXArray(raw, entry.shape, type: UInt16.self).view(dtype: .bfloat16)
            case "F32": return MLXArray(raw, entry.shape, type: Float.self)
            default: return nil
            }
        }
        let shouldRetain = configuration.retainAllResidentWeights
            || (!coreMLHeadEnabled && name.hasPrefix("language_model.lm_head."))
            || (retainAttentionWeights
                && (name.contains(".linear_attn.") || name.contains(".self_attn.")))
        if shouldRetain, let made {
            retainedWeights[name] = made
        }
        return made
    }

    /// Apply the base quantized linear layer and its optional LoRA update.
    private func linear(_ x: MLXArray, _ stem: String, bits: Int) -> MLXArray? {
        guard let w = weight("\(stem).weight"),
              let s = weight("\(stem).scales"),
              let b = weight("\(stem).biases")
        else { return nil }
        let base = quantizedMatmul(
            x.asType(.bfloat16), w, scales: s, biases: b, transpose: true,
            groupSize: configuration.groupSize, bits: bits, mode: .affine)

        guard let prepared = preparedLoRA(stem) else { return base }
        let delta = matmul(matmul(x.asType(.float16), prepared.a), prepared.b)
        return base + delta.asType(base.dtype)
    }

    /// LoRA matrices are converted, transposed, scaled, and cached on first use.
    private struct PreparedLoRA {
        let a: MLXArray
        let b: MLXArray
    }
    private var loraCache: [String: PreparedLoRA] = [:]

    private func preparedLoRA(_ stem: String) -> PreparedLoRA? {
        if let cached = loraCache[stem] { return cached }
        guard let lora,
              let aEntry = lora.entries["\(stem).lora_A"],
              let bEntry = lora.entries["\(stem).lora_B"],
              let aPointer = lora.pointer(to: "\(stem).lora_A"),
              let bPointer = lora.pointer(to: "\(stem).lora_B")
        else { return nil }

        func loadAdapter(_ entry: SafetensorsShard.Entry,
                         _ pointer: UnsafeRawPointer) -> MLXArray? {
            let raw = UnsafeRawBufferPointer(start: pointer, count: entry.byteCount)
            switch entry.dtype {
            case "F16": return MLXArray(raw, entry.shape, type: Float16.self)
            case "F32": return MLXArray(raw, entry.shape, type: Float.self)
            default: return nil
            }
        }
        guard let a = loadAdapter(aEntry, aPointer),
              let bb = loadAdapter(bEntry, bPointer)
        else { return nil }
        // float16 throughout, as the Python implementation does.
        let prepared = PreparedLoRA(
            a: a.asType(.float16).transposed(),
            b: (Float16(2.0) * bb.asType(.float16)).transposed())
        // Materialised here so the graph that produced them is not replayed per token.
        eval(prepared.a, prepared.b)
        loraCache[stem] = prepared
        return prepared
    }

    private func norm(_ x: MLXArray, _ name: String) -> MLXArray? {
        guard let w = weight(name) else { return nil }
        return MLXFast.rmsNorm(x.asType(.bfloat16), weight: w, eps: 1e-6)
    }

    private func rmsNormalise(_ x: MLXArray) -> MLXArray {
        let squares = mean(x.asType(.float32) * x.asType(.float32), axis: -1, keepDims: true)
        return (x.asType(.float32) * rsqrt(squares + 1e-6)).asType(x.dtype)
    }

    // MARK: - Layers

    /// Copy just the rows named, straight out of the mapping.
    ///
    /// `weight(_:)` builds an `MLXArray` over a whole tensor, and building one **copies**
    /// — mlx-c routes it to `allocator::malloc` plus `std::copy`. For a table indexed one
    /// row at a time that is the wrong shape by five orders of magnitude.
    private func rows(_ name: String, _ indices: [Int32]) -> MLXArray? {
        guard let entry = weights.entry(name), let base = weights.pointer(to: name),
              entry.shape.count == 2, entry.shape[0] > 0
        else { return nil }
        let columns = entry.shape[1]
        let rowBytes = entry.byteCount / entry.shape[0]
        var gathered = [UInt8](repeating: 0, count: rowBytes * indices.count)
        gathered.withUnsafeMutableBytes { destination in
            for (slot, row) in indices.enumerated() {
                guard row >= 0, Int(row) < entry.shape[0] else { continue }
                memcpy(destination.baseAddress!.advanced(by: slot * rowBytes),
                       base.advanced(by: Int(row) * rowBytes), rowBytes)
            }
        }
        let shape = [indices.count, columns]
        return gathered.withUnsafeBytes { raw -> MLXArray? in
            switch entry.dtype {
            case "U32", "I32": return MLXArray(raw, shape, type: UInt32.self)
            case "BF16": return MLXArray(raw, shape, type: UInt16.self).view(dtype: .bfloat16)
            case "F32": return MLXArray(raw, shape, type: Float.self)
            default: return nil
            }
        }
    }

    /// Gather only the embedding rows needed by the current token batch.
    private func embed(_ tokens: MLXArray) -> MLXArray? {
        let stem = "language_model.model.embed_tokens"
        let ids = tokens.reshaped([-1]).asArray(Int32.self)
        guard let w = rows("\(stem).weight", ids),
              let s = rows("\(stem).scales", ids),
              let b = rows("\(stem).biases", ids)
        else { return nil }
        // Dequantize only the gathered rows rather than the full embedding table.
        let out = dequantized(w, scales: s, biases: b,
                              groupSize: configuration.groupSize, bits: configuration.bits,
                              mode: .affine)
        return out.reshaped(tokens.shape + [out.dim(-1)])
    }

    /// Grouped-query attention with partial RoPE and an output gate.
    ///
    /// The gate is the second half of `q_proj`; RoPE covers 64 of each 256-dimensional
    /// head and uses a base of 10,000,000.
    private func fullAttention(_ x: MLXArray, layer: Int, state: State) -> MLXArray? {
        let stem = "language_model.model.layers.\(layer).self_attn"
        let heads = 16, kvHeads = 2, headDim = 256
        let (batch, length) = (x.dim(0), x.dim(1))

        guard let qOut = linear(x, "\(stem).q_proj", bits: 4),
              let kOut = linear(x, "\(stem).k_proj", bits: 4),
              let vOut = linear(x, "\(stem).v_proj", bits: 4),
              let qNorm = weight("\(stem).q_norm.weight"),
              let kNorm = weight("\(stem).k_norm.weight")
        else { return nil }

        let split = qOut.reshaped([batch, length, heads, 2 * headDim])
        var queries = split[.ellipsis, 0 ..< headDim]
        let gate = split[.ellipsis, headDim ..< (2 * headDim)]
            .reshaped([batch, length, heads * headDim])

        var keys = kOut.reshaped([batch, length, kvHeads, headDim])
        var values = vOut.reshaped([batch, length, kvHeads, headDim])

        queries = MLXFast.rmsNorm(queries, weight: qNorm, eps: 1e-6).transposed(0, 2, 1, 3)
        keys = MLXFast.rmsNorm(keys, weight: kNorm, eps: 1e-6).transposed(0, 2, 1, 3)
        values = values.transposed(0, 2, 1, 3)

        let offset = state.offset
        func rope(_ a: MLXArray) -> MLXArray {
            MLXFast.RoPE(a, dimensions: 64, traditional: false,
                         base: 10_000_000, scale: 1, offset: offset)
        }
        queries = rope(queries)
        keys = rope(keys)

        if let pastKeys = state.keys[layer], let pastValues = state.values[layer] {
            keys = concatenated([pastKeys, keys], axis: 2)
            values = concatenated([pastValues, values], axis: 2)
        }
        state.keys[layer] = keys
        state.values[layer] = values

        // A single query attending to everything before it needs no mask; passing a
        // square one sized for the prompt is how a working prefill breaks decode.
        let total = keys.dim(2)
        var mask: MLXArray? = nil
        if length > 1 {
            var entries = [Float](repeating: 0, count: length * total)
            for row in 0 ..< length {
                for column in (offset + row + 1) ..< total {
                    entries[row * total + column] = -Float.greatestFiniteMagnitude
                }
            }
            mask = MLXArray(entries, [length, total]).asType(queries.dtype)
        }

        let attended = MLXFast.scaledDotProductAttention(
            queries: queries, keys: keys, values: values, scale: 1.0 / 16.0, mask: mask)
        let merged = attended.transposed(0, 2, 1, 3)
            .reshaped([batch, length, heads * headDim])
        return linear(merged * sigmoid(gate), "\(stem).o_proj", bits: 4)
    }

    /// Gated DeltaNet: depthwise causal convolution, then a gated delta-rule recurrence
    /// over a float32 state.
    ///
    /// `q` and `k` carry 16 heads against `v`'s 32 and are matched by **consecutive**
    /// duplication — h0, h0, h1, h1 — not by tiling. Both run; one is correct.
    private func linearAttention(_ x: MLXArray, layer: Int, state: State) -> MLXArray? {
        let stem = "language_model.model.layers.\(layer).linear_attn"
        let kHeads = 16, vHeads = 32, headDim = 128
        let keyDim = kHeads * headDim, valueDim = vHeads * headDim
        let convDim = keyDim * 2 + valueDim, kernel = 4
        let (batch, length) = (x.dim(0), x.dim(1))

        guard let qkv = linear(x, "\(stem).in_proj_qkv", bits: 4),
              let zRaw = linear(x, "\(stem).in_proj_z", bits: 4),
              let aRaw = linear(x, "\(stem).in_proj_a", bits: 4),
              let bRaw = linear(x, "\(stem).in_proj_b", bits: 4),
              let convWeight = weight("\(stem).conv1d.weight"),
              let aLog = weight("\(stem).A_log"),
              let dtBias = weight("\(stem).dt_bias"),
              let normWeight = weight("\(stem).norm.weight")
        else { return nil }

        let history = state.convolution[layer]?.asType(qkv.dtype)
            ?? MLXArray.zeros([batch, kernel - 1, convDim]).asType(qkv.dtype)
        let tail = concatenated([history, qkv], axis: 1)

        // `conv1d` accumulates internally in float32. Preserve that precision for
        // numerically consistent recurrent state updates.
        let convolved = conv1d(tail, convWeight, groups: convDim)
        let activated = convolved * sigmoid(convolved)
        state.convolution[layer] = tail[0..., (tail.dim(1) - (kernel - 1))...]

        var q = activated[.ellipsis, 0 ..< keyDim].reshaped([batch, length, kHeads, headDim])
        var k = activated[.ellipsis, keyDim ..< (2 * keyDim)]
            .reshaped([batch, length, kHeads, headDim])
        let v = activated[.ellipsis, (2 * keyDim) ..< convDim]
            .reshaped([batch, length, vHeads, headDim])

        // q is scaled by Dk⁻¹ and k by Dk^-0.5 — not the same factor.
        let inverseScale = Float(pow(Double(headDim), -0.5))
        q = rmsNormalise(q) * (inverseScale * inverseScale)
        k = rmsNormalise(k) * inverseScale
        q = repeated(q, count: vHeads / kHeads, axis: -2)
        k = repeated(k, count: vHeads / kHeads, axis: -2)

        let beta = sigmoid(bRaw.asType(.float32))
        let shifted = aRaw.asType(.float32) + dtBias.asType(.float32)
        let g = MLX.exp(-MLX.exp(aLog.asType(.float32)) * MLX.log(MLX.exp(shifted) + 1))

        var recurrent = state.recurrent[layer]?.asType(.float32)
            ?? MLXArray.zeros([batch, vHeads, headDim, headDim], dtype: .float32)
        var outputs: [MLXArray] = []
        // Use matrix contractions for the recurrent updates to avoid materializing large
        // broadcast-multiply intermediates:
        //
        //     (recurrent * kt[..., None, :]).sum(-1)     2 MiB temporary, reduced to 16 KB
        //     matmul(recurrent, kt[..., None])           16 KB, straight out of a GEMV
        //
        // GEMV accumulation order can differ from elementwise reduction in the last bits.
        for t in 0 ..< length {
            let qt = q[0..., t].asType(.float32).expandedDimensions(axis: -1)
            let kt = k[0..., t].asType(.float32).expandedDimensions(axis: -1)
            let vt = v[0..., t].asType(.float32)
            let decay = g[0..., t].expandedDimensions(axis: -1).expandedDimensions(axis: -1)

            recurrent = recurrent * decay
            let memory = matmul(recurrent, kt).squeezed(axis: -1)
            let delta = (vt - memory) * beta[0..., t].expandedDimensions(axis: -1)
            // The outer product has to materialise 2 MiB whichever way it is written, so
            // this one stays as it was.
            recurrent = recurrent
                + kt.swappedAxes(-1, -2) * delta.expandedDimensions(axis: -1)
            outputs.append(matmul(recurrent, qt).squeezed(axis: -1))
        }
        state.recurrent[layer] = recurrent

        let stacked = MLX.stacked(outputs, axis: 1).asType(x.dtype)
        let z = zRaw.reshaped([batch, length, vHeads, headDim])
        // RMSNormGated, in float32 as trained.
        let normalised = MLXFast.rmsNorm(stacked, weight: normWeight, eps: 1e-6)
            .asType(.float32)
        let gated = (sigmoid(z.asType(.float32)) * z.asType(.float32) * normalised)
            .asType(x.dtype)
        return linear(gated.reshaped([batch, length, valueDim]), "\(stem).out_proj", bits: 4)
    }

    /// Router, K streamed experts, and the shared expert.
    ///
    /// Apply softmax over all expert logits before top-k selection, then renormalize the
    /// selected scores.
    /// Routed experts for **one or more** tokens.
    ///
    /// The single-token form this replaces treated the K experts as the batch axis, every
    /// one of them seeing the same input. With T tokens there are T*K (token, expert)
    /// pairs and each needs its own input row, so the batch axis becomes T*K and the
    /// expert for each row is named by `rhsIndices`.
    ///
    /// For a batch, read the union of experts selected by its tokens. The union may be
    /// smaller than T*K when tokens share expert selections.
    private func moe(_ x: MLXArray, layer: Int, state: State) -> MLXArray? {
        let prefix = "language_model.model.layers.\(layer).mlp"
        let k = configuration.activeExperts
        let tokens = x.dim(1)

        if tokens >= Self.fullLayerPrefillMinimum {
            return moeFullLayer(x, layer: layer, state: state, prefix: prefix, k: k, tokens: tokens)
        }

        let ids: [Int]
        var scores: MLXArray

        // Predictions replace the normal gate during single-token decode. Prefill uses
        // the normal gate because its multi-token routing shape differs.
        if tokens == 1, let predicted = state.pregate.predicted[layer],
           let predictedScores = state.pregate.scores[layer], predicted.count == k {
            ids = predicted
            // Rebuilt here, from k floats. The alternative — keeping the slice of the
            // staging array — would hold that array and its graph across the step.
            scores = MLXArray(predictedScores, [1, 1, k])
        } else {
            guard let logits = linear(x, "\(prefix).gate", bits: 8) else { return nil }
            let gates = softmax(logits.asType(.float32), axis: -1)
            let chosen = argPartition(-gates, kth: k - 1, axis: -1)[.ellipsis, 0 ..< k]
            scores = takeAlong(gates, chosen, axis: -1)
            scores = scores / scores.sum(axis: -1, keepDims: true)

            // Expert IDs must reach the CPU before their file reads can be issued.
            let kind: StepProfile.Phase =
                configuration.isLinear(layer) ? .syncLinear : .syncFull
            ids = profile.measure(kind) { () -> [Int] in
                eval(chosen, scores)
                return chosen.reshaped([-1]).asArray(Int32.self).map { Int($0) }
            }
        }
        guard ids.count == tokens * k else { return nil }

        // Captured for this layer's own head, which runs at the step boundary and
        // predicts for `layer + 1`. The last position is the right one for both: a
        // prefill's final position is what the first decode step continues from.
        //
        // Store the routes actually used, including predictions, as inputs for the next
        // pregate evaluation. Tensor reshaping is deferred until after graph evaluation.
        if let pregate, pregate.owners.contains(layer) {
            state.pregate.moeInput[layer] = x
            state.pregate.executed[layer] = ids.suffix(k).map { Int32($0) }
        }

        // Distinct experts in first-appearance order, and where each pair's expert landed.
        var rowOf: [Int: Int] = [:]
        var union: [Int] = []
        for id in ids where rowOf[id] == nil {
            rowOf[id] = union.count
            union.append(id)
        }
        profile.recordRouting(union: union.count, pairs: ids.count)

        // How much of what this layer needs was already asked for. **This is the number
        // that bounds the whole scheme** — the read can only be hidden for blocks the
        // guess anticipated — and it is free to collect, because both sets are already
        // on the CPU. Recorded before the routing is overwritten below.
        if configuration.routingGuess != .none {
            let guessed = state.prefetched[layer] ?? []
            profile.recordPrefetch(
                hits: union.count { guessed.contains($0) }, total: union.count)
        }
        // What executed, for the next step to guess from. The union rather than the
        // last position's pair: during prefill that is strictly more informative, and
        // during decode they are the same thing.
        var history = state.recentRouting[layer] ?? []
        history.insert(union, at: 0)
        state.recentRouting[layer] = Array(history.prefix(configuration.routingLookback))

        guard let pool = try? experts.load(layer: layer, experts: union),
              let rows = try? ids.map({ id -> Int32 in
                  guard let row = rowOf[id] else { throw Failure.layerFailed(layer) }
                  return Int32(row)
              })
        else { return nil }

        // **The single-token form is kept separate, and it is not premature.** Generalising
        // this for batching added a `reshaped` on top of a broadcast — a stride-0 view that
        // a reshape has to materialise — and decode runs this on every token of every
        // layer. Batching is used only by prefill; making decode pay for it was a
        // regression, not a simplification.
        let expanded = x.reshaped([tokens, 1, configuration.hidden]).asType(.bfloat16)
        let inputs = tokens == 1
            ? broadcast(expanded, to: [k, 1, configuration.hidden])
            : broadcast(expanded, to: [tokens, k, configuration.hidden])
                .reshaped([tokens * k, 1, configuration.hidden])
        let indices = MLXArray(rows)

        func project(_ input: MLXArray, _ offset: Int) -> MLXArray {
            gatherQuantizedMM(
                input, pool[offset], scales: pool[offset + 1], biases: pool[offset + 2],
                rhsIndices: indices, transpose: true,
                groupSize: configuration.groupSize, bits: configuration.bits,
                // **An assertion about the input, not a request**, so it is true only
                // when it is true. With one token the union is built in first-appearance
                // order from k distinct experts, making the rows exactly `0 ..< k` and
                // ascending; with more, rows repeat and are unordered, and claiming
                // otherwise would silently gather the wrong experts.
                mode: .affine, sortedIndices: tokens == 1)
        }
        let gate = project(inputs, ExpertLoader.gate)
        let up = project(inputs, ExpertLoader.up)
        let routed = project(gate * sigmoid(gate) * up, ExpertLoader.down)
        let weighted = routed
            * scores.reshaped([tokens * k, 1, 1]).asType(routed.dtype)
        let combined = weighted
            .reshaped([tokens, k, configuration.hidden])
            .sum(axis: 1)

        // Compute the shared expert after routed expert loading to avoid competing for
        // memory bandwidth during the reads.
        guard let sharedGate = linear(x, "\(prefix).shared_expert_gate", bits: 8),
              let sg = linear(x, "\(prefix).shared_expert.gate_proj", bits: 4),
              let su = linear(x, "\(prefix).shared_expert.up_proj", bits: 4),
              let shared = linear(sg * sigmoid(sg) * su,
                                  "\(prefix).shared_expert.down_proj", bits: 4)
        else { return nil }
        return combined.reshaped(shared.shape) + sigmoid(sharedGate) * shared
    }

    /// Prefill for a long chunk: the layer file is read once, and expert ids
    /// index that stack directly. The ids still come back to the CPU afterwards
    /// so the pregate feature sees what actually ran; the read no longer waits
    /// on that sync.
    private func moeFullLayer(_ x: MLXArray, layer: Int, state: State,
                              prefix: String, k: Int, tokens: Int) -> MLXArray? {
        guard let pool = try? experts.loadFullLayer(layer: layer),
              let logits = linear(x, "\(prefix).gate", bits: 8)
        else { return nil }
        let gates = softmax(logits.asType(.float32), axis: -1)
        let chosen = argPartition(-gates, kth: k - 1, axis: -1)[.ellipsis, 0 ..< k]
        var scores = takeAlong(gates, chosen, axis: -1)
        scores = scores / scores.sum(axis: -1, keepDims: true)

        let kind: StepProfile.Phase =
            configuration.isLinear(layer) ? .syncLinear : .syncFull
        let ids: [Int] = profile.measure(kind) {
            eval(chosen, scores)
            return chosen.reshaped([-1]).asArray(Int32.self).map { Int($0) }
        }
        guard ids.count == tokens * k else { return nil }
        if let pregate, pregate.owners.contains(layer) {
            state.pregate.moeInput[layer] = x
            state.pregate.executed[layer] = ids.suffix(k).map { Int32($0) }
        }
        var rowOf: [Int: Int] = [:]
        var union: [Int] = []
        for id in ids where rowOf[id] == nil {
            rowOf[id] = union.count
            union.append(id)
        }
        profile.recordRouting(union: union.count, pairs: ids.count)
        var history = state.recentRouting[layer] ?? []
        history.insert(union, at: 0)
        state.recentRouting[layer] = Array(history.prefix(configuration.routingLookback))

        let expanded = x.reshaped([tokens, 1, configuration.hidden]).asType(.bfloat16)
        let inputs = broadcast(expanded, to: [tokens, k, configuration.hidden])
            .reshaped([tokens * k, 1, configuration.hidden])
        // Row `e` of the full stack is expert `e`, so the router index is the gather index.
        let indices = chosen.reshaped([tokens * k])
        func project(_ input: MLXArray, _ offset: Int) -> MLXArray {
            gatherQuantizedMM(
                input, pool[offset], scales: pool[offset + 1], biases: pool[offset + 2],
                rhsIndices: indices, transpose: true,
                groupSize: configuration.groupSize, bits: configuration.bits,
                mode: .affine, sortedIndices: false)
        }
        let gate = project(inputs, ExpertLoader.gate)
        let up = project(inputs, ExpertLoader.up)
        let routed = project(gate * sigmoid(gate) * up, ExpertLoader.down)
        let weighted = routed * scores.reshaped([tokens * k, 1, 1]).asType(routed.dtype)
        let combined = weighted.reshaped([tokens, k, configuration.hidden]).sum(axis: 1)
        guard let sharedGate = linear(x, "\(prefix).shared_expert_gate", bits: 8),
              let sg = linear(x, "\(prefix).shared_expert.gate_proj", bits: 4),
              let su = linear(x, "\(prefix).shared_expert.up_proj", bits: 4),
              let shared = linear(sg * sigmoid(sg) * su,
                                  "\(prefix).shared_expert.down_proj", bits: 4)
        else { return nil }
        return combined.reshaped(shared.shape) + sigmoid(sharedGate) * shared
    }

    /// Run all routing heads as one batch and return the expert IDs for the next step.
    ///
    /// Returns the expert set per consuming layer, which is what a prefetch needs.
    @discardableResult
    private func stagePregate(state: State) -> [Int: [Int]] {
        guard let pregate else { return [:] }
        let k = configuration.activeExperts

        var inputs: [MLXArray] = []
        var executed: [[Int32]] = [], previous: [[Int32]] = []
        inputs.reserveCapacity(pregate.owners.count)
        for owner in pregate.owners {
            // A pruned or otherwise absent owner would desynchronise the stack from
            // `owners`, so the whole staging is skipped rather than silently shifted.
            //
            // Skipping has to *clear* the predictions, not just decline to write new
            // ones. `advance()` deliberately keeps `predicted` alive across the step
            // boundary — that is how a prediction made at step N reaches step N+1 — so
            // leaving it untouched here would hand step N+1 the routing computed for
            // step N, one token stale, with nothing in any log to say so.
            guard let input = state.pregate.moeInput[owner],
                  let ran = state.pregate.executed[owner]
            else {
                state.pregate.predicted = [:]
                state.pregate.scores = [:]
                state.pregate.advance()
                return [:]
            }
            // Sliced here rather than at capture time: by now `eval(h)` has run, so this
            // reshapes a materialised array instead of adding a node to a live graph.
            // The last position is the right one for a prefill chunk too — it is what
            // the first decode step continues from.
            let vector = input.ndim == 1
                ? input
                : (input.dim(1) == 1 ? input.reshaped([-1]) : input[0, -1])
            inputs.append(vector)
            executed.append(ran)
            // Use a zero row when no previous-token routing is available.
            previous.append(state.pregate.previous[owner] ?? [])
        }

        let logits = pregate.logits(
            inputs: stacked(inputs, axis: 0),
            executed: pregate.oneHotRows(executed),
            previous: pregate.oneHotRows(previous))

        // Apply the same softmax, top-k selection, and score normalization as the router.
        let gates = softmax(logits, axis: -1)
        let chosen = argPartition(-gates, kth: k - 1, axis: -1)[.ellipsis, 0 ..< k]
        var weights = takeAlong(gates, chosen, axis: -1)
        weights = weights / weights.sum(axis: -1, keepDims: true)

        // The one barrier, and it brings back both halves. Pulling the weights across
        // too costs 66 floats and means nothing MLX-shaped is carried into the next
        // step — a view into `weights` would have kept the array, and its graph, alive
        // for every one of the thirty-three layers that held one.
        let (flat, weighting) = profile.measure(.pregateStage) { () -> ([Int32], [Float]) in
            eval(chosen, weights)
            return (chosen.reshaped([-1]).asArray(Int32.self),
                    weights.reshaped([-1]).asArray(Float.self))
        }

        var sets: [Int: [Int]] = [:]
        for (index, consumer) in pregate.consumers.enumerated() {
            let range = (index * k) ..< ((index + 1) * k)
            let slice = flat[range].map { Int($0) }
            state.pregate.predicted[consumer] = slice
            state.pregate.scores[consumer] = Array(weighting[range])
            sets[consumer] = Array(Set(slice)).sorted()
        }

        state.pregate.advance()
        return sets
    }

    // MARK: - Forward

    /// Run one model step and evaluate the resulting graph once at the end.
    func step(tokens: MLXArray, state: State) throws -> MLXArray {
        try step(tokens: tokens, state: state, needsLogits: true)!
    }

    /// One step, with the option of **not** projecting to the vocabulary.
    ///
    /// During prefill, intermediate positions do not need vocabulary logits. This option
    /// skips that projection when the result will not be consumed.
    func step(tokens: MLXArray, state: State, needsLogits: Bool) throws -> MLXArray? {
        guard var h = embed(tokens) else { throw Failure.tensorMissing("embed_tokens") }
        // Checkpoint weights use bfloat16; keep activations in the same format.
        h = h.asType(.bfloat16)

        let batchedPrefill = tokens.dim(1) >= Self.fullLayerPrefillMinimum
        for layer in 0 ..< configuration.layers {
            if batchedPrefill && layer % 10 == 0 {
                print("[35b layer] \(layer) T=\(tokens.dim(1))")
            }
            let stem = "language_model.model.layers.\(layer)"
            guard let normed = norm(h, "\(stem).input_layernorm.weight") else {
                throw Failure.layerFailed(layer)
            }
            let attended = configuration.isLinear(layer)
                ? linearAttention(normed, layer: layer, state: state)
                : fullAttention(normed, layer: layer, state: state)
            guard let attended else { throw Failure.layerFailed(layer) }
            h = h + attended

            guard let post = norm(h, "\(stem).post_attention_layernorm.weight"),
                  let mixed = moe(post, layer: layer, state: state)
            else { throw Failure.layerFailed(layer) }
            h = h + mixed
            // A full-layer expert stack is 432 MiB. Evaluating here drops it
            // before the next layer's read; holding all forty would be 17 GB.
            if batchedPrefill {
                eval(h)
                if let held = state.pregate.moeInput[layer] {
                    let last = held[0, -1]
                    eval(last)
                    state.pregate.moeInput[layer] = last
                }
            }
        }
        eval(h)

        // Run routing predictions after graph evaluation; they consume hidden states
        // produced by this step and provide expert IDs for the next step.
        switch configuration.routingGuess {
        case .none:
            break
        case .trainedHeads:
            if tokens.dim(1) > 1 && !needsLogits {
                // Batched prefill never consumes the one-token-ahead head predictions.
                // Reuse the real routing as a read hint and avoid loading/running 132 MiB
                // of heads until the final prefill batch, whose prediction is consumed
                // by the first decode step.
                state.prefetched = state.recentRouting.mapValues { Array(Set($0.joined())) }
                for (layer, chosen) in state.prefetched {
                    experts.prefetch(layer: layer, experts: chosen)
                }
                break
            }
            state.prefetched = [:]
            for (layer, chosen) in stagePregate(state: state) {
                state.prefetched[layer] = chosen
                experts.prefetch(layer: layer, experts: chosen)
            }
            // The official heads cover consumers 7...39. Preserve their exact routing,
            // but prefetch the uncovered first layers from their last real routing.
            // This is advice only: misses still run the router and read the right files.
            for layer in 0 ..< configuration.layers where state.prefetched[layer] == nil {
                guard let chosen = state.recentRouting[layer]?.first else { continue }
                state.prefetched[layer] = chosen
                experts.prefetch(layer: layer, experts: chosen)
            }
        case .recentRouting:
            // Snapshotted, not read live: `recentRouting` is rewritten layer by layer as
            // the next step runs, so scoring the hit rate against it would compare a
            // layer's routing with its own and report a meaningless 100%.
            state.prefetched = state.recentRouting.mapValues { Array(Set($0.joined())) }
            for (layer, chosen) in state.prefetched {
                experts.prefetch(layer: layer, experts: chosen)
            }
        }

        state.offset += tokens.dim(1)
        profile.countStep()
        guard needsLogits else { return nil }
        guard let final = norm(h, "language_model.model.norm.weight") else {
            throw Failure.tensorMissing("model.norm")
        }
        if coreMLHeadEnabled {
            // Core ML consumes only the final position. This is 4 KiB instead of the
            // 248,320-element logits vector, and avoids retaining MLX's 286 MiB head.
            let last = final[0, -1].reshaped([1, configuration.hidden])
            eval(last)
            return last
        }
        guard let logits = linear(final, "language_model.lm_head", bits: 4) else {
            throw Failure.tensorMissing("lm_head")
        }
        return logits
    }

    /// Argmax over the last position's logits.
    func greedy(_ output: MLXArray) async throws -> Int32 {
        if coreMLHeadEnabled, let coreMLHead,
           output.dim(output.ndim - 1) == configuration.hidden {
            let coreMLToken = try await profile.measure(.vocabularyHead) {
                try await coreMLHead.predict(output)
            }
            if !coreMLHeadValidated {
                guard let logits = linear(output, "language_model.lm_head", bits: 4) else {
                    throw Failure.tensorMissing("lm_head")
                }
                let mlxToken = Self.mlxGreedy(logits)
                if mlxToken != coreMLToken {
                    coreMLHeadEnabled = false
                    print("[coreml-head] parity FAIL coreml=\(coreMLToken) mlx=\(mlxToken); falling back")
                    return mlxToken
                }
                coreMLHeadValidated = true
                retainedWeights = retainedWeights.filter {
                    !$0.key.hasPrefix("language_model.lm_head.")
                }
                print("[coreml-head] parity PASS token=\(coreMLToken)")
            }
            return coreMLToken
        }
        return Self.mlxGreedy(output)
    }

    private static func mlxGreedy(_ logits: MLXArray) -> Int32 {
        // `-1`, not `0`: with a batched forward the interesting logits are the last
        // position's. Identical for a single token, which is why this was never wrong.
        let final = logits.ndim == 2 ? logits[0] : logits[0, -1]
        let choice = argMax(final.asType(.float32), axis: -1)
        eval(choice)
        return choice.item(Int32.self)
    }

    /// Greedy decode. Prompt and generation take the same path — one token at a time —
    /// which costs a little on prefill and buys a single code path.
    func generate(promptIds: [Int32], stop: StopCondition,
                  state: State? = nil,
                  onToken: @escaping (Int32) -> Bool) throws -> [Int32]
    {
        let state = state ?? State()
        var produced: [Int32] = []
        var next: Int32 = 0

        for (index, token) in promptIds.enumerated() {
            let logits = try step(tokens: MLXArray([token], [1, 1]), state: state)
            if index == promptIds.count - 1 {
                let choice = argMax(logits[0, 0].asType(.float32), axis: -1)
                eval(choice)
                next = choice.item(Int32.self)
            }
        }

        while true {
            switch stop.evaluate(token: next, produced: produced.count) {
            case .hitEndToken, .hitLimit:
                return produced
            case .keepGoing:
                break
            }
            produced.append(next)
            // The callback decides whether to continue, so cancellation is the caller's
            // to express rather than something this loop has to be told about.
            guard onToken(next) else { return produced }

            let logits = try step(tokens: MLXArray([next], [1, 1]), state: state)
            let choice = argMax(logits[0, 0].asType(.float32), axis: -1)
            eval(choice)
            next = choice.item(Int32.self)
        }
    }
}