Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
10.5 KB
Raw Blame
import Edge0Core
import Foundation
import MLX

public struct Edge0GenerationResult: Sendable {
    public let text: String
    public let generatedTokenCount: Int
    public let elapsedSeconds: TimeInterval
    public let prefillSeconds: TimeInterval
    public let prefillTokensPerSecond: Double
    public let timeToFirstTokenSeconds: TimeInterval
    public let decodeTokensPerSecond: Double
    public let peakMemoryBytes: Int
}

private enum Edge0ChatTemplate8B {
    static func firstTurn(_ text: String, thinking: Bool) -> String {
        "<role>SYSTEM</role>detailed thinking \(thinking ? "on" : "off")<|role_end|>" +
        "<role>HUMAN</role>\(text)<|role_end|>" + assistantPrefix(thinking: thinking)
    }

    static func nextTurn(_ text: String, thinking: Bool) -> String {
        "<|role_end|><role>HUMAN</role>\(text)<|role_end|>" +
        assistantPrefix(thinking: thinking)
    }

    private static func assistantPrefix(thinking: Bool) -> String {
        "<role>ASSISTANT</role>\n" + (thinking ? "<think>" : "<think></think>")
    }
}

/// Product-facing, stateful chat entry point. The model math and cache lifetime
/// remain owned by StreamingEdge0Model8B; this type only mirrors the official
/// chat template and official sampling loop.
public final class Edge0ChatEngine: @unchecked Sendable {
    private let tokenizer: Edge0Tokenizer8B
    private let model: StreamingEdge0Model8B
    private var hasConversationContext = false
    private var tokenCount = 0

    /// Short prompts do not amortize loading all 128 experts per layer.
    private static let fullLayerPrefillMinimum = 128

    private static var runtimeDevice: Device {
        #if targetEnvironment(simulator)
        .cpu
        #else
        .gpu
        #endif
    }

    public init(modelURL: URL, progress: (String) -> Void = { _ in }) throws {
        let built = try Device.withDefaultDevice(Self.runtimeDevice) {
            let configuration = try Edge0Configuration8B.load(from: modelURL.appendingPathComponent("config.json"))
            let tokenizer = try Edge0Tokenizer8B(contentsOf: modelURL.appendingPathComponent("tokenizer.json"))
            let store = try ExpertTensorStore(
                modelURL: modelURL.appendingPathComponent("model.safetensors"),
                expertCount: configuration.numExperts
            )
            let model = try StreamingEdge0Model8B(
                configuration: configuration, store: store,
                loraWeightsURL: modelURL.appendingPathComponent(
                    "lora_edge0_8b.safetensors"),
                prerouterWeightsURL: modelURL.appendingPathComponent(
                    "prerouter_edge0_8b.safetensors"),
                progress: progress)
            progress("prewarming official decode path")
            try model.prewarm()
            return (tokenizer, model)
        }
        tokenizer = built.0
        model = built.1
    }

    public func reset() {
        model.reset()
        hasConversationContext = false
        tokenCount = 0
    }

    public func reply(
        to userText: String,
        maxTokens: Int = 2048,
        thinking: Bool = false,
        seed: UInt64? = nil,
        onText: @escaping @Sendable (String) -> Void = { _ in },
        shouldContinue: @escaping @Sendable () -> Bool = { true }
    ) throws -> Edge0GenerationResult {
        try Device.withDefaultDevice(Self.runtimeDevice) {
            try generateReply(to: userText, maxTokens: maxTokens,
                              thinking: thinking, seed: seed,
                              onText: onText, shouldContinue: shouldContinue)
        }
    }

    private func generateReply(
        to userText: String, maxTokens: Int, thinking: Bool, seed: UInt64?,
        onText: @escaping @Sendable (String) -> Void,
        shouldContinue: @escaping @Sendable () -> Bool
    ) throws -> Edge0GenerationResult {
        let trimmed = userText.trimmingCharacters(in: .whitespacesAndNewlines)
        guard !trimmed.isEmpty else { throw M1Error.invalid("Message cannot be empty") }
        guard maxTokens > 0 else { throw M1Error.invalid("maxTokens must be positive") }

        let prompt = hasConversationContext
            ? Edge0ChatTemplate8B.nextTurn(trimmed, thinking: thinking)
            : Edge0ChatTemplate8B.firstTurn(trimmed, thinking: thinking)
        let promptIDs = try tokenizer.encode(prompt)
        guard !promptIDs.isEmpty else { throw M1Error.invalid("Tokenizer returned an empty prompt") }

        let started = Date()
        guard shouldContinue() else {
            model.reset(); hasConversationContext = false; tokenCount = 0
            throw CancellationError()
        }
        var output: StreamingModelOutput?
        // Full-layer prefill loads every expert in a layer. That cost is almost
        // independent of length, so a short chat prompt pays for the whole 4 GB
        // expert set. Below this cutoff, walk the same top-8 decode path.
        let streamingPrefill = promptIDs.count < Self.fullLayerPrefillMinimum
        if streamingPrefill {
            for id in promptIDs {
                guard shouldContinue() else {
                    model.reset(); hasConversationContext = false; tokenCount = 0
                    throw CancellationError()
                }
                output = try model(tokenID: id)
            }
        } else {
            let prefillChunk = 2048
            for start in stride(from: 0, to: promptIDs.count, by: prefillChunk) {
                guard shouldContinue() else {
                    model.reset(); hasConversationContext = false; tokenCount = 0
                    throw CancellationError()
                }
                let end = min(start + prefillChunk, promptIDs.count)
                let chunk = Array(promptIDs[start..<end])
                if chunk.count > 1 {
                    output = try model.prefill(tokenIDs: chunk)
                } else if let id = chunk.first {
                    output = try model(tokenID: id)
                }
            }
        }
        let prefillFinished = Date()
        tokenCount += promptIDs.count

        var generated: [Int] = []
        var samplingHistory = promptIDs
        var peakMemory = output?.peakMemory ?? 0
        var firstTokenAt: Date?
        var decodeStartedAt: Date?
        while generated.count < maxTokens, let current = output {
            if !shouldContinue() {
                if generated.isEmpty {
                    model.reset(); hasConversationContext = false; tokenCount = 0
                    throw CancellationError()
                }
                break
            }
            let next = try sample(current.logits, history: samplingHistory,
                                  firstToken: generated.isEmpty, seed: seed.map {
                                      $0 &+ UInt64(generated.count)
                                  })
            if next == tokenizer.endOfTurnTokenID { break }
            generated.append(next)
            samplingHistory.append(next)
            onText(try tokenizer.decode(generated))
            if firstTokenAt == nil {
                firstTokenAt = Date()
                decodeStartedAt = firstTokenAt
            }
            if generated.count == maxTokens { break }
            output = try model(tokenID: next)
            peakMemory = max(peakMemory, output?.peakMemory ?? 0)
            tokenCount += 1
        }
        hasConversationContext = true

        let finished = Date()
        let firstTokenSeconds = firstTokenAt?.timeIntervalSince(started)
            ?? finished.timeIntervalSince(started)
        let decodeTokenCount = max(0, generated.count - 1)
        let decodeSeconds = decodeStartedAt.map { finished.timeIntervalSince($0) } ?? 0
        let prefillSeconds = prefillFinished.timeIntervalSince(started)
        let decodeTokensPerSecond = decodeSeconds > 0
            ? Double(decodeTokenCount) / decodeSeconds : 0
        print(String(
            format: "[8b prefill] path=%@ tokens=%d total=%.2fs rate=%.2f tok/s",
            streamingPrefill ? "streaming-top8" : "full-layer",
            promptIDs.count,
            prefillSeconds,
            Double(promptIDs.count) / max(0.000_001, prefillSeconds)))
        print(String(
            format: "[8b decode] tokens=%d total=%.2fs rate=%.2f tok/s peak=%.0f MB",
            decodeTokenCount,
            decodeSeconds,
            decodeTokensPerSecond,
            Double(peakMemory) / 1_048_576))

        return Edge0GenerationResult(
            text: try tokenizer.decode(generated),
            generatedTokenCount: generated.count,
            elapsedSeconds: finished.timeIntervalSince(started),
            prefillSeconds: prefillSeconds,
            prefillTokensPerSecond: Double(promptIDs.count) /
                max(0.000_001, prefillSeconds),
            timeToFirstTokenSeconds: firstTokenSeconds,
            decodeTokensPerSecond: decodeTokensPerSecond,
            peakMemoryBytes: peakMemory
        )
    }

    /// Official Edge0 policy: first token greedy, then temperature 0.7,
    /// top-k 64, top-p 0.95 and HF-style repetition penalty 1.1.
    private func sample(_ input: MLXArray, history: [Int],
                        firstToken: Bool, seed: UInt64?) throws -> Int {
        let logits = input.flattened().asType(.float32)
        if firstToken { return argMax(logits).item(Int32.self).intValue }

        if !history.isEmpty {
            let unique = Array(Set(history)).sorted()
            let ids = MLXArray(unique.map(Int32.init))
            let values = logits[ids]
            logits[ids] = which(values .> 0, values / 1.1, values * 1.1)
        }
        let scaled = logits / 0.7
        let k = min(64, scaled.dim(0))
        let candidateIDs = argPartition(-scaled, kth: k - 1)[..<k]
        let candidateValues = takeAlong(scaled, candidateIDs, axis: 0)
        let order = argSort(-candidateValues)
        let rankedIDs = takeAlong(candidateIDs, order, axis: 0)
        let rankedValues = takeAlong(candidateValues, order, axis: 0)
        let cumulative = cumsum(softmax(rankedValues), axis: 0)
        let retained = (cumulative .<= 0.95).asType(.int32).sum()
        let retainedCount = maximum(retained, MLXArray(Int32(1)))
        let thresholdIndex = (retainedCount - 1).reshaped([1])
        let threshold = takeAlong(rankedValues, thresholdIndex, axis: 0)
        let masked = which(rankedValues .< threshold,
                           MLXArray(-Float.infinity), rankedValues)
        let local = MLXRandom.categorical(
            masked, key: seed.map(MLXRandom.key)).item(UInt32.self)
        return rankedIDs[Int(local)].item(Int32.self).intValue
    }
}

private extension Int32 {
    var intValue: Int { Int(self) }
}