Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
11.9 KB
Raw Blame
import Darwin
import Foundation
import MLX

/// Product-facing wrapper around the streaming Edge0 35B implementation.
///
/// This engine deliberately lives beside `Edge0ChatEngine`: selecting 35B creates only
/// this model, while selecting 8B continues through the existing engine unchanged.
public final class Edge0ChatEngine35B: @unchecked Sendable {
    public static let modelFolderName = "repacked"

    private let directory: URL
    private let model: Edge0Model35B
    private let tokenizer: Tokenizer
    private let instructions: String
    private let systemPrefix: [Int32]
    private let state = Edge0Model35B.State()

    private var processed: [Int32] = []
    private var cachedPrefix: PrefixCache?

    public init(
        modelURL: URL,
        instructions: String = "You are a helpful assistant.",
        progress: (String) -> Void = { _ in }
    ) throws {
        directory = modelURL
        self.instructions = instructions

        // Configure MLX memory use for the selected 35B runtime.
        let highMem = CommandLine.arguments.contains("--high-mem-35b")
        Memory.cacheLimit = (highMem ? 1536 : 384) * 1_048_576
        print(String(
            format: "[mem] cacheLimit=%dMiB highMem=%@ available=%dMiB",
            Memory.cacheLimit / 1_048_576,
            highMem ? "true" : "false",
            os_proc_available_memory() / 1_048_576))
        progress("mapping Edge0 35B weights")
        var configuration = Edge0Model35B.Configuration()
        configuration.round6 = CommandLine.arguments.contains("--round6")
        if configuration.round6 {
            configuration.activeExperts = 2
        }
        configuration.retainAllResidentWeights = highMem
        model = try Edge0Model35B(directory: modelURL, configuration: configuration)
        guard model.availableLayers == model.configuration.layers else {
            throw Edge035BFailure.incompleteExperts(
                found: model.availableLayers, expected: model.configuration.layers)
        }
        progress("loading Edge0 35B tokenizer")
        tokenizer = try Tokenizer(url: modelURL.appendingPathComponent("tokenizer.bin"))

        if instructions.isEmpty {
            systemPrefix = []
        } else {
            let text = try ChatTemplate.render(
                [.init(role: "system", content: instructions)],
                addGenerationPrompt: false)
            // The first user role marker is fixed too. Persisting its state saves those
            // tokens on every new conversation; `PrefixCache.matches` still verifies the
            // encoded prefix against the complete prompt before it is trusted.
            systemPrefix = tokenizer.encode(text + ChatTemplate.userOpening)
        }
        cachedPrefix = PrefixCache.read(from: Self.prefixURL(
            directory: modelURL, fingerprint: model.configuration.fingerprint))
        try model.prepareForFirstTurn()
        progress("Edge0 35B ready")
    }

    public func reset() {
        state.reset()
        processed.removeAll(keepingCapacity: true)
    }

    public func reply(
        to userText: String,
        maxTokens: Int = 512,
        thinking: Bool = true,
        onText: @escaping @Sendable (String) -> Void = { _ in },
        shouldContinue: @escaping @Sendable () -> Bool = { true }
    ) async throws -> Edge0GenerationResult {
        let prompt = userText.trimmingCharacters(in: .whitespacesAndNewlines)
        guard !prompt.isEmpty else { throw Edge035BFailure.invalid("Message cannot be empty") }
        guard maxTokens > 0 else { throw Edge035BFailure.invalid("maxTokens must be positive") }

        let continuing = !processed.isEmpty
        let ids: [Int32]
        if continuing {
            ids = tokenizer.encode(ChatTemplate.continuation(user: prompt, thinking: thinking))
        } else {
            var messages: [ChatTemplate.Message] = []
            if !instructions.isEmpty {
                messages.append(.init(role: "system", content: instructions))
            }
            messages.append(.init(role: "user", content: prompt))
            ids = tokenizer.encode(try ChatTemplate.render(messages, thinking: thinking))
        }
        guard !ids.isEmpty else { throw Edge035BFailure.invalid("Tokenizer returned an empty prompt") }
        print("[35b] thinking=\(thinking ? "on" : "off")")

        GPU.resetPeakMemory()
        let started = Date()
        let thermalAtStart = ProcessInfo.processInfo.thermalState
        var start = 0
        if !continuing {
            state.reset()
            processed.removeAll(keepingCapacity: true)
            if let cache = cachedPrefix, cache.matches(ids) {
                restore(cache.state, into: state)
                processed = cache.tokens
                start = cache.tokens.count
            }
        }

        var next: Int32 = 0
        // Later turns already have attention weights resident. Keep that cache on
        // for a continued prefill so those projections are not copied again.
        // The first prefill still copies each of them once; retaining earlier
        // only adds dirty pages with nothing to reuse inside that single pass.
        if continuing { model.beginDecodeCaching() }
        model.profile.reset()
        // An older cache may contain only the system message. Restore it, compute just
        // the newly-added user opening, then atomically replace it with the longer cache.
        let capturePoint = !continuing && start < systemPrefix.count
            && ids.starts(with: systemPrefix) ? systemPrefix.count : nil
        var index = start
        while index < ids.count {
            guard shouldContinue() else { return try cancelled() }
            var size = min(Edge0Model35B.maximumBatch, ids.count - index)
            if let capturePoint, index < capturePoint {
                size = min(size, capturePoint - index)
            }
            let slice = Array(ids[index ..< index + size])
            let isLast = index + size == ids.count
            let logits = try model.step(
                tokens: MLXArray(slice, [1, size]), state: state, needsLogits: isLast)
            if isLast, let logits { next = try await model.greedy(logits) }
            processed.append(contentsOf: slice)
            index += size

            if let capturePoint, index == capturePoint {
                let captured = PrefixCache(tokens: systemPrefix, state: state.snapshot())
                cachedPrefix = captured
                captured.write(to: Self.prefixURL(
                    directory: directory, fingerprint: model.configuration.fingerprint))
            }
        }
        let prefillFinished = Date()
        let prefillSeconds = prefillFinished.timeIntervalSince(started)
        let prefillTokens = ids.count - start
        print(String(
            format: "[35b prefill] tokens=%d total=%.2fs rate=%.2f tok/s %@",
            prefillTokens,
            prefillSeconds,
            Double(prefillTokens) / max(0.000_001, prefillSeconds),
            model.profile.report))
        model.beginDecodeCaching()
        model.profile.reset()

        let stop = StopCondition(maximumTokens: maxTokens)
        var raw = ""
        var shown = ""
        var produced = 0
        var firstTokenAt: Date?
        var decodeStartedAt: Date?

        while true {
            switch stop.evaluate(token: next, produced: produced) {
            case .hitEndToken, .hitLimit:
                let finished = Date()
                let decodeSeconds = decodeStartedAt.map {
                    finished.timeIntervalSince($0)
                } ?? 0
                let footprint = Self.physicalFootprintBytes()
                print(String(
                    format: "[35b decode] tokens=%d total=%.2fs rate=%.2f tok/s thermal=%@→%@ mlxPeak=%.0fMB footprint=%.0fMB available=%dMB %@",
                    max(0, produced - 1),
                    decodeSeconds,
                    Double(max(0, produced - 1)) / max(0.000_001, decodeSeconds),
                    Self.thermalName(thermalAtStart),
                    Self.thermalName(ProcessInfo.processInfo.thermalState),
                    Double(Memory.peakMemory) / 1_048_576,
                    Double(footprint) / 1_048_576,
                    os_proc_available_memory() / 1_048_576,
                    model.profile.report))
                shown = thinking
                    ? ThinkingBlock.settled(raw)
                    : ThinkingBlock.settledReply(raw)
                if !shown.isEmpty { onText(shown) }
                return Edge0GenerationResult(
                    text: shown,
                    generatedTokenCount: produced,
                    elapsedSeconds: finished.timeIntervalSince(started),
                    prefillSeconds: prefillSeconds,
                    prefillTokensPerSecond: Double(prefillTokens) /
                        max(0.000_001, prefillSeconds),
                    timeToFirstTokenSeconds: firstTokenAt?.timeIntervalSince(started)
                        ?? finished.timeIntervalSince(started),
                    decodeTokensPerSecond: decodeStartedAt.map {
                        Double(max(0, produced - 1)) /
                            max(0.000_001, finished.timeIntervalSince($0))
                    } ?? 0,
                    peakMemoryBytes: max(Memory.peakMemory, footprint))
            case .keepGoing:
                break
            }

            guard shouldContinue() else { return try cancelled() }
            if firstTokenAt == nil {
                firstTokenAt = Date()
                decodeStartedAt = firstTokenAt
            }
            raw += tokenizer.decode([next])
            let visible = thinking
                ? ThinkingBlock.visible(in: raw)
                : ThinkingBlock.visibleReply(in: raw)
            if let visible, visible != shown {
                shown = visible
                onText(shown)
            }
            produced += 1

            let logits = try model.step(tokens: MLXArray([next], [1, 1]), state: state)
            processed.append(next)
            next = try await model.greedy(logits)
        }
    }

    private func cancelled<T>() throws -> T {
        reset()
        throw CancellationError()
    }

    private func restore(_ source: Edge0Model35B.State, into target: Edge0Model35B.State) {
        let snapshot = source.snapshot()
        target.convolution = snapshot.convolution
        target.recurrent = snapshot.recurrent
        target.keys = snapshot.keys
        target.values = snapshot.values
        target.offset = snapshot.offset
    }

    private static func prefixURL(directory: URL, fingerprint: String) -> URL {
        PrefixCache.fileURL(in: directory, fingerprint: fingerprint)
    }

    private static func thermalName(_ state: ProcessInfo.ThermalState) -> String {
        switch state {
        case .nominal: "nominal"
        case .fair: "fair"
        case .serious: "serious"
        case .critical: "critical"
        @unknown default: "unknown"
        }
    }

    /// Process physical footprint (jetsam-relevant), not just MLX allocator peak.
    private static func physicalFootprintBytes() -> Int {
        var info = task_vm_info_data_t()
        var count = mach_msg_type_number_t(
            MemoryLayout<task_vm_info_data_t>.stride / MemoryLayout<natural_t>.stride)
        let result = withUnsafeMutablePointer(to: &info) {
            $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) {
                task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count)
            }
        }
        guard result == KERN_SUCCESS else { return 0 }
        return Int(info.phys_footprint)
    }
}

private enum Edge035BFailure: LocalizedError {
    case incompleteExperts(found: Int, expected: Int)
    case invalid(String)

    var errorDescription: String? {
        switch self {
        case .incompleteExperts(let found, let expected):
            return "Edge0 35B expert files are incomplete (found \(found), expected \(expected))"
        case .invalid(let message):
            return message
        }
    }
}