Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
22.9 KB
Raw Blame
import Edge0MLX
import Darwin
import Foundation
import SwiftUI

private struct ChatMessage: Identifiable, Sendable {
    enum Role: Sendable { case user, assistant }
    let id: UUID
    let role: Role
    var text: String
    var detail: String?

    init(id: UUID = UUID(), role: Role, text: String, detail: String?) {
        self.id = id; self.role = role; self.text = text; self.detail = detail
    }
}

private enum LocalModel: String, CaseIterable, Identifiable, Sendable {
    case edge8
    case edge35b

    var id: String { rawValue }
    var title: String {
        switch self {
        case .edge8: "Edge0 8B"
        case .edge35b: "Edge0 35B"
        }
    }
    var folderName: String {
        self == .edge8 ? "Edge0-8B-A1B-preview" : Edge0ChatEngine35B.modelFolderName
    }
}

private actor ChatRuntime {
    private enum LoadedEngine {
        case edge8(Edge0ChatEngine)
        case edge35b(Edge0ChatEngine35B)
    }
    private var engine: LoadedEngine?

    func load(_ model: LocalModel, modelURL: URL) throws {
        // Do not keep both models alive while switching; 35B is memory constrained.
        engine = nil
        switch model {
        case .edge8:
            engine = .edge8(try Edge0ChatEngine(modelURL: modelURL))
        case .edge35b:
            engine = .edge35b(try Edge0ChatEngine35B(modelURL: modelURL))
        }
    }

    func reply(to text: String,
               maxTokens: Int? = nil,
               thinking: Bool,
               onText: @escaping @Sendable (String) -> Void) async throws -> Edge0GenerationResult {
        let shouldContinue = { @Sendable in
            !withUnsafeCurrentTask { $0?.isCancelled ?? false }
        }
        switch engine {
        case .edge8(let engine):
            if let maxTokens {
                return try engine.reply(
                    to: text, maxTokens: maxTokens, thinking: thinking,
                    onText: onText, shouldContinue: shouldContinue)
            }
            return try engine.reply(
                to: text, thinking: thinking, onText: onText, shouldContinue: shouldContinue)
        case .edge35b(let engine):
            if let maxTokens {
                return try await engine.reply(
                    to: text, maxTokens: maxTokens, thinking: thinking,
                    onText: onText, shouldContinue: shouldContinue)
            }
            return try await engine.reply(
                to: text, thinking: thinking, onText: onText, shouldContinue: shouldContinue)
        case nil:
            throw RuntimeFailure.modelNotLoaded
        }
    }

    func reset() {
        switch engine {
        case .edge8(let engine): engine.reset()
        case .edge35b(let engine): engine.reset()
        case nil: break
        }
    }

    func unload() { engine = nil }

    private enum RuntimeFailure: LocalizedError {
        case modelNotLoaded
        var errorDescription: String? { "Model is not loaded" }
    }
}

@MainActor
private final class ChatViewModel: ObservableObject {
    enum Phase: Equatable { case choosing, ready, preparing, generating, failed(String) }

    @Published var input = ""
    @Published private(set) var messages: [ChatMessage] = []
    @Published private(set) var phase: Phase = .choosing
    @Published private(set) var selectedModel: LocalModel?

    private let runtime = ChatRuntime()
    private var generationTask: Task<Void, Never>?

    var isRunning: Bool { phase == .preparing || phase == .generating }
    /// Off unless the process is launched with `--thinking`. `--no-thinking` forces it off.
    private let thinkingEnabled: Bool = {
        let args = CommandLine.arguments
        if args.contains("--no-thinking") { return false }
        return args.contains("--thinking")
    }()
    var statusText: String? {
        switch phase {
        case .choosing: nil
        case .ready: nil
        case .preparing: "Loading \(selectedModel?.title ?? "local model")…"
        case .generating: "\(selectedModel?.title ?? "Model") is generating…"
        case .failed(let message): message
        }
    }

    func isInstalled(_ model: LocalModel) -> Bool {
        modelIsComplete(model, at: resolveModelURL(model))
    }

    private func modelIsComplete(_ model: LocalModel, at url: URL) -> Bool {
        switch model {
        case .edge8:
            return FileManager.default.fileExists(
                atPath: url.appendingPathComponent("model.safetensors").path)
        case .edge35b:
            return FileManager.default.fileExists(
                atPath: url.appendingPathComponent("model.safetensors.index.json").path)
                && FileManager.default.fileExists(
                    atPath: url.appendingPathComponent("tokenizer.bin").path)
                && FileManager.default.fileExists(
                    atPath: url.appendingPathComponent("experts-L39.bin").path)
        }
    }

    func choose(_ model: LocalModel) {
        guard !isRunning else { return }
        selectedModel = model
        messages.removeAll()
        input = ""
        phase = .preparing
        let url = resolveModelURL(model)
        Task {
            do {
                try await runtime.load(model, modelURL: url)
                if phase == .preparing { phase = .ready }
            } catch {
                phase = .failed(Self.friendlyMessage(for: error))
            }
        }
    }

    func chooseAnotherModel() {
        guard !isRunning else { return }
        messages.removeAll()
        input = ""
        selectedModel = nil
        phase = .choosing
        Task { await runtime.unload() }
    }

    func useSuggestion(_ text: String, maxTokens: Int? = nil) {
        input = text
        send(maxTokens: maxTokens)
    }

    func send(maxTokens: Int? = nil) {
        let text = input.trimmingCharacters(in: .whitespacesAndNewlines)
        guard !text.isEmpty, !isRunning else { return }
        input = ""
        messages.append(ChatMessage(role: .user, text: text, detail: nil))
        let assistantID = UUID()
        messages.append(ChatMessage(id: assistantID, role: .assistant,
                                    text: "", detail: nil))
        phase = .generating
        generationTask = Task {
            do {
                let result = try await runtime.reply(
                    to: text,
                    maxTokens: maxTokens,
                    thinking: thinkingEnabled,
                    onText: { [weak self] partial in
                        Task { @MainActor in
                            guard let self,
                                  let index = self.messages.firstIndex(where: { $0.id == assistantID })
                            else { return }
                            self.messages[index].text = partial
                        }
                    })
                let detail = String(
                    format: "%d tokens · TTFT %.1fs · prefill %.1f tok/s · decode %.2f tok/s · peak %.0f MB",
                    result.generatedTokenCount, result.timeToFirstTokenSeconds,
                    result.prefillTokensPerSecond,
                    result.decodeTokensPerSecond,
                    Double(result.peakMemoryBytes) / 1_048_576
                )
                if let index = messages.firstIndex(where: { $0.id == assistantID }) {
                    messages[index].text = result.text.isEmpty ? "(The model returned an empty response)" : result.text
                    messages[index].detail = detail
                }
                print("[generation] \(detail)")
                if CommandLine.arguments.contains("--bench-second"),
                   CommandLine.arguments.contains("--bench-35b") {
                    print("[bench] second turn")
                    let second = try await runtime.reply(
                        to: "Summarize it in one sentence.",
                        maxTokens: 64,
                        thinking: thinkingEnabled,
                        onText: { _ in })
                    let secondDetail = String(
                        format: "%d tokens · TTFT %.1fs · prefill %.1f tok/s · decode %.2f tok/s · peak %.0f MB",
                        second.generatedTokenCount, second.timeToFirstTokenSeconds,
                        second.prefillTokensPerSecond,
                        second.decodeTokensPerSecond,
                        Double(second.peakMemoryBytes) / 1_048_576
                    )
                    print("[generation turn2] \(secondDetail)")
                }
                if CommandLine.arguments.contains("--bench-8b")
                    || CommandLine.arguments.contains("--bench-35b") {
                    var info = task_vm_info_data_t()
                    var count = mach_msg_type_number_t(
                        MemoryLayout<task_vm_info_data_t>.stride / MemoryLayout<natural_t>.stride)
                    let status = withUnsafeMutablePointer(to: &info) {
                        $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) {
                            task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count)
                        }
                    }
                    print("[bench memory] endFootprintMiB=\(status == KERN_SUCCESS ? Double(info.phys_footprint) / 1_048_576 : -1) thermal=\(ProcessInfo.processInfo.thermalState.rawValue)")
                    print("[bench timing] ttftSeconds=\(result.timeToFirstTokenSeconds) prefillSeconds=\(result.prefillSeconds) prefillTokPerSec=\(result.prefillTokensPerSecond) decodeTokPerSec=\(result.decodeTokensPerSecond)")
                }
                phase = .ready
                if CommandLine.arguments.contains("--bench-8b")
                    || CommandLine.arguments.contains("--bench-35b")
                    || CommandLine.arguments.contains("--smoke-35b") {
                    fflush(nil)
                    exit(0)
                }
            } catch is CancellationError {
                phase = .ready
            } catch {
                phase = .failed(Self.friendlyMessage(for: error))
                if CommandLine.arguments.contains("--bench-8b")
                    || CommandLine.arguments.contains("--bench-35b")
                    || CommandLine.arguments.contains("--smoke-35b") {
                    fflush(stdout)
                    exit(1)
                }
            }
            generationTask = nil
        }
    }

    func stop() { generationTask?.cancel() }

    func newConversation() {
        guard !isRunning, selectedModel != nil else { return }
        messages.removeAll()
        input = ""
        phase = .ready
        Task { await runtime.reset() }
    }

    /// Sideload releases ship without weights. File Sharing exposes Documents, so a
    /// copied model there wins; a model baked into the bundle remains the fallback.
    private func resolveModelURL(_ model: LocalModel) -> URL {
        if let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first {
            let nested = documents
                .appendingPathComponent("Models", isDirectory: true)
                .appendingPathComponent(model.folderName, isDirectory: true)
            if modelIsComplete(model, at: nested) { return nested }
            let direct = documents.appendingPathComponent(model.folderName, isDirectory: true)
            if modelIsComplete(model, at: direct) { return direct }
        }
        let models = Bundle.main.resourceURL?
            .appendingPathComponent("Models", isDirectory: true)
            ?? Bundle.main.bundleURL.appendingPathComponent("Models", isDirectory: true)
        return models.appendingPathComponent(model.folderName, isDirectory: true)
    }

    private static func friendlyMessage(for error: Error) -> String {
        let description = error.localizedDescription
        if description.contains("doesn’t exist") || description.contains("No such file")
            || description.contains("incomplete") {
            return "Model not included in this build"
        }
        return "Run failed: \(description)"
    }
}

struct ContentView: View {
    @StateObject private var chat = ChatViewModel()
    @FocusState private var inputFocused: Bool

    private let suggestions = [
        "What is artificial intelligence?",
        "How do LLMs understand context?",
        "Tell me about renewable energy",
        "Write a short poem about autumn",
    ]

    var body: some View {
        ZStack {
            Color.black.ignoresSafeArea()
            if chat.selectedModel == nil {
                modelPicker
            } else {
                VStack(spacing: 0) {
                    header
                    conversation
                }
            }
        }
        .preferredColorScheme(.dark)
        .safeAreaInset(edge: .bottom, spacing: 0) {
            if chat.selectedModel != nil { composer }
        }
        .task {
            // Keeps the normal launch picker intact while allowing a repeatable
            // on-device smoke/benchmark run from `devicectl`.
            let smoke = CommandLine.arguments.contains("--smoke-35b")
            let bench8 = CommandLine.arguments.contains("--bench-8b")
            let bench35 = CommandLine.arguments.contains("--bench-35b")
            let benchmark = bench8 || bench35
            let target: LocalModel =
                bench8 || CommandLine.arguments.contains("--model=8b") ? .edge8 : .edge35b
            if chat.selectedModel == nil,
               (CommandLine.arguments.contains("--model=35b")
                || CommandLine.arguments.contains("--model=8b")
                || smoke || benchmark),
               chat.isInstalled(target) {
                print("[bench] loading \(target.title)")
                print("[bench] thermalAtLaunch=\(ProcessInfo.processInfo.thermalState.rawValue)")
                chat.choose(target)
                if smoke || benchmark {
                    while chat.phase == .preparing {
                        try? await Task.sleep(for: .milliseconds(100))
                    }
                    if case .failed(let message) = chat.phase {
                        print("[bench] load failed: \(message)")
                    } else if chat.phase == .ready {
                        let long = CommandLine.arguments.contains("--bench-long")
                        let sentence = "Streaming inference overlaps disk bandwidth with compute. Each mixture-of-experts layer activates only a few experts."
                        let prompt = smoke
                            ? "Reply with this only: 35B is running"
                            : (long
                                ? Array(repeating: sentence, count: 80).joined()
                                : "What is artificial intelligence?")
                        chat.useSuggestion(prompt, maxTokens: long ? 8 : (benchmark ? 64 : nil))
                    }
                }
            }
        }
    }

    private var modelPicker: some View {
        ScrollView {
            VStack(alignment: .leading, spacing: 22) {
                Spacer(minLength: 54)
                Text("Choose a local model")
                    .font(.system(size: 34, weight: .bold, design: .rounded))

                ForEach(LocalModel.allCases) { model in
                    let installed = chat.isInstalled(model)
                    Button { chat.choose(model) } label: {
                        HStack(spacing: 16) {
                            Image(systemName: model == .edge8 ? "bolt.fill" : "brain.head.profile")
                                .font(.system(size: 23, weight: .semibold))
                                .frame(width: 48, height: 48)
                                .background(.white.opacity(0.09), in: RoundedRectangle(cornerRadius: 14))
                            VStack(alignment: .leading, spacing: 5) {
                                Text(model.title).font(.headline)
                                if !installed {
                                    Text("Not included")
                                        .font(.caption2)
                                        .foregroundStyle(.orange)
                                }
                            }
                            Spacer()
                            Image(systemName: "chevron.right")
                                .foregroundStyle(.secondary)
                        }
                        .padding(18)
                        .background(.white.opacity(0.07), in: RoundedRectangle(cornerRadius: 22))
                    }
                    .buttonStyle(.plain)
                    .disabled(!installed)
                    .opacity(installed ? 1 : 0.62)
                }

                Spacer(minLength: 40)
            }
            .padding(.horizontal, 22)
            .frame(maxWidth: 620)
            .frame(maxWidth: .infinity)
        }
    }

    private var header: some View {
        HStack(spacing: 10) {
            Text(chat.selectedModel?.title ?? "Edge0")
                .font(.headline.weight(.semibold))
            Spacer()
            Button(action: chat.chooseAnotherModel) {
                Image(systemName: "arrow.left.arrow.right")
                    .font(.system(size: 15, weight: .semibold))
                    .frame(width: 38, height: 38)
                    .background(.white.opacity(0.07), in: Circle())
            }
            .foregroundStyle(.white)
            .disabled(chat.isRunning)
            .accessibilityLabel("Switch model")
            Button(action: chat.newConversation) {
                Image(systemName: "square.and.pencil")
                    .font(.system(size: 17, weight: .semibold))
                    .frame(width: 38, height: 38)
                    .background(.white.opacity(0.07), in: Circle())
            }
            .foregroundStyle(.white)
            .disabled(chat.isRunning)
            .accessibilityLabel("New chat")
        }
        .padding(.horizontal, 20)
        .padding(.top, 8)
        .padding(.bottom, 10)
    }

    @ViewBuilder
    private var conversation: some View {
        if chat.messages.isEmpty {
            welcome
        } else {
            ScrollViewReader { proxy in
                ScrollView {
                    LazyVStack(spacing: 18) {
                        ForEach(chat.messages) { message in
                            messageBubble(message).id(message.id)
                        }
                        if let status = chat.statusText { statusRow(status).id("status") }
                    }
                    .padding(.horizontal, 18)
                    .padding(.top, 16)
                    .padding(.bottom, 24)
                }
                .scrollDismissesKeyboard(.interactively)
                .onChange(of: chat.messages.count) {
                    withAnimation { proxy.scrollTo(chat.messages.last?.id, anchor: .bottom) }
                }
                .onChange(of: chat.phase) {
                    withAnimation { proxy.scrollTo("status", anchor: .bottom) }
                }
            }
        }
    }

    private var welcome: some View {
        ScrollView {
            VStack(spacing: 18) {
                Spacer(minLength: 90)
                Text("edge0")
                    .font(.system(size: 58, weight: .medium, design: .rounded))
                    .tracking(-3)
                Text("Private AI, running locally on your iPhone")
                    .font(.subheadline)
                    .foregroundStyle(.secondary)
                    .multilineTextAlignment(.center)
                VStack(spacing: 10) {
                    ForEach(suggestions, id: \.self) { suggestion in
                        Button(suggestion) { chat.useSuggestion(suggestion) }
                            .font(.subheadline.weight(.medium))
                            .foregroundStyle(.white.opacity(0.88))
                            .padding(.horizontal, 16)
                            .padding(.vertical, 11)
                            .background(.white.opacity(0.07), in: Capsule())
                    }
                }
                .padding(.top, 8)
                if let status = chat.statusText { statusRow(status) }
                Spacer(minLength: 120)
            }
            .frame(maxWidth: .infinity)
            .padding(.horizontal, 24)
        }
    }

    private func messageBubble(_ message: ChatMessage) -> some View {
        VStack(alignment: message.role == .user ? .trailing : .leading, spacing: 6) {
            Text(message.text)
                .font(.body)
                .foregroundStyle(.white.opacity(0.94))
                .textSelection(.enabled)
                .padding(.horizontal, 15)
                .padding(.vertical, 12)
                .background(
                    message.role == .user ? Color.white.opacity(0.14) : Color.white.opacity(0.07),
                    in: RoundedRectangle(cornerRadius: 19, style: .continuous)
                )
            if let detail = message.detail {
                Text(detail)
                    .font(.caption2.monospacedDigit())
                    .foregroundStyle(.secondary)
                    .padding(.horizontal, 6)
            }
        }
        .frame(maxWidth: .infinity, alignment: message.role == .user ? .trailing : .leading)
    }

    private func statusRow(_ text: String) -> some View {
        HStack(spacing: 10) {
            if chat.isRunning { ProgressView().controlSize(.small) }
            Text(text)
                .font(.footnote)
                .foregroundStyle(chat.isRunning ? Color.secondary : Color.red.opacity(0.9))
        }
        .frame(maxWidth: .infinity, alignment: .leading)
        .padding(.horizontal, 6)
    }

    private var composer: some View {
        HStack(alignment: .bottom, spacing: 10) {
            TextField("Ask anything…", text: $chat.input, axis: .vertical)
                .lineLimit(1...5)
                .focused($inputFocused)
                .submitLabel(.send)
                .onSubmit { chat.send() }
                .padding(.horizontal, 16)
                .padding(.vertical, 13)
                .background(.white.opacity(0.08), in: RoundedRectangle(cornerRadius: 22, style: .continuous))

            Button {
                if chat.isRunning, chat.phase == .generating {
                    chat.stop()
                } else {
                    chat.send()
                }
                inputFocused = false
            } label: {
                Image(systemName: chat.phase == .generating ? "stop.fill" : "arrow.up")
                    .font(.system(size: 17, weight: .bold))
                    .foregroundStyle(.black)
                    .frame(width: 45, height: 45)
                    .background(.white, in: Circle())
            }
            .disabled(chat.phase == .preparing ||
                      chat.phase == .choosing ||
                      (chat.phase != .generating &&
                       chat.input.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty))
            .opacity(chat.phase == .preparing ? 0.5 : 1)
            .accessibilityLabel(chat.phase == .generating ? "Stop" : "Send")
        }
        .padding(.horizontal, 16)
        .padding(.top, 10)
        .padding(.bottom, 8)
        .background(.ultraThinMaterial)
    }
}