Skip to content

Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook

47.8 KB
Raw Blame
import Foundation
import Metal
import TurboFieldfare
import Synchronization

final class GenerationTaskRegistry: Sendable {
    private struct Entry: Sendable {
        let id: UUID
        var task: Task<Void, Never>?
    }

    private let state = Mutex<Entry?>(nil)

    var hasActiveGeneration: Bool { state.withLock { $0 != nil } }

    func reserve(_ id: UUID) -> Bool {
        state.withLock { entry in
            guard entry == nil else { return false }
            entry = Entry(id: id, task: nil)
            return true
        }
    }

    func attach(_ task: Task<Void, Never>, to id: UUID) {
        let shouldCancel = state.withLock { entry -> Bool in
            guard entry?.id == id else { return true }
            entry?.task = task
            return false
        }
        if shouldCancel { task.cancel() }
    }

    func take(_ id: UUID) -> Task<Void, Never>? {
        state.withLock { entry in
            guard entry?.id == id else { return nil }
            defer { entry = nil }
            return entry?.task
        }
    }

    func takeCurrent() -> Task<Void, Never>? {
        state.withLock { entry in
            defer { entry = nil }
            return entry?.task
        }
    }

    func clear(_ id: UUID) {
        state.withLock { entry in
            if entry?.id == id { entry = nil }
        }
    }

}

/// Real-model inference client for the Mac app. Wraps the same raw-completion
/// loop the CLI uses (`runRawCompletion`, BOS + verbatim encode, no chat
/// template) behind the `AppInferenceClient` event stream, with an explicit
/// load lifecycle so the resident weights stay warm across generations.
public final class RealInferenceClient: AppModelLifecycleClient, @unchecked Sendable {
    private let session: RealInferenceSession
    /// Bytes of image tower held mapped, readable without awaiting the session.
    public var currentVisionTowerBytes: UInt64? {
        session.towerBytes.withLock { $0 }
    }

    private let memorySampler: AppMemorySampler
    private let generationTasks = GenerationTaskRegistry()

    public init(memorySampler: AppMemorySampler = AppMemorySampler()) {
        self.memorySampler = memorySampler
        self.session = RealInferenceSession()
    }

    public func ensureLoaded(modelDirectory: URL,
                             maxContextTokens: Int,
                             options: AppRuntimeOptions,
                             forceLogitsHead: Bool,
                             onState: @escaping @Sendable (AppModelLoadState) -> Void) async throws {
        try await session.ensureLoaded(
            key: SessionLoadKey(directory: modelDirectory.standardizedFileURL,
                                maxContext: maxContextTokens,
                                options: options,
                                forceLogitsHead: forceLogitsHead),
            onState: onState)
    }

    public func unload() async {
        await session.unload()
    }

    /// Drops the KV so the next turn starts a fresh lineage. The model stays
    /// loaded: this ends a conversation, it does not unload ~1.6 GB.
    public func resetConversation() async {
        await session.resetConversation()
    }

    /// In-process, so there is no stale-epoch window to guard: the caller is
    /// the only writer. The epoch is accepted and ignored; the decode service
    /// holds the gate that uses it.
    public func resetConversation(epoch: UUID) async throws {
        await session.resetConversation()
    }

    /// Replays a stored conversation into the KV and returns the token count it
    /// holds afterwards.
    ///
    /// `onProgress` receives the same `.prefill` events a live turn reports, so
    /// a reopen shows the progress a first turn would; the decode service
    /// forwards them and the app animates its gauge from them.
    /// In-process, so `epoch` has no gate to open and `maxContextTokens` is
    /// already fixed by the loaded runner; both are accepted and ignored, the
    /// same way `resetConversation(epoch:)` treats its epoch. The decode
    /// service holds the gate that uses them.
    public func restoreConversation(
        _ lineage: AppConversationLineage,
        epoch: UUID = UUID(),
        options: AppRuntimeOptions,
        maxContextTokens: Int = 0,
        onPrefillProgress: @escaping @Sendable (Int, Int) -> Void = { _, _ in }
    ) async throws -> Int {
        return try await session.restoreConversation(
            MultimodalConversationLineage(
                tokenIDs: lineage.tokenIDs,
                images: try lineage.images.map {
                    // Bounded by the lineage before the range is built. Both
                    // figures arrive over the socket in the service, and a
                    // sign check alone let a pair near `Int.max` overflow the
                    // addition and trap the process instead of failing the
                    // restore. `lineage.tokenIDs.count - tokenLowerBound`
                    // cannot overflow once the bound is non-negative.
                    guard $0.tokenLowerBound >= 0, $0.tokenCount > 0,
                          $0.tokenCount <= lineage.tokenIDs.count - $0.tokenLowerBound
                    else {
                        throw AppInferenceError.conversationRestoreFailed(
                            "a stored image claims \($0.tokenCount) tokens at "
                                + "offset \($0.tokenLowerBound) of a "
                                + "\(lineage.tokenIDs.count)-token conversation")
                    }
                    return MultimodalReplayImage(
                        tokenRange: $0.tokenLowerBound
                            ..< ($0.tokenLowerBound + $0.tokenCount),
                        pixelsURL: $0.fileURL,
                        expectedDigest: $0.expectedDigest)
                },
                uncommittedBoundary: lineage.boundaryTokenIDs,
                boundaryNeedsReplay: lineage.boundaryNeedsReplay),
            options: options) { progress in
                guard case .prefill(let done, let total) = progress else { return }
                onPrefillProgress(done, total)
            }
    }

    /// Whether a conversation is still open on the session.
    ///
    /// Exists so a test can assert that `unload()` released it. It cannot be
    /// inferred from the outside, and not releasing it means unload freed
    /// nothing at all — the conversation holds the model, the runner, the
    /// scratch and the tower.
    public var hasOpenConversation: Bool {
        get async { await session.hasConversation }
    }

    /// Tokens the open conversation's KV holds, or zero when none is open.
    public var conversationTokenCount: Int {
        get async { await session.currentConversationTokens }
    }

    /// The same figure without awaiting the session, for the decode service's
    /// writer thread.
    public var currentConversationTokens: Int {
        session.conversationTokens.withLock { $0 }
    }

    public func generate(_ request: AppGenerationRequest) -> AsyncThrowingStream<AppInferenceEvent, Error> {
        AsyncThrowingStream { continuation in
            let generationID = UUID()
            guard generationTasks.reserve(generationID) else {
                continuation.yield(.failed(.generationInFlight, partial: nil))
                continuation.finish(throwing: AppInferenceError.generationInFlight)
                return
            }
            let task = Task { [self] in
                await session.run(request: request,
                                  memorySampler: memorySampler,
                                  continuation: continuation)
                generationTasks.clear(generationID)
            }
            generationTasks.attach(task, to: generationID)

            continuation.onTermination = { [generationTasks] _ in
                generationTasks.take(generationID)?.cancel()
            }
        }
    }

    public func cancel() {
        generationTasks.takeCurrent()?.cancel()
    }

    /// Ends the turn at the next token boundary and keeps what it produced.
    ///
    /// Only decode has token boundaries. A stop during prefill or an image
    /// encode has nothing to stop at, so it cancels instead and the
    /// conversation rewinds the turn — otherwise Stop did nothing at all until
    /// the first token, which on a long prompt is a long time.
    public func stop() {
        session.stopRequested.withLock { $0 = true }
        guard session.decodeBegan.withLock({ $0 }) else {
            generationTasks.takeCurrent()?.cancel()
            return
        }
    }

}

struct SessionLoadKey: Equatable, Sendable {
    var directory: URL
    var maxContext: Int
    var options: AppRuntimeOptions
    var forceLogitsHead: Bool

    init(directory: URL,
         maxContext: Int,
         options: AppRuntimeOptions,
         forceLogitsHead: Bool = false) {
        self.directory = directory.standardizedFileURL
        self.maxContext = maxContext
        self.options = options
        self.forceLogitsHead = forceLogitsHead
    }
}

struct TokenizerDirectoryCache: Equatable, Sendable {
    private(set) var directory: URL?

    func shouldReload(for modelDirectory: URL) -> Bool {
        directory != modelDirectory.standardizedFileURL
    }

    mutating func markLoaded(for modelDirectory: URL) {
        directory = modelDirectory.standardizedFileURL
    }

    mutating func clear() {
        directory = nil
    }
}

/// Owns the loaded model and serializes load / unload / generate. All Metal
/// command-buffer waits happen inside this actor, off the main actor; one
/// cooperative-pool thread is occupied for the duration of a generation,
/// which is acceptable for the app's single session. The 8 GB rule lives
/// here: a reload releases the loaded model, runner, and scratch before constructing
/// replacements, so two models are never alive at once.
actor RealInferenceSession {
    private var loadedKey: SessionLoadKey?
    private var ctx: MetalContext?
    private var tokenizer: GFTokenizer?
    private var tokenizerDirectoryCache = TokenizerDirectoryCache()
    private var runner: RealForwardRunner?
    private var scratch: RawCompletionScratch?
    private var model: Model?
    /// The open conversation, when the app is in chat mode. Nil for the
    /// one-shot path, and dropped by any load, unload, or explicit reset.
    private var conversation: MultimodalConversation?
    /// Bytes of image tower held mapped right now, published outside the actor
    /// so a reader does not have to await it mid-decode.
    nonisolated let towerBytes = Mutex<UInt64?>(nil)
    /// Tokens the open conversation's KV holds, published outside the actor for
    /// the same reason `towerBytes` is: the decode service's writer thread
    /// stamps it on the turn's terminal event and cannot await this actor
    /// mid-decode.
    nonisolated let conversationTokens = Mutex<Int>(0)
    /// Set by Stop, read by the decode loop at each token boundary.
    ///
    /// Cancelling the task instead throws out of `runRawCompletion`'s loop
    /// before its `shouldStop` is consulted, and the conversation then rewinds
    /// the whole turn — measured: a stop after 13 tokens left the KV at zero.
    /// Stopping cooperatively keeps what was produced and lets the next turn
    /// continue from it.
    nonisolated let stopRequested = Mutex<Bool>(false)
    /// Whether the run has reached decode. Before that there are no token
    /// boundaries to stop at, and half a prefilled prompt is not a turn worth
    /// keeping — so a stop there has to cancel and let the conversation rewind.
    nonisolated let decodeBegan = Mutex<Bool>(false)

    private var visionRuntime: VisionRuntime? {
        didSet { publishTowerBytes() }
    }
    private var visionRuntimeError: Error?

    /// Called after anything that maps or releases tower regions.
    private func publishTowerBytes() {
        let value = visionRuntime.map { UInt64($0.retainedWeightBytes) }
        towerBytes.withLock { $0 = value }
    }

    /// A reader the decode loop can call from wherever it runs. The `Mutex` is
    /// a non-copyable stored property, so it is reached through `self` rather
    /// than captured.
    private nonisolated func stopFlagReader() -> @Sendable () -> Bool {
        { [weak self] in self?.stopRequested.withLock { $0 } ?? false }
    }

    func resetConversation() async {
        // Same hazard as `unload()`: `runner.reset()` under a live decode either
        // aborts that turn mid-stream or lets two turns drive one runner.
        if let conversation { await conversation.invalidate() }
        conversation = nil
        runner?.reset()
        conversationTokens.withLock { $0 = 0 }
    }

    /// Rebuilds a stored conversation's KV and returns how many tokens it holds.
    ///
    /// Always onto a conversation of its own, never onto whatever was open: the
    /// caller has just reset, and restoring into a live lineage is refused
    /// downstream anyway. The returned count is read back from the conversation
    /// rather than from the request, so the gauge shows what the KV has and not
    /// what the caller hoped it would have.
    func restoreConversation(
        _ lineage: MultimodalConversationLineage,
        options: AppRuntimeOptions,
        onProgress: @escaping @Sendable (RawDecodeProgress) -> Void
    ) async throws -> Int {
        var prefillConfig = options.prefillConfig
        // A replay is a prefill and nothing else; with the app's prefill
        // toggle off it ran at decode speed and outlived the restore deadline.
        if let coerced = prefillConfig.coercedForReplay() {
            prefillConfig = coerced
        }
        // Image spans only run under chunked prefill, and the app's prefill
        // toggle can select `.off`. Coerce rather than refuse a conversation
        // the user can see: whether its images replay is not a performance
        // preference.
        if !lineage.images.isEmpty,
           let coerced = prefillConfig.coercedForImagePrompt() {
            prefillConfig = coerced
        }
        await resetConversation()
        let conversation = try conversationForRestore(options: options)
        do {
            try await conversation.restore(
                lineage: lineage,
                prefillConfig: prefillConfig,
                checkCancellation: { try Task.checkCancellation() },
                onProgress: onProgress)
        } catch let error as MultimodalConversationError {
            // A refused restore leaves the KV empty and this session usable, so
            // the conversation is dropped rather than kept as an inert one the
            // next turn would resume onto.
            self.conversation = nil
            runner?.reset()
            conversationTokens.withLock { $0 = 0 }
            throw AppInferenceError.conversationRestoreFailed("\(error)")
        }
        publishTowerBytes()
        let count = await conversation.kvTokenCount
        conversationTokens.withLock { $0 = count }
        return count
    }

    private func conversationForRestore(
        options: AppRuntimeOptions
    ) throws -> MultimodalConversation {
        guard let model, let ctx, let tokenizer, let runner, let scratch else {
            throw AppInferenceError.modelLoadFailed("session lost its loaded state")
        }
        runner.reset()
        // The same waiver `conversationForTurn` takes, for the same reason: one
        // conversation and one generation at a time is this session's
        // invariant, not the type system's.
        nonisolated(unsafe) let sharedModel = model
        nonisolated(unsafe) let sharedVision = visionRuntime
        let created = MultimodalConversation(
            model: sharedModel, context: ctx, tokenizer: tokenizer, runner: runner,
            scratch: scratch, visionRuntime: sharedVision,
            visionRuntimeError: visionRuntimeError,
            visionResidency: options.visionResidencyPolicy,
            maxContext: runner.maxContext)
        conversation = created
        return created
    }

    var hasConversation: Bool { conversation != nil }

    var currentConversationTokens: Int {
        get async {
            let count = await conversation?.kvTokenCount ?? 0
            conversationTokens.withLock { $0 = count }
            return count
        }
    }

    func ensureLoaded(key: SessionLoadKey,
                      onState: @Sendable (AppModelLoadState) -> Void) async throws {
        if loadedKey == key, runner != nil { return }

        conversation = nil
        conversationTokens.withLock { $0 = 0 }
        runner = nil
        scratch = nil
        model = nil
        loadedKey = nil
        visionRuntime = nil
        visionRuntimeError = nil

        let start = Date()
        do {
            onState(.loading(.validatingDirectory))
            let manifest = key.directory.appendingPathComponent("manifest.json")
            guard FileManager.default.fileExists(atPath: manifest.path) else {
                throw AppInferenceError.modelNotFound(key.directory.path)
            }

            onState(.loading(.tokenizer))
            if tokenizer == nil || tokenizerDirectoryCache.shouldReload(for: key.directory) {
                do {
                    tokenizer = try await Self.loadTokenizer(for: key.directory)
                    tokenizerDirectoryCache.markLoaded(for: key.directory)
                } catch {
                    throw AppInferenceError.tokenizerUnavailable("\(error)")
                }
            }
            try Task.checkCancellation()

            onState(.loading(.verifyingWeights))
            let runtimeConfiguration = try key.options.resolvedRuntimeConfiguration(
                forceLogitsHead: key.forceLogitsHead)
            let context: MetalContext
            if let ctx {
                context = ctx
            } else {
                context = try MetalContext()
                ctx = context
            }
            let loadedModel = try Model.load(
                directoryURL: key.directory,
                device: context.device,
                streamingMode: .pread(slotCount: runtimeConfiguration.expertCacheSlots),
                expertCachePolicy: runtimeConfiguration.modelExpertCachePolicy,
                integrityPolicy: key.options.modelVerification.runtimeValue)
            try Task.checkCancellation()

            onState(.loading(.preparingRunner))
            let loadedRunner = try RealForwardRunner(
                model: loadedModel,
                context: context,
                maxContext: key.maxContext,
                runtimeConfiguration: runtimeConfiguration)
            let loadedScratch = try RawCompletionScratch(context: context,
                                                         vocab: loadedModel.config.vocabSize)
            try Task.checkCancellation()

            let loadedVisionRuntime: VisionRuntime?
            let loadedVisionRuntimeError: Error?
            do {
                let runtime = try VisionRuntime.open(
                    textModelURL: key.directory,
                    context: context)
                // Keep Ready maps the tower during the load rather than on the
                // first image, so the wait is where the user asked for it.
                if key.options.visionResidencyPolicy == .keepReady {
                    onState(.loading(.mappingImageTower))
                    try runtime.prewarmWeightRegions()
                }
                loadedVisionRuntime = runtime
                loadedVisionRuntimeError = nil
            } catch {
                // A missing or invalid pack only means images are unavailable;
                // the reason travels to the first image turn.
                loadedVisionRuntime = nil
                loadedVisionRuntimeError = error
            }
            try Task.checkCancellation()

            runner = loadedRunner
            scratch = loadedScratch
            model = loadedModel
            loadedKey = key
            visionRuntime = loadedVisionRuntime
            visionRuntimeError = loadedVisionRuntimeError
            onState(.ready(modelDirectory: key.directory,
                           loadSeconds: Date().timeIntervalSince(start)))
        } catch is CancellationError {
            throw CancellationError()
        } catch let appError as AppInferenceError {
            onState(.failed(appError))
            throw appError
        } catch {
            let appError = AppInferenceError.modelLoadFailed("\(error)")
            onState(.failed(appError))
            throw appError
        }
    }

    private static func loadTokenizer(for modelDirectory: URL) async throws -> GFTokenizer {
        try await GFTokenizer.load(forModelDirectory: modelDirectory)
    }

    static func forceLogitsHead(for request: AppGenerationRequest) -> Bool {
        !request.isPureGreedy
    }

    static func generationConfig(for request: AppGenerationRequest,
                                 maxNewTokens: Int? = nil) -> GenerationConfig {
        GenerationConfig(maxNewTokens: maxNewTokens ?? request.maxNewTokens,
                         temperature: request.temperature,
                         topK: request.topK,
                         topP: request.topP,
                         repetitionPenalty: request.repetitionPenalty)
    }

    static func effectiveMaxNewTokens(requested: Int,
                                      promptTokenCount: Int,
                                      maxContext: Int) -> Int {
        min(requested, max(0, maxContext - promptTokenCount))
    }

    func unload() async {
        // Awaited, not just dropped. `MultimodalConversation` holds the model,
        // runner, scratch and vision runtime as strong `let`s, and a turn in
        // flight holds its own reference — so nilling this while a decode runs
        // frees nothing and the next load builds a second set beside the live
        // one. `invalidate()` is what waits for that decode.
        if let conversation { await conversation.invalidate() }
        conversation = nil
        conversationTokens.withLock { $0 = 0 }
        visionRuntime = nil
        visionRuntimeError = nil
        model = nil
        runner = nil
        scratch = nil
        tokenizer = nil
        tokenizerDirectoryCache.clear()
        loadedKey = nil
    }

    /// The single-prompt path: reset the KV and prefill the whole rendered
    /// prompt. Unchanged behaviour, moved out of `run` so the conversational
    /// path sits beside it rather than inside it.
    private func runOneShot(
        request: AppGenerationRequest,
        runner: RealForwardRunner,
        tokenizer: GFTokenizer,
        ctx: MetalContext,
        scratch: RawCompletionScratch,
        prefillConfig: PrefillRuntimeConfig,
        progress: ProgressState,
        memorySampler: AppMemorySampler,
        report: @escaping @Sendable (RawDecodeProgress) -> Void
    ) async throws -> TurnOutcome {
            let promptIds: [Int32]
            let multimodalInput: MultimodalPrefillInput?
            if request.imageAttachments.isEmpty {
                let renderedPrompt = try tokenizer.applyChatTemplate([
                    GFTokenizer.Message(role: .user, content: request.prompt)
                ])
                promptIds = tokenizer.encode(renderedPrompt, addBOS: false)
                multimodalInput = nil
            } else {
                guard let visionRuntime, let model else {
                    throw AppInferenceError.invalidRequest(
                        "Image support is unavailable: "
                            + (visionRuntimeError.map(String.init(describing:))
                                ?? "the image companion pack is not installed"))
                }
                var features: [UUID: VisionFeatures] = [:]
                features.reserveCapacity(request.imageAttachments.count)
                for attachment in request.imageAttachments {
                    try Task.checkCancellation()
                    // The file may have changed between selection and send.
                    let actualDigest = try Sha256Verifier.hashFile(
                        at: attachment.fileURL, chunkBytes: 256 * 1_024)
                    guard actualDigest == attachment.sha256 else {
                        throw AppInferenceError.invalidRequest(
                            "Image \(attachment.displayName) changed after selection.")
                    }
                    defer { publishTowerBytes() }
                    features[attachment.id] = try visionRuntime.encodeImage(
                        at: attachment.fileURL,
                        languageModel: model,
                        residencyPolicy: request.runtimeOptions.visionResidencyPolicy,
                        checkCancellation: { try Task.checkCancellation() })
                }
                var content = request.imageAttachments.map {
                    MultimodalContentPart.image(id: $0.id)
                }
                if !request.prompt.isEmpty { content.append(.text(request.prompt)) }
                let input = try MultimodalPromptRenderer.render(
                    messages: [MultimodalMessage(role: .user, content: content)],
                    featuresByID: features,
                    tokenizer: tokenizer)
                promptIds = input.effectiveTokenIDs
                multimodalInput = input
            }
            progress.promptTokenCount = promptIds.count
            guard promptIds.count < runner.maxContext else {
                throw AppInferenceError.contextOverflow(prompt: promptIds.count,
                                                        maxNew: request.maxNewTokens,
                                                        maxContext: runner.maxContext)
            }
            memorySampler.resetPeak()
            _ = memorySampler.sample()
            let config = Self.generationConfig(
                for: request,
                maxNewTokens: Self.effectiveMaxNewTokens(
                    requested: request.maxNewTokens,
                    promptTokenCount: promptIds.count,
                    maxContext: runner.maxContext))
            runner.reset()
            progress.prefillStart = Date()


        let result = try await runRawCompletion(
            producer: runner, tokenizer: tokenizer, promptIds: promptIds,
            multimodalInput: multimodalInput,
            config: config, context: ctx, scratch: scratch,
            prefillConfig: prefillConfig,
            shouldStop: stopFlagReader(),
            onProgress: report)
        return TurnOutcome(
            reason: result.reason, prefillSeconds: result.prefillSeconds,
            decodeSeconds: result.decodeSeconds, newTokens: result.newTokens,
            computedPrefillTokens: result.computedPrefillTokens)
    }

    /// What both turn paths report back, so the terminal diagnostics do not
    /// have to know which one ran.
    struct TurnOutcome {
        let reason: StopReason
        let prefillSeconds: Double
        let decodeSeconds: Double
        let newTokens: Int
        /// Nil on the single-prompt path: nothing was retained to reuse.
        var cachedTokens: Int?
        var computedPrefillTokens: Int?
        var conversationTokens: Int?
        /// What the turn put into the KV. Nil on the single-prompt path, which
        /// keeps nothing and therefore has nothing to record.
        var promptTokenIDs: [Int32]?
        var generatedTokenIDs: [Int32]?
        var boundaryTokenIDs: [Int32]?
        var boundaryNeedsReplay: Bool?
    }

    /// The open conversation, or a new one on the same runner.
    ///
    /// Built lazily rather than at load: a load that is never followed by a
    /// conversational turn should not reset the runner, and the residency
    /// policy belongs to the turn that asks for it.
    private func conversationForTurn(
        _ request: AppGenerationRequest
    ) throws -> MultimodalConversation {
        if let conversation { return conversation }
        guard let model, let ctx, let tokenizer, let runner, let scratch else {
            throw AppInferenceError.modelLoadFailed("session lost its loaded state")
        }
        // A conversation starts from an empty KV. The runner may be carrying a
        // one-shot generation's tokens, and resuming a first turn onto those
        // would put a prompt the user never sent in front of this one.
        runner.reset()
        // The same waiver `TurboFieldfareModelSession` takes, and it holds for
        // the same reason: `Model` and `VisionRuntime` are not Sendable because
        // two conversations encoding at once would race the tower's per-encode
        // scratch. What prevents that here is this session's own invariant —
        // one conversation, and one generation at a time behind
        // `GenerationTaskRegistry` and the decode service's serial command loop
        // — not the type system. Anything that lets two turns run at once has
        // to make `VisionRuntime` an actor first.
        nonisolated(unsafe) let sharedModel = model
        nonisolated(unsafe) let sharedVision = visionRuntime
        let created = MultimodalConversation(
            model: sharedModel, context: ctx, tokenizer: tokenizer, runner: runner,
            scratch: scratch, visionRuntime: sharedVision,
            visionRuntimeError: visionRuntimeError,
            visionResidency: request.runtimeOptions.visionResidencyPolicy,
            maxContext: runner.maxContext)
        conversation = created
        return created
    }

    private func runConversationTurn(
        request: AppGenerationRequest,
        prefillConfig: PrefillRuntimeConfig,
        progress: ProgressState,
        memorySampler: AppMemorySampler,
        report: @escaping @Sendable (RawDecodeProgress) -> Void
    ) async throws -> TurnOutcome {
        let conversation = try conversationForTurn(request)
        var parts: [MultimodalContinuationPart] = []
        var imageURLs: [URL] = []
        for attachment in request.imageAttachments {
            try Task.checkCancellation()
            // Same check the one-shot path makes: the file was hashed when it
            // was staged, and a file rewritten since then is a different image
            // than the one the user attached.
            let actualDigest = try Sha256Verifier.hashFile(
                at: attachment.fileURL, chunkBytes: 256 * 1_024)
            guard actualDigest == attachment.sha256 else {
                throw AppInferenceError.invalidRequest(
                    "Image \(attachment.displayName) changed after selection.")
            }
            parts.append(.image)
            imageURLs.append(attachment.fileURL)
        }
        if !request.prompt.isEmpty { parts.append(.text(request.prompt)) }

        memorySampler.resetPeak()
        _ = memorySampler.sample()
        progress.prefillStart = Date()
        defer { publishTowerBytes() }
        do {
            // `maxNewTokens` is clamped inside the conversation against what the
            // KV has room for, so it is passed through unmodified here.
            let turn = try await conversation.send(
                parts: parts, images: imageURLs,
                config: Self.generationConfig(
                    for: request, maxNewTokens: request.maxNewTokens),
                prefillConfig: prefillConfig,
                checkCancellation: { try Task.checkCancellation() },
                shouldStop: stopFlagReader(),
                onProgress: report)
            progress.promptTokenCount = turn.promptTokens
            // The conversation's own count. `promptTokens + completionTokens`
            // is one too many whenever a run stops on max tokens or is
            // cancelled: that final token is held outside the KV for the next
            // turn to replay.
            conversationTokens.withLock { $0 = turn.kvTokens }
            return TurnOutcome(
                reason: turn.reason, prefillSeconds: turn.prefillSeconds,
                decodeSeconds: turn.decodeSeconds, newTokens: turn.completionTokens,
                cachedTokens: turn.cachedTokens,
                computedPrefillTokens: turn.computedPrefillTokens,
                conversationTokens: turn.kvTokens,
                promptTokenIDs: turn.promptTokenIDs,
                generatedTokenIDs: turn.generatedTokenIDs,
                boundaryTokenIDs: turn.uncommittedBoundaryTokenIDs,
                boundaryNeedsReplay: turn.boundaryNeedsReplay)
        } catch let error as MultimodalConversationError {
            // Mapped rather than flattened: a lineage that broke can only be
            // cleared, an exhausted context is the user's to act on, and an
            // unavailable image names the pack that failed to open. Reporting
            // all three as one generic failure is how a chat becomes
            // undiagnosable.
            switch error {
            case .lineageBroken, .lineageRecoveryFailed:
                throw AppInferenceError.conversationLineageLost("\(error)")
            case .contextExhausted(let prompt, let reserve, let maxContext):
                // Not `contextOverflow`: that message names the user's max
                // response, and the guard that refused this turn does not look
                // at it. Reporting a figure the check never used is how a user
                // lowers their reply length and sees no change.
                throw AppInferenceError.conversationContextExhausted(
                    prompt: prompt, reserve: reserve, maxContext: maxContext)
            case .imageUnavailable, .invalidReplayImage, .invalidReplayToken:
                throw AppInferenceError.invalidRequest("\(error)")
            case .closed, .busy, .emptyTurn, .lineageNotEmpty:
                throw AppInferenceError.unknown("\(error)")
            }
        }
    }

    func run(request: AppGenerationRequest,
             memorySampler: AppMemorySampler,
             continuation: AsyncThrowingStream<AppInferenceEvent, Error>.Continuation) async {
        // Cleared here, before either branch and before anything a stop could
        // race. Resetting them inside the conversational branch left the
        // single-prompt path with a flag nothing ever lowered — after one Stop
        // every later generation returned exactly one token — and left a stale
        // `decodeBegan` that swallowed a Stop pressed while the next turn was
        // still hashing its images.
        stopRequested.withLock { $0 = false }
        decodeBegan.withLock { $0 = false }
        var prefillConfig = request.runtimeOptions.prefillConfig
        // Image spans only run under chunked prefill, and the app's prefill
        // toggle can select `.off`. Coerce rather than fail after the encodes:
        // whether images work is not a performance preference.
        if !request.imageAttachments.isEmpty,
           let coerced = prefillConfig.coercedForImagePrompt() {
            prefillConfig = coerced
        }
        let progress = ProgressState()
        do {
            try request.validate()
            let requestKey = SessionLoadKey(
                directory: request.modelDirectory.standardizedFileURL,
                maxContext: request.maxContextTokens,
                options: request.runtimeOptions,
                forceLogitsHead: Self.forceLogitsHead(for: request))
            guard let loadedKey else { throw AppInferenceError.modelNotLoaded }
            guard loadedKey == requestKey else { throw AppInferenceError.reloadRequired }
            guard let runner, let tokenizer, let ctx, let scratch else {
                throw AppInferenceError.modelLoadFailed("session lost its loaded state")
            }
            let executedPrefillMode: PrefillExecutedMode =
                prefillConfig.mode == .chunked ? .chunked : .off
            let prefillDiagnostics = PrefillExecutionDiagnostics(config: prefillConfig,
                                                                 executedMode: executedPrefillMode,
                                                                 kvStorageMode: .fp16)

            let report: @Sendable (RawDecodeProgress) -> Void = { event in
                switch event {
                case .prefill(let done, let total):
                    if done == total {
                        // From here there are token boundaries to stop at.
                        self.decodeBegan.withLock { $0 = true }
                        progress.decodeStart = Date()
                        progress.countersAtDecodeStart = RunnerCounterSnapshot(runner)
                    }
                    continuation.yield(.prefillProgress(done: done, total: total))
                case .token(let index, _, let delta):
                    if progress.firstTokenDate == nil { progress.firstTokenDate = Date() }
                    progress.generated = index + 1
                    if index % 8 == 0 { _ = memorySampler.sample() }
                    continuation.yield(.token(AppTokenEvent(
                        index: index,
                        textDelta: delta,
                        elapsedDecodeSeconds: progress.elapsedDecodeSeconds)))
                case .tail(let text):
                    continuation.yield(.token(AppTokenEvent(
                        index: max(progress.generated - 1, 0),
                        textDelta: text,
                        elapsedDecodeSeconds: progress.elapsedDecodeSeconds)))
                }
            }

            let outcome: TurnOutcome
            if request.continuesConversation {
                outcome = try await runConversationTurn(
                    request: request, prefillConfig: prefillConfig,
                    progress: progress, memorySampler: memorySampler,
                    report: report)
            } else {
                outcome = try await runOneShot(
                    request: request, runner: runner, tokenizer: tokenizer,
                    ctx: ctx, scratch: scratch, prefillConfig: prefillConfig,
                    progress: progress, memorySampler: memorySampler,
                    report: report)
            }
            let result = outcome

            let diagnostics = makeDiagnostics(request: request,
                                              memorySampler: memorySampler,
                                              progress: progress,
                                              stopReason: Self.stopReason(result.reason),
                                              prefillSeconds: result.prefillSeconds,
                                              decodeSeconds: result.decodeSeconds,
                                              generated: result.newTokens,
                                              cachedTokens: result.cachedTokens,
                                              computedPrefillTokens: result.computedPrefillTokens,
                                              conversationTokens: result.conversationTokens,
                                              turnRecord: result,
                                              prefill: prefillDiagnostics)
            continuation.yield(.finished(diagnostics))
            continuation.finish()
        } catch is CancellationError {
            let diagnostics = makeDiagnostics(request: request,
                                              memorySampler: memorySampler,
                                              progress: progress,
                                              stopReason: .cancelled,
                                              prefillSeconds: progress.elapsedPrefillSeconds,
                                              decodeSeconds: progress.elapsedDecodeSeconds,
                                              generated: progress.generated,
                                              prefill: PrefillExecutionDiagnostics(
                                                config: prefillConfig,
                                                executedMode: prefillConfig.mode == .chunked
                                                    ? PrefillExecutedMode.chunked : .off,
                                                kvStorageMode: .fp16))
            continuation.yield(.cancelled(diagnostics))
            continuation.finish(throwing: AppInferenceError.cancelled)
        } catch let prefillError as PrefillError {
            let diagnostics = Self.prefillFailureDiagnostics(config: prefillConfig,
                                                             kvStorageMode: .fp16,
                                                             reason: prefillError.description)
            failGeneration(.unknown(prefillError.description),
                           request: request,
                           memorySampler: memorySampler,
                           progress: progress,
                           continuation: continuation,
                           prefill: diagnostics,
                           forcePartialDiagnostics: true)
        } catch let appError as AppInferenceError {
            failGeneration(appError, request: request, memorySampler: memorySampler,
                           progress: progress, continuation: continuation)
        } catch {
            failGeneration(.unknown("\(error)"), request: request, memorySampler: memorySampler,
                           progress: progress, continuation: continuation)
        }
    }

    private func failGeneration(_ error: AppInferenceError,
                                request: AppGenerationRequest,
                                memorySampler: AppMemorySampler,
                                progress: ProgressState,
                                continuation: AsyncThrowingStream<AppInferenceEvent, Error>.Continuation,
                                prefill: PrefillExecutionDiagnostics? = nil,
                                forcePartialDiagnostics: Bool = false) {
        let partial = progress.generated > 0 || forcePartialDiagnostics
            ? makeDiagnostics(request: request, memorySampler: memorySampler,
                              progress: progress, stopReason: .failed,
                              prefillSeconds: progress.elapsedPrefillSeconds,
                              decodeSeconds: progress.elapsedDecodeSeconds,
                              generated: progress.generated,
                              prefill: prefill)
            : nil
        continuation.yield(.failed(error, partial: partial))
        continuation.finish(throwing: error)
    }

    private func makeDiagnostics(request: AppGenerationRequest,
                                 memorySampler: AppMemorySampler,
                                 progress: ProgressState,
                                 stopReason: AppStopReason,
                                 prefillSeconds: Double? = nil,
                                 decodeSeconds: Double,
                                 generated: Int,
                                 cachedTokens: Int? = nil,
                                 computedPrefillTokens: Int? = nil,
                                 conversationTokens: Int? = nil,
                                 turnRecord: TurnOutcome? = nil,
                                 prefill: PrefillExecutionDiagnostics? = nil,
                                 errorDescription: String? = nil) -> AppDiagnostics {
        _ = memorySampler.sample()
        let ttft: Double?
        if let first = progress.firstTokenDate, let start = progress.decodeStart {
            ttft = first.timeIntervalSince(start)
        } else {
            ttft = nil
        }
        return AppDiagnostics(
            generatedTokens: generated,
            stopReason: stopReason,
            promptTokenCount: progress.promptTokenCount,
            cachedPromptTokens: cachedTokens,
            computedPrefillTokens: computedPrefillTokens,
            conversationTokens: conversationTokens,
            promptTokenIDs: turnRecord?.promptTokenIDs,
            generatedTokenIDs: turnRecord?.generatedTokenIDs,
            boundaryTokenIDs: turnRecord?.boundaryTokenIDs,
            boundaryNeedsReplay: turnRecord?.boundaryNeedsReplay,
            prefillSeconds: prefillSeconds,
            timeToFirstTokenSeconds: ttft,
            decodeSeconds: decodeSeconds,
            tokensPerSecond: decodeSeconds > 0 ? Double(generated) / decodeSeconds : 0,
            peakMemoryBytes: memorySampler.peakBytes,
            visionTowerMappedBytes: visionRuntime.map { UInt64($0.retainedWeightBytes) },
            runtimeOptions: request.runtimeOptions,
            prefill: prefill,
            runner: runnerDiagnostics(progress: progress, generated: generated))
    }

    /// Per-token buckets as diffs of the runner's cumulative counters from the
    /// decode start (excludes prefill), divided by the decode forward count.
    /// The forward count is `generated - 1`: each loop iteration that continues
    /// ends with one `produce`; the final sampled token never runs a forward.
    private func runnerDiagnostics(progress: ProgressState, generated: Int) -> AppRunnerDiagnostics? {
        guard let runner, let base = progress.countersAtDecodeStart, generated > 1 else { return nil }
        let now = RunnerCounterSnapshot(runner)
        let forwards = Double(generated - 1)
        func ms(_ end: UInt64, _ start: UInt64) -> Double {
            Double(end &- start) / 1_000_000 / forwards
        }
        return AppRunnerDiagnostics(
            cb1MillisecondsPerToken: ms(now.cb1, base.cb1),
            ioMillisecondsPerToken: ms(now.io, base.io),
            cb2MillisecondsPerToken: ms(now.cb2, base.cb2),
            headMillisecondsPerToken: ms(now.head, base.head),
            rdadviseMillisecondsPerToken: ms(now.rdadvise, base.rdadvise),
            rdadviseCallsPerToken: Double(now.rdadviseCalls &- base.rdadviseCalls) / forwards,
            rdadviseMegabytesPerToken: Double(now.rdadviseBytes &- base.rdadviseBytes) / 1_048_576.0 / forwards,
            rdadviseSkippedPerToken: Double(now.rdadviseSkipped &- base.rdadviseSkipped) / forwards,
            rdadviseFailures: now.rdadviseFailures &- base.rdadviseFailures)
    }

    private static func stopReason(_ reason: StopReason) -> AppStopReason {
        switch reason {
        case .eos: return .eos
        case .endOfTurn: return .endOfTurn
        case .maxTokens: return .maxTokens
        case .stopString: return .stopString
        case .cancelled: return .cancelled
        case .toolCalls: return .toolCalls
        }
    }

    internal static func prefillFailureDiagnostics(config: PrefillRuntimeConfig,
                                                   kvStorageMode: PrefillKVStorageMode,
                                                   reason: String) -> PrefillExecutionDiagnostics {
        PrefillExecutionDiagnostics.unsupported(config: config,
                                                kvStorageMode: kvStorageMode,
                                                reason: reason)
    }
}

/// Mutable per-generation state shared between the progress callback and the
/// surrounding actor method. Single-threaded: the callback runs synchronously
/// inside `runRawCompletion` on the session actor's task.
private final class ProgressState: @unchecked Sendable {
    var generated = 0
    var promptTokenCount: Int?
    var prefillStart: Date?
    var decodeStart: Date?
    var firstTokenDate: Date?
    var countersAtDecodeStart: RunnerCounterSnapshot?

    var elapsedDecodeSeconds: Double {
        guard let decodeStart else { return 0 }
        return Date().timeIntervalSince(decodeStart)
    }

    var elapsedPrefillSeconds: Double? {
        guard let prefillStart else { return nil }
        let end = decodeStart ?? Date()
        return max(end.timeIntervalSince(prefillStart), 0)
    }
}

private struct RunnerCounterSnapshot {
    let cb1: UInt64
    let io: UInt64
    let cb2: UInt64
    let head: UInt64
    let rdadvise: UInt64
    let rdadviseCalls: UInt64
    let rdadviseBytes: UInt64
    let rdadviseFailures: UInt64
    let rdadviseSkipped: UInt64

    init(_ runner: RealForwardRunner) {
        cb1 = runner.totalCb1Nanos
        io = runner.totalIoNanos
        cb2 = runner.totalCb2Nanos
        head = runner.totalHeadNanos &+ runner.totalHeadFusedNanos
        rdadvise = runner.totalRDAdviseNanos
        rdadviseCalls = runner.totalRDAdviseCalls
        rdadviseBytes = runner.totalRDAdviseBytes
        rdadviseFailures = runner.totalRDAdviseFailures
        rdadviseSkipped = runner.totalRDAdviseSkipped
    }
}