Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook
Swift apple-silicongemmagemma4gemma4-26b-a4bgpgpullmllm-inferencelocal-aimacosmetalon-device-aion-device-llmswift
import Foundation
/// Decides whether an incoming generate may join the open conversation.
///
/// Extracted from the service's command loop so the decision is testable
/// without a socket, a model, or a Metal device. The loop cannot express the
/// boundary cases this holds — a turn from a replaced conversation, a turn that
/// skips or repeats a position, a one-shot arriving while a lineage is open —
/// and every one of them ends with a message the user never sent being appended
/// to a KV, which no later check can detect.
public struct DecodeConversationGate: Equatable, Sendable {
/// Why a turn cannot run. Typed rather than a string so a test pins the
/// case, and the loop owns the wording.
public enum Rejection: Error, Equatable, Sendable {
/// The turn names a conversation that is no longer the open one.
case staleConversation(requested: UUID, open: UUID?)
/// The turn's position does not follow the committed turns.
case outOfOrderTurn(requested: Int?, committed: Int)
/// A one-shot generate arrived while a conversation was open. Running
/// it would reset the KV under a lineage the app still believes in.
case oneShotDuringConversation(open: UUID)
}
public enum Admission: Equatable, Sendable {
/// No conversation is open: reset the KV and prefill the whole prompt.
case oneShot
/// Append to the open lineage at `index`.
case turn(epoch: UUID, index: Int)
}
public private(set) var openEpoch: UUID?
public private(set) var committedTurns = 0
public init() {}
/// Starts a new lineage. The caller drops the KV; this only records that it
/// did.
public mutating func reset(to epoch: UUID) {
openEpoch = epoch
committedTurns = 0
}
/// Opens a lineage that already has turns behind it: a stored conversation
/// whose token IDs are back in the KV.
///
/// Opening it at zero, as a reset does, would reject the reopened
/// conversation's very next turn as out of order and keep rejecting every
/// turn after it. A negative count is clamped rather than trusted: the
/// figure arrives over a socket.
public mutating func restore(to epoch: UUID, committedTurns: Int) {
openEpoch = epoch
self.committedTurns = max(0, committedTurns)
}
/// Ends any lineage. Both unload and load reach here: each releases or
/// rebuilds the runner and the KV, so the tokens the epoch named are gone,
/// and a turn resuming onto them would resume onto nothing.
public mutating func endLineage() {
openEpoch = nil
committedTurns = 0
}
/// A restore was asked for and did not complete.
///
/// The session drops the KV it held before it prefills the requested
/// record, so by the time a restore can fail — a digest that does not
/// match, a token outside the vocabulary — the previous lineage's tokens
/// are already gone. Leaving that epoch open admitted its next turn onto an
/// empty cache as an opening turn, with the caller's transcript showing
/// every exchange the model could no longer see. Fails closed for a
/// restore refused before the session was touched too: the app treats
/// every failed replay as the end of what it held, and the two sides must
/// not disagree about which epoch is open.
public mutating func restoreFailed() {
endLineage()
}
public func admit(_ request: DecodeGenerationRequest) -> Result<Admission, Rejection> {
guard let requested = request.conversationEpoch else {
if let openEpoch { return .failure(.oneShotDuringConversation(open: openEpoch)) }
return .success(.oneShot)
}
guard requested == openEpoch else {
return .failure(.staleConversation(requested: requested, open: openEpoch))
}
guard request.turnIndex == committedTurns else {
return .failure(.outOfOrderTurn(requested: request.turnIndex,
committed: committedTurns))
}
return .success(.turn(epoch: requested, index: committedTurns))
}
/// Records that a turn actually ran. Called after the generation, not at
/// admission: a turn rejected downstream, or one that failed before
/// touching the KV, must not advance the position the next turn has to
/// match.
public mutating func commit(_ admission: Admission) {
guard case .turn(let epoch, _) = admission, epoch == openEpoch else { return }
committedTurns += 1
}
}