Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook
Swift apple-silicongemmagemma4gemma4-26b-a4bgpgpullmllm-inferencelocal-aimacosmetalon-device-aion-device-llmswift
public enum RuntimeHeadPath: String, Codable, Sendable {
case fusedRows = "fused-rows"
case logits
}
public enum RuntimePrefillPolicy: String, Codable, Sendable {
case off
case chunked
}
public enum RuntimePrefillAttentionPath: String, Codable, Sendable {
case causalTiled = "causal-tiled"
case fullTensorOps2DPreferred = "full-tensorops-2d-preferred"
case fullTensorOps2DValidityV2 = "full-tensorops-2d-validity-v2"
}
public enum RuntimeExpertCachePolicy: String, Codable, Sendable {
case lfu
case lru
}
public struct RuntimeConfiguration: Sendable, Equatable {
public static let allowedExpertCacheSlots = [8, 16, 24, 32]
public static let allowedPrefillChunkTokens = PrefillRuntimeConfig.allowedChunkTokens
public static let minimumExpertCacheSlotsForChunkedPrefill = 16
/// The one rendering shared by every help text and every rejection that
/// names one of the arrays above, so neither can name a value the guard
/// does not accept: the hardcoded "32, 64, or 128" outlived the widening of
/// the allowed set and told users 256 was illegal while the guard accepted
/// it. It lives here, beside the arrays, because the CLI's usage and the
/// server's usage and rejections all render the same sets.
///
/// `alsoAccepting` appends parse-level aliases after the integers, so
/// `--prefill-chunk-tokens` renders "32, 64, 128, 256, or auto". They come
/// last because they are not members of the array the guards test, and
/// because the help line's integers are read back by
/// `usageNamesExactlyTheAllowedValues`.
public static func allowedValueList(_ values: [Int],
alsoAccepting aliases: [String] = []) -> String {
let words = values.map(String.init) + aliases
switch words.count {
case 0: return ""
case 1: return words[0]
case 2: return "\(words[0]) or \(words[1])"
default:
return words.dropLast().joined(separator: ", ") + ", or " + words[words.count - 1]
}
}
public let expertCacheSlots: Int
public let expertCachePolicy: RuntimeExpertCachePolicy
public let rdadvisePolicy: RDAdvicePolicyMode
public let prefillPolicy: RuntimePrefillPolicy
public let prefillChunkTokens: Int
public let prefillAttentionPath: RuntimePrefillAttentionPath
public let headPath: RuntimeHeadPath
public init(expertCacheSlots: Int = 16,
expertCachePolicy: RuntimeExpertCachePolicy = .lfu,
rdadvisePolicy: RDAdvicePolicyMode = .off,
prefillEnabled: Bool = true,
prefillChunkTokens: Int = 128,
prefillAttentionPath: RuntimePrefillAttentionPath = .fullTensorOps2DPreferred,
forceLogitsHead: Bool = false) {
precondition(Self.allowedExpertCacheSlots.contains(expertCacheSlots),
"unsupported expert-cache slot count")
precondition(Self.allowedPrefillChunkTokens.contains(prefillChunkTokens),
"unsupported prefill chunk size")
self.expertCacheSlots = expertCacheSlots
self.expertCachePolicy = expertCachePolicy
self.rdadvisePolicy = rdadvisePolicy
self.prefillPolicy = prefillEnabled ? .chunked : .off
self.prefillChunkTokens = prefillChunkTokens
self.prefillAttentionPath = prefillAttentionPath
self.headPath = forceLogitsHead ? .logits : .fusedRows
}
public static var production: RuntimeConfiguration {
RuntimeConfiguration()
}
public var fp16RingEnabled: Bool { true }
public var rdadviseEnabled: Bool { rdadvisePolicy != .off }
public var prefillConfig: PrefillRuntimeConfig {
switch prefillPolicy {
case .off:
return .off
case .chunked:
return .production(chunkTokens: prefillChunkTokens)
}
}
public var modelExpertCachePolicy: ExpertCachePolicy {
expertCachePolicy == .lru ? .lru : .lfu
}
}