Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook
Swift apple-silicongemmagemma4gemma4-26b-a4bgpgpullmllm-inferencelocal-aimacosmetalon-device-aion-device-llmswift
import Foundation
enum ServerLog {
static func accepted(id: String, streaming: Bool) {
write("request \(id) accepted streaming=\(streaming)")
}
static func prepared(id: String, promptTokens: Int?) {
let count = promptTokens.map(String.init) ?? "backend-managed"
write("request \(id) prepared prompt=\(count)")
}
static func queued(id: String) {
write("request \(id) queued")
}
static func generating(id: String) {
write("request \(id) generating")
}
static func completed(id: String,
duration: Duration,
completion: ServerCompletion) {
write(completedMessage(id: id, duration: duration, completion: completion))
}
/// Split out so the throughput arithmetic can be checked without a log sink
/// and without a decode. `pp_tok_s` divides by the tokens actually
/// prefilled: a prompt-cache hit contributes no prefill work, so counting
/// cached tokens as prefilled would report a rate the runtime never
/// achieved. Carries counts and durations only, never content.
static func completedMessage(id: String,
duration: Duration,
completion: ServerCompletion) -> String {
let usage = completion.usage
// Clamped because both counts come from the runtime: a cached count
// above the prompt would otherwise log a negative token count and a
// negative rate rather than the zero work that was done.
let computedPrefillTokens = max(
usage.promptTokens - usage.promptTokensDetails.cachedTokens, 0)
let prefillTokensPerSecond = completion.prefillSeconds > 0
? Double(computedPrefillTokens) / completion.prefillSeconds
: 0
let decodeTokensPerSecond = completion.decodeSeconds > 0
? Double(usage.completionTokens) / completion.decodeSeconds
: 0
return "request \(id) completed in \(format(duration)) "
+ "prompt=\(usage.promptTokens) "
+ "cached=\(usage.promptTokensDetails.cachedTokens) "
+ "completion=\(usage.completionTokens) "
+ "pp=\(formatSeconds(completion.prefillSeconds)) "
+ "pp_tok_s=\(formatRate(prefillTokensPerSecond)) "
+ "tg=\(formatSeconds(completion.decodeSeconds)) "
+ "tg_tok_s=\(formatRate(decodeTokensPerSecond)) "
+ "finish=\(completion.finishReason)"
}
static func visionPackInvalid(at url: URL, error: Error) {
write("vision pack at \(url.path) is invalid: \(String(reflecting: error))")
}
static func visionRuntimeUnsupported(at url: URL, error: Error) {
write("vision runtime for pack at \(url.path) is unsupported: "
+ String(reflecting: error))
}
static func visionRuntimeUnsupported() {
write("vision runtime is unsupported: the image tower requires an M2 or newer Mac")
}
/// The session is an actor, so generation is serialized: this line always
/// belongs to the request between the preceding `generating` and the
/// following `completed`. Carries a reason code only, never prompt content.
static func promptCacheMiss(reason: ServerPromptCacheMissReason) {
write(promptCacheMissMessage(reason: reason))
}
static func promptCacheMissMessage(reason: ServerPromptCacheMissReason) -> String {
"prompt cache miss reason=\(reason.rawValue)"
}
static func promptCachePublication(_ result: ServerPromptCachePublishResult) {
if let message = promptCachePublicationMessage(result) { write(message) }
}
static func promptCachePublicationMessage(_ result: ServerPromptCachePublishResult) -> String? {
guard case .rejected(let reason) = result else { return nil }
return "prompt cache publication rejected reason=\(reason.rawValue)"
}
/// A request whose client stopped listening. Kept distinct from `failed`
/// because a cancel is not a server fault and should not read as one, and
/// added because excluding it from logging altogether left the request's last
/// line as `generating` forever: an operator could not tell a running request
/// from an abandoned one, or from a crashed one. Carries no content.
static func cancelled(id: String, phase: String, duration: Duration) {
write(cancelledMessage(id: id, phase: phase, duration: duration))
}
/// Split out so the wording can be checked without a log sink: this line
/// must stay distinguishable from `failed`, must name the phase, and must
/// carry no prompt or generated content.
static func cancelledMessage(id: String, phase: String, duration: Duration) -> String {
"request \(id) cancelled by client in \(format(duration)) phase=\(phase)"
}
static func failed(id: String,
phase: String,
status: UInt,
error: Error) {
write("request \(id) failed phase=\(phase) status=\(status) "
+ "error=\(String(reflecting: error))")
}
private static func format(_ duration: Duration) -> String {
let seconds = Double(duration.components.seconds)
+ Double(duration.components.attoseconds) / 1e18
return formatSeconds(seconds)
}
private static func formatSeconds(_ seconds: Double) -> String {
String(format: "%.3fs", seconds)
}
private static func formatRate(_ rate: Double) -> String {
String(format: "%.3f", rate)
}
private static func write(_ message: String) {
let line = "[\(Date().formatted(.iso8601))] \(message)\n"
FileHandle.standardError.write(Data(line.utf8))
}
}