Skip to content

Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook

5.7 KB
Raw Blame
import Foundation

enum ServerLog {
    static func accepted(id: String, streaming: Bool) {
        write("request \(id) accepted streaming=\(streaming)")
    }

    static func prepared(id: String, promptTokens: Int?) {
        let count = promptTokens.map(String.init) ?? "backend-managed"
        write("request \(id) prepared prompt=\(count)")
    }

    static func queued(id: String) {
        write("request \(id) queued")
    }

    static func generating(id: String) {
        write("request \(id) generating")
    }

    static func completed(id: String,
                          duration: Duration,
                          completion: ServerCompletion) {
        write(completedMessage(id: id, duration: duration, completion: completion))
    }

    /// Split out so the throughput arithmetic can be checked without a log sink
    /// and without a decode. `pp_tok_s` divides by the tokens actually
    /// prefilled: a prompt-cache hit contributes no prefill work, so counting
    /// cached tokens as prefilled would report a rate the runtime never
    /// achieved. Carries counts and durations only, never content.
    static func completedMessage(id: String,
                                 duration: Duration,
                                 completion: ServerCompletion) -> String {
        let usage = completion.usage
        // Clamped because both counts come from the runtime: a cached count
        // above the prompt would otherwise log a negative token count and a
        // negative rate rather than the zero work that was done.
        let computedPrefillTokens = max(
            usage.promptTokens - usage.promptTokensDetails.cachedTokens, 0)
        let prefillTokensPerSecond = completion.prefillSeconds > 0
            ? Double(computedPrefillTokens) / completion.prefillSeconds
            : 0
        let decodeTokensPerSecond = completion.decodeSeconds > 0
            ? Double(usage.completionTokens) / completion.decodeSeconds
            : 0
        return "request \(id) completed in \(format(duration)) "
            + "prompt=\(usage.promptTokens) "
            + "cached=\(usage.promptTokensDetails.cachedTokens) "
            + "completion=\(usage.completionTokens) "
            + "pp=\(formatSeconds(completion.prefillSeconds)) "
            + "pp_tok_s=\(formatRate(prefillTokensPerSecond)) "
            + "tg=\(formatSeconds(completion.decodeSeconds)) "
            + "tg_tok_s=\(formatRate(decodeTokensPerSecond)) "
            + "finish=\(completion.finishReason)"
    }

    static func visionPackInvalid(at url: URL, error: Error) {
        write("vision pack at \(url.path) is invalid: \(String(reflecting: error))")
    }

    static func visionRuntimeUnsupported(at url: URL, error: Error) {
        write("vision runtime for pack at \(url.path) is unsupported: "
            + String(reflecting: error))
    }

    static func visionRuntimeUnsupported() {
        write("vision runtime is unsupported: the image tower requires an M2 or newer Mac")
    }

    /// The session is an actor, so generation is serialized: this line always
    /// belongs to the request between the preceding `generating` and the
    /// following `completed`. Carries a reason code only, never prompt content.
    static func promptCacheMiss(reason: ServerPromptCacheMissReason) {
        write(promptCacheMissMessage(reason: reason))
    }

    static func promptCacheMissMessage(reason: ServerPromptCacheMissReason) -> String {
        "prompt cache miss reason=\(reason.rawValue)"
    }

    static func promptCachePublication(_ result: ServerPromptCachePublishResult) {
        if let message = promptCachePublicationMessage(result) { write(message) }
    }

    static func promptCachePublicationMessage(_ result: ServerPromptCachePublishResult) -> String? {
        guard case .rejected(let reason) = result else { return nil }
        return "prompt cache publication rejected reason=\(reason.rawValue)"
    }

    /// A request whose client stopped listening. Kept distinct from `failed`
    /// because a cancel is not a server fault and should not read as one, and
    /// added because excluding it from logging altogether left the request's last
    /// line as `generating` forever: an operator could not tell a running request
    /// from an abandoned one, or from a crashed one. Carries no content.
    static func cancelled(id: String, phase: String, duration: Duration) {
        write(cancelledMessage(id: id, phase: phase, duration: duration))
    }

    /// Split out so the wording can be checked without a log sink: this line
    /// must stay distinguishable from `failed`, must name the phase, and must
    /// carry no prompt or generated content.
    static func cancelledMessage(id: String, phase: String, duration: Duration) -> String {
        "request \(id) cancelled by client in \(format(duration)) phase=\(phase)"
    }

    static func failed(id: String,
                       phase: String,
                       status: UInt,
                       error: Error) {
        write("request \(id) failed phase=\(phase) status=\(status) "
            + "error=\(String(reflecting: error))")
    }

    private static func format(_ duration: Duration) -> String {
        let seconds = Double(duration.components.seconds)
            + Double(duration.components.attoseconds) / 1e18
        return formatSeconds(seconds)
    }

    private static func formatSeconds(_ seconds: Double) -> String {
        String(format: "%.3fs", seconds)
    }

    private static func formatRate(_ rate: Double) -> String {
        String(format: "%.3f", rate)
    }

    private static func write(_ message: String) {
        let line = "[\(Date().formatted(.iso8601))] \(message)\n"
        FileHandle.standardError.write(Data(line.utf8))
    }
}