Skip to content

Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook

12.7 KB
Raw Blame
import Foundation
import TurboFieldfare

public struct ServerArguments: Equatable, Sendable {
    public let model: String
    public let port: Int
    public let modelID: String
    public let maxContext: Int
    public let queueLimit: Int
    public let promptCacheMode: ServerPromptCacheMode
    public let expertCacheSlots: Int
    public let expertCachePolicy: RuntimeExpertCachePolicy
    public let prefillPolicy: RuntimePrefillPolicy
    public let prefillChunkTokens: Int
    public let rdadvisePolicy: RDAdvicePolicyMode
    public let visionPack: String?
    public let visionResidency: VisionResidencyPolicy

    public static let allowedMaxContext = [
        4_096, 8_192, 16_384, 32_768, 65_536, 98_304, 131_072, 196_608, 262_144,
    ]
    public static let unbackedContextOverrideVariable = "TURBO_FIELDFARE_ALLOW_UNBACKED_CONTEXT"

    public static let usage = """
    usage: TurboFieldfareServer --model <completed .gturbo directory> [options]

      --model <dir>              Required model directory.
      --vision-pack <dir>        Vision companion pack (default beside text model).
      --vision-residency <on-demand|keep-ready>
                                 Routed-expert residency during vision (default on-demand).
      --port <1...65535>         Loopback port (default 8080).
      --model-id <id>            API model identifier (default gemma-4-26b-a4b-it).
      --max-context <tokens>     4096, 8192, 16384, 32768, 65536, 98304,
                                 131072, 196608, or 262144 (default 16384).
                                 Contexts exceeding the host memory budget are refused.
                                 TURBO_FIELDFARE_ALLOW_UNBACKED_CONTEXT=1 overrides this.
      --queue-limit <count>      Maximum queued requests (default 4).
      --prompt-cache-mode <off|single-prefix>
                                 Prompt KV reuse mode (default single-prefix).
      --expert-cache-slots <n>   Expert-cache slots: \(RuntimeConfiguration.allowedValueList(RuntimeConfiguration.allowedExpertCacheSlots)) (default 16).
      --expert-cache-policy <s>  Expert-cache policy: lfu or lru (default lfu).
      --prefill on|off           Enable or disable chunked prompt prefill (default on).
                                 Chunked prefill requires 16 or more cache slots.
      --prefill-chunk-tokens <n|auto>
                                 Prefill chunk size: \(RuntimeConfiguration.allowedValueList(RuntimeConfiguration.allowedPrefillChunkTokens, alsoAccepting: ["auto"]))
                                 (default 128). Each chunk re-reads the routed
                                 expert pool, so larger chunks read less; auto
                                 runs at the cap, 256, which prefills every
                                 prompt in the same spans a per-request size
                                 would. Prefill scratch is sized from the chunk,
                                 so the cap holds about 32.5 MB of it against
                                 16.4 MB at 128.
      --rdadvise <s>             Read-advice policy: off, default, bounded, or adaptive
                                 (default off).
      --help                     Show this help.
    """

    // Mirrors the CLI's runtime flags so both binaries accept the same options
    // with the same validation, instead of the server pinning production
    // defaults. RuntimeConfiguration traps on unsupported values, so every
    // bound is checked here before the initializer runs.
    public func resolvedRuntimeConfiguration(
        forceLogitsHead: Bool = true
    ) throws -> RuntimeConfiguration {
        guard RuntimeConfiguration.allowedExpertCacheSlots.contains(expertCacheSlots) else {
            throw ServerArgumentError.notAllowed(
                flag: "--expert-cache-slots",
                allowed: RuntimeConfiguration.allowedExpertCacheSlots)
        }
        guard RuntimeConfiguration.allowedPrefillChunkTokens.contains(prefillChunkTokens) else {
            throw ServerArgumentError.notAllowed(
                flag: "--prefill-chunk-tokens",
                allowed: RuntimeConfiguration.allowedPrefillChunkTokens,
                alsoAccepting: ["auto"])
        }
        guard prefillPolicy == .off
                || expertCacheSlots >= RuntimeConfiguration.minimumExpertCacheSlotsForChunkedPrefill
        else {
            throw ServerArgumentError.invalid(
                "--expert-cache-slots \(expertCacheSlots) requires --prefill off")
        }
        return RuntimeConfiguration(
            expertCacheSlots: expertCacheSlots,
            expertCachePolicy: expertCachePolicy,
            rdadvisePolicy: rdadvisePolicy,
            prefillEnabled: prefillPolicy == .chunked,
            prefillChunkTokens: prefillChunkTokens,
            forceLogitsHead: forceLogitsHead)
    }

    public static func parse(
        _ input: [String],
        hostMemoryBytes: UInt64 = ContextAdmission.hostMemoryBytes,
        environment: [String: String] = ProcessInfo.processInfo.environment
    ) throws -> ServerArguments {
        var model: String?
        var port = 8080
        var modelID = "gemma-4-26b-a4b-it"
        var maxContext = 16_384
        var queueLimit = 4
        var promptCacheMode: ServerPromptCacheMode = .singlePrefix
        var visionPack: String?
        var visionResidency: VisionResidencyPolicy = .onDemand
        var expertCacheSlots = 16
        var expertCachePolicy = RuntimeExpertCachePolicy.lfu
        var prefillPolicy = RuntimePrefillPolicy.chunked
        var prefillChunkTokens = 128
        var rdadvisePolicy = RDAdvicePolicyMode.off
        var index = 0
        while index < input.count {
            let flag = input[index]
            if flag == "--help" || flag == "-h" { throw ServerArgumentError.help }
            guard index + 1 < input.count else {
                throw ServerArgumentError.invalid("\(flag) requires a value")
            }
            let value = input[index + 1]
            index += 2
            switch flag {
            case "--model":
                model = value
            case "--port":
                guard let parsed = Int(value), (1...65_535).contains(parsed) else {
                    throw ServerArgumentError.invalid("--port must be between 1 and 65535")
                }
                port = parsed
            case "--model-id":
                guard !value.isEmpty else {
                    throw ServerArgumentError.invalid("--model-id must not be empty")
                }
                modelID = value
            case "--max-context":
                guard let parsed = Int(value),
                      allowedMaxContext.contains(parsed) else {
                    throw ServerArgumentError.invalid("--max-context is not supported")
                }
                maxContext = parsed
            case "--queue-limit":
                guard let parsed = Int(value), parsed > 0 else {
                    throw ServerArgumentError.invalid("--queue-limit must be positive")
                }
                queueLimit = parsed
            case "--prompt-cache-mode":
                guard let parsed = ServerPromptCacheMode(rawValue: value) else {
                    throw ServerArgumentError.invalid(
                        "--prompt-cache-mode must be off or single-prefix")
                }
                promptCacheMode = parsed
            case "--vision-pack":
                visionPack = value
            case "--vision-residency":
                guard let parsed = VisionResidencyPolicy(rawValue: value) else {
                    throw ServerArgumentError.invalid(
                        "--vision-residency must be on-demand or keep-ready")
                }
                visionResidency = parsed
            case "--expert-cache-slots":
                guard let parsed = Int(value),
                      RuntimeConfiguration.allowedExpertCacheSlots.contains(parsed) else {
                    throw ServerArgumentError.notAllowed(
                        flag: flag,
                        allowed: RuntimeConfiguration.allowedExpertCacheSlots)
                }
                expertCacheSlots = parsed
            case "--expert-cache-policy":
                guard let parsed = RuntimeExpertCachePolicy(rawValue: value) else {
                    throw ServerArgumentError.invalid("--expert-cache-policy must be lfu or lru")
                }
                expertCachePolicy = parsed
            case "--prefill":
                switch value {
                case "on": prefillPolicy = .chunked
                case "off": prefillPolicy = .off
                default: throw ServerArgumentError.invalid("--prefill must be on or off")
                }
            case "--prefill-chunk-tokens":
                // `auto` is an alias for the cap here, and nothing downstream
                // learns it was spelled that way. A per-request size is the
                // smallest allowed size that covers the span, so the cap
                // prefills every prompt in exactly the spans that size would,
                // and the KV ring is sized from the cap either way. The prefill
                // scratch is the one thing a per-request size changes: it is
                // allocated from the chunk, about 125.5 KB per token over a
                // fixed 328 KB, and the
                // server would reallocate it on every size change.
                if value == "auto" {
                    prefillChunkTokens = PrefillRuntimeConfig.maxChunkTokens
                    break
                }
                guard let parsed = Int(value),
                      RuntimeConfiguration.allowedPrefillChunkTokens.contains(parsed) else {
                    throw ServerArgumentError.notAllowed(
                        flag: flag,
                        allowed: RuntimeConfiguration.allowedPrefillChunkTokens,
                        alsoAccepting: ["auto"])
                }
                prefillChunkTokens = parsed
            case "--rdadvise":
                guard let parsed = RDAdvicePolicyMode(rawValue: value) else {
                    throw ServerArgumentError.invalid(
                        "--rdadvise must be off, default, bounded, or adaptive")
                }
                rdadvisePolicy = parsed
            default:
                throw ServerArgumentError.invalid("unknown flag: \(flag)")
            }
        }
        guard let model else { throw ServerArgumentError.invalid("--model is required") }
        // Refused here rather than at load: the KV allocation for a context the
        // host cannot back fails deep inside the runtime, after the model has
        // already started loading, with an allocator error that says nothing
        // about how much memory the choice actually needs.
        let config = ArchConfig.gemma4_26B_A4B
        if case .needsMemory = ContextAdmission.availability(config: config,
                                                             maxContext: maxContext,
                                                             hostMemoryBytes: hostMemoryBytes, expertCacheSlots: expertCacheSlots),
           environment[unbackedContextOverrideVariable] != "1" {
            throw ServerArgumentError.invalid(
                ContextAdmission.needDescription(config: config,
                                                 maxContext: maxContext,
                                                 hostMemoryBytes: hostMemoryBytes, expertCacheSlots: expertCacheSlots)
                    + " Set \(unbackedContextOverrideVariable)=1 to start anyway.")
        }
        return ServerArguments(model: model,
                               port: port,
                               modelID: modelID,
                               maxContext: maxContext,
                               queueLimit: queueLimit,
                               promptCacheMode: promptCacheMode,
                               expertCacheSlots: expertCacheSlots,
                               expertCachePolicy: expertCachePolicy,
                               prefillPolicy: prefillPolicy,
                               prefillChunkTokens: prefillChunkTokens,
                               rdadvisePolicy: rdadvisePolicy,
                               visionPack: visionPack,
                               visionResidency: visionResidency)
    }
}

public enum ServerArgumentError: Error, Equatable, CustomStringConvertible {
    case help
    case invalid(String)

    public var description: String {
        switch self {
        case .help: "help"
        case .invalid(let message): message
        }
    }
}

extension ServerArgumentError {
    static func notAllowed(flag: String,
                           allowed: [Int],
                           alsoAccepting aliases: [String] = []) -> ServerArgumentError {
        .invalid("\(flag) must be "
            + RuntimeConfiguration.allowedValueList(allowed, alsoAccepting: aliases))
    }
}