Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook
Swift apple-silicongemmagemma4gemma4-26b-a4bgpgpullmllm-inferencelocal-aimacosmetalon-device-aion-device-llmswift
import CryptoKit
import Foundation
import TurboFieldfare
public enum ServerInferenceEvent: Equatable, Sendable {
case content(String)
case toolCall(ParsedToolCall)
}
public struct ServerCompletion: Equatable, Sendable {
public let content: String
public let toolCalls: [ParsedToolCall]
public let finishReason: String
public let usage: OpenAIUsage
/// Wall time the runtime spent prefilling the prompt and decoding the
/// completion, as `runRawCompletion` already measures them around those two
/// phases; nothing is timed here. Defaulted to zero so a backend that does
/// not decode through the runtime — the test doubles, the compatibility
/// harness — reports no timing rather than a fabricated one.
public let prefillSeconds: Double
public let decodeSeconds: Double
public init(content: String,
toolCalls: [ParsedToolCall],
finishReason: String,
usage: OpenAIUsage,
prefillSeconds: Double = 0,
decodeSeconds: Double = 0) {
self.content = content
self.toolCalls = toolCalls
self.finishReason = finishReason
self.usage = usage
self.prefillSeconds = prefillSeconds
self.decodeSeconds = decodeSeconds
}
}
enum StructuredOutputFailureKind: String, Equatable, Sendable {
case decoderConsume = "decoder_consume"
case decoderFinish = "decoder_finish"
case orphanToolResponse = "orphan_tool_response"
}
enum StructuredOutputFailureCause: String, Equatable, Sendable {
case malformed
case unknownTool = "unknown_tool"
case oversized
case unexpected
case none
static func classify(_ error: Error) -> Self {
guard let parserError = error as? GemmaToolCallParserError else {
return .unexpected
}
switch parserError {
case .malformed: return .malformed
case .unknownTool: return .unknownTool
case .oversized: return .oversized
}
}
}
struct StructuredOutputFailureDiagnostics: Equatable, Sendable {
let renderedPromptTokens: Int
let effectivePromptTokens: Int
let resultPromptTokens: Int
let cachedPromptTokens: Int
let computedPrefillTokens: Int
let completionTokens: Int
let maxCompletionTokens: Int
let rawStop: String
let kvPosition: Int
let kvBackedTokens: Int
let boundaryTokens: Int
let decodedCalls: Int
let visibleBytes: Int
let stopStringMatched: Bool
let toolStartCount: Int
let toolEndCount: Int
let toolResponseCount: Int
let toolResponseEndCount: Int
let lastToolStartOffset: Int
let lastToolEndOffset: Int
let lastToolResponseOffset: Int
let lastToolResponseEndOffset: Int
let effectiveCountMatchesResult: Bool
let effectivePrefixMatchesKV: Bool
let kvPositionMatchesHistory: Bool
let completionCountMatchesHistory: Bool
let prefillAccountingMatches: Bool
let renderedPromptHash: String
let effectivePromptHash: String
let generatedHash: String
init(
renderedPromptIDs: [Int32],
effectivePromptIDs: [Int32],
result: RawDecodeResult,
maxCompletionTokens: Int,
decodedCalls: Int,
visibleBytes: Int,
stopStringMatched: Bool,
toolStartID: Int32,
toolEndID: Int32,
toolResponseID: Int32,
toolResponseEndID: Int32
) {
let safePrefillCount = min(
max(result.prefillTokens, 0),
result.kvBackedTokenIDs.count)
let committedGenerated = result.kvBackedTokenIDs.dropFirst(safePrefillCount)
let boundary = result.uncommittedBoundaryTokenIDs[...]
let generatedSegments = [committedGenerated, boundary]
var toolStartCount = 0
var toolEndCount = 0
var toolResponseCount = 0
var toolResponseEndCount = 0
var lastToolStartOffset = -1
var lastToolEndOffset = -1
var lastToolResponseOffset = -1
var lastToolResponseEndOffset = -1
var offset = 0
for segment in generatedSegments {
for tokenID in segment {
if tokenID == toolStartID {
toolStartCount += 1
lastToolStartOffset = offset
}
if tokenID == toolEndID {
toolEndCount += 1
lastToolEndOffset = offset
}
if tokenID == toolResponseID {
toolResponseCount += 1
lastToolResponseOffset = offset
}
if tokenID == toolResponseEndID {
toolResponseEndCount += 1
lastToolResponseEndOffset = offset
}
offset += 1
}
}
let (prefillAccounted, prefillOverflow) = result.cachedPromptTokens
.addingReportingOverflow(result.computedPrefillTokens)
self.renderedPromptTokens = renderedPromptIDs.count
self.effectivePromptTokens = effectivePromptIDs.count
self.resultPromptTokens = result.prefillTokens
self.cachedPromptTokens = result.cachedPromptTokens
self.computedPrefillTokens = result.computedPrefillTokens
self.completionTokens = result.newTokens
self.maxCompletionTokens = maxCompletionTokens
self.rawStop = Self.rawStop(result.reason)
self.kvPosition = result.kvPosition
self.kvBackedTokens = result.kvBackedTokenIDs.count
self.boundaryTokens = result.uncommittedBoundaryTokenIDs.count
self.decodedCalls = decodedCalls
self.visibleBytes = visibleBytes
self.stopStringMatched = stopStringMatched
self.toolStartCount = toolStartCount
self.toolEndCount = toolEndCount
self.toolResponseCount = toolResponseCount
self.toolResponseEndCount = toolResponseEndCount
self.lastToolStartOffset = lastToolStartOffset
self.lastToolEndOffset = lastToolEndOffset
self.lastToolResponseOffset = lastToolResponseOffset
self.lastToolResponseEndOffset = lastToolResponseEndOffset
self.effectiveCountMatchesResult = effectivePromptIDs.count == result.prefillTokens
self.effectivePrefixMatchesKV = result.kvBackedTokenIDs.count >= effectivePromptIDs.count
&& result.kvBackedTokenIDs.prefix(effectivePromptIDs.count)
.elementsEqual(effectivePromptIDs)
self.kvPositionMatchesHistory = result.kvPosition == result.kvBackedTokenIDs.count
self.completionCountMatchesHistory = offset == result.newTokens
self.prefillAccountingMatches = !prefillOverflow
&& prefillAccounted == result.prefillTokens
self.renderedPromptHash = Self.i32leSHA256([renderedPromptIDs[...]])
self.effectivePromptHash = Self.i32leSHA256([effectivePromptIDs[...]])
self.generatedHash = Self.i32leSHA256(generatedSegments)
}
static func i32leSHA256(_ segments: [ArraySlice<Int32>]) -> String {
var hasher = SHA256()
var bytes = Data()
bytes.reserveCapacity(4_096)
for segment in segments {
for tokenID in segment {
var littleEndian = UInt32(bitPattern: tokenID).littleEndian
withUnsafeBytes(of: &littleEndian) {
bytes.append(contentsOf: $0)
}
if bytes.count == 4_096 {
hasher.update(data: bytes)
bytes.removeAll(keepingCapacity: true)
}
}
}
if !bytes.isEmpty { hasher.update(data: bytes) }
return hasher.finalize().map { String(format: "%02x", $0) }.joined()
}
var logDescription: String {
[
"rendered_prompt_tokens=\(renderedPromptTokens)",
"effective_prompt_tokens=\(effectivePromptTokens)",
"result_prompt_tokens=\(resultPromptTokens)",
"cached_prompt_tokens=\(cachedPromptTokens)",
"computed_prefill_tokens=\(computedPrefillTokens)",
"completion_tokens=\(completionTokens)",
"max_completion_tokens=\(maxCompletionTokens)",
"raw_stop=\(rawStop)",
"kv_position=\(kvPosition)",
"kv_backed_tokens=\(kvBackedTokens)",
"boundary_tokens=\(boundaryTokens)",
"decoded_calls=\(decodedCalls)",
"visible_bytes=\(visibleBytes)",
"stop_string_matched=\(stopStringMatched)",
"tool_start_count=\(toolStartCount)",
"tool_end_count=\(toolEndCount)",
"tool_response_count=\(toolResponseCount)",
"tool_response_end_count=\(toolResponseEndCount)",
"last_tool_start_offset=\(lastToolStartOffset)",
"last_tool_end_offset=\(lastToolEndOffset)",
"last_tool_response_offset=\(lastToolResponseOffset)",
"last_tool_response_end_offset=\(lastToolResponseEndOffset)",
"effective_count_matches_result=\(effectiveCountMatchesResult)",
"effective_prefix_matches_kv=\(effectivePrefixMatchesKV)",
"kv_position_matches_history=\(kvPositionMatchesHistory)",
"completion_count_matches_history=\(completionCountMatchesHistory)",
"prefill_accounting_matches=\(prefillAccountingMatches)",
"rendered_prompt_i32le_sha256=\(renderedPromptHash)",
"effective_prompt_i32le_sha256=\(effectivePromptHash)",
"generated_i32le_sha256=\(generatedHash)",
].joined(separator: " ")
}
private static func rawStop(_ reason: StopReason) -> String {
switch reason {
case .eos: "eos"
case .endOfTurn: "end_of_turn"
case .maxTokens: "max_tokens"
case .stopString: "stop_string"
case .cancelled: "cancelled"
case .toolCalls: "tool_calls"
}
}
}
struct StructuredOutputFailure: Error, CustomDebugStringConvertible, Sendable {
let kind: StructuredOutputFailureKind
let cause: StructuredOutputFailureCause
let diagnostics: StructuredOutputFailureDiagnostics
var debugDescription: String {
"structured_output_failure kind=\(kind.rawValue) "
+ "cause=\(cause.rawValue) \(diagnostics.logDescription)"
}
}
public struct ServerPreparedRequest: Sendable {
public let request: ValidatedChatRequest
fileprivate let promptIDs: [Int32]?
public var promptTokenCount: Int? { promptIDs?.count }
init(request: ValidatedChatRequest, promptIDs: [Int32]? = nil) {
self.request = request
self.promptIDs = promptIDs
}
}
public protocol ServerInferenceBackend: Sendable {
func prepare(_ request: ValidatedChatRequest) async throws -> ServerPreparedRequest
func generate(_ request: ValidatedChatRequest,
onEvent: @escaping @Sendable (ServerInferenceEvent) -> Void) async throws -> ServerCompletion
func generate(_ prepared: ServerPreparedRequest,
onEvent: @escaping @Sendable (ServerInferenceEvent) -> Void) async throws -> ServerCompletion
}
public extension ServerInferenceBackend {
func prepare(_ request: ValidatedChatRequest) async throws -> ServerPreparedRequest {
ServerPreparedRequest(request: request)
}
func generate(
_ prepared: ServerPreparedRequest,
onEvent: @escaping @Sendable (ServerInferenceEvent) -> Void
) async throws -> ServerCompletion {
try await generate(prepared.request, onEvent: onEvent)
}
}
public actor ServerCoordinator {
private struct Waiter {
let id: UUID
let continuation: CheckedContinuation<Void, Error>
}
private let queueLimit: Int
private var admittedCount = 0
private var active = false
private var waiters: [Waiter] = []
private var shuttingDown = false
public init(queueLimit: Int) {
self.queueLimit = queueLimit
}
public func run<T: Sendable>(
onQueued: @escaping @Sendable () -> Void = {},
_ operation: @escaping @Sendable () async throws -> T
) async throws -> T {
try await runPreparing(
onQueued: onQueued,
prepare: { () },
operation: { _ in try await operation() })
}
func runPreparing<Prepared: Sendable, T: Sendable>(
onQueued: @escaping @Sendable () -> Void = {},
prepare: @escaping @Sendable () async throws -> Prepared,
operation: @escaping @Sendable (Prepared) async throws -> T
) async throws -> T {
try Task.checkCancellation()
guard !shuttingDown else { throw CancellationError() }
guard admittedCount <= queueLimit else { throw ServerRequestError.queueFull }
admittedCount += 1
defer { admittedCount -= 1 }
let prepared = try await prepare()
try Task.checkCancellation()
try await acquire(onQueued: onQueued)
defer { release() }
return try await operation(prepared)
}
private func acquire(onQueued: @escaping @Sendable () -> Void) async throws {
try Task.checkCancellation()
guard !shuttingDown else { throw CancellationError() }
if !active {
active = true
return
}
guard waiters.count < queueLimit else { throw ServerRequestError.queueFull }
onQueued()
let id = UUID()
try await withTaskCancellationHandler {
try await withCheckedThrowingContinuation { continuation in
waiters.append(Waiter(id: id, continuation: continuation))
}
} onCancel: {
Task { await self.cancelWaiter(id) }
}
if Task.isCancelled {
release()
throw CancellationError()
}
}
private func cancelWaiter(_ id: UUID) {
guard let index = waiters.firstIndex(where: { $0.id == id }) else { return }
let waiter = waiters.remove(at: index)
waiter.continuation.resume(throwing: CancellationError())
}
private func release() {
if waiters.isEmpty {
active = false
} else {
waiters.removeFirst().continuation.resume()
}
}
public func shutdown() {
shuttingDown = true
let queued = waiters
waiters.removeAll()
for waiter in queued {
waiter.continuation.resume(throwing: CancellationError())
}
}
public var queuedCount: Int { waiters.count }
public var isActive: Bool { active }
}
/// How the server reads the images of a request: one plan per image, and the
/// encode reads the plan the count came from.
///
/// Planning opens the file, sniffs it, parses its metadata and — for a JPEG —
/// walks its markers. Counting with a plan and then letting `encodeImage(at:)`
/// re-plan internally did all of that a second time per image, and the two
/// opens saw two files: one rewritten in between produced a count the laid-out
/// span no longer matched, and the turn died on `placeholderMismatch` naming
/// nothing the client did.
///
/// Both multimodal paths go through here so the pattern has one implementation
/// to keep correct rather than one per path.
enum ServerRequestImages {
/// An image planned once: the count that lays out its placeholder span,
/// and — while the plan behind that count is still open — the bytes the
/// count was taken from.
struct Planned {
let url: URL
let softTokenCount: Int
let plan: VisionImagePlan?
}
/// How many of a request's images keep their plan open at once.
///
/// A plan owns the descriptor its ImageIO source reads through, and what
/// bounds a request's image count is the context budget alone — an
/// 8,192-token context admits some 2,700 single-token images. One
/// descriptor each would exhaust the process table and take the listening
/// socket down with it, so past this many the plan is released after
/// counting and remade at encode time. Real requests carry a handful of
/// images and never reach it.
static let maximumOpenPlans = 32
/// Plans every image before any of them is encoded.
///
/// Two reasons for the order: the counts that lay out the placeholder spans
/// then come from the same open files the encodes read, and a request whose
/// last image cannot be read at all is refused before the tower has run on
/// the images ahead of it.
static func plans(
for urls: [URL],
with preprocessor: Gemma4ImagePreprocessor,
maximumOpenPlans: Int = ServerRequestImages.maximumOpenPlans,
checkCancellation: () throws -> Void = {}
) throws -> [Planned] {
var planned: [Planned] = []
planned.reserveCapacity(urls.count)
for url in urls {
// Planning a JPEG walks its whole scan, so an abandoned request
// stops at the next image rather than reading all of them first.
try checkCancellation()
let plan = try preprocessor.plan(fileURL: url)
planned.append(Planned(
url: url,
softTokenCount: plan.geometry.softTokenCount,
plan: planned.count < maximumOpenPlans ? plan : nil))
}
return planned
}
/// What an image's encode reads through.
enum Source {
/// The plan the image's count was taken from, still open.
case plan(VisionImagePlan)
/// The file again, for an image the open bound released.
case reopened(URL)
}
/// An image is encoded from the plan its count was taken from. Only one
/// whose plan the open bound released is read a second time.
///
/// Returned rather than dispatched through caller-supplied closures:
/// Swift 6.2's region isolation rejects an actor-isolated closure that
/// captures a session's `VisionRuntime` and `Model` and is handed to a
/// nonisolated callee, which broke the build on Xcode 26.0-26.3.
static func source(for image: Planned) -> Source {
guard let plan = image.plan else { return .reopened(image.url) }
return .plan(plan)
}
/// Every image of a request planned, each still paired with the id its
/// features are filed under.
///
/// The pairing is the reason this is not two statements at the call site:
/// `[UUID: URL]` has no order, so the ids and the plans have to be taken
/// from one snapshot. Pairing them from two walks of the dictionary files
/// one image's features under another's id, and when both project to the
/// same soft-token count nothing downstream can notice.
///
/// Planning every image before the caller encodes any is the other half:
/// the spans are laid out from the same open files the encodes read, and a
/// request whose last image cannot be read at all is refused before the
/// tower has run on the ones ahead of it.
static func plannedEntries(
_ imageFiles: [UUID: URL],
with preprocessor: Gemma4ImagePreprocessor,
maximumOpenPlans: Int = ServerRequestImages.maximumOpenPlans,
checkCancellation: () throws -> Void = {}
) throws -> [(id: UUID, image: Planned)] {
let entries = Array(imageFiles)
let planned = try plans(
for: entries.map { $0.value },
with: preprocessor,
maximumOpenPlans: maximumOpenPlans,
checkCancellation: checkCancellation)
return zip(entries, planned).map { (id: $0.key, image: $1) }
}
/// A file the caller supplied that cannot be read is a bad request, not a
/// server fault, and it stays one wherever the read fails. Admission reads
/// dimensions with stream verification off; the encode re-plans with it on,
/// so a truncated upload is admitted and only fails the second read.
/// Without this that second failure reached the generic handler as a 500,
/// which official clients retry and 4xx they do not.
static func requestError(forUnreadable error: any Error) -> any Error {
if error is ServerRequestError || error is CancellationError { return error }
return ServerRequestError.invalid(
message: "image could not be read: \(error)",
param: "messages", code: "invalid_image")
}
}
public actor ServerModelSession: ServerInferenceBackend {
private let context: MetalContext
private let model: Model
private let tokenizer: GFTokenizer
private let runner: RealForwardRunner
private let scratch: RawCompletionScratch
private let prefillConfig: PrefillRuntimeConfig
private let maxContext: Int
private let promptCacheMode: ServerPromptCacheMode
private let promptCacheDomain: ServerPromptCacheDomain
private var promptCache = ServerPromptCache()
public nonisolated let visionCapability: String
private let visionRuntime: VisionRuntime?
private let visionResidencyPolicy: VisionResidencyPolicy
/// What the cached KV was actually produced by. Six configuration fields
/// plus every `TURBO_FIELDFARE_*` variable in the environment: those select
/// kernels - about nineteen vision switches alone - and one of them set to
/// `0` silently picks a different attention path. Leaving them out meant a
/// prefix built by one kernel set could be reused under another, with
/// nothing recording that anything had changed.
static func runtimeIdentityString(
runtime: RuntimeConfiguration,
environment: [String: String] = ProcessInfo.processInfo.environment
) -> String {
let configuration = [
String(runtime.expertCacheSlots),
runtime.expertCachePolicy.rawValue,
runtime.rdadvisePolicy.rawValue,
runtime.prefillPolicy.rawValue,
String(runtime.prefillChunkTokens),
runtime.headPath.rawValue,
]
let switches = environment
.filter { $0.key.hasPrefix("TURBO_FIELDFARE_") }
.map { "\($0.key)=\($0.value)" }
.sorted()
return (configuration + switches).joined(separator: ":")
}
public static func load(modelDirectory: URL,
maxContext: Int,
visionPackURL: URL? = nil,
visionResidencyPolicy: VisionResidencyPolicy = .onDemand,
promptCacheMode: ServerPromptCacheMode = .singlePrefix,
runtimeConfiguration: RuntimeConfiguration) async throws -> ServerModelSession {
let tokenizerFolder = GFTokenizer.tokenizerFolder(forModelDirectory: modelDirectory)
guard let tokenizerFolder else {
throw GFTokenizerError.missingToolTemplate
}
let templateURL = tokenizerFolder.appendingPathComponent("chat_template.jinja")
guard FileManager.default.fileExists(atPath: templateURL.path) else {
throw GFTokenizerError.missingToolTemplate
}
let tokenizer = try await GFTokenizer.load(from: tokenizerFolder)
let context = try MetalContext()
let runtime = runtimeConfiguration
let model = try Model.load(
directoryURL: modelDirectory,
device: context.device,
streamingMode: .pread(slotCount: runtime.expertCacheSlots),
expertCachePolicy: runtime.modelExpertCachePolicy,
integrityPolicy: .fullSha256)
let runner = try RealForwardRunner(model: model,
context: context,
maxContext: maxContext,
runtimeConfiguration: runtime)
let scratch = try RawCompletionScratch(context: context, vocab: model.config.vocabSize)
let templateDigest = SHA256.hash(data: try Data(contentsOf: templateURL))
.map { String(format: "%02x", $0) }
.joined()
let runtimeIdentity = Self.runtimeIdentityString(runtime: runtime)
let runtimeDigest = SHA256.hash(data: Data(runtimeIdentity.utf8))
.map { String(format: "%02x", $0) }
.joined()
let promptCacheDomain = ServerPromptCacheDomain(
modelID: model.modelID,
sourceSnapshotHash: model.sourceSnapshotHash,
runtimeProfileHash: runtimeDigest,
maximumContext: maxContext,
kvStorage: PrefillKVStorageMode.fp16.rawValue,
fp16RingEnabled: runtime.fp16RingEnabled,
templateSHA256: templateDigest)
let visionRuntime: VisionRuntime?
let visionCapability: String
// An explicit pack path is an operator's statement that the pack is
// there; a typo must fail loudly at startup, not serve text with vision
// silently missing.
if let visionPackURL,
!FileManager.default.fileExists(atPath: visionPackURL.path) {
throw VisionPackError.packNotFound(visionPackURL.path)
}
// A model directory without the .gturbo suffix has no companion
// location; that only means vision is missing, never that a
// text-serving load must fail.
let resolvedVisionPackURL = visionPackURL
?? (try? VisionPackLocation.companionURL(forTextModel: modelDirectory))
if let hardwareCapability = Self.hardwareVisionCapability(
supportsVisionRuntime: VisionRuntime.isSupported(on: context.device)) {
visionRuntime = nil
visionCapability = hardwareCapability
ServerLog.visionRuntimeUnsupported()
} else if let resolvedVisionPackURL,
FileManager.default.fileExists(atPath: resolvedVisionPackURL.path) {
do {
visionRuntime = try VisionRuntime.open(
textModelURL: modelDirectory,
context: context,
visionPackURL: resolvedVisionPackURL)
visionCapability = "ready"
} catch {
visionRuntime = nil
visionCapability = Self.unavailableVisionCapability(for: error)
if visionCapability == "unsupported" {
ServerLog.visionRuntimeUnsupported(
at: resolvedVisionPackURL, error: error)
} else {
ServerLog.visionPackInvalid(at: resolvedVisionPackURL, error: error)
}
}
} else {
visionRuntime = nil
visionCapability = "missing"
}
return ServerModelSession(context: context,
model: model,
tokenizer: tokenizer,
runner: runner,
scratch: scratch,
prefillConfig: runtime.prefillConfig,
maxContext: maxContext,
promptCacheMode: promptCacheMode,
promptCacheDomain: promptCacheDomain,
visionRuntime: visionRuntime,
visionCapability: visionCapability,
visionResidencyPolicy: visionResidencyPolicy)
}
static func hardwareVisionCapability(
supportsVisionRuntime: Bool
) -> String? {
supportsVisionRuntime ? nil : "unsupported"
}
static func unavailableVisionCapability(for error: Error) -> String {
guard let runtimeError = error as? VisionRuntimeError else {
return "invalid"
}
if case .unsupportedKernel = runtimeError {
return "unsupported"
}
return "invalid"
}
private init(context: MetalContext,
model: Model,
tokenizer: GFTokenizer,
runner: RealForwardRunner,
scratch: RawCompletionScratch,
prefillConfig: PrefillRuntimeConfig,
maxContext: Int,
promptCacheMode: ServerPromptCacheMode,
promptCacheDomain: ServerPromptCacheDomain,
visionRuntime: VisionRuntime?,
visionCapability: String,
visionResidencyPolicy: VisionResidencyPolicy) {
self.context = context
self.model = model
self.tokenizer = tokenizer
self.runner = runner
self.scratch = scratch
self.prefillConfig = prefillConfig
self.maxContext = maxContext
self.promptCacheMode = promptCacheMode
self.promptCacheDomain = promptCacheDomain
self.visionRuntime = visionRuntime
self.visionResidencyPolicy = visionResidencyPolicy
self.visionCapability = visionCapability
}
public func generate(
_ request: ValidatedChatRequest,
onEvent: @escaping @Sendable (ServerInferenceEvent) -> Void
) async throws -> ServerCompletion {
let prepared = try await prepare(request)
return try await generate(prepared, onEvent: onEvent)
}
public func prepare(_ request: ValidatedChatRequest) async throws -> ServerPreparedRequest {
if !request.imageFiles.isEmpty {
let softTokens = try imageSoftTokenCounts(request)
try validateImageTokenBudget(request, softTokens: softTokens)
}
return ServerPreparedRequest(
request: request,
promptIDs: request.multimodalMessages == nil ? try renderPrompt(request) : nil)
}
private func imagePreprocessor(_ visionRuntime: VisionRuntime) -> Gemma4ImagePreprocessor {
Gemma4ImagePreprocessor(device: context.device, config: visionRuntime.config)
}
/// The encode side of a planned image, bound to this session's runtime.
private func encodeTurnImage(
_ image: ServerRequestImages.Planned, visionRuntime: VisionRuntime
) throws -> VisionFeatures {
switch ServerRequestImages.source(for: image) {
case .plan(let plan):
return try visionRuntime.encodeImage(
plan: plan,
languageModel: model,
residencyPolicy: visionResidencyPolicy,
checkCancellation: { try Task.checkCancellation() })
case .reopened(let url):
return try visionRuntime.encodeImage(
at: url,
languageModel: model,
residencyPolicy: visionResidencyPolicy,
checkCancellation: { try Task.checkCancellation() })
}
}
/// Projected token count per image from headers alone; no pixel is decoded
/// and no scan is walked.
private func imageSoftTokenCounts(
_ request: ValidatedChatRequest
) throws -> [Int] {
guard let visionRuntime else {
throw ServerRequestError.invalid(
message: "image support is unavailable",
param: "messages", code: "vision_unavailable")
}
let preprocessor = imagePreprocessor(visionRuntime)
var counts: [Int] = []
counts.reserveCapacity(request.imageFiles.count)
for url in request.imageFiles.values {
do {
let geometry = try preprocessor.admissionGeometry(fileURL: url)
counts.append(geometry.softTokenCount)
} catch {
throw ServerRequestImages.requestError(forUnreadable: error)
}
}
return counts
}
/// Rejects an image request that cannot fit the context before any pixel is
/// decoded or any tower runs.
private func validateImageTokenBudget(
_ request: ValidatedChatRequest, softTokens: [Int]
) throws {
let textTokens = try renderPrompt(request).count
guard ServerImageTokenBudget.fits(
softTokenCounts: softTokens,
textTokens: textTokens,
maxContext: maxContext) else {
throw ServerImageTokenBudget.rejection(
imageCount: request.imageFiles.count,
softTokenCounts: softTokens,
textTokens: textTokens,
maxContext: maxContext)
}
}
/// The trailing user turn rendered as a continuation, with only its own
/// images encoded. Token counts come from geometry, so the turn's shape is
/// known before any encode.
private func multimodalContinuation(
request: ValidatedChatRequest,
cachedMessageCount: Int
) async throws -> MultimodalPrefillInput {
guard let messages = request.multimodalMessages, let visionRuntime,
messages.count == cachedMessageCount + 2,
let turn = messages.last, turn.role == .user,
turn.toolCalls.isEmpty, turn.toolCallID == nil else {
throw MultimodalPromptRendererError.placeholderMismatch
}
var parts: [MultimodalContinuationPart] = []
var urls: [URL] = []
for part in turn.content {
switch part {
case .text(let value):
parts.append(.text(value))
case .image(let id):
guard let url = request.imageFiles[id] else {
throw MultimodalPromptRendererError.missingImage(id)
}
parts.append(.image)
urls.append(url)
}
}
let images = try ServerRequestImages.plans(
for: urls,
with: imagePreprocessor(visionRuntime),
checkCancellation: { try Task.checkCancellation() })
let bridge = try tokenizer.encodeMultimodalUserContinuation(
textAndImages: parts, imageTokenCounts: images.map(\.softTokenCount))
var spans: [MultimodalImageSpan] = []
for (range, image) in zip(bridge.imageTokenRanges, images) {
try Task.checkCancellation()
let features = try encodeTurnImage(image, visionRuntime: visionRuntime)
guard features.tokenCount == range.count else {
// Geometry disagreed with the encode; refuse rather than inject
// a span of the wrong length.
throw MultimodalPromptRendererError.placeholderMismatch
}
spans.append(MultimodalImageSpan(tokenRange: range, features: features))
}
return try MultimodalPrefillInput(
effectiveTokenIDs: bridge.effectiveTokenIDs,
embeddingTokenIDs: bridge.embeddingTokenIDs,
imageSpans: spans)
}
private func renderMultimodal(
_ request: ValidatedChatRequest
) async throws -> MultimodalPrefillInput {
guard let messages = request.multimodalMessages, let visionRuntime else {
throw ServerRequestError.invalid(
message: "image support is unavailable",
param: "messages", code: "vision_unavailable")
}
do {
var features: [UUID: VisionFeatures] = [:]
// Scoped, so the plans — up to `maximumOpenPlans` live ImageIO
// descriptors — are released before the render walks the whole
// history, rather than held across it.
do {
let planned: [(id: UUID, image: ServerRequestImages.Planned)]
do {
planned = try ServerRequestImages.plannedEntries(
request.imageFiles,
with: imagePreprocessor(visionRuntime),
checkCancellation: { try Task.checkCancellation() })
} catch {
throw ServerRequestImages.requestError(forUnreadable: error)
}
features.reserveCapacity(planned.count)
for entry in planned {
try Task.checkCancellation()
features[entry.id] = try encodeTurnImage(
entry.image, visionRuntime: visionRuntime)
}
}
return try MultimodalPromptRenderer.render(
messages: messages,
featuresByID: features,
tokenizer: tokenizer,
tools: request.tools)
} catch let error as MultimodalPromptRendererError {
throw Self.clientError(for: error) ?? error
}
}
/// Renderer failures a client can cause with its own message content are
/// request rejections, not server errors; the rest stay opaque and surface
/// as 500s because they mean the server's own layout went wrong.
private static func clientError(
for error: MultimodalPromptRendererError
) -> ServerRequestError? {
switch error {
case .reservedImageMarker:
.invalid(
message: "message text contains a reserved image placeholder token",
param: "messages", code: "invalid_message")
case .placeholderMismatch:
.invalid(
message: "message content could not be aligned with its images",
param: "messages", code: "invalid_message")
case .emptyMessages, .emptyContent:
.invalid(
message: "multimodal messages must not be empty",
param: "messages", code: "invalid_message")
case .missingImage, .unexpectedImage:
nil
}
}
public func generate(
_ prepared: ServerPreparedRequest,
onEvent: @escaping @Sendable (ServerInferenceEvent) -> Void
) async throws -> ServerCompletion {
let request = prepared.request
var completionStarted = false
var completed = false
defer {
// Rendering, planning and image encoding touch no KV state, so a
// request that fails or is cancelled before the completion starts
// leaves the cache exactly as the last completed request left it.
// Image encoding is seconds long, so this window is not rare.
if completionStarted, !completed {
promptCache.invalidate()
runner.reset()
}
}
let needsToolTemplate = usesToolTemplate(request)
let isMultimodal = request.multimodalMessages != nil
if isMultimodal, visionRuntime == nil {
throw ServerRequestError.invalid(
message: "image support is unavailable",
param: "messages", code: "vision_unavailable")
}
// Matched before any image is encoded. A multimodal hit continues from
// the cached KV, so the images already inside that prefix are never
// re-encoded; only a full-prefill miss needs them.
let renderedTextIDs: [Int32]? = isMultimodal
? nil : try (prepared.promptIDs ?? renderPrompt(request))
var cacheMatch: ServerPromptCacheMatch = .miss
if promptCacheMode == .singlePrefix {
cacheMatch = promptCache.match(
domain: promptCacheDomain,
request: request,
renderedPromptIDs: renderedTextIDs,
tokenizer: tokenizer)
}
let effectivePromptIDs: [Int32]
let completionStart: RawCompletionStart
var multimodalInput: MultimodalPrefillInput?
switch cacheMatch {
case .hit(let effective, let cached):
effectivePromptIDs = effective
completionStart = .resume(cachedPromptTokens: cached)
multimodalInput = nil
case .renderThenResume(let cached):
// Build the new turn as a continuation on the cached KV rather than
// re-rendering history: the KV holds tokens the model generated, and
// a fresh render re-tokenises that assistant text, so the two cannot
// be compared. Only this turn's images are encoded.
do {
let bridge = try await multimodalContinuation(
request: request,
cachedMessageCount: promptCache.inputMessageCount ?? 0)
effectivePromptIDs = (promptCache.kvBackedTokenIDs ?? [])
+ bridge.effectiveTokenIDs
multimodalInput = bridge
completionStart = .resume(cachedPromptTokens: cached)
} catch is CancellationError {
// A disconnect mid-plan or mid-encode is not a bridge failure:
// the cached prefix is untouched and stays valid for the retry.
throw CancellationError()
} catch {
ServerLog.promptCacheMiss(reason: .continuationBridgeUnavailable)
promptCache.invalidate()
let rendered = try await renderMultimodal(request)
effectivePromptIDs = rendered.effectiveTokenIDs
multimodalInput = rendered
completionStart = .reset
}
case .miss(let reason):
if promptCacheMode == .singlePrefix {
ServerLog.promptCacheMiss(reason: reason)
}
promptCache.invalidate()
if let renderedTextIDs {
effectivePromptIDs = renderedTextIDs
multimodalInput = nil
} else {
let rendered = try await renderMultimodal(request)
effectivePromptIDs = rendered.effectiveTokenIDs
multimodalInput = rendered
}
completionStart = .reset
}
// What the template renders versus what actually prefills: on a text
// cache hit the two genuinely differ, and that difference is the point
// of the rendered/effective split in the failure diagnostics. A
// multimodal resume has no full render to compare against, so the
// effective ids stand in.
let renderedPromptIDs = renderedTextIDs ?? effectivePromptIDs
guard effectivePromptIDs.count < maxContext else {
throw ServerRequestError.invalid(
message: "effective prompt exceeds the configured context",
param: "messages",
code: "context_length_exceeded")
}
var config = request.generationConfig
config.maxNewTokens = min(
request.maximumCompletionTokens,
maxContext - effectivePromptIDs.count)
config.stopStrings = []
let decoder = needsToolTemplate
? StructuredAssistantDecoder(
tokenizer: tokenizer,
allowedTools: Set(request.tools.map(\.name)))
: nil
var stopMatcher = StreamingStopMatcher(stops: request.generationConfig.stopStrings)
var content = ""
var calls: [ParsedToolCall] = []
var decodingError: Error?
var shouldStop = false
completionStarted = true
let result = try await runRawCompletion(
producer: runner,
tokenizer: tokenizer,
promptIds: effectivePromptIDs,
multimodalInput: multimodalInput,
config: config,
context: context,
scratch: scratch,
prefillConfig: prefillConfig,
start: completionStart,
shouldStop: { shouldStop }) { progress in
guard decodingError == nil else { return }
do {
func handle(_ events: [StructuredAssistantEvent]) {
for event in events {
switch event {
case .content(let text):
let visible = stopMatcher.push(text)
if !visible.isEmpty {
content += visible
onEvent(.content(visible))
}
if stopMatcher.isStopped { shouldStop = true }
case .toolCall(let call):
calls.append(call)
onEvent(.toolCall(call))
}
}
}
switch progress {
case .prefill:
break
case .token(_, let tokenID, let delta):
let events = if let decoder {
try decoder.consume(tokenID: tokenID, delta: delta)
} else {
delta.isEmpty ? [] : [StructuredAssistantEvent.content(delta)]
}
handle(events)
case .tail(let text):
// The flush tail is not tied to a token ID, so it must
// go through the decoder's channel state explicitly;
// appending it directly would leak text held back
// inside the thought channel or a tool call.
let events = if let decoder {
try decoder.consumeTail(text)
} else {
text.isEmpty ? [] : [StructuredAssistantEvent.content(text)]
}
handle(events)
}
} catch {
decodingError = error
shouldStop = true
}
}
func structuredFailure(
kind: StructuredOutputFailureKind,
cause: StructuredOutputFailureCause
) -> StructuredOutputFailure {
StructuredOutputFailure(
kind: kind,
cause: cause,
diagnostics: StructuredOutputFailureDiagnostics(
renderedPromptIDs: renderedPromptIDs ?? effectivePromptIDs,
effectivePromptIDs: effectivePromptIDs,
result: result,
maxCompletionTokens: config.maxNewTokens,
decodedCalls: calls.count,
visibleBytes: content.utf8.count,
stopStringMatched: stopMatcher.isStopped,
toolStartID: tokenizer.toolCallStartID,
toolEndID: tokenizer.toolCallEndID,
toolResponseID: tokenizer.toolResponseID,
toolResponseEndID: tokenizer.toolResponseEndID))
}
if let decodingError {
throw structuredFailure(
kind: .decoderConsume,
cause: .classify(decodingError))
}
do {
try decoder?.finish()
} catch {
throw structuredFailure(
kind: .decoderFinish,
cause: .classify(error))
}
if needsToolTemplate, result.reason == .toolCalls, calls.isEmpty {
throw structuredFailure(kind: .orphanToolResponse, cause: .none)
}
let tail = stopMatcher.finish()
if !tail.isEmpty {
content += tail
onEvent(.content(tail))
}
let reason: String
if !calls.isEmpty {
reason = "tool_calls"
} else if result.reason == .maxTokens {
reason = "length"
} else {
reason = "stop"
}
// Publishing an image turn is safe now that the entry carries per-message
// image identity: a later turn whose images differ is refused by
// `imagesDiverged` rather than resumed onto a KV built from a different
// picture.
if promptCacheMode == .singlePrefix {
let publication = promptCache.publish(
domain: promptCacheDomain,
request: request,
content: content,
calls: calls,
result: result,
stopStringFiltered: stopMatcher.isStopped)
ServerLog.promptCachePublication(publication)
}
completed = true
return ServerCompletion(
content: content,
toolCalls: calls,
finishReason: reason,
usage: OpenAIUsage(promptTokens: result.prefillTokens,
completionTokens: result.newTokens,
totalTokens: result.prefillTokens + result.newTokens,
cachedTokens: result.cachedPromptTokens),
prefillSeconds: result.prefillSeconds,
decodeSeconds: result.decodeSeconds)
}
private func renderPrompt(_ request: ValidatedChatRequest) throws -> [Int32] {
let promptIDs: [Int32]
if usesToolTemplate(request) {
promptIDs = try tokenizer.encodeToolChat(
messages: request.messages,
tools: request.tools)
} else {
let rendered = try tokenizer.applyChatTemplate(request.messages)
promptIDs = tokenizer.encode(rendered, addBOS: false)
}
guard promptIDs.count < maxContext else {
throw ServerRequestError.invalid(
message: "prompt exceeds the configured context",
param: "messages",
code: "context_length_exceeded")
}
return promptIDs
}
private func usesToolTemplate(_ request: ValidatedChatRequest) -> Bool {
!request.tools.isEmpty || request.messages.contains {
$0.role == .developer || $0.role == .tool || !$0.toolCalls.isEmpty
}
}
}