Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook
Swift apple-silicongemmagemma4gemma4-26b-a4bgpgpullmllm-inferencelocal-aimacosmetalon-device-aion-device-llmswift
import Foundation
import Tokenizers
/// The pinned Gemma 4 detokenization pipeline, without HF's cleanup pass.
///
/// `tokenizer.json` declares `decoder: Sequence[Replace("▁" -> " "),
/// ByteFallback, Fuse]`. swift-transformers runs that sequence and then applies
/// `clean_up_tokenization_spaces`, which defaults to **true** when the key is
/// absent — and Gemma's `tokenizer_config.json` omits it.
///
/// That cleanup pass is a legacy heuristic for whitespace tokenizers and is
/// wrong for a metaspace + byte-fallback BPE, which is lossless by
/// construction. It rewrites text the model deliberately produced:
///
/// ```text
/// "he said ' ok ' now" -> "he said'ok'now"
/// "step 1 . done" -> "step 1. done"
/// " . indented" -> " . indented"
/// ```
///
/// So `decode(encode(x)) != x`. It also breaks streaming: because the pass runs
/// over the whole string on every call, appending a token can rewrite text that
/// was already emitted, which no append-only stream can represent.
///
/// This type reproduces the declared decoder sequence and stops there. Both the
/// batch (`GFTokenizer.decode`) and streaming (`GFDetokenizer`) paths go through
/// it, so they agree by construction.
enum GemmaDecoding {
/// `<0xXX>` byte-fallback token, e.g. `<0xE2>`.
static func isByteFallback(_ token: String) -> Bool {
token.count == 6
&& token.hasPrefix("<0x")
&& token.hasSuffix(">")
&& token.dropFirst(3).dropLast().allSatisfy { $0.isHexDigit }
}
static func byteValue(_ token: String) -> UInt8? {
guard isByteFallback(token) else { return nil }
return UInt8(token.dropFirst(3).dropLast(), radix: 16)
}
/// One token's contribution to the output: the `Replace` decoder's
/// `"▁" -> " "` substitution. Byte-fallback tokens are handled by
/// `ByteFallbackRun` instead — they only decode as a complete run.
static func fragment(_ token: String) -> String {
// Runs once per generated token; most tokens contain no "▁", and the
// guard returns them without Foundation bridging or an allocation.
guard token.unicodeScalars.contains(sentencePieceUnderline) else { return token }
var scalars = String.UnicodeScalarView()
for scalar in token.unicodeScalars {
scalars.append(scalar == sentencePieceUnderline ? " " : scalar)
}
return String(scalars)
}
private static let sentencePieceUnderline: Unicode.Scalar = "\u{2581}"
}
/// A run of `<0xXX>` byte-fallback tokens, decoded with the reference HF
/// `tokenizers` ByteFallback semantics: the run commits as a whole — valid
/// UTF-8 becomes its text, anything else (including an incomplete trailing
/// sequence) becomes one U+FFFD per byte of the run. The swift-transformers
/// port differs here (it decodes invalid runs leniently and drops a trailing
/// run outright — issue #58); we follow the Rust reference.
///
/// Streaming: while the run is valid so far, nothing can be emitted, because
/// one more byte can invalidate the whole run retroactively. The moment the
/// run becomes invalid its fate is sealed — every byte, past and future,
/// decodes to exactly one U+FFFD — so an invalid run streams replacement
/// characters live rather than freezing the stream, and buffers nothing.
struct ByteFallbackRun {
private var bytes: [UInt8] = []
private var poisoned = false
/// Continuation bytes still owed by the current lead, and the allowed
/// range for the next one (RFC 3629 constrains the first continuation
/// after E0/ED/F0/F4 leads).
private var pendingContinuations = 0
private var nextContinuation: ClosedRange<UInt8> = 0x80...0xBF
/// Text this byte contributes to the stream: `""` while the run is valid
/// so far, replacement characters once it can no longer become valid.
mutating func push(_ byte: UInt8) -> String {
if poisoned { return "\u{FFFD}" }
bytes.append(byte)
if accept(byte) { return "" }
poisoned = true
defer { bytes.removeAll(keepingCapacity: true) }
return String(repeating: "\u{FFFD}", count: bytes.count)
}
/// Close the run: a non-byte token follows, or the stream ends.
mutating func commit() -> String {
defer {
bytes.removeAll(keepingCapacity: true)
poisoned = false
pendingContinuations = 0
nextContinuation = 0x80...0xBF
}
guard !poisoned, !bytes.isEmpty else { return "" }
return String(bytes: bytes, encoding: .utf8)
?? String(repeating: "\u{FFFD}", count: bytes.count)
}
/// Advance the incremental UTF-8 validator. `false` means no suffix can
/// ever make the run valid again.
private mutating func accept(_ byte: UInt8) -> Bool {
if pendingContinuations > 0 {
guard nextContinuation.contains(byte) else { return false }
pendingContinuations -= 1
nextContinuation = 0x80...0xBF
return true
}
switch byte {
case 0x00...0x7F: break
case 0xC2...0xDF: pendingContinuations = 1
case 0xE0: pendingContinuations = 2; nextContinuation = 0xA0...0xBF
case 0xE1...0xEC, 0xEE, 0xEF: pendingContinuations = 2
case 0xED: pendingContinuations = 2; nextContinuation = 0x80...0x9F
case 0xF0: pendingContinuations = 3; nextContinuation = 0x90...0xBF
case 0xF1...0xF3: pendingContinuations = 3
case 0xF4: pendingContinuations = 3; nextContinuation = 0x80...0x8F
default: return false // stray continuation, overlong lead, or > U+10FFFF
}
return true
}
}