Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook
Swift apple-silicongemmagemma4gemma4-26b-a4bgpgpullmllm-inferencelocal-aimacosmetalon-device-aion-device-llmswift
import Foundation
import Metal
import Testing
@testable import TurboFieldfare
@testable import TurboFieldfareFormat
extension ModelLoaderTests {
@Test func loadsValidDirectory() throws {
let dir = try Self.writeToySynthetic()
defer { try? FileManager.default.removeItem(at: dir) }
let device = try #require(MTLCreateSystemDefaultDevice())
let model = try Model.load(
directoryURL: dir, device: device,
expecting: .gemma4Toy())
let embed = model.embedding
#expect(embed.length == UInt64(1024 * 64 / 2))
#expect(embed.shape.0 == 1024 && embed.shape.1 == 64)
let norm = model.finalNorm
#expect(norm.length == UInt64(64 * 2))
// Tied lm_head returns the same view as embedding.
#expect(model.lmHead.offset == model.embedding.offset)
#expect(model.lmHead.length == model.embedding.length)
}
@Test func loadsLayoutDeclaredNonstandardLayerFilenames() throws {
let dir = try Self.writeToySynthetic()
defer { try? FileManager.default.removeItem(at: dir) }
let packed = dir.appendingPathComponent("packed_experts")
let layoutURL = packed.appendingPathComponent("layout.json")
var layout = try JSONSerialization.jsonObject(
with: Data(contentsOf: layoutURL)) as! [String: Any]
var layers = layout["layers"] as! [[String: Any]]
let manifestURL = dir.appendingPathComponent("manifest.json")
var manifest = try JSONSerialization.jsonObject(
with: Data(contentsOf: manifestURL)) as! [String: Any]
var files = manifest["files"] as! [String: [String: Any]]
for index in layers.indices {
let oldName = layers[index]["file"] as! String
let newName = "experts_\(index).bin"
try FileManager.default.moveItem(
at: packed.appendingPathComponent(oldName),
to: packed.appendingPathComponent(newName))
layers[index]["file"] = newName
files["packed_experts/\(newName)"] = files.removeValue(
forKey: "packed_experts/\(oldName)")
}
layout["layers"] = layers
let layoutData = try JSONSerialization.data(
withJSONObject: layout, options: [.sortedKeys, .withoutEscapingSlashes])
try layoutData.write(to: layoutURL)
files["packed_experts/layout.json"]?["size"] = layoutData.count
files["packed_experts/layout.json"]?["sha256"] =
try Sha256Verifier.hashFile(at: layoutURL)
manifest["files"] = files
try JSONSerialization.data(
withJSONObject: manifest, options: [.sortedKeys, .withoutEscapingSlashes])
.write(to: manifestURL)
let device = try #require(MTLCreateSystemDefaultDevice())
let model = try Model.load(
directoryURL: dir, device: device, expecting: .gemma4Toy())
let expert = try model.routedExpert(layer: 1, expert: 2)
let bytes = expert.buffer.contents().advanced(by: Int(expert.offset))
.assumingMemoryBound(to: UInt8.self)
#expect(bytes[0] == 1)
#expect(bytes[1] == 2)
}
@Test func residentBytesAreReadableFromBuffer() throws {
let dir = try Self.writeToySynthetic()
defer { try? FileManager.default.removeItem(at: dir) }
let device = try #require(MTLCreateSystemDefaultDevice())
let model = try Model.load(
directoryURL: dir, device: device,
expecting: .gemma4Toy())
let norm = model.finalNorm
let contents = norm.buffer.contents()
// Norm region was patterned 0xC0 | (i & 0x3F).
for i in 0..<Int(norm.length) {
let got = contents.load(fromByteOffset: Int(norm.offset) + i, as: UInt8.self)
#expect(got == UInt8(0xC0 | (i & 0x3F)), "norm byte \(i)")
}
}
@Test func missingManifestFailsPartialInstall() throws {
let dir = try Self.writeToySynthetic()
defer { try? FileManager.default.removeItem(at: dir) }
try FileManager.default.removeItem(at: dir.appendingPathComponent("manifest.json"))
let device = try #require(MTLCreateSystemDefaultDevice())
#expect {
_ = try Model.load(
directoryURL: dir, device: device,
expecting: .gemma4Toy())
} throws: { error in
if case ModelError.partialInstall = error { return true }
return false
}
}
@Test func mismatchedShaFailsChecksumMismatch() throws {
let dir = try Self.writeToySynthetic()
defer { try? FileManager.default.removeItem(at: dir) }
// Flip one byte at the very end of the resident region (not inside
// the index, which the loader reads earlier and would error
// differently). Manifest sha was computed before this corruption.
let url = dir.appendingPathComponent("model_weights.bin")
var data = try Data(contentsOf: url)
data[data.count - 1] ^= 0xFF
try data.write(to: url)
let device = try #require(MTLCreateSystemDefaultDevice())
#expect {
_ = try Model.load(
directoryURL: dir, device: device,
expecting: .gemma4Toy())
} throws: { error in
if case ModelError.checksumMismatch = error { return true }
return false
}
}
@Test func malformedZeroPayloadResidentEntryThrowsWithoutTrap() throws {
let dir = try Self.writeToySynthetic()
defer { try? FileManager.default.removeItem(at: dir) }
let weightsURL = dir.appendingPathComponent("model_weights.bin")
var weights = try Data(contentsOf: weightsURL)
let firstEntry = GTurboFormatV1.residentHeaderBytes
weights.replaceSubrange((firstEntry + 8)..<(firstEntry + 24),
with: repeatElement(UInt8(0), count: 16))
try weights.write(to: weightsURL)
let manifestURL = dir.appendingPathComponent("manifest.json")
var manifest = try JSONSerialization.jsonObject(
with: Data(contentsOf: manifestURL)) as! [String: Any]
var files = manifest["files"] as! [String: [String: Any]]
files["model_weights.bin"]?["sha256"] = try Sha256Verifier.hashFile(at: weightsURL)
manifest["files"] = files
try JSONSerialization.data(withJSONObject: manifest, options: [.sortedKeys])
.write(to: manifestURL)
let device = try #require(MTLCreateSystemDefaultDevice())
#expect {
_ = try Model.load(directoryURL: dir, device: device, expecting: .gemma4Toy())
} throws: { error in
if case ModelError.indexCorrupt(let detail) = error {
return detail.contains("primary payload is empty")
}
return false
}
}
@Test func integrityPoliciesExposeIdenticalResidentAndRoutedBytes() throws {
let dir = try Self.writeToySynthetic()
defer { try? FileManager.default.removeItem(at: dir) }
try Self.writeVerifiedInstallReceipt(directoryURL: dir)
let device = try #require(MTLCreateSystemDefaultDevice())
let full = try Model.load(
directoryURL: dir,
device: device,
expecting: .gemma4Toy(),
integrityPolicy: .fullSha256)
let trusted = try Model.load(
directoryURL: dir,
device: device,
expecting: .gemma4Toy(),
integrityPolicy: .sizeCheckTrustedReceipt)
#expect(full.embedding.length == trusted.embedding.length)
let fullEmbedding = full.embedding.buffer.contents().advanced(by: Int(full.embedding.offset))
let trustedEmbedding = trusted.embedding.buffer.contents().advanced(
by: Int(trusted.embedding.offset))
#expect(memcmp(fullEmbedding, trustedEmbedding, Int(full.embedding.length)) == 0)
let fullExpert = try full.routedExpert(layer: 0, expert: 0)
let trustedExpert = try trusted.routedExpert(layer: 0, expert: 0)
#expect(fullExpert.length == trustedExpert.length)
let fullExpertBytes = fullExpert.buffer.contents().advanced(by: Int(fullExpert.offset))
let trustedExpertBytes = trustedExpert.buffer.contents().advanced(by: Int(trustedExpert.offset))
#expect(memcmp(fullExpertBytes, trustedExpertBytes, Int(fullExpert.length)) == 0)
}
@Test func nonPageAlignedExpertStrideFailsAtManifest() throws {
let dir = try Self.writeToySynthetic()
defer { try? FileManager.default.removeItem(at: dir) }
let manifestURL = dir.appendingPathComponent("manifest.json")
var root =
try JSONSerialization.jsonObject(
with: Data(contentsOf: manifestURL)) as! [String: Any]
root["expertStride"] = 1024
let data = try JSONSerialization.data(withJSONObject: root)
try data.write(to: manifestURL)
let device = try #require(MTLCreateSystemDefaultDevice())
#expect {
_ = try Model.load(
directoryURL: dir, device: device,
expecting: .gemma4Toy())
} throws: { error in
if case ModelError.expertStrideNotPageAligned = error { return true }
return false
}
}
}