Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook
Swift apple-silicongemmagemma4gemma4-26b-a4bgpgpullmllm-inferencelocal-aimacosmetalon-device-aion-device-llmswift
import Darwin
import Foundation
import TurboFieldfareDecodeProtocol
/// Why a wait for a decode-service event ended without one.
public enum DecodeServiceTransportError: Error, CustomStringConvertible {
/// The service stopped answering while still connected: no frame of any
/// kind arrived within the deadline. A closed socket surfaces as the
/// reader's own error instead.
case timedOut(seconds: Double)
/// The connection is already known to be dead; a new one must be built.
case connectionLost(underlying: String)
public var description: String {
switch self {
case .timedOut(let seconds):
"the decode service stopped responding after \(Int(seconds))s"
case .connectionLost(let underlying):
"the decode service connection is gone: \(underlying)"
}
}
}
/// Demultiplexes the service's event stream onto the callers waiting for it.
///
/// Every wait is bounded and cancellable. A blocking wait meant that a service
/// which stayed alive but stopped answering — its writer thread died, or a
/// command buffer wedged — left the app in `.running` forever with Cancel
/// inert, because nothing downstream could observe that no event was coming.
final class DecodeServiceResponseRouter: @unchecked Sendable {
private struct Waiter {
let requestID: UUID
let continuation: CheckedContinuation<DecodeServiceEvent, Error>
}
private let lock = NSLock()
private var pending: [UUID: [DecodeServiceEvent]] = [:]
private var waiters: [UUID: Waiter] = [:]
private var terminalError: Error?
/// Requests whose waiter gave up. Their later events are dropped rather
/// than buffered: a request that was cancelled or timed out is never waited
/// on again, so buffering its remaining events grew the map for the life of
/// the connection.
private var abandoned: [UUID] = []
private static let abandonedMemory = 32
/// Called once, off the reader thread, when the stream ends for any reason.
/// The client uses it to tear the connection down rather than keep handing
/// out handles that can only fail.
private let onTerminate: @Sendable (DecodeServiceResponseRouter, Error) -> Void
private let output: FileHandle
init(
output: FileHandle,
onTerminate: @escaping @Sendable (DecodeServiceResponseRouter, Error) -> Void
= { _, _ in }
) {
self.onTerminate = onTerminate
self.output = output
let reader = Thread { [weak self] in
self?.readFrames(from: output)
}
reader.name = "TurboFieldfare.DecodeService.ResponseRouter"
reader.qualityOfService = .userInitiated
reader.start()
}
/// Closes the stream this router reads.
///
/// The reader owns a `dup` of the socket, so closing the client's write
/// handle does not wake it: against a service that is alive but no longer
/// answering commands — the case the deadlines exist for — the thread would
/// otherwise stay blocked in `read` for the life of the process, holding a
/// descriptor nothing will ever close.
func closeStream() {
// `shutdown` first. Closing a descriptor another thread is already
// blocked reading is unspecified by POSIX — measured here it does wake
// the reader, but this is called precisely when a thread is parked in
// `read`, and shutting the socket down removes the reliance. On a pipe
// it fails with ENOTSOCK and costs nothing.
shutdown(output.fileDescriptor, SHUT_RDWR)
try? output.close()
}
/// True once the stream has ended. The connection cannot recover from this;
/// only a new one can.
var isTerminated: Bool {
lock.lock()
defer { lock.unlock() }
return terminalError != nil
}
func next(matching requestID: UUID, timeout: Duration = .seconds(300)) async throws
-> DecodeServiceEvent {
let waiterID = UUID()
let timeoutSeconds = Double(timeout.components.seconds)
+ Double(timeout.components.attoseconds) / 1e18
// Scheduled by libdispatch rather than the cooperative pool: this
// deadline exists precisely for the moments when the machine is
// saturated, which is when a cooperative timer is starved worst.
let deadline = DispatchSource.makeTimerSource(
queue: DispatchQueue.global(qos: .utility))
deadline.schedule(deadline: .now() + timeoutSeconds)
deadline.setEventHandler { [self] in
fail(waiterID, with: DecodeServiceTransportError.timedOut(
seconds: timeoutSeconds))
}
deadline.resume()
defer { deadline.cancel() }
return try await withTaskCancellationHandler {
try await withCheckedThrowingContinuation { continuation in
register(Waiter(requestID: requestID, continuation: continuation),
id: waiterID)
}
} onCancel: {
self.fail(waiterID, with: CancellationError())
}
}
private func register(_ waiter: Waiter, id: UUID) {
lock.lock()
// Cancellation before the wait even starts. `withTaskCancellationHandler`
// runs its handler immediately when the task is already cancelled, which
// is before this call, so `fail` had nothing to find and the waiter
// registered here would have sat until its deadline — 900 s on a load,
// which is the hang this router exists to end. The same applies to a
// cancel landing between the handler being installed and this lock.
if Task.isCancelled {
pending.removeValue(forKey: waiter.requestID)
abandoned.append(waiter.requestID)
if abandoned.count > Self.abandonedMemory { abandoned.removeFirst() }
lock.unlock()
waiter.continuation.resume(throwing: CancellationError())
return
}
// A frame that arrived before this call, or a stream that has already
// ended, resolves the wait without it ever being stored.
if var events = pending[waiter.requestID], !events.isEmpty {
let event = events.removeFirst()
if events.isEmpty {
pending.removeValue(forKey: waiter.requestID)
} else {
pending[waiter.requestID] = events
}
lock.unlock()
waiter.continuation.resume(returning: event)
return
}
if let terminalError {
lock.unlock()
waiter.continuation.resume(throwing: terminalError)
return
}
waiters[id] = waiter
lock.unlock()
}
/// Resolves a waiter that timed out or was cancelled. A waiter already
/// resumed by an arriving frame is simply absent.
private func fail(_ id: UUID, with error: Error) {
lock.lock()
let waiter = waiters.removeValue(forKey: id)
if let waiter {
pending.removeValue(forKey: waiter.requestID)
abandoned.append(waiter.requestID)
if abandoned.count > Self.abandonedMemory { abandoned.removeFirst() }
}
lock.unlock()
waiter?.continuation.resume(throwing: error)
}
private func deliver(_ event: DecodeServiceEvent) {
lock.lock()
if let entry = waiters.first(where: { $0.value.requestID == event.generationID }) {
waiters.removeValue(forKey: entry.key)
lock.unlock()
entry.value.continuation.resume(returning: event)
return
}
guard !abandoned.contains(event.generationID) else {
lock.unlock()
return
}
pending[event.generationID, default: []].append(event)
lock.unlock()
}
private func terminate(with error: Error) {
lock.lock()
guard terminalError == nil else {
lock.unlock()
return
}
terminalError = error
// Events that arrived before the stream ended are still answers; a
// service that replies and then exits is ordinary. Only waiters with
// nothing buffered learn the connection is gone.
var delivered: [(Waiter, DecodeServiceEvent)] = []
var stranded: [Waiter] = []
for waiter in waiters.values {
if var events = pending[waiter.requestID], !events.isEmpty {
let event = events.removeFirst()
if events.isEmpty {
pending.removeValue(forKey: waiter.requestID)
} else {
pending[waiter.requestID] = events
}
delivered.append((waiter, event))
} else {
stranded.append(waiter)
}
}
waiters.removeAll()
lock.unlock()
for (waiter, event) in delivered {
waiter.continuation.resume(returning: event)
}
for waiter in stranded {
waiter.continuation.resume(throwing: error)
}
onTerminate(self, error)
}
private func readFrames(from output: FileHandle) {
do {
while true {
deliver(try DecodeFrameCodec.read(
DecodeServiceEvent.self, from: output))
}
} catch {
terminate(with: error)
}
}
}