Skip to content

Gemma 4 26B-A4B inference in ~2 GB of RAM on any M-series MacBook

17.6 KB
Raw Blame
import Foundation
import NIOCore

struct ParsedChatRequestBody {
    let json: Data
    let stagedImages: [String: ServerStagedImage]
    let lease: ServerAttachmentLease?
}

final class StreamingChatRequestBody {
    static let maximumWireBytes = 96 * 1_024 * 1_024
    static let maximumSanitizedBytes = 5 * 1_024 * 1_024
    /// JSON nesting is decoded recursively downstream, on the event-loop thread
    /// whose stack is the 512 KiB pthread default, so deep nesting exhausts it
    /// and kills the process rather than failing the request. Foundation only
    /// rejects beyond 512, which is already past the crash threshold.
    static let maximumContainerDepth = 64

    private enum ContainerKind { case object, array }
    private struct Container {
        let kind: ContainerKind
        let contextKey: String?
        var currentKey: String?
    }

    private enum StringMode {
        /// An object key: copied through and collected so the key can be
        /// decoded when the string closes.
        case key
        /// Any other string: copied through untouched.
        case plain
        /// A messages[].content[].image_url.url value before its data-URL
        /// header's comma has decided between staging and passthrough.
        case urlPrefix
        /// The base64 payload of an image data URL, streamed to disk.
        case staging
    }

    private struct StringState {
        var mode: StringMode
        /// A backslash was seen in `key`/`plain` mode, where escapes are copied
        /// through raw; the flag only stops an escaped quote from terminating
        /// the string.
        var escaped = false
        var escape = JSONStringEscapeDecoder()
        var keyBytes: [UInt8] = []
        /// The exact wire bytes of a `urlPrefix` value, re-emitted verbatim if
        /// the value turns out not to be an image data URL.
        var rawPrefix: [UInt8] = []
        /// The same prefix with JSON escapes decoded, which is what the header
        /// match and the comma detection must run on.
        var decodedPrefix: [UInt8] = []
        var writer: StreamingBase64FileWriter?
        var replacementToken: String?
    }

    private var wireBytes = 0
    private var json = Data()
    private var containers: [Container] = []
    private var string: StringState?
    private var stagedImages: [String: ServerStagedImage] = [:]
    private var totalImageBytes = 0
    private var lease: ServerAttachmentLease?
    /// Where this request stages its images. Injectable so a test can own its
    /// staging root instead of sharing one with every other server in the
    /// process.
    private let attachmentRoot: URL
    /// Staging writes decoded image bytes to disk, so a server that cannot
    /// serve images must refuse them here — before any byte is staged — not
    /// after the request has queued behind a generation.
    private let visionCapability: String

    init(attachmentRoot: URL = ServerAttachmentDirectory.root,
         visionCapability: String = "ready") {
        self.attachmentRoot = attachmentRoot
        self.visionCapability = visionCapability
    }

    func feed(_ buffer: inout ByteBuffer) throws {
        guard let bytes = buffer.readBytes(length: buffer.readableBytes) else { return }
        wireBytes += bytes.count
        guard wireBytes <= Self.maximumWireBytes else {
            throw ServerRequestError.invalid(
                message: "request body is too large",
                param: nil, code: "request_too_large")
        }
        var index = 0
        while index < bytes.count {
            if let mode = string?.mode {
                switch mode {
                case .key, .plain:
                    index = try consumeCopiedString(bytes, from: index)
                case .urlPrefix:
                    index = try consumeURLPrefix(bytes, at: index)
                case .staging:
                    index = try consumeStagingPayload(bytes, from: index)
                }
            } else {
                index = try consumeStructural(bytes, at: index)
            }
        }
    }

    func finish() throws -> ParsedChatRequestBody {
        guard string == nil else {
            throw ServerRequestError.invalid(
                message: "unterminated JSON string",
                param: nil, code: "invalid_json")
        }
        guard json.count <= Self.maximumSanitizedBytes else {
            throw ServerRequestError.invalid(
                message: "non-image request content is too large",
                param: nil, code: "request_too_large")
        }
        return ParsedChatRequestBody(
            json: json,
            stagedImages: stagedImages,
            lease: lease)
    }

    /// Only a URL at exactly messages[].content[].image_url.url is intercepted.
    /// An image_url-shaped object anywhere else — a tool schema, a text field —
    /// is someone else's data and must pass through byte-identical.
    private var isMessageImageURLValue: Bool {
        containers.count == 6
            && containers[0].kind == .object && containers[0].contextKey == nil
            && containers[1].kind == .array && containers[1].contextKey == "messages"
            && containers[2].kind == .object
            && containers[3].kind == .array && containers[3].contextKey == "content"
            && containers[4].kind == .object
            && containers[5].kind == .object
            && containers[5].contextKey == "image_url"
            && containers[5].currentKey == "url"
    }

    private func consumeStructural(_ bytes: [UInt8], at index: Int) throws -> Int {
        let byte = bytes[index]
        if byte == UInt8(ascii: "\"") {
            try append(byte)
            if isMessageImageURLValue {
                string = StringState(mode: .urlPrefix)
            } else if containers.last?.kind == .object,
                      containers.last?.currentKey == nil {
                string = StringState(mode: .key)
            } else {
                string = StringState(mode: .plain)
            }
            return index + 1
        }
        try append(byte)
        switch byte {
        case UInt8(ascii: "{"), UInt8(ascii: "["):
            guard containers.count < Self.maximumContainerDepth else {
                throw ServerRequestError.invalid(
                    message: "request JSON nests deeper than "
                        + "\(Self.maximumContainerDepth) levels",
                    param: nil, code: "invalid_request_body")
            }
            containers.append(Container(
                kind: byte == UInt8(ascii: "{") ? .object : .array,
                contextKey: containers.last?.kind == .object
                    ? containers.last?.currentKey : containers.last?.contextKey,
                currentKey: nil))
        case UInt8(ascii: "}"), UInt8(ascii: "]"):
            if !containers.isEmpty { containers.removeLast() }
        case UInt8(ascii: ","):
            if !containers.isEmpty, containers[containers.count - 1].kind == .object {
                containers[containers.count - 1].currentKey = nil
            }
        default:
            break
        }
        return index + 1
    }

    /// A `key` or `plain` string: whole spans up to the next quote or backslash
    /// are copied at once, so the sanitized body never pays a per-byte state
    /// machine for its bulk.
    private func consumeCopiedString(_ bytes: [UInt8], from start: Int) throws -> Int {
        var state = string!
        var index = start
        if state.escaped {
            state.escaped = false
            if state.mode == .key { state.keyBytes.append(bytes[index]) }
            try append(bytes[index])
            string = state
            return index + 1
        }
        let end = Self.nextQuoteOrBackslash(bytes, from: index)
        if end > index {
            try append(contentsOf: bytes[index..<end])
            if state.mode == .key {
                state.keyBytes.append(contentsOf: bytes[index..<end])
            }
            index = end
        }
        guard index < bytes.count else {
            string = state
            return index
        }
        let byte = bytes[index]
        try append(byte)
        if byte == UInt8(ascii: "\\") {
            state.escaped = true
            if state.mode == .key { state.keyBytes.append(byte) }
            string = state
            return index + 1
        }
        if state.mode == .key, !containers.isEmpty,
           containers[containers.count - 1].kind == .object {
            let quoted = Data([UInt8(ascii: "\"")] + state.keyBytes + [UInt8(ascii: "\"")])
            containers[containers.count - 1].currentKey =
                try JSONDecoder().decode(String.self, from: quoted)
        }
        string = nil
        return index + 1
    }

    /// The head of an image URL, one byte at a time — it is at most 64 decoded
    /// bytes before it either becomes a data-URL payload or passes through.
    /// Escapes are decoded because a spec-valid serializer may write the same
    /// URL as `data:image\/png;...`; the raw bytes are kept alongside so a
    /// passthrough re-emits exactly what arrived.
    private func consumeURLPrefix(_ bytes: [UInt8], at index: Int) throws -> Int {
        var state = string!
        let byte = bytes[index]
        if state.escape.isActive {
            state.rawPrefix.append(byte)
            let decoded = try state.escape.consume(byte)
            try acceptPrefix(decoded: decoded, state: &state)
            return index + 1
        }
        if byte == UInt8(ascii: "\"") {
            try append(contentsOf: state.rawPrefix)
            try append(byte)
            string = nil
            return index + 1
        }
        if byte == UInt8(ascii: "\\") {
            state.rawPrefix.append(byte)
            state.escape.begin()
            string = state
            return index + 1
        }
        state.rawPrefix.append(byte)
        try acceptPrefix(decoded: [byte], state: &state)
        return index + 1
    }

    private func acceptPrefix(decoded: [UInt8], state: inout StringState) throws {
        for byte in decoded where byte == UInt8(ascii: ",") {
            let header = String(decoding: state.decodedPrefix, as: UTF8.self)
                .lowercased()
            if header.hasPrefix("data:image/") {
                guard visionCapability == "ready" else {
                    throw ServerRequestError.invalid(
                        message: visionCapability == "disabled"
                            ? "image support is disabled"
                            : "image support is unavailable",
                        param: "messages", code: "vision_unavailable")
                }
                guard ServerAttachmentStore.allowedDataURLHeaders.contains(header) else {
                    throw ServerRequestError.invalid(
                        message: "image_url must be a JPEG, PNG, HEIC, or HEIF base64 data URL",
                        param: "messages", code: "unsupported_image")
                }
                guard stagedImages.count < ServerAttachmentStore.maximumRequestImages else {
                    throw ServerRequestError.invalid(
                        message: "at most \(ServerAttachmentStore.maximumRequestImages) images "
                            + "per request",
                        param: "messages", code: "too_many_images")
                }
                let lease = try attachmentLease()
                let token = UUID().uuidString.lowercased()
                let writer = try StreamingBase64FileWriter(
                    directoryURL: lease.directoryURL)
                try append(contentsOf: Array("turbofieldfare-attachment:\(token)".utf8))
                string = StringState(mode: .staging,
                                     writer: writer,
                                     replacementToken: token)
            } else {
                try append(contentsOf: state.rawPrefix)
                string = StringState(mode: .plain)
            }
            return
        }
        state.decodedPrefix.append(contentsOf: decoded)
        if state.decodedPrefix.count > 64 {
            try append(contentsOf: state.rawPrefix)
            string = StringState(mode: .plain)
            return
        }
        string = state
    }

    /// The base64 payload: whole spans up to the next quote or backslash go to
    /// the writer at once. Escaped payload bytes are decoded and fed
    /// individually; anything that decodes outside the base64 alphabet fails in
    /// the writer exactly as it would have arrived raw.
    private func consumeStagingPayload(_ bytes: [UInt8], from start: Int) throws -> Int {
        var state = string!
        if state.escape.isActive {
            let decoded = try state.escape.consume(bytes[start])
            if !decoded.isEmpty { try state.writer!.feed(contentsOf: decoded) }
            string = state
            return start + 1
        }
        let byte = bytes[start]
        if byte == UInt8(ascii: "\"") {
            let staged = try state.writer!.finish()
            totalImageBytes += staged.encodedBytes
            guard totalImageBytes <= ServerAttachmentStore.maximumRequestImageBytes else {
                throw ServerRequestError.invalid(
                    message: "inline image bytes exceed the server limit",
                    param: "messages", code: "image_too_large")
            }
            stagedImages[state.replacementToken!] = staged
            try append(byte)
            string = nil
            return start + 1
        }
        if byte == UInt8(ascii: "\\") {
            state.escape.begin()
            string = state
            return start + 1
        }
        let end = Self.nextQuoteOrBackslash(bytes, from: start + 1)
        try state.writer!.feed(contentsOf: bytes[start..<end])
        string = state
        return end
    }

    private static func nextQuoteOrBackslash(_ bytes: [UInt8], from start: Int) -> Int {
        var index = start
        while index < bytes.count {
            let byte = bytes[index]
            if byte == UInt8(ascii: "\"") || byte == UInt8(ascii: "\\") { return index }
            index += 1
        }
        return index
    }

    private func attachmentLease() throws -> ServerAttachmentLease {
        if let lease { return lease }
        let created = try ServerAttachmentDirectory.makeLease(in: attachmentRoot)
        lease = created
        return created
    }

    private func append(_ byte: UInt8) throws {
        json.append(byte)
        try checkSanitizedSize()
    }

    private func append(contentsOf bytes: some Sequence<UInt8>) throws {
        json.append(contentsOf: bytes)
        try checkSanitizedSize()
    }

    private func checkSanitizedSize() throws {
        guard json.count <= Self.maximumSanitizedBytes else {
            throw ServerRequestError.invalid(
                message: "non-image request content is too large",
                param: nil, code: "request_too_large")
        }
    }
}

/// Decodes JSON string escape sequences inside an intercepted image URL, where
/// the sanitized body keeps a replacement token and the raw bytes are gone —
/// the URL has to be matched and staged from what the escapes mean, not how
/// they were written.
private struct JSONStringEscapeDecoder {
    private enum State {
        case idle
        case started
        case unicode([UInt8])
    }

    private var state = State.idle

    var isActive: Bool {
        if case .idle = state { return false }
        return true
    }

    /// Enters the sequence; the caller has just consumed the backslash.
    mutating func begin() {
        state = .started
    }

    /// Feeds the next raw byte of the sequence. Returns the decoded bytes once
    /// the sequence completes, and an empty array while it is still open.
    mutating func consume(_ byte: UInt8) throws -> [UInt8] {
        switch state {
        case .idle:
            preconditionFailure("consume called outside an escape sequence")
        case .started:
            switch byte {
            case UInt8(ascii: "\""), UInt8(ascii: "\\"), UInt8(ascii: "/"):
                state = .idle
                return [byte]
            case UInt8(ascii: "b"): state = .idle; return [8]
            case UInt8(ascii: "f"): state = .idle; return [12]
            case UInt8(ascii: "n"): state = .idle; return [10]
            case UInt8(ascii: "r"): state = .idle; return [13]
            case UInt8(ascii: "t"): state = .idle; return [9]
            case UInt8(ascii: "u"):
                state = .unicode([])
                return []
            default:
                throw Self.invalidEscape
            }
        case .unicode(var digits):
            guard Self.hexValue(byte) != nil else { throw Self.invalidEscape }
            digits.append(byte)
            guard digits.count == 4 else {
                state = .unicode(digits)
                return []
            }
            state = .idle
            let value = digits.reduce(UInt32(0)) { $0 << 4 | UInt32(Self.hexValue($1)!) }
            // Surrogate pairs only encode characters no data URL may contain,
            // so they are refused rather than paired across two sequences.
            guard let scalar = Unicode.Scalar(value) else { throw Self.invalidEscape }
            return Array(String(scalar).utf8)
        }
    }

    private static let invalidEscape = ServerRequestError.invalid(
        message: "image_url contains an unsupported escape sequence",
        param: "messages", code: "invalid_image_url")

    private static func hexValue(_ byte: UInt8) -> UInt8? {
        switch byte {
        case UInt8(ascii: "0")...UInt8(ascii: "9"): byte - UInt8(ascii: "0")
        case UInt8(ascii: "a")...UInt8(ascii: "f"): byte - UInt8(ascii: "a") + 10
        case UInt8(ascii: "A")...UInt8(ascii: "F"): byte - UInt8(ascii: "A") + 10
        default: nil
        }
    }
}