Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

360 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
"""MLX inference backend for Apple Silicon; drop-in InferenceBackend replacement using mlx-lm/mlx-vlm."""

import copy
import difflib
import functools
import hashlib
import importlib
import inspect
import math
import os
import re
import shutil
import sys
import threading
import time
from collections import Counter, OrderedDict
from contextlib import ExitStack, closing, contextmanager, nullcontext
from typing import Optional, Generator
from core.inference.worker import RowRefused
from core.inference import context_refusal
from core.inference.incremental_decode import IncrementalDecode
from core.inference.native_tool_tokens import (
    NativeToolTokenDecoder,
    closes_an_open_envelope,
    decoder_preserves_token,
    reasoning_control_tokens,
    stop_token_text,
)
from core.inference.runtime_context import (
    MAX_REQUESTABLE_CONTEXT,
    UNSET_GENERATION_BUDGET,
    generation_budget_for_window,
    runtime_context_length,
)
from core.inference.chat_template_helpers import (
    # Aliased to this module's historic names; bodies moved to the shared helper (#10092).
    alternating_turns,
    count_structured_images as _count_vlm_images,
    count_structured_videos as _count_vlm_videos,
    detect_reasoning_channel_markers,
    make_reasoning_normalizer,
    messages_have_tool_history as _vlm_messages_have_tool_history,
    messages_with_attached_image,
    neutralize_for_render,
    normalize_reasoning_snapshots,
    prompt_opens_reasoning_channel,
    strip_open_reasoning_prefill,
    ThoughtUnresumableError,
    trailing_assistant_resume_kind,
    trailing_assistant_text,
    vlm_prompt_issue as _vlm_prompt_issue,
)
from core.inference.mcp_images import (
    image_marker_parts,
    pixels_in_marker_order,
    top_up_image_markers,
    trim_image_turns,
)
from utils.models.model_config import (
    is_audio_input_type,
    load_mlx_adapter_processor,
    load_mlx_adapter_tokenizer,
)
from utils.utils import is_metal_queue_dead
from loggers import get_logger

logger = get_logger(__name__)


# mlx-vlm prefix reuse: a forward is not shape-invariant, so reuse must prefill on mlx-vlm's grid

VLM_PREFILL_STEP = 2048
VLM_PROMPT_CACHE_ENTRIES = 6


def _prefill_layer_step(model, step):
    if not step:
        return None
    try:
        from unsloth_zoo.mlx.offload import wide_prefill_step
        wide = wide_prefill_step(model)
    except Exception:
        return None
    own = vlm_prefill_step()
    return own if wide == own and step % own == 0 else None


def vlm_prefill_step():
    """mlx-vlm's own step, so an unreused request prefills as mlx-vlm does. It divides every
    boundary, so a step no grid can be built on falls back here rather than reaching a request."""
    try:
        from mlx_vlm.generate.common import DEFAULT_PREFILL_STEP_SIZE
        step = int(DEFAULT_PREFILL_STEP_SIZE)
    except Exception:
        return VLM_PREFILL_STEP
    return step if step > 0 else VLM_PREFILL_STEP


def shape_stable_prefix(
    token_count,
    origin = 0,
    step = VLM_PREFILL_STEP,
):
    """Rows whole chunks produced from ``origin``; mlx-vlm holds the last token back."""
    rows = token_count - 1
    if rows < origin:
        return 0
    return origin + (rows - origin) // step * step


def _is_cache(value):
    """Every mlx-vlm cache class carries ``state``; read off the type so no property runs."""
    return hasattr(value, "__dict__") and hasattr(type(value), "state")


def _is_opaque_cache(value):
    """Attributes in __slots__ are invisible to vars(), so the copy must stop, not return the original."""
    return not hasattr(value, "__dict__") and hasattr(type(value), "state")


def _rebuilt_tuple(original, items):
    """Rebuild as the tuple's own type: TurboQuant dispatches on its state class."""
    rebuild = getattr(type(original), "_make", None)
    return rebuild(items) if rebuild is not None else tuple(items)


def _copy_value(value, mx):
    if isinstance(value, mx.array):
        # Safe to share: MLX writes in place only into an unreferenced buffer; slices are copied out.
        return mx.contiguous(value)
    if isinstance(value, list):
        return [_copy_value(item, mx) for item in value]
    if isinstance(value, tuple):
        return _rebuilt_tuple(value, (_copy_value(item, mx) for item in value))
    if isinstance(value, dict):
        return {key: _copy_value(item, mx) for key, item in value.items()}
    if _is_cache(value):
        duplicate = copy.copy(value)
        for name, item in list(vars(duplicate).items()):
            setattr(duplicate, name, _copy_value(item, mx))
        return duplicate
    if _is_opaque_cache(value):
        raise TypeError(f"{type(value).__name__} cannot be copied")
    return value


def _arrays(value, mx):
    if isinstance(value, mx.array):
        yield value
    elif isinstance(value, (list, tuple)):
        for item in value:
            yield from _arrays(item, mx)
    elif isinstance(value, dict):
        for item in value.values():
            yield from _arrays(item, mx)
    elif _is_cache(value):
        for item in vars(value).values():
            yield from _arrays(item, mx)


def _release_value(value, mx):
    if isinstance(value, mx.array):
        return None
    if isinstance(value, list):
        value[:] = [_release_value(item, mx) for item in value]
    elif isinstance(value, tuple):
        return _rebuilt_tuple(value, (_release_value(item, mx) for item in value))
    elif isinstance(value, dict):
        for key, item in list(value.items()):
            value[key] = _release_value(item, mx)
    elif _is_cache(value):
        for name, item in list(vars(value).items()):
            setattr(value, name, _release_value(item, mx))
    return value


def release_cache_entries(entries):
    import mlx.core as mx
    entries[:] = [_release_value(entry, mx) for entry in entries]


def copy_cache_entries(entries):
    import mlx.core as mx

    for entry in entries:
        if not (_is_cache(entry) or isinstance(entry, (list, tuple))):
            raise TypeError(f"{type(entry).__name__} cannot be copied")
    copies = [_copy_value(entry, mx) for entry in entries]
    # Not mx.eval: a wait between prefill chunks idles the GPU.
    mx.async_eval(list(_arrays(copies, mx)))
    return copies


def cache_entries_nbytes(entries):
    import mlx.core as mx
    return sum(array.nbytes for array in _arrays(entries, mx))


def cache_entries_offset(entries):
    """Tokens the entries hold, else None: only ``offset`` counts prompt rows."""
    for entry in entries:
        offset = getattr(entry, "offset", None)
        if isinstance(offset, int):
            return offset
        nested = vars(entry).values() if _is_cache(entry) else entry
        if isinstance(nested, (list, tuple)) or _is_cache(entry):
            found = cache_entries_offset(
                [item for item in nested if _is_cache(item) or isinstance(item, (list, tuple))]
            )
            if found is not None:
                return found
    return None


def media_prefix_end(token_ids, media_token_ids):
    """Rows through the last media placeholder, 0 if none: what mlx-vlm resumes past."""
    media = {int(token) for token in media_token_ids}
    if not media:
        return 0
    for index in range(len(token_ids) - 1, -1, -1):
        if int(token_ids[index]) in media:
            return index + 1
    return 0


def _chunk_rows(args, kwargs):
    for chunk in (kwargs.get("inputs_embeds"), kwargs.get("inputs"), args[0] if args else None):
        shape = getattr(chunk, "shape", None)
        if shape and len(shape) > 1:
            return shape[1]
    return 0


def _place_per_layer_inputs(kwargs, rows, start):
    """Slice ``per_layer_inputs`` to this forward's own rows: mlx-vlm computes it for the whole
    uncached prompt and the model slices it at an offset a resume shifts. A single-token forward
    gets none and computes its own, as the reference model does."""
    inputs = kwargs.get("per_layer_inputs")
    shape = getattr(inputs, "shape", None)
    if not shape:
        return
    if rows == 1:
        kwargs.pop("per_layer_inputs")
    elif rows > 1 and shape[1] != rows and 0 <= start and start + rows <= shape[1]:
        kwargs["per_layer_inputs"] = inputs[:, start : start + rows]


class _ForwardRecord:
    __slots__ = (
        "forwards",
        "capture_at",
        "capture_rows",
        "snapshot",
        "tail_at",
        "tail",
        "resume_offset",
        "on_resume",
    )

    def __init__(self):
        self.forwards = 0
        self.capture_at = 0
        self.capture_rows = 0
        self.snapshot = None
        self.tail_at = 0
        self.tail = None
        self.resume_offset = None
        self.on_resume = None


_RECORDING_CLASSES = {}


def _prompt_wide_position_ids(args, kwargs):
    """Whether ``position_ids`` spans more than the chunk being fed: mlx-vlm primes mRoPE
    prompt-wide and not every family slices the kwarg (glm4v feeds it through as is), so a
    prompt-wide one is withheld for the model's own primed state."""
    shape = getattr(kwargs.get("position_ids"), "shape", None)
    if not shape:
        return False
    chunk = kwargs.get("inputs", args[0] if args else None)
    chunk_shape = getattr(chunk, "shape", None) or getattr(
        kwargs.get("inputs_embeds"), "shape", None
    )
    return bool(chunk_shape) and len(chunk_shape) > 1 and shape[-1] != chunk_shape[1]


def _forward_capturing(base, model, args, kwargs, cache, rows):
    from unsloth_zoo.mlx.offload import capture_prefill_rows

    def copy(entry):
        try:
            return copy_cache_entries([entry])[0]
        except Exception as exc:
            logger.info("MLX VLM prompt cache: cache layout not copied (%s)", exc)

    with capture_prefill_rows(rows, copy) as captured:
        output = base.__call__(model, *args, **kwargs)
    snapshot = [captured.get(id(entry)) for entry in cache]
    return output, snapshot if all(entry is not None for entry in snapshot) else None


def _recording_class(base):
    """``base`` with a forward that counts itself and copies the cache on cue."""
    cls = _RECORDING_CLASSES.get(base)
    if cls is not None:
        return cls

    def __call__(self, *args, **kwargs):
        # A plain attribute: nn.Module stores dicts and lists in its module tree.
        record = self.__dict__.get("_studio_forward_record")
        cache = kwargs.get("cache")
        if record is not None:
            offset = cache_entries_offset(cache) if cache is not None else None
            if record.resume_offset is None:
                record.resume_offset = offset or 0
                if record.on_resume is not None:
                    record.on_resume(record.resume_offset)
            # Only after a resume: a Qwen VL model fed no positions reuses the previous request's.
            if record.resume_offset and _prompt_wide_position_ids(args, kwargs):
                kwargs.pop("position_ids")
            _place_per_layer_inputs(
                kwargs, _chunk_rows(args, kwargs), (offset or 0) - record.resume_offset
            )
        # As the next forward begins, not as the boundary one returns: generate_step converts the cache between the
        # two (KV quantization).
        boundary = record is not None and record.forwards == record.capture_at
        if (
            record is not None
            and record.forwards
            and (boundary and not record.capture_rows or record.forwards == record.tail_at)
            and cache is not None
        ):
            try:
                copied = copy_cache_entries(cache)
            except Exception as exc:
                logger.info("MLX VLM prompt cache: cache layout not copied (%s)", exc)
            else:
                if boundary:
                    record.snapshot = copied
                else:
                    record.tail = copied
        if boundary and record.capture_rows and cache is not None:
            output, record.snapshot = _forward_capturing(
                base, self, args, kwargs, cache, record.capture_rows
            )
        else:
            output = base.__call__(self, *args, **kwargs)
        if record is not None:
            record.forwards += 1
        return output

    cls = type(f"Recording{base.__name__}", (base,), {"__call__": __call__, "_studio_base": base})
    _RECORDING_CLASSES[base] = cls
    return cls


def _offset_aware_policy(host):
    """``chunked_prefill_policy`` of ``host`` fed only the token types past the cache: with the
    vision block already there, the suffix is text and may chunk."""
    base_policy = type(host).chunked_prefill_policy

    def policy(
        *,
        prompt_cache = None,
        prefill_kwargs = None,
        **rest,
    ):
        offset = cache_entries_offset(prompt_cache) if prompt_cache is not None else 0
        if offset and prefill_kwargs:
            prefill_kwargs = dict(prefill_kwargs)
            for key in ("mm_token_type_ids", "token_type_ids"):
                shape = getattr(prefill_kwargs.get(key), "shape", None)
                if shape and len(shape) > 1 and shape[1] > offset:
                    prefill_kwargs[key] = prefill_kwargs[key][:, offset:]
        return base_policy(host, prompt_cache = prompt_cache, prefill_kwargs = prefill_kwargs, **rest)

    return policy


class RecordingForward:
    def __init__(self, language_model):
        self._language_model = language_model
        self.record = _ForwardRecord()
        self._base = None

    def __enter__(self):
        model = self._language_model
        base = getattr(type(model), "_studio_base", type(model))
        self._base = base
        model.__class__ = _recording_class(base)
        model.__dict__["_studio_forward_record"] = self.record
        return self

    def __exit__(self, *_exc):
        model = self._language_model
        model.__dict__.pop("_studio_forward_record", None)
        model.__class__ = self._base
        return False

    def unrecorded(self, *args, **kwargs):
        """A forward the record does not count: rows prepared outside the generation."""
        return self._base.__call__(self._language_model, *args, **kwargs)


def _cache_classes(name):
    modules = (sys.modules.get(m) for m in ("mlx_vlm.models.cache", "mlx_lm.models.cache"))
    return tuple(getattr(m, name) for m in modules if hasattr(m, name))


def _rows_in_order(entry, classes):
    """Rows are positions 0..offset-1: plain KV, or a ring that has not dropped a row yet."""
    kv, rings = classes
    if entry.keys is None:
        return False
    return type(entry) in kv or (
        type(entry) in rings and entry.offset == entry._idx == entry.keys.shape[2]
    )


def _shared_kv_pairs(entries, base, classes, pairs):
    """All or none: False when ``base`` cannot serve every in-order entry."""
    if isinstance(entries, (list, tuple)):
        if not isinstance(base, (list, tuple)) or len(entries) != len(base):
            return False
        return all(_shared_kv_pairs(e, b, classes, pairs) for e, b in zip(entries, base))
    if type(entries) is not type(base):
        return False
    if type(entries) in classes[0] + classes[1]:
        if not _rows_in_order(entries, classes):
            return True
        if not _rows_in_order(base, classes) or entries.offset > base.offset:
            return False
        if entries.keys.shape[2] > base.keys.shape[2]:
            return False
        pairs.append((entries, base))
        return True
    nested = getattr(entries, "caches", None)
    if isinstance(nested, (list, tuple)):
        return _shared_kv_pairs(nested, base.caches, classes, pairs)
    return True


def share_kv_rows(entries, base):
    """Point in-order entries at ``base``'s leading rows; returns bytes now read through ``base``."""
    pairs = []
    classes = (_cache_classes("KVCache"), _cache_classes("RotatingKVCache"))
    if not _shared_kv_pairs(entries, base, classes, pairs):
        return 0
    shared = 0
    for entry, source in pairs:
        rows = entry.keys.shape[2]
        entry.keys = source.keys[..., :rows, :]
        entry.values = source.values[..., :rows, :]
        shared += entry.keys.nbytes + entry.values.nbytes
    return shared


def kv_skeleton(entries):
    """``entries`` with in-order KV rows left out (``offset`` kept) and recurrent state copied;
    None for any other entry (quantized, wrapped)."""
    import mlx.core as mx

    classes = (_cache_classes("KVCache"), _cache_classes("RotatingKVCache"))
    recurrent = _cache_classes("ArraysCache")
    skeleton = []
    for entry in entries:
        if type(entry) in classes[0] + classes[1] and _rows_in_order(entry, classes):
            entry = copy.copy(entry)
            entry.keys = entry.values = None
        elif isinstance(entry, recurrent):
            entry = _copy_value(entry, mx)
        else:
            return None
        skeleton.append(entry)
    mx.async_eval(list(_arrays(skeleton, mx)))
    return skeleton


def fill_kv_skeleton(skeleton, final):
    """Point ``skeleton``'s KV entries at the leading rows of the same row's ``final`` cache;
    False when ``final`` does not hold them from row 0 (a wrapped ring)."""
    kv = _cache_classes("KVCache") + _cache_classes("RotatingKVCache")
    if len(final) != len(skeleton):
        return False
    pairs = [(entry, source) for entry, source in zip(skeleton, final) if type(entry) in kv]
    if not all(
        type(source) is type(entry)
        and source.keys is not None
        and getattr(source, "_idx", source.offset) == source.offset >= entry.offset
        for entry, source in pairs
    ):
        return False
    for entry, source in pairs:
        entry.keys = source.keys[..., : entry.offset, :]
        entry.values = source.values[..., : entry.offset, :]
    return True


def compact_sliding_windows(entries):
    """Drop ring rows past the window after a multi-row update; no later update reads them."""
    rings = _cache_classes("RotatingKVCache")
    pending = list(entries)
    compacted = []
    while pending:
        entry = pending.pop()
        nested = entry if isinstance(entry, (list, tuple)) else getattr(entry, "caches", None)
        if isinstance(nested, (list, tuple)):
            pending.extend(nested)
        elif type(entry) in rings and entry.keys is not None:
            excess = entry.keys.shape[2] - entry.max_size
            if excess > 0 and entry._idx == entry.keys.shape[2]:
                entry.keys = entry._trim(excess, entry.keys)
                entry.values = entry._trim(excess, entry.values)
                entry._idx = entry.max_size
                compacted.extend((entry.keys, entry.values))
    if compacted:
        import mlx.core as mx
        mx.async_eval(compacted)


class VLMPromptSnapshotStore:
    """LRU boundary snapshots under a byte budget; a shorter prefix reads KV rows from its base.
    A replay (one per key) serves only a prompt of that exact length and reads through no base.

    Those over the budget are held for one chat's next lookup only; a boundary that serves is
    handed over uncopied, since that request banks successors a kept copy would sit beside."""

    def __init__(
        self,
        max_bytes,
        max_entries = VLM_PROMPT_CACHE_ENTRIES,
        holds = True,
    ):
        self._max_bytes = max_bytes
        self._max_entries = max_entries
        self._entries = OrderedDict()
        self._bases = {}
        self._replays = {}
        self.nbytes = 0
        # One request's boundary and replay.
        self._held = VLMPromptSnapshotStore(math.inf, 2, holds = False) if holds else None

    def __len__(self):
        return len(self._entries)

    def __contains__(self, item):
        return item in self._entries or (self._held is not None and item in self._held)

    def lookup(
        self,
        key,
        token_ids,
        limit,
        takes = False,
    ):
        """Longest stored prefix ending by ``limit``, or the replay holding all but the last token.
        A held replay that serves stays held unless the caller ``takes`` it, as a boundary is."""
        entries, n = self._lookup(key, token_ids, limit)
        if not self._held:
            return entries, n
        held, held_n = self._held._lookup(key, token_ids, limit)
        if held_n <= n:
            self._held.clear()
            return entries, n
        if takes or not self._held._is_replay((key, tuple(token_ids[:held_n]))):
            self._held.clear()
        return held, held_n

    def release_held(self):
        if self._held is not None:
            self._held.clear()

    def over_budget(self, entries):
        return cache_entries_nbytes(entries) > self._max_bytes

    def held_entries(self, key):
        """``key``'s held snapshots as (prefix ids, entries, replay), shortest first."""
        held = self._held._entries.items() if self._held else ()
        return sorted(
            (
                (ids, entries, self._held._is_replay((stored, ids)))
                for (stored, ids), (entries, _nbytes) in held
                if stored == key
            ),
            key = lambda item: len(item[0]),
        )

    def _lookup(self, key, token_ids, limit):
        best = None
        for item, (entries, _nbytes) in self._entries.items():
            stored_key, prefix = item
            n = len(prefix)
            fits = n == len(token_ids) - 1 if self._is_replay(item) else n <= limit
            if stored_key != key or n == 0 or not fits:
                continue
            if best is not None and n <= len(best[0]):
                continue
            if tuple(token_ids[:n]) == prefix:
                best = (prefix, entries)
        if best is None:
            return None, 0
        self._touch((key, best[0]))
        return best[1], len(best[0])

    def store(
        self,
        key,
        prefix_ids,
        entries,
        replay = False,
        hold = True,
    ):
        compact_sliding_windows(entries)
        nbytes = cache_entries_nbytes(entries)
        if nbytes > self._max_bytes:
            held = self._held is not None and hold
            logger.debug(
                "MLX VLM prompt cache: %s %.2f GB snapshot over the %.2f GB budget",
                "holding" if held else "skipping",
                nbytes / 1e9,
                self._max_bytes / 1e9,
            )
            if not held:
                return False
            if any(other[0] != key for other in self._held._entries):
                self._held.clear()
            return self._held.store(key, prefix_ids, entries, replay)
        item = (key, tuple(prefix_ids))
        replay = replay or self._is_replay(item)
        readers = [other for other, base in self._bases.items() if base == item]
        self._pop(item)
        self._entries[item] = (entries, nbytes)
        self.nbytes += nbytes
        replaced = self._replays.get(key)
        if replay:
            self._replays[key] = item
        longer = [other for other in reversed(self._entries) if _extends(other, item)]
        base = self._bases.get(longer[0], longer[0]) if longer and not replay else item
        rebased = {
            other
            for other in list(self._entries)
            if other != base
            and other not in (replaced, self._replays.get(key))
            and (other == item or _extends(item, other))
            and self._rebase(other, base)
        }
        for other in readers:
            if other not in rebased:
                self.discard(other)
        if replay and replaced not in (None, item):
            self._hand_down(replaced)
            self.discard(replaced)
        self._touch(item)
        while self._entries and (
            self.nbytes > self._max_bytes or len(self._entries) > self._max_entries
        ):
            self.discard(next(iter(self._entries)))
        return item in self._entries

    def _rebase(self, item, base):
        entries, nbytes = self._entries[item]
        shared = share_kv_rows(entries, self._entries[base][0])
        if shared:
            owned = cache_entries_nbytes(entries) - shared
            self._entries[item] = (entries, owned)
            self.nbytes += owned - nbytes
            self._bases[item] = base
        return bool(shared)

    def _hand_down(self, item):
        """Re-home ``item``'s readers first, else an edit's new replay drops the old branch."""
        readers = [other for other, base in self._bases.items() if base == item]
        for other in sorted(readers, key = lambda other: len(other[1]), reverse = True):
            del self._bases[other]
            longer = [
                longest
                for longest in reversed(self._entries)
                if longest != item and self._bases.get(longest) != item and _extends(longest, other)
            ]
            longer.sort(key = lambda longest: len(longest[1]), reverse = True)
            if longer and self._rebase(other, self._bases.get(longer[0], longer[0])):
                self._touch(other)
                continue
            entries, nbytes = self._entries[other]
            try:
                entries = copy_cache_entries(entries)
            except Exception as exc:
                logger.info("MLX VLM prompt cache: snapshot not handed down (%s)", exc)
                self.discard(other)
                continue
            owned = cache_entries_nbytes(entries)
            self._entries[other] = (entries, owned)
            self.nbytes += owned - nbytes
            self._entries.move_to_end(other)

    def _touch(self, item):
        self._entries.move_to_end(item)
        base = self._bases.get(item)
        if base is not None:
            # Base is evicted last.
            self._entries.move_to_end(base)

    def _is_replay(self, item):
        return self._replays.get(item[0]) == item

    def _pop(self, item):
        dropped = self._entries.pop(item, None)
        self._bases.pop(item, None)
        if self._is_replay(item):
            del self._replays[item[0]]
        if dropped is not None:
            self.nbytes -= dropped[1]

    def discard(self, item):
        self._pop(item)
        for other in [other for other, base in self._bases.items() if base == item]:
            self.discard(other)
        if self._held is not None:
            self._held.discard(item)

    def retain(self, item):
        keep = (item, self._bases.get(item))
        for other in [other for other in self._entries if other not in keep]:
            self.discard(other)
        if self._held is not None:
            self._held.retain(item)

    def retain_key(self, key):
        """Held snapshots go too: no prompt has matched them yet."""
        for other in [other for other in self._entries if other[0] != key]:
            self.discard(other)
        self.release_held()

    def resume_entries(self):
        """The newest turn's longest boundary snapshot, then its replay."""
        newest = next(reversed(self._entries), None)
        if newest is None:
            return []
        key = newest[0]
        replay = self._replays.get(key)
        boundaries = [i for i in self._entries if i[0] == key and not self._is_replay(i)]
        # _touch keeps the longest last, but after an edit that can be an older base the replay's
        # turn no longer extends: prefer the longest one it does.
        fits = [i for i in boundaries if replay is not None and replay[1][: len(i[1])] == i[1]]
        boundary = max(fits, key = lambda i: len(i[1])) if fits else next(reversed(boundaries), None)
        return [
            (item[0], list(item[1]), self._entries[item][0], self._is_replay(item))
            for item in (boundary, replay)
            if item is not None
        ]

    def clear(self):
        self._entries.clear()
        self._bases.clear()
        self._replays.clear()
        self.nbytes = 0
        self.release_held()


def _extends(item, other):
    (key, prefix), (other_key, other_prefix) = item, other
    return (
        key == other_key
        and len(prefix) > len(other_prefix)
        and prefix[: len(other_prefix)] == other_prefix
    )


class VLMPromptCacheSession:
    """The ``prompt_cache_state`` one generation hands to mlx-vlm: ``find_prefix_length`` picks the
    snapshot to resume from, swaps ``cache`` to it and arms the capture, and ``finish`` stores
    it. ``releases_unserved`` frees the rest, the headroom a media pass needs. A ``media_block``
    prefills the rows through the last vision token for models that cannot chunk them, becoming
    the grid's origin, and ``policy_hosts`` are the models whose ``chunked_prefill_policy`` would
    then refuse to chunk the suffix."""

    def __init__(
        self,
        store,
        key,
        language_model,
        make_cache,
        media_token_ids = (),
        releases_unserved = False,
        media_block = None,
        policy_hosts = (),
        step = VLM_PREFILL_STEP,
        window = None,
    ):
        self.step = step
        self.window = window or step
        self._store = store
        self._key = key
        self._media_token_ids = tuple(media_token_ids)
        self._releases_unserved = releases_unserved
        self.media_block = media_block
        self._origin = 0
        # Produced for this request: prefill work rather than reuse, for the stats.
        self.produced_tokens = 0
        self.produced_seconds = 0.0
        self._policy_hosts = tuple(policy_hosts)
        self._forward = RecordingForward(language_model)
        self.cache = make_cache()
        self.reused_tokens = 0
        self._token_ids = None
        self._media_end = 0
        self._handed_over = False
        self._rehold = None

    def __enter__(self):
        self._forward.__enter__()
        self._patched_hosts = [
            host for host in self._policy_hosts if hasattr(type(host), "chunked_prefill_policy")
        ]
        for host in self._patched_hosts:
            host.chunked_prefill_policy = _offset_aware_policy(host)
        return self

    def __exit__(self, *exc):
        try:
            for host in self._patched_hosts:
                # policy_hosts is (model, language_model), which name one object twice when the wrapper exposes none,
                # so unpatch idempotently rather than del twice.
                host.__dict__.pop("chunked_prefill_policy", None)
        finally:
            # The class goes back regardless: one left wrapped serves every later request.
            self._forward.__exit__(*exc)
        return False

    def find_prefix_length(self, token_ids):
        token_ids = list(token_ids)
        self._token_ids = token_ids
        origin = self.media_block.rows(token_ids) if self.media_block is not None else 0
        self._origin = origin
        boundary = shape_stable_prefix(len(token_ids), origin, self.step)
        self._media_end = media_prefix_end(token_ids, self._media_token_ids)
        record = self._forward.record
        if boundary < self._media_end:
            # Inside the media span: the prompt has to grow past it first.
            self._keep(None)
            return 0
        entries, prefix_len = self._store.lookup(self._key, token_ids, boundary)
        self._handed_over = (
            entries is not None and (self._key, tuple(token_ids[:prefix_len])) not in self._store
        )
        if self._handed_over and prefix_len == boundary:
            # Banking no boundary past it, the request holds again what served it, from its final cache.
            try:
                self._rehold = kv_skeleton(entries)
            except Exception as exc:
                logger.info(
                    "MLX VLM prompt cache: served snapshot not kept to hold again (%s)", exc
                )
        if entries is None and origin:
            # Produced here instead, in the one square forward the vision tokens need.
            self._keep(None)
            started = time.perf_counter()
            try:
                entries, prefix_len = self.media_block.prefill(self._forward, origin), origin
                self.produced_tokens = origin
            except Exception as exc:
                # Prefilled by mlx-vlm as before, off this grid: no capture.
                logger.info("MLX VLM prompt cache: media block not prefilled (%s)", exc)
                self._keep(None)
                return 0
            finally:
                # Spent either way, and outside mlx-vlm's prompt timer.
                self.produced_seconds = time.perf_counter() - started
        if entries is not None:
            # The copy is taken at the first forward, once mlx-vlm has resumed.
            self.cache = entries
            record.on_resume = self._detach_served
        self._keep((self._key, tuple(token_ids[:prefix_len])) if prefix_len else None)
        rows = boundary - prefix_len
        record.capture_at, record.capture_rows = (
            divmod(rows, self.window) if rows >= 0 else (rows, 0)
        )
        if prefix_len <= boundary < len(token_ids) - 1:
            record.tail_at = record.capture_at + 1
        self.reused_tokens = prefix_len
        return prefix_len

    def _keep(self, item):
        # At lookup, before the media pass a declined request starts.
        if self._releases_unserved:
            self._store.retain(item)

    def _detach_served(self, offset):
        prefix_ids = self._token_ids[: self.reused_tokens]
        if offset != self.reused_tokens:
            self._rehold = None
            # Declined: freed now, before the fresh-cache request's media pass evaluates.
            self._store.discard((self._key, tuple(prefix_ids)))
            try:
                release_cache_entries(self.cache)
            except Exception as exc:
                logger.info("MLX VLM prompt cache: declined snapshot not released (%s)", exc)
            return
        if self._handed_over:
            return
        # About to be advanced by this generation: the store keeps a copy.
        try:
            copy = copy_cache_entries(self.cache)
        except Exception as exc:
            # Costs the next turn its reuse, never this answer.
            logger.info("MLX VLM prompt cache: served snapshot not kept (%s)", exc)
            self._store.discard((self._key, tuple(prefix_ids)))
            return
        # Held only in place of what served it, never beside the generation that produced it.
        kept = (self._key, tuple(prefix_ids)) in self._store
        self._store.store(self._key, prefix_ids, copy, hold = kept)

    def update(self, _token_ids, _cache):
        """mlx-vlm's after-generation hook; ``finish`` stores, as it also runs on a consumer that
        stops reading early."""

    def finish(self):
        if self._token_ids is None:
            return False
        record = self._forward.record
        stored = self._store_boundary(record.snapshot)
        stored = self._store_rehold() or stored
        replay = len(self._token_ids) - 1
        if (
            record.tail is not None
            and record.resume_offset == self.reused_tokens
            and cache_entries_offset(record.tail) == replay
        ):
            stored = (
                self._store.store(self._key, self._token_ids[:replay], record.tail, replay = True)
                or stored
            )
        return stored

    def _store_rehold(self):
        skeleton, self._rehold = self._rehold, None
        try:
            if not skeleton or not fill_kv_skeleton(skeleton, self.cache):
                return False
            import mlx.core as mx
            mx.eval(list(_arrays(skeleton, mx)))
        except Exception as exc:
            logger.info("MLX VLM prompt cache: served snapshot not held again (%s)", exc)
            return False
        return self._store.store(self._key, self._token_ids[: self.reused_tokens], skeleton)

    def _store_boundary(self, snapshot):
        if snapshot is None:
            return False
        # Read off the snapshot: a declined offer captures an earlier boundary.
        held = cache_entries_offset(snapshot)
        if (
            not held
            or held < self._origin
            or (held - self._origin) % self.step
            or held < self._media_end
            or held > shape_stable_prefix(len(self._token_ids), self._origin, self.step)
        ):
            logger.debug("MLX VLM prompt cache: snapshot holds %r rows, not stored", held)
            return False
        return self._store.store(self._key, self._token_ids[:held], snapshot)


class VLMBatchRowCache:
    """The ``prompt_cache_state`` one batched row hands to unsloth-zoo, on the single path's grid
    and store: ``open`` resumes the longest snapshot ending by the row's boundary and names the
    chunk ends to bank, the boundary and the replay; ``checkpoint`` stores each. With no
    ``store`` the row prefills cold on the same cache layout; ``releases_unserved`` frees what
    does not serve it, as the single path does. A row takes what is held uncopied. Over the
    budget (with ``alone``) a row keeps only a skeleton, filled in ``retire`` from the cache
    unsloth-zoo hands back if ``alone()`` then; a copy would sit beside its decode."""

    def __init__(
        self,
        store,
        key,
        make_cache,
        media_token_ids = (),
        releases_unserved = False,
        step = VLM_PREFILL_STEP,
        alone = None,
    ):
        self._store = store
        self._key = key
        self._make_cache = make_cache
        self._media_token_ids = tuple(media_token_ids)
        self._releases_unserved = releases_unserved
        self._step = step
        self._alone = alone
        self._token_ids = []
        self._replay = None
        self._skeletons = []

    def open(self, token_ids):
        self._token_ids = token_ids = list(token_ids)
        boundary = shape_stable_prefix(len(token_ids), 0, self._step)
        media_end = media_prefix_end(token_ids, self._media_token_ids)
        if self._store is None:
            return self._make_cache(), ()
        if boundary < media_end:
            self._keep(None)
            return self._make_cache(), ()
        cache, handed = None, False
        held = self._store.held_entries(self._key) if self._alone is not None else ()
        entries, prefix = self._store.lookup(self._key, token_ids, boundary, takes = True)
        if entries is not None and prefix >= media_end:
            if (self._key, tuple(token_ids[:prefix])) not in self._store:
                # Handed over by the store: the row's only copy.
                cache, handed = entries, True
            else:
                try:
                    # The row advances its cache in place; the store keeps its own.
                    cache = copy_cache_entries(entries)
                except Exception as exc:
                    logger.info(
                        "MLX VLM prompt cache: snapshot not copied for a batched row (%s)", exc
                    )
        if cache is None:
            cache, prefix = self._make_cache(), 0
        self._keep((self._key, tuple(token_ids[:prefix])) if prefix else None)
        replay = len(token_ids) - 1
        self._replay = replay if replay != boundary else None
        lengths = [length for length in (boundary, replay) if length > prefix]
        if handed and boundary <= prefix:
            # Banking no boundary past it, the row holds again what served it and the prefixes it extends.
            self._skeletons = [
                (len(ids), skeleton, is_replay)
                for ids, held_entries, is_replay in held
                if len(ids) <= prefix and (skeleton := kv_skeleton(held_entries))
            ]
        return cache, lengths

    def _keep(self, item):
        if self._releases_unserved:
            self._store.retain(item)

    def checkpoint(self, token_count, cache):
        # Runs inside the batch's prefill: a failure costs a later turn its reuse, never this row.
        try:
            replay = token_count == self._replay
            skeleton = (
                self._alone is not None and self._store.over_budget(cache) and kv_skeleton(cache)
            )
            if skeleton:
                self._skeletons.append((token_count, skeleton, replay))
                return
            self._store.store(
                self._key,
                self._token_ids[:token_count],
                copy_cache_entries(cache),
                replay = replay,
                hold = False,
            )
        except Exception as exc:
            logger.info("MLX VLM prompt cache: batched row snapshot not stored (%s)", exc)

    def retire(self, take):
        skeletons, self._skeletons = self._skeletons, []
        if not skeletons or not self._alone():
            return
        try:
            import mlx.core as mx

            final = take()
            if not all(fill_kv_skeleton(entries, final) for _count, entries, _replay in skeletons):
                return
            mx.eval([list(_arrays(entries, mx)) for _count, entries, _replay in skeletons])
            for token_count, entries, replay in skeletons:
                self._store.store(self._key, self._token_ids[:token_count], entries, replay = replay)
        except Exception as exc:
            logger.info("MLX VLM prompt cache: batched row snapshot not held (%s)", exc)


def _mlx_adapter_modules(model):
    """Return bypassable adapter entries and unsupported wrapper paths."""
    adapters = []
    unsupported = []
    for path, module in model.named_modules():
        if not path or not (hasattr(module, "lora_a") and hasattr(module, "lora_b")):
            continue
        base = getattr(module, "linear", None)
        if base is None:
            base = getattr(module, "embedding", None)
        if base is None:
            unsupported.append(path)
        else:
            adapters.append((path, module, base))
    return adapters, unsupported


_MLX_FUSION_UNAVAILABLE = set()


def _mlx_fusion_unavailable(name, error):
    """Log once per cause: this is a per-request path, so repeating would flood."""
    key = (name, type(error).__name__, str(error))
    if key not in _MLX_FUSION_UNAVAILABLE:
        _MLX_FUSION_UNAVAILABLE.add(key)
        logger.warning(
            "Optional MLX fusion %s is unavailable, continuing with native inference: %s: %s",
            name,
            type(error).__name__,
            error,
        )


def _mlx_inference_patch(name):
    """An optional Zoo fusion helper, or None. Never raises: these are a throughput
    optimization, so no Zoo state may fail a load or a request that worked without it."""
    try:
        patches = importlib.import_module("unsloth_zoo.mlx.inference")
    except ModuleNotFoundError as error:
        if error.name != "unsloth_zoo.mlx.inference":
            _mlx_fusion_unavailable(name, error)  # a transitive import inside Zoo failed
        return None
    except Exception as error:
        # A partial or skewed install raises ImportError rather than ModuleNotFoundError.
        _mlx_fusion_unavailable(name, error)
        return None
    return getattr(patches, name, None)


@contextmanager
def _mlx_optional_fusion(name, model, *args):
    """Hold an optional Zoo fusion scope, or yield the model unfused. Guard ENTRY only:
    an open scope must unwind through the ExitStack as an unguarded `with` would, or an
    interrupted generation stops restoring the module tree. A dead Metal queue is not guarded."""
    patch = _mlx_inference_patch(name)
    with ExitStack() as scope:
        active = model
        if patch is not None:
            try:
                entered = scope.enter_context(patch(model, *args))
            except Exception as error:
                if is_metal_queue_dead(error):
                    raise
                _mlx_fusion_unavailable(name, error)
            else:
                if entered is not None:
                    active = entered
        yield active


def _mlx_scope_refused(name, error):
    if is_metal_queue_dead(error):
        raise error
    _mlx_fusion_unavailable(name, error)


# What Studio entered by hand before Zoo's registry; a Zoo without it keeps exactly these.
_MLX_LEGACY_SCOPES = (
    "fused_moe_gate_up",
    "fused_decode_conv_silu",
    "fused_residual_norm",
    "fused_moe_router",
    "fused_moe_routed_experts",
    "fused_residual_norm_handoff",
    "fused_packed_gated_delta",
)


def _fast_verify_kwargs(fast_verify):
    return {} if fast_verify is None else {"fast_verify": fast_verify}


@contextmanager
def _mlx_inference_scopes(
    model,
    call,
    int8_prefill,
    fast_verify = None,
):
    """Hold the Zoo scopes Zoo picks for `call` (its call traits), so a new Zoo scope needs no Studio change.
    A Zoo without the registry keeps the scopes Studio listed before it (none for the text batch).
    `fast_verify` is the load's settled decision (None under a Zoo without it): a refusal here would drop every scope."""
    compose = _mlx_inference_patch("inference_scopes")
    with ExitStack() as scope:
        if compose is None and "batch" not in call:
            for name in _MLX_LEGACY_SCOPES:
                scope.enter_context(_mlx_optional_fusion(name, model))
            if "speculative" in call:
                scope.enter_context(_mlx_optional_fusion("exact_speculative_verify", model))
        elif compose is not None:
            try:
                scope.enter_context(
                    compose(
                        model,
                        call,
                        int8_prefill = int8_prefill,
                        on_error = _mlx_scope_refused,
                        **_fast_verify_kwargs(fast_verify),
                    )
                )
            except Exception as error:
                if is_metal_queue_dead(error):
                    raise
                _mlx_fusion_unavailable("inference_scopes", error)
        yield model


def _mlx_warm_inference(
    model,
    int8_prefill,
    fast_verify = None,
):
    """Zoo's warm-up generation, so the first request skips the scopes' kernel builds and first-use
    checks. A Zoo without it, or a failed warm-up, leaves them to the first request."""
    warm = _mlx_inference_patch("warm_inference")
    if warm is None:
        return
    try:
        warm(model, int8_prefill = int8_prefill, **_fast_verify_kwargs(fast_verify))
    except Exception as error:
        if is_metal_queue_dead(error):
            raise
        logger.warning(
            "MLX warm-up failed, so the first request runs it: %s: %s",
            type(error).__name__,
            error,
        )


def _int8_prefill_zoo():
    """unsloth_zoo's MLX inference module when it can run int8 prefill, else None."""
    try:
        zoo = importlib.import_module("unsloth_zoo.mlx.inference")
        if "int8_prefill" in inspect.signature(zoo.nax_quantized_linear).parameters:
            return zoo
    except Exception:
        pass
    return None


def _int8_prefill_status(requested, model):
    """Whether a load runs int8 prefill: asked for, and the loaded model can use it here."""
    status = {"requested": bool(requested), "active": False, "reason": "", "scope": None}
    if not requested:
        return status
    zoo = _int8_prefill_zoo()
    available = getattr(zoo, "int8_prefill_available", None)
    if available is None:
        status["reason"] = "unsupported_zoo"
        return status
    try:
        verdict = available(model)
    except Exception as exc:
        if is_metal_queue_dead(exc):
            raise
        logger.warning("MLX int8 prefill check failed: %s", exc)
        status["reason"] = "probe_failed"
        return status
    status["reason"] = verdict.reason
    if verdict.available:
        status["active"] = True
        status["scope"] = zoo.nax_quantized_linear
    else:
        logger.info("MLX int8 prefill not applied: %s", verdict.reason)
    return status


def _ane_prefill_zoo():
    """unsloth_zoo's MLX inference module when it can run Neural Engine prefill, else None."""
    try:
        zoo = importlib.import_module("unsloth_zoo.mlx.inference")
        if "ane_prefill" in inspect.signature(zoo.ane_prefill_mlp).parameters:
            return zoo
    except Exception:
        pass
    return None


def _ane_prefill_status(requested, model):
    """Whether a load runs Neural Engine prefill: asked for, and the loaded model can use it here. Read again after
    the warm-up, whose first prepare settles which layers attach."""
    status = {"requested": bool(requested), "active": False, "reason": "", "scope": None}
    if not requested:
        return status
    zoo = _ane_prefill_zoo()
    if zoo is None:
        status["reason"] = "unsupported_zoo"
        return status
    try:
        verdict = zoo.ane_prefill_available(model)
    except Exception as exc:
        if is_metal_queue_dead(exc):
            raise
        logger.warning("MLX Neural Engine prefill check failed: %s", exc)
        status["reason"] = "probe_failed"
        return status
    status["reason"] = verdict.reason
    if verdict.available:
        status["active"] = True
        status["scope"] = zoo.ane_prefill_mlp
    else:
        logger.info("MLX Neural Engine prefill not applied: %s", verdict.reason)
    return status


def _fast_verify_zoo():
    """unsloth_zoo's MLX inference module when it can run Fast Verify, else None."""
    try:
        zoo = importlib.import_module("unsloth_zoo.mlx.inference")
        if "fast_verify" in inspect.signature(zoo.inference_scopes).parameters:
            return zoo
    except Exception:
        pass
    return None


def _fast_verify_status(requested, model):
    """Whether a load builds any drafter for Fast Verify: asked for, and the loaded model can use it here."""
    status = {"requested": bool(requested), "active": False, "reason": ""}
    if not requested:
        return status
    zoo = _fast_verify_zoo()
    if zoo is None:
        status["reason"] = "unsupported_zoo"
        return status
    try:
        verdict = zoo.fast_verify_available(model)
    except Exception as exc:
        if is_metal_queue_dead(exc):
            raise
        logger.warning("MLX Fast Verify check failed: %s", exc)
        status["reason"] = "probe_failed"
        return status
    status["reason"] = verdict.reason
    status["active"] = verdict.available
    if not verdict.available:
        logger.info("MLX Fast Verify not applied: %s", verdict.reason)
    return status


def _settle_fast_verify(status, model, drafter):
    """Pin the load's decision on `model` once its drafter is built, so every later Zoo call agrees,
    including those that take no argument and would read UNSLOTH_MLX_FAST_VERIFY. Without a drafter
    it is off: Zoo's scopes serve only speculative calls, so a plain decode has nothing to gain."""
    zoo = _fast_verify_zoo()
    if zoo is None:
        return status
    decided = status["active"] and drafter is not None
    try:
        zoo.pin_fast_verify(model, decided)
    except ValueError:
        # A drafter that failed after pinning its target holds it until reload, so generations must follow.
        logger.warning("MLX Fast Verify stays on without a drafter: a failed build pinned it")
        return status
    if decided == status["active"]:
        return status
    return {**status, "active": False, "reason": "no_drafter"}


_MLX_OFFLOAD_PARTS = (("ple", "PLE tables"), ("experts", "quantized routed experts"))


def normalize_mlx_offload(value):
    """An offload knob as a load reads it: ``"auto"`` (also null), the fraction read from disk, or
    None for off."""
    if value in (None, "auto"):
        return "auto"
    return None if value == "off" else float(value)


def mlx_resolve_offload(
    model_dir,
    knobs,
    *,
    pinned,
    load_in_4bit: bool,
    retains_history: bool,
    kv_bits = None,
    vision = None,
    drafter = None,
) -> tuple:
    """The ``(ple, expert)`` fractions a load of *model_dir* reads from disk (None for off), from
    knobs as ``normalize_mlx_offload`` gives them: an ``"auto"`` knob decided by unsloth_zoo's 50/50
    rule on this load's pricing, and off where that cannot be planned; any other knob as given;
    then the planned prefill step for experts on disk, else None. ``vision=True`` prices the
    mlx-vlm route a speculative text load takes; ``drafter``, a ``(path, builtin)`` to make room for."""
    fixed = tuple(None if knob == "auto" else knob for knob in knobs) + (None,)
    if "auto" not in knobs or not model_dir:
        return fixed
    from core.inference.mlx_memory import mlx_offload_plan, mlx_offload_planner

    if mlx_offload_planner() is None:
        return fixed
    budget = mlx_memory_budget(retains_history = retains_history)
    ceiling = pinned or mlx_checkpoint_window(model_dir)
    if budget is None or not ceiling:
        return fixed
    if kv_bits is not None and mlx_kv_quant_is_refused(model_dir, kv_bits):
        kv_bits = None
    with mlx_rng_preserved():
        try:
            plan = mlx_offload_plan(
                model_dir,
                budget_bytes = budget,
                max_ctx = int(ceiling),
                context = pinned or None,
                ple = knobs[0],
                experts = knobs[1],
                kv_bits = kv_bits,
                load_in_4bit = load_in_4bit,
                vision = vision,
                drafter = drafter,
            )
        except Exception as exc:
            logger.info("MLX auto offload not planned for %s: %s", model_dir, exc)
            return fixed
    return fixed if plan is None else plan


# Most of each offloadable part an auto drafter may add to disk (break-even was ~5 points on sampled decode, #15342).
MLX_AUTO_DRAFTER_ROOM = 0.04


def mlx_resolve_drafter_offload(model_dir, knobs, drafter, *, auto: bool, **priced) -> tuple:
    """``(plan, baseline)``: ``mlx_resolve_offload``'s plan with room made for *drafter*, and for an *auto*
    drafter the target-only plan its costless check measures against. An explicit drafter keeps its room
    unless it is priced not to fit there; an auto drafter gets room only when no part puts more than
    ``MLX_AUTO_DRAFTER_ROOM`` more of itself on disk and it then fits costlessly; otherwise, the target-only plan."""
    if not drafter:
        return mlx_resolve_offload(model_dir, knobs, **priced), None
    from types import SimpleNamespace

    def fits(plan, **kwargs):
        return mlx_drafter_fit(
            model_dir,
            mlx_checkpoint_window(model_dir),
            priced["pinned"],
            SimpleNamespace(path = drafter[0], builtin = drafter[1]),
            load_in_4bit = priced["load_in_4bit"],
            kv_bits = priced.get("kv_bits"),
            **kwargs,
            **mlx_offload_route(plan),
        )[0]

    room = mlx_resolve_offload(model_dir, knobs, drafter = drafter, **priced)
    if not auto:
        # Room a chosen drafter does not fit even under MLX's own cap would only slow the load.
        if "auto" in knobs and fits(room, reserve = False) is False:
            return mlx_resolve_offload(model_dir, knobs, **priced), None
        return room, None
    alone = mlx_resolve_offload(model_dir, knobs, **priced)
    if room == alone:
        return room, alone
    if all(
        (more or 0) - (less or 0) <= MLX_AUTO_DRAFTER_ROOM
        for more, less in zip(room[:2], alone[:2])
    ) and fits(room, costless = True, baseline = mlx_offload_route(alone)):
        return room, alone
    return alone, alone


def mlx_offload_route(plan) -> dict:
    """The ``offload`` / ``prefill_chunk`` pricing kwargs of a ``(ple, experts, prefill_step)`` plan, both
    always present so they override a priced route."""
    return {"offload": tuple(plan[:2]) if any(plan[:2]) else None, "prefill_chunk": plan[2]}


def mlx_checkpoint_window(model_dir) -> Optional[int]:
    from types import SimpleNamespace

    from core.inference.mlx_memory import _snapshot_config

    try:
        config = _snapshot_config(model_dir)
    except Exception:
        return None
    if config is None:
        return None
    native = mlx_native_context_length(SimpleNamespace(config = config))
    return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)


def _takes_prefill_step(target) -> bool:
    fields = getattr(target, "__dataclass_fields__", None)
    if fields is not None:
        return "prefill_step_size" in fields
    try:
        params = inspect.signature(target).parameters
    except (TypeError, ValueError):
        return False
    return "prefill_step_size" in params or any(p.kind is p.VAR_KEYWORD for p in params.values())


def _mlx_offload_request(knobs, plan):
    """The offload a load asks for: the ``(ple, experts)`` knobs as sent, the fraction each wants,
    the ``FastMLXModel.from_pretrained`` kwargs, and why none is passed when unsloth_zoo cannot
    offload. unsloth_zoo hands an ``"auto"`` knob to *plan* once it has the checkpoint (the one it
    resolved and downloaded), and never does for a load it cannot offload; with no *plan* it is off."""
    from core.inference.mlx_memory import mlx_offload_planner

    requested = dict(zip(("ple", "experts"), knobs))
    wanted = {part: normalize_mlx_offload(value) for part, value in requested.items()}
    if plan is None:
        wanted = {part: None if value == "auto" else value for part, value in wanted.items()}
    offload = {
        "requested": requested,
        "wanted": wanted,
        "reason": None,
        "kwargs": {},
        "prefill_step": None,
    }
    if not any(wanted.values()):
        return offload
    if mlx_offload_planner() is None:
        offload["reason"] = "the installed unsloth_zoo cannot read weights from disk"
    elif "auto" in wanted.values():

        def decide(model_dir):
            *fractions, step = plan(model_dir, tuple(wanted.values()))
            # A repeat call sees fractions, not "auto", and plans no step: keep the first one.
            if not offload["decided"]:
                offload["prefill_step"] = step
            wanted.update(zip(wanted, fractions))
            offload["decided"] = True
            return dict(wanted)

        offload["decided"] = False
        offload["kwargs"]["offload"] = decide
    else:
        offload["kwargs"]["offload"] = dict(wanted)
    return offload


def _load_offloading(load, source, load_kwargs, offload):
    """``load(source, **load_kwargs)``, again without offload when unsloth_zoo refuses it for this load
    (an adapter, a bitsandbytes checkpoint, runtime quantization); ``offload`` keeps the reason."""
    try:
        return load(source, **load_kwargs)
    except ValueError as exc:
        # unsloth_zoo's offload refusals all open so.
        if not offload["kwargs"] or not str(exc).startswith("Unsloth: offload "):
            raise
        logger.info("MLX offload not applied to %s: %s", source, exc)
        offload["reason"] = str(exc)
    load_kwargs.pop("offload", None)
    offload["kwargs"] = {}
    return load(source, **load_kwargs)


def _mlx_offload_status(offload, model):
    """Per knob the fraction read from disk, the request and why it is not applied; and the bytes
    the loaded model reads from disk."""
    reports = getattr(model, "_unsloth_offload", None) or {}
    status = {"disk_bytes": sum(report["on_disk_bytes"] for report in reports.values()) or None}
    for part, what in _MLX_OFFLOAD_PARTS:
        applied = offload["wanted"][part] if part in reports else None
        reason = None
        if offload["wanted"][part] not in (None, "auto") and applied is None:
            reason = offload["reason"] or (
                "unsloth_zoo cannot offload this load (an adapter, a bitsandbytes checkpoint or "
                "runtime quantization)"
                if offload.get("decided") is False
                else f"the model has no {what} unsloth_zoo can read from disk"
            )
        status[part] = {
            "applied": applied,
            "requested": offload["requested"][part],
            "reason": reason,
        }
    return status


# Keyed by checkpoint fingerprint: the panel asks again on every model switch.
_int8_prefill_checkpoint_cache: dict = {}
_int8_prefill_checkpoint_lock = threading.Lock()


def mlx_int8_prefill_checkpoint_status(model_dir) -> tuple[bool, str]:
    """(available, reason) of int8 prefill for a downloaded MLX checkpoint, without loading it."""
    check = getattr(_int8_prefill_zoo(), "int8_prefill_checkpoint_available", None)
    if check is None:
        return False, "unsupported_zoo"
    try:
        from core.inference.mlx_memory import _checkpoint_fingerprint
        key = (os.path.realpath(model_dir), _checkpoint_fingerprint(model_dir))
    except Exception:
        key = None
    with _int8_prefill_checkpoint_lock:
        if key in _int8_prefill_checkpoint_cache:
            return _int8_prefill_checkpoint_cache[key]
    try:
        verdict = check(model_dir)
        answer = (bool(verdict.available), verdict.reason)
    except Exception as exc:
        logger.debug("MLX int8 prefill checkpoint check failed for %s: %s", model_dir, exc)
        return False, "probe_failed"
    if key is not None and answer[1] != "probe_failed":
        with _int8_prefill_checkpoint_lock:
            _int8_prefill_checkpoint_cache[key] = answer
            while len(_int8_prefill_checkpoint_cache) > 32:
                _int8_prefill_checkpoint_cache.pop(next(iter(_int8_prefill_checkpoint_cache)))
    return answer


_ane_prefill_checkpoint_cache: dict = {}
_ane_prefill_checkpoint_lock = threading.Lock()


def mlx_ane_prefill_checkpoint_status(model_dir) -> tuple[bool, str]:
    """(available, reason) of Neural Engine prefill for a downloaded MLX checkpoint, without loading it."""
    check = getattr(_ane_prefill_zoo(), "ane_prefill_checkpoint_available", None)
    if check is None:
        return False, "unsupported_zoo"
    try:
        from core.inference.mlx_memory import _checkpoint_fingerprint
        key = (os.path.realpath(model_dir), _checkpoint_fingerprint(model_dir))
    except Exception:
        key = None
    with _ane_prefill_checkpoint_lock:
        if key in _ane_prefill_checkpoint_cache:
            return _ane_prefill_checkpoint_cache[key]
    try:
        verdict = check(model_dir)
        answer = (bool(verdict.available), verdict.reason)
    except Exception as exc:
        logger.debug("MLX Neural Engine prefill checkpoint check failed for %s: %s", model_dir, exc)
        return False, "probe_failed"
    if key is not None and answer[1] != "probe_failed":
        with _ane_prefill_checkpoint_lock:
            _ane_prefill_checkpoint_cache[key] = answer
            while len(_ane_prefill_checkpoint_cache) > 32:
                _ane_prefill_checkpoint_cache.pop(next(iter(_ane_prefill_checkpoint_cache)))
    return answer


def mlx_fast_verify_checkpoint_status(model_dir) -> tuple[bool, str]:
    """(available, reason) of Fast Verify for a downloaded MLX checkpoint, without loading it."""
    zoo = _fast_verify_zoo()
    if zoo is None:
        return False, "unsupported_zoo"
    try:
        verdict = zoo.fast_verify_checkpoint_available(model_dir)
    except Exception as exc:
        logger.debug("MLX Fast Verify checkpoint check failed for %s: %s", model_dir, exc)
        return False, "probe_failed"
    return bool(verdict.available), verdict.reason


def _vlm_generation_context():
    import mlx.core as mx
    from mlx_vlm.generate import generation_stream
    return mx.stream(generation_stream)


def _iter_vlm_responses(responses):
    iterator = iter(responses)
    try:
        while True:
            # Token views outside VLM's inner step must use its generation stream too.
            with _vlm_generation_context():
                try:
                    response = next(iterator)
                except StopIteration:
                    return
            yield response
    finally:
        close = getattr(iterator, "close", None)
        if close is not None:
            with _vlm_generation_context():
                close()


@contextmanager
def _temporary_mlx_adapter_state(model, use_adapter):
    """Select base or adapter modules for one request, then restore the tree."""
    if use_adapter is None:
        yield
        return
    if isinstance(use_adapter, str):
        raise NotImplementedError(
            "Unsloth MLX: named adapter selection is not supported; use True for "
            "the loaded adapter or False for the base model."
        )
    if use_adapter is not True and use_adapter is not False:
        raise TypeError("Unsloth MLX: use_adapter must be None, True, False, or a string.")

    adapters, unsupported = _mlx_adapter_modules(model)
    if use_adapter is True:
        if not adapters and not unsupported:
            logger.warning("MLX adapter requested, but the active model has no adapter layers")
        yield
        return
    if unsupported:
        raise RuntimeError(
            "Unsloth MLX: cannot disable adapter layers without their base modules: "
            + ", ".join(unsupported[:5])
        )
    if not adapters:
        yield
        return

    from mlx.utils import tree_unflatten

    base_modules = tree_unflatten([(path, base) for path, _, base in adapters])
    adapter_modules = tree_unflatten([(path, wrapper) for path, wrapper, _ in adapters])
    try:
        model.update_modules(base_modules)
        yield
    finally:
        model.update_modules(adapter_modules)


def _mlx_vlm_model_config(model):
    """Return the loaded MLX model config and its type, preferring whichever of config / _config
    actually carries a model_type."""

    def _model_type(cfg):
        return cfg.get("model_type") if isinstance(cfg, dict) else getattr(cfg, "model_type", None)

    configs = [
        cfg
        for cfg in (getattr(model, "config", None), getattr(model, "_config", None))
        if cfg is not None
    ]
    for cfg in configs:
        model_type = _model_type(cfg)
        if model_type is not None:
            return cfg, model_type
    return (configs[0] if configs else None), None


# Matched on the context term (mlx-lm spells it four ways); not max_length/n_sequences
_MLX_CONTEXT_KEY = re.compile(
    r"(?:position(?:s|_embeddings)|(?:^|_)ctx|context_len(?:gth)?"
    r"|seq(?:uence)?_len(?:gth)?|model_max_length)$"
)
# A vision tower's token count reads like a length but bounds one image, not the window.
_MLX_CONTEXT_KEY_EXCLUDE = re.compile(r"(?:^|_)image_")
# Below one cache block is not a window; above, configs carry sentinel "unlimited" lengths.
_MLX_MIN_PLAUSIBLE_CONTEXT = 256
_MLX_MAX_PLAUSIBLE_CONTEXT = 1 << 24


def _mlx_config_candidates(model):
    """Config-ish objects that may carry text-tower dims, most specific first."""
    language_model = getattr(model, "language_model", None)
    config, _ = _mlx_vlm_model_config(model)
    text_config = None
    if config is not None:
        text_config = (
            config.get("text_config")
            if isinstance(config, dict)
            else getattr(config, "text_config", None)
        )
    return [
        cfg
        for cfg in (
            getattr(language_model, "args", None),
            text_config,
            getattr(model, "args", None),
            config,
        )
        if cfg is not None
    ]


def _positive_int(value):
    if isinstance(value, bool):
        return None
    try:
        value_int = int(value)
    except (TypeError, ValueError, OverflowError):
        # json.loads accepts Infinity: unusable, but must not abort the load.
        return None
    return value_int if value_int > 0 else None


def mlx_native_context_length(model):
    """Takes the widest length across every config, since a stub text config can understate the window."""
    lengths = []
    for cfg in _mlx_config_candidates(model):
        try:
            items = cfg.items() if isinstance(cfg, dict) else vars(cfg).items()
        except TypeError:
            continue
        lengths += [
            length
            for name, value in items
            if _MLX_CONTEXT_KEY.search(str(name))
            and not _MLX_CONTEXT_KEY_EXCLUDE.search(str(name))
            and (length := _positive_int(value)) is not None
            and _MLX_MIN_PLAUSIBLE_CONTEXT <= length <= _MLX_MAX_PLAUSIBLE_CONTEXT
        ]
    return max(lengths) if lengths else None


def _ascii_registry_key(value):
    """An mlx-vlm registry key lowered within ASCII, else None. Not `casefold`: it folds non-ASCII
    onto ASCII, so "ſmolvlm" would reach `smolvlm`."""
    if not isinstance(value, str) or not value.isascii():
        return None
    return value.lower()


def _resumes_thought(messages, continue_final_message) -> bool:
    return bool(continue_final_message) and (
        trailing_assistant_resume_kind(messages) == "reasoning_content"
    )


def _think_prefix(prompt, special_tokens, messages, continue_final_message, **kwargs) -> str:
    """``detect_think_prefill``; refuses a resumed thought whose ``</think>`` decoding would strip."""
    from core.inference.chat_template_helpers import detect_think_prefill

    resumes = _resumes_thought(messages, continue_final_message)
    prefix = detect_think_prefill(prompt, special_tokens, resumes_thought = resumes, **kwargs)
    if resumes and not prefix:
        raise ThoughtUnresumableError()
    return prefix


def _render_registered_vlm_prompt(
    processor,
    model,
    messages,
    num_images,
    num_audios = 0,
    continue_final_message = False,
    tools = None,
):
    """Resumes a partial from the swept messages, since a raw partial could close the turn."""
    from mlx_vlm import prompt_utils

    config, model_type = _mlx_vlm_model_config(model)
    if config is None:
        return None
    model_config = getattr(prompt_utils, "MODEL_CONFIG", {})
    if model_type not in model_config:
        # ASCII fold: casefold would route 'ſmolvlm' to `smolvlm`
        folded = _ascii_registry_key(model_type)
        matches = (
            [
                key
                for key in model_config
                if isinstance(key, str) and _ascii_registry_key(key) == folded
            ]
            if folded is not None
            else []
        )
        # An ambiguous fold is not evidence for either key: stay fail-closed.
        if len(matches) != 1:
            return None
        canonical = matches[0]
        # Copy: mutating the loaded config would leak into capability and export logic
        config = dict(config) if isinstance(config, dict) else dict(config.__dict__)
        config["model_type"] = canonical

    # mlx-vlm's formatters carry no reasoning opener to reopen a thought on.
    if _resumes_thought(messages, continue_final_message):
        return None
    # Recovery path: sweeps the caller's original list rather than reusing a copy (#7066).
    swept, tools, _ = neutralize_for_render(processor, messages, tools)
    partial = trailing_assistant_text(swept) if continue_final_message else None
    rendered = prompt_utils.apply_chat_template(
        processor,
        config,
        swept[:-1] if partial else swept,
        add_generation_prompt = True,
        num_images = num_images,
        num_audios = num_audios,
        **({"tools": tools} if tools else {}),
    )
    if isinstance(rendered, str) and rendered.strip():
        # A prefilled open "<think>" would resume the answer inside the reasoning block.
        return f"{strip_open_reasoning_prefill(rendered)}{partial}" if partial else rendered
    raise RuntimeError("mlx-vlm's registered renderer returned an empty prompt.")


# Rate the chat route decodes uploads to; mlx-vlm does not resample arrays.
_AUDIO_INPUT_SAMPLE_RATE = 16000
_AUDIO_PROBE_MESSAGES = [{"role": "user", "content": "audio"}]
# Same turn with and without an image part, so a diff isolates the image marker.
_IMAGE_PROBE_MESSAGES = [
    {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "hi"}]}
]
# A pair differing only in image count, so a diff isolates the marker; the question is distinctive.
_MULTI_IMAGE_PROBE_TEXT = "what is in these"
_MULTI_IMAGE_PROBE_MESSAGES = [
    [
        {
            "role": "user",
            "content": [{"type": "image"}] * count
            + [{"type": "text", "text": _MULTI_IMAGE_PROBE_TEXT}],
        }
    ]
    for count in (1, 2)
]
_TEXT_PROBE_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
_VIDEO_PROBE_MESSAGES = [
    {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "hi"}]}
]


def _mlx_vlm_decodes_video() -> bool:
    """Both halves of the decoder, ``load_video`` (0.5.0+) and cv2, so a clip missing either is
    refused by name rather than by a ModuleNotFoundError mid-stream."""
    try:
        from mlx_vlm import utils as vlm_utils
    except ImportError:
        return False
    if not callable(getattr(vlm_utils, "load_video", None)):
        return False
    try:
        import cv2  # noqa: F401
    except ImportError:
        return False
    return True


def _video_placeholder(processor):
    """The model's own video placeholder, when it names one."""
    for source in (processor, getattr(processor, "tokenizer", None)):
        token = getattr(source, "video_token", None)
        if isinstance(token, str) and token:
            return token
    return None


def _marks_video(target, processor) -> bool:
    """Whether ``target`` renders a video part as a marker: a render that merely differs from the
    text one can still be prose, as in the image probe."""
    from core.inference.chat_template_helpers import apply_chat_template_for_generation

    marked = apply_chat_template_for_generation(target, _VIDEO_PROBE_MESSAGES)
    if not marked or marked == apply_chat_template_for_generation(target, _TEXT_PROBE_MESSAGES):
        return False
    if _vlm_prompt_issue(marked, _VIDEO_PROBE_MESSAGES):
        return False
    placeholder = _video_placeholder(processor)
    return placeholder is None or placeholder in marked


def _video_render_target(processor):
    """What renders a turn with video. mlx-vlm's Gemma 4 processor rebuilds each message from media
    counts and drops a video part it holds no clip for; its tokenizer carries the same template and
    renders the part as written. A processor that keeps the marker keeps rendering, since some
    deliberately differ from their tokenizer (MiniCPM-V's non-thinking prefix)."""
    from core.inference.chat_template_helpers import chat_render_target

    target = chat_render_target(processor)
    tokenizer = getattr(processor, "tokenizer", None)
    template = getattr(processor, "chat_template", None)
    if (
        target is not processor
        or not template
        or getattr(tokenizer, "chat_template", None) != template
    ):
        return target
    try:
        drops = not _marks_video(processor, processor)
    except Exception:
        drops = True
    return tokenizer if drops else processor


def _mlx_lm_reads_images(model) -> bool:
    """An image model unsloth-zoo loaded through mlx-lm, with an unsloth-zoo that prefills its images."""
    if getattr(model, "_unsloth_supports_images", None) is not True:
        return False
    try:
        from unsloth_zoo.mlx.generate import prefill_image_prompt  # noqa: F401
    except ImportError:
        return False
    return True


def _mlx_reads_video(processor) -> bool:
    """Processor carries a video component AND the template places a video marker."""
    if processor is None or getattr(processor, "video_processor", None) is None:
        return False
    if not _mlx_vlm_decodes_video():
        return False
    try:
        return _marks_video(_video_render_target(processor), processor)
    except BaseException:
        # A load must never fail on a probe, matching _image_marker_survives.
        return False


def _write_video_clip(video_b64: str) -> str:
    import base64
    import tempfile

    from utils.media_playlist import refuse_playlist

    data = base64.b64decode(video_b64)
    # cv2.VideoCapture and mlx-vlm both open this through FFmpeg, which follows playlist entries.
    refuse_playlist(data)
    with tempfile.NamedTemporaryFile(prefix = "unsloth-video-", delete = False) as handle:
        handle.write(data)
        return handle.name


# mlx-vlm holds two native-resolution RGB copies of the sampled frames before the processor resizes.
_VIDEO_DECODE_BUDGET_BYTES = 2 << 30


# load_video's own defaults, unchanged 0.5.0 to 0.7.0, for a release that settles a knob nowhere.
_VIDEO_SAMPLING_FALLBACK = {"fps": 2.0, "min_frames": 4}


def _sampled(name, value):
    return _VIDEO_SAMPLING_FALLBACK[name] if value is None else value


def _declared_video_sampling(processor) -> dict:
    """Read the way mlx-vlm's ``processor_video_sampling`` reads it, so both branches agree."""
    component = getattr(processor, "video_processor", None)
    if component is None:
        return {}
    for owner in (component, processor):
        hook = getattr(owner, "video_sampling_defaults", None)
        if callable(hook):
            declared = hook()
            return dict(declared) if isinstance(declared, dict) else vars(declared)
    return {name: getattr(component, name, None) for name in ("fps", "min_frames", "max_frames")}


def _video_sampling_target(processor) -> tuple[float, int, Optional[int]]:
    """mlx-vlm's (fps, min_frames, nframes); older releases pass only ``fps`` to ``load_video``."""
    import inspect

    from mlx_vlm import utils as vlm_utils

    resolve = getattr(vlm_utils, "resolve_video_sampling", None)
    if resolve is not None:
        sampling = resolve(processor, {})
        # Unset is unreachable today; the fallback keeps the rate divisible for a future release.
        return (
            _sampled("fps", sampling.fps),
            _sampled("min_frames", sampling.min_frames),
            sampling.nframes,
        )
    # Below 0.7.0 prepare_inputs forwards only fps, so nothing else applies the model's rate
    defaults = inspect.signature(vlm_utils.load_video).parameters

    def _default(name):
        parameter = defaults.get(name)
        if parameter is None or parameter.default is inspect.Parameter.empty:
            return None
        return parameter.default

    declared_fps = _declared_video_sampling(processor).get("fps")
    return (
        _sampled("fps", _default("fps") if declared_fps is None else declared_fps),
        _sampled("min_frames", _default("min_frames")),
        None,
    )


class _ClipsTooLarge(RuntimeError):
    pass


def _video_frame_rate(
    clip_path: str,
    processor,
    clips: int = 1,
) -> Optional[float]:
    """The sampling ``fps`` keeping the decoded stack within this clip's share of the budget; None
    when the clip has no readable frame geometry or count, which mlx-vlm then refuses itself."""
    import cv2

    capture = cv2.VideoCapture(clip_path)
    try:
        if not capture.isOpened():
            return None
        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
        total_frames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
        native_fps = capture.get(cv2.CAP_PROP_FPS)
    finally:
        capture.release()
    if min(width, height, total_frames) <= 0:
        return None
    if not native_fps or native_fps <= 0:
        # The one measurement mlx-vlm substitutes rather than refusing; match its substitution.
        native_fps = 1.0
    wanted_fps, min_frames, nframes = _video_sampling_target(processor)
    # mlx-vlm holds every decoded clip until the processor runs, so the clips share one budget.
    budget = _VIDEO_DECODE_BUDGET_BYTES // clips
    affordable = budget // (2 * 3 * width * height)
    needed = min_frames if nframes is None else nframes
    if affordable < needed:
        raise _ClipsTooLarge(
            f"The video's {width}x{height} frames are too large to decode: {needed} of them "
            f"exceed the {budget >> 20} MiB frame budget."
        )
    return min(wanted_fps, affordable * native_fps / total_frames)


def _video_sampling_fps(
    clip_paths,
    processor,
    rows = 1,
) -> Optional[float]:
    """One rate for every clip, since mlx-vlm before 0.7.3 applies a single ``fps`` to all.

    ``rows`` copies of the clips share the decode budget when a batch decodes them together.
    """
    rates = [_video_frame_rate(path, processor, len(clip_paths) * rows) for path in clip_paths]
    return min((rate for rate in rates if rate is not None), default = None)


def _discard_video_clip(path: str) -> None:
    try:
        os.remove(path)
    except OSError:
        pass


_SPEECH_REPLY_SYSTEM_PROMPT = "Respond with interleaved text and audio."


def _mlx_lm_serves(model_type) -> bool:
    from importlib.util import find_spec

    from mlx_lm.utils import MODEL_REMAPPING
    return find_spec(f"mlx_lm.models.{MODEL_REMAPPING.get(model_type, model_type)}") is not None


def _zoo_declares_multimodal(model_name, hf_token = None) -> bool:
    """unsloth_zoo's verdict from mlx-vlm's own configs, for mlx-vlm-only families the transformers
    registry does not know (an audio-only speech model, a tower under another name)."""
    try:
        from utils.models.model_config import _read_raw_config_json

        config = _read_raw_config_json(model_name, hf_token)
        model_type = (config or {}).get("model_type")
        # A type mlx-lm serves keeps its text path, and skips zoo's scan of every mlx-vlm family.
        if not isinstance(model_type, str) or not model_type or _mlx_lm_serves(model_type):
            return False
        from unsloth_zoo.mlx.loader import _is_vlm

        return _is_vlm(config)
    except Exception as exc:  # noqa: BLE001 -- detection must never fail a load
        logger.debug("unsloth_zoo multimodal detection skipped for %s (%s)", model_name, exc)
        return False


def _zoo_builds_text_on_mlx_vlm(model_name, hf_token = None) -> bool:
    """A text family only mlx-vlm ships (Raon-1 Mini): unsloth_zoo builds it there, so it is served there."""
    try:
        from utils.models.model_config import _read_raw_config_json
        from unsloth_zoo.mlx.loader import _mlx_vlm_only_text_model

        config = _read_raw_config_json(model_name, hf_token) or {}
        return bool(_mlx_vlm_only_text_model(config, config.get("model_type") or ""))
    except Exception as exc:  # noqa: BLE001 -- detection must never fail a load
        logger.debug("unsloth_zoo text-route detection skipped for %s (%s)", model_name, exc)
        return False


def _classify_mlx_audio_type(
    model,
    processor,
    is_vision,
    config_audio_type = None,
):
    """The probe may only retract audio_vlm once it has run, else the pre-load audio_type stands."""

    def _probe_says_no():
        # Authoritative for audio_vlm only; anything else passes through.
        return None if config_audio_type == "audio_vlm" else config_audio_type

    if not is_vision or processor is None:
        return config_audio_type
    # A model load must never fail on a probe; BaseException since any escape aborts the load
    try:
        from unsloth_zoo.mlx.utils import (
            audio_extractor_sampling_rate,
            audio_input_capability,
        )

        if audio_extractor_sampling_rate(processor) != _AUDIO_INPUT_SAMPLE_RATE:
            logger.info(
                "MLX audio input unavailable: feature extractor is not %d Hz, and "
                "the chat route decodes uploads to that rate.",
                _AUDIO_INPUT_SAMPLE_RATE,
            )
            return _probe_says_no()
        args = (processor, model, _AUDIO_PROBE_MESSAGES, 0)
        marked = _render_registered_vlm_prompt(*args, num_audios = 1)
        if not marked or marked == _render_registered_vlm_prompt(*args, num_audios = 0):
            logger.info(
                "MLX audio input unavailable: mlx-vlm's renderer for this family "
                "places no audio marker."
            )
            return _probe_says_no()
        capability = audio_input_capability(model, processor, texts = marked)
        if capability.capable:
            return "audio_vlm"
        logger.info("MLX audio input unavailable for this model: %s", capability.reason)
        return _probe_says_no()
    except BaseException as exc:
        logger.info(
            "MLX audio capability check did not run (%s); keeping the pre-load "
            "classification %r. Audio input needs an unsloth_zoo release providing "
            "audio_input_capability.",
            type(exc).__name__,
            config_audio_type,
        )
    return config_audio_type


def _mlx_config_field(model, name):
    """Reads a config field from a dict or an object, since a getattr-only read misses dict configs."""
    for cfg in (getattr(model, "config", None), getattr(model, "_config", None)):
        value = cfg.get(name) if isinstance(cfg, dict) else getattr(cfg, name, None)
        if value is not None:
            return value
    return None


def _mlx_stop_token_ids(tokenizer, model = None):
    """Prefers mlx_vlm's stopping criteria: tokenizer and config stop ids disagree on some repos."""
    for source in (
        getattr(getattr(tokenizer, "stopping_criteria", None), "eos_token_ids", None),
        _mlx_config_field(model, "eos_token_id"),
        getattr(tokenizer, "eos_token_ids", None),
        getattr(tokenizer, "eos_token_id", None),
    ):
        if source is None:
            continue
        if isinstance(source, (list, tuple, set, frozenset)):
            # Empty/unset falls through (not an answer); a bare id is kept since 0 is a valid token
            if not source:
                continue
            return tuple(source)
        # int() so a numpy scalar cannot raise here, inside a caller's finally.
        return (int(source),)
    return ()


def _mlx_stop_sequences(stop):
    """The sequences a request asked to stop on, as a list. An empty one is dropped rather than
    matched: it is found at position 0 of every reply and would end each turn before its first
    token."""
    return [x for x in ([stop] if isinstance(stop, str) else stop or []) if x]


def _rebuilt_detokenizer(detokenizer):
    """Rebuilds the naive detokenizer from its tokenizer, since copying it raises on some versions."""
    tokenizer = getattr(detokenizer, "_tokenizer", None)
    if tokenizer is None:
        return None
    try:
        return type(detokenizer)(tokenizer)
    except Exception as exc:  # noqa: BLE001 -- third-party detokenizers vary; caller falls back
        logger.debug("MLX streaming detokenizer could not be rebuilt (%s)", exc)
        return None


def _mlx_stream_detokenizer(source):
    """Takes a fresh read of the detokenizer where possible; copying raises on the naive kind."""
    detokenizer = getattr(source, "detokenizer", None)
    if detokenizer is None:
        return None
    independent = getattr(source, "detokenizer", None)
    if independent is None or independent is detokenizer:
        try:
            independent = copy.copy(detokenizer)
        except Exception as exc:  # noqa: BLE001 -- third-party detokenizers vary; caller falls back
            # mlx-vlm < 0.6.0 cannot copy the naive detokenizer; rebuild it as __copy__ would
            independent = _rebuilt_detokenizer(detokenizer)
            if independent is None:
                logger.debug("MLX streaming detokenizer unavailable (%s)", exc)
                return None
    try:
        independent.reset()
    except Exception as exc:  # noqa: BLE001 -- as above
        logger.debug("MLX streaming detokenizer unavailable (%s)", exc)
        return None
    return independent


@functools.cache
def _eager_spm_detokenizer_class():
    from mlx_lm.tokenizer_utils import SPMStreamingDetokenizer
    class EagerSPMStreamingDetokenizer(SPMStreamingDetokenizer):
        def __init__(
            self,
            tokenizer,
            trim_space = True,
        ):
            self._trims_first_space = trim_space
            super().__init__(tokenizer, trim_space = trim_space)

        def reset(self):
            super().reset()
            self.trim_space = self._trims_first_space

        # mlx-lm trims every flush while the text is still empty; only the first may be trimmed.
        def _try_flush(self, force = False):
            pending = self._unflushed
            super()._try_flush(force)
            if pending and not self._unflushed:
                self.trim_space = False

        def add_token(
            self,
            token,
            skip_special_token_ids = (),
        ):
            if token not in skip_special_token_ids:
                super().add_token(token)

    return EagerSPMStreamingDetokenizer


class _SpmPieces(list):
    @property
    def vocab(self):  # mlx-lm 0.31.x
        return {piece: index for index, piece in enumerate(self)}

    def convert_ids_to_tokens(self, ids):  # mlx-lm >= 0.32
        return [self[index] for index in ids]


def _stream_spm_tokens_eagerly(processor):
    """mlx-vlm's SPM detokenizer holds each piece until the next word starts; mlx-lm's does not."""
    held = getattr(processor, "detokenizer", None)
    if (type(held).__module__, type(held).__name__) != (
        "mlx_vlm.tokenizer_utils",
        "SPMStreamingDetokenizer",
    ):
        return
    # Reusing mlx-vlm's pieces skips a second ~1 s walk of a 262k vocab.
    pieces = getattr(held, "tokenmap", None)
    if pieces and None not in pieces:
        source = _SpmPieces(pieces)
    else:
        source = getattr(processor, "tokenizer", processor)
    try:
        processor.detokenizer = _eager_spm_detokenizer_class()(source, trim_space = held.trim_space)
    except Exception as exc:  # noqa: BLE001 -- mlx-vlm's still streams the same text, only later
        logger.debug("MLX streaming detokenizer kept as loaded (%s)", exc)


class _MlxStreamedText:
    """A reply rebuilt from the ids the runtime sampled, for turns carrying native controls.

    A streaming detokenizer holds a piece until the ones after it settle, so a control reaches
    ``response.text`` inside a later segment, with nothing left to say which characters came from
    it: deciding suppression there deletes prose that merely spells a control, and re-decoding the
    id on its own step writes a second copy ahead of the text still held.

    ``generation_tokens``, not a finish reason, says which ids the reply owes -- the supported
    mlx-vlm floor has no such field, and a turn broken out of on a stop token or the token limit
    reports a position no yield covered. Hence also ``finish``, run by the caller once the runtime
    stops yielding.

    Stop ids are held rather than written, since both runtimes stop before their detokenizer sees
    one, and released only to close a tool envelope. The run is held whole: the ids read as ending
    a turn come from the model config, which lists more than the one the runtime stopped on.

    Neither a denoised block, which names one id for many, nor a missing detokenizer leaves ids to
    rebuild from; the runtime's own text stands and the allowlist cannot speak for it."""

    def __init__(self, detokenizer, decoder, tokenizer, stop_ids):
        self._detokenizer = detokenizer
        self._decoder = decoder
        self._tokenizer = tokenizer
        self._stop_ids = stop_ids
        self._held_stops = []
        self._filled = 0

    def feed(self, response) -> str:
        if self._detokenizer is None:
            # Every id but the stop id is already in the text, so only the last can still be a closer owed.
            stop_id = getattr(response, "token", None)
            self._held_stops = (
                [int(stop_id)] if stop_id is not None and int(stop_id) in self._stop_ids else []
            )
            return getattr(response, "text", None) or ""
        filled = getattr(response, "generation_tokens", None)
        if filled is not None:
            filled = int(filled)
            if filled <= self._filled:
                return ""
            several = filled > self._filled + 1
            self._filled = filled
            if several:
                return self._drain() + (getattr(response, "text", None) or "")
        token_id = getattr(response, "token", None)
        if token_id is None:
            return ""
        return self.feed_token(int(token_id))

    def feed_token(self, token_id: int) -> str:
        """One sampled id, for a runtime that reports ids one at a time."""
        if token_id in self._stop_ids:
            self._held_stops.append(token_id)
            return ""
        if self._decoder.suppresses(token_id):
            return ""
        if self._held_stops:
            # Text followed them, so they ended nothing.
            held, self._held_stops = self._held_stops, []
            for stop_id in held:
                if not self._decoder.suppresses(stop_id):
                    self._detokenizer.add_token(stop_id)
        self._detokenizer.add_token(token_id)
        return self._detokenizer.last_segment

    def finish(self, sampled: str) -> str:
        """The text still held once the runtime stops yielding."""
        if self._detokenizer is None:
            return self._closing_stop(sampled)
        segment = self._drain()
        return segment + self._closing_stop(sampled + segment)

    def _drain(self) -> str:
        self._detokenizer.finalize()
        return self._detokenizer.last_segment

    def _closing_stop(self, settled: str) -> str:
        held, self._held_stops = self._held_stops, []
        closing = ""
        for stop_id in held:
            if not self._decoder.keeps(stop_id):
                continue
            text = stop_token_text(self._tokenizer, stop_id) or ""
            if settled.endswith(text):
                continue
            # Nested markup ends on a closer each, so the run can owe more than one.
            if closes_an_open_envelope(settled + text, text):
                closing += text
        return closing


def _require_row_detokenizer(streamed):
    """A batch row feeds ids one at a time, so unlike the single path it has no runtime text to fall back on."""
    if streamed is not None and streamed._detokenizer is None:
        raise RowRefused("no independent detokenizer to decode this row's ids")


class _ReplyText:
    """A reply's decode, fed one id at a time or called as ``decode(ids)`` on an append-only list.
    With a native ``decoder`` a runtime may stop on an allowlisted control (<|end_message|>): the
    trailing stop id stays only when it closes an open envelope, judged with the prefill, since a
    restored <think> lives in the prompt."""

    def __init__(
        self,
        tokenizer = None,
        decode_kwargs = None,
        *,
        decoder = None,
        stop_ids = (),
        prefix = "",
    ):
        self._tokenizer = tokenizer
        self._decode_kwargs = decode_kwargs
        self._decoder = decoder
        self._stop_ids = stop_ids
        self._prefix = prefix
        self._reset()

    def _reset(self):
        decoder = self._decoder
        if decoder is None:
            self._text = IncrementalDecode(self._tokenizer, self._decode_kwargs)
        elif callable(getattr(decoder, "incremental", None)):
            self._text = decoder.incremental()
        else:
            self._text = IncrementalDecode(None, fallback = decoder.decode)
        self._count = 0
        self.text = ""

    def feed(self, token_id) -> str:
        self._count += 1
        before = self._text.text
        whole = self._text.feed(token_id)
        if self._decoder is not None and token_id in self._stop_ids:
            if not closes_an_open_envelope(self._prefix + whole, whole[len(before) :]):
                whole = before
        self.text = whole
        return whole

    def __call__(self, ids) -> str:
        if len(ids) == self._count + 1:
            return self.feed(ids[-1])
        if len(ids) != self._count:
            self._reset()
            for token_id in ids:
                self.feed(token_id)
        return self.text


def _mlx_stop_cut(text: str, stops) -> tuple[int, bool]:
    """Holds back text that could grow into a stop; a rewriting caller must withhold until turn ends"""
    resolved = text.rstrip("\ufffd")
    cut = len(resolved)
    for sequence in stops:
        found = resolved.find(sequence)
        if found != -1:
            cut = min(cut, found)
    if cut < len(resolved):
        return cut, True
    held = 0
    for sequence in stops:
        for size in range(min(len(sequence) - 1, len(resolved)), held, -1):
            if resolved.endswith(sequence[:size]):
                held = size
                break
    return len(resolved) - held, False


def _mlx_finish_reason(response, stop_ids, generated_n, max_tokens):
    """At the limit, a stop token can look like exhaustion, so the last token's identity decides."""
    reason = getattr(response, "finish_reason", None)
    if reason in ("stop", "length"):
        return reason
    if generated_n < max_tokens:
        return "stop"
    token = getattr(response, "token", None)
    return "stop" if token is not None and token in tuple(stop_ids) else "length"


def _build_generation_stats(
    prompt_n,
    prompt_tps,
    gen_n,
    gen_tps,
    cached_n = 0,
    finish_reason = None,
    drafts = None,
):
    """Map mlx stream stats onto the usage/timings shape llama-server emits, plus the reason
    generation ended. ``drafts`` is a speculative reply's (drafted, accepted) token counts."""
    prompt_n = int(prompt_n or 0)
    gen_n = int(gen_n or 0)
    cached_n = int(cached_n or 0)
    prompt_tps = float(prompt_tps or 0.0)
    gen_tps = float(gen_tps or 0.0)
    prompt_ms = (prompt_n / prompt_tps * 1000.0) if prompt_tps > 0 else 0.0
    predicted_ms = (gen_n / gen_tps * 1000.0) if gen_tps > 0 else 0.0
    total_prompt_n = prompt_n + cached_n
    return {
        "usage": {
            "prompt_tokens": total_prompt_n,
            "completion_tokens": gen_n,
            "total_tokens": total_prompt_n + gen_n,
            # Report cached tokens in the OpenAI field as llama-server does, not only timings.cache_n
            "prompt_tokens_details": {"cached_tokens": cached_n},
        },
        "timings": {
            "prompt_n": prompt_n,
            "prompt_ms": prompt_ms,
            "prompt_per_token_ms": (prompt_ms / prompt_n) if prompt_n > 0 else 0.0,
            "prompt_per_second": prompt_tps,
            "predicted_n": gen_n,
            "predicted_ms": predicted_ms,
            "predicted_per_token_ms": (predicted_ms / gen_n) if gen_n > 0 else 0.0,
            "predicted_per_second": gen_tps,
            "cache_n": cached_n,
            **({} if drafts is None else {"draft_n": drafts[0], "draft_n_accepted": drafts[1]}),
        },
        # Latched at generation exit so a later cancel cannot rewrite the finish reason
        "finish_reason": finish_reason,
    }


PROMPT_CACHE_ENTRIES = 6

# Every bit width mx.quantize supports; unrelated to llama.cpp's cache_type_kv names.
MLX_KV_BITS_CHOICES = (8, 6, 5, 4, 3, 2)
MLX_TURBOQUANT_BITS_CHOICES = (4, 3.5, 3, 2)
# Quantization group size; a head dim that is not a multiple makes mx.quantize raise.
MLX_KV_GROUP_SIZE = 64
MLX_PREFILL_CHUNK_FALLBACK = 2048


def _generation_step(*, vision: bool, drafted: bool):
    """The function that would run this generation's prefill."""
    if vision:
        from mlx_vlm.generate.ar import generate_step as vlm_generate_step
        return vlm_generate_step
    if drafted:
        from mlx_lm.generate import speculative_generate_step
        return speculative_generate_step
    from mlx_lm.generate import generate_step

    return generate_step


def _generation_default(setting: str, fallback: int, *, vision: bool, drafted: bool) -> int:
    """A prefill setting as the runtime that would run generation defaults to it."""
    import inspect
    try:
        found = (
            inspect.signature(_generation_step(vision = vision, drafted = drafted))
            .parameters[setting]
            .default
        )
        if isinstance(found, bool) or not isinstance(found, int) or found <= 0:
            raise ValueError(f"the runtime states no usable {setting}")
        return found
    except Exception as exc:
        logger.debug("Cannot read the runtime's %s: %s", setting, exc)
        return fallback


def mlx_vlm_snapshot_store_available() -> bool:
    """Whether a vision load can build the snapshot store, which decides two things about it."""
    if _prompt_cache_max_bytes() <= 0:
        return False
    try:
        from mlx_vlm.generate import GenerationResult
    except Exception as exc:
        logger.debug("Cannot tell whether mlx-vlm carries the snapshot grid: %s", exc)
        return False
    return hasattr(GenerationResult, "cached_tokens")


def mlx_vlm_pins_the_prefill_step(config) -> bool:
    """Whether EVERY request to this vision load prefills on the pinned grid."""
    if not mlx_vlm_snapshot_store_available():
        return False
    return bool(MLXInferenceBackend._vlm_media_token_ids(config))


def mlx_prefill_chunk(
    *,
    vision: bool = False,
    drafted: bool = False,
    config = None,
) -> int:
    """Most tokens one prefill step takes; the last step of a prompt may be shorter."""
    if vision and mlx_vlm_pins_the_prefill_step(config):
        return vlm_prefill_step()
    return _generation_default(
        "prefill_step_size", MLX_PREFILL_CHUNK_FALLBACK, vision = vision, drafted = drafted
    )


# Surfaced with the resolved setting so an API client sees the reuse cost too.
MLX_KV_QUANT_NO_REUSE = (
    "The installed mlx-lm cannot measure a quantized cache entry, so prompt-cache "
    "reuse across turns is disabled while this is on."
)
# The cap _configure_memory_limits installs, so a fitted context names a load MLX would allocate.
MLX_MEMORY_LIMIT_FRACTION = 0.85


MLX_TURBOQUANT_TEXT_LOAD = (
    "TurboQuant is not available for this model: it could not be loaded through the runtime "
    "TurboQuant requires. The model is served without it."
)


def _turboquant_refusal(*, is_vision, is_distributed, is_lora):
    """Why a text model cannot take the mlx-vlm route TurboQuant lives on, else ""."""
    if is_vision:
        return ""
    if is_distributed:
        return "TurboQuant is not available under distributed inference. Run on one device, or turn it off."
    if is_lora:
        return "TurboQuant is not available for a LoRA adapter on a text model. Merge it first, or turn TurboQuant off."
    return ""


MLX_KV_QUANT_AUTO = "auto"
# TurboQuant's widths repeat mx.quantize's, so the scheme rides in the value: "4" alone is ambiguous.
MLX_TURBOQUANT_PREFIX = "tq-"


def _kv_quant_choice(bits):
    return str(int(bits)) if float(bits).is_integer() else str(float(bits))


MLX_KV_QUANT_CHOICES = (
    (MLX_KV_QUANT_AUTO,)
    + tuple(_kv_quant_choice(bits) for bits in MLX_KV_BITS_CHOICES)
    + tuple(MLX_TURBOQUANT_PREFIX + _kv_quant_choice(bits) for bits in MLX_TURBOQUANT_BITS_CHOICES)
)


def parse_mlx_kv_quant(value):
    """The (bit width, TurboQuant) pair for a choice. Total: an unknown one reads as auto."""

    if value is None:
        return None, False
    text = str(value).strip().lower()
    if text in ("", MLX_KV_QUANT_AUTO):
        return None, False
    if text not in MLX_KV_QUANT_CHOICES:
        logger.warning(
            "MLX kv quantization %r unsupported (choose %s); ignoring",
            value,
            " or ".join(MLX_KV_QUANT_CHOICES),
        )
        return None, False
    turboquant = text.startswith(MLX_TURBOQUANT_PREFIX)
    width = float(text.removeprefix(MLX_TURBOQUANT_PREFIX))
    return (int(width) if width.is_integer() else width), turboquant


def encode_mlx_kv_quant(bits, turboquant = False):
    if bits is None:
        return MLX_KV_QUANT_AUTO
    return (MLX_TURBOQUANT_PREFIX if turboquant else "") + _kv_quant_choice(bits)


def _kv_entry_nbytes(entry):
    """Reads the nbytes property directly, since upstream's cache admission sums that same property."""
    try:
        return int(entry.nbytes)
    except Exception:
        return None


def _normalize_mlx_kv_bits(value):
    """Supported bit width, or None when unset or out of domain."""
    if value is None:
        return None
    try:
        bits = int(value)
    except (TypeError, ValueError):
        logger.warning("MLX kv_bits=%r is not an integer; ignoring", value)
        return None
    if bits not in MLX_KV_BITS_CHOICES:
        logger.warning(
            "MLX kv_bits=%s unsupported (choose %s); ignoring",
            bits,
            " or ".join(str(b) for b in MLX_KV_BITS_CHOICES),
        )
        return None
    return bits


def _mlx_rng_key_words():
    """A key that is not two words is logged before declining, as the rewind cannot work on it."""
    try:
        import mlx.core as mx
        words = mx.random.state[0].tolist()
    except Exception:
        return None
    if len(words) != 2:
        logger.warning(
            "MLX exposes a %d-word random key; Unsloth can only rewind the "
            "two-word form, so sizing a load will not restore the PRNG and "
            "sampling after a load may differ from an unsized run.",
            len(words),
        )
        return None
    return _as_uint32_pair(int(words[0]), int(words[1]))


def _as_uint32_pair(high, low):
    """Range-checked, not masked: masking an out-of-range word would silently restore a wrong key."""
    converted = []
    for word in (high, low):
        if not -(2**31) <= word < 2**32:
            logger.warning(
                "MLX exposed a random key word of %d, which is not a 32-bit "
                "word; sizing a load will not restore the PRNG and sampling "
                "after a load may differ from an unsized run.",
                word,
            )
            return None
        converted.append(word & 0xFFFFFFFF)
    return (converted[0], converted[1])


def _restore_mlx_rng_key(words):
    """Unguarded on purpose: the range check in _as_uint32_pair removes the only way it can raise."""
    if words is None:
        return
    import mlx.core as mx

    pair = _as_uint32_pair(int(words[0]), int(words[1]))
    if pair is None:
        return
    mx.random.seed((pair[0] << 32) | pair[1])


@contextmanager
def mlx_rng_preserved():
    """Leave the PRNG where it was found around weightless builds."""
    rng_key = _mlx_rng_key_words()
    try:
        yield
    finally:
        _restore_mlx_rng_key(rng_key)


def _kv_entry_windowed(entry):
    return any(getattr(entry, name, None) is not None for name in ("max_size", "window_size"))


def _allocate_empty_quantized_exactly(cls):
    """First block at mx.quantize's ``dim * bits // 32``; upstream's ``dim // (32 // bits)`` is a word too wide at 3/5/6 bits."""
    original = cls.update_and_fetch
    if getattr(original, "_unsloth_exact_empty_alloc", False):
        return

    @functools.wraps(original)
    def update_and_fetch(self, keys, values):
        if self.keys is None:
            import mlx.core as mx

            batch, heads, steps, _ = keys.shape
            shape = (batch, heads, (self.step + steps - 1) // self.step * self.step)

            def block(dim):
                return (
                    mx.zeros((*shape, dim * self.bits // 32), dtype = mx.uint32),
                    mx.zeros((*shape, dim // self.group_size), dtype = keys.dtype),
                    mx.zeros((*shape, dim // self.group_size), dtype = keys.dtype),
                )

            self.keys, self.values = block(keys.shape[-1]), block(values.shape[-1])
        return original(self, keys, values)

    update_and_fetch._unsloth_exact_empty_alloc = True
    cls.update_and_fetch = update_and_fetch


def _kv_quant_targets(entries):
    """``(indices, held)``: windowed entries keep their ring; past two entries the last
    full-attention one stays float, as mlx-vlm's should_quantize_kv_layer does."""
    targets = [
        index
        for index, entry in enumerate(entries)
        if getattr(entry, "to_quantized", None) is not None and not _kv_entry_windowed(entry)
    ]
    if len(entries) > 2 and targets:
        targets.pop()
        return targets, 1
    return targets, 0


def _quantize_kv_entries(entries, bits):
    for index in _kv_quant_targets(entries)[0]:
        entry = entries[index]
        entries[index] = entry.to_quantized(group_size = MLX_KV_GROUP_SIZE, bits = bits)
        if getattr(entry, "keys", True) is None:
            _allocate_empty_quantized_exactly(type(entries[index]))
    return entries


def _kv_quant_probe(
    language_model,
    entries,
    bits,
    evaluate = True,
):
    """``(converted, skipped, failure, retainable)``: runs a token through the converted cache,
    since static proxies (declared head_dim, window spelling) proved wrong both ways. Unevaluated,
    the same graph is only built, which is what a weightless tower affords."""
    import mlx.core as mx

    settle = mx.eval if evaluate else (lambda _arrays: None)

    targets, held = _kv_quant_targets(entries)
    skipped = len(entries) - len(targets) - held
    if not targets:
        # Verdict already known, so skip the cost of a full model call.
        return 0, skipped, None, True

    # The forward passes below draw random numbers, so keep sampled output stable.
    rng_key = _mlx_rng_key_words()
    try:
        try:
            language_model(mx.array([[0]]), cache = entries)
            settle([getattr(entry, "state", None) for entry in entries])
        except Exception as exc:
            if is_metal_queue_dead(exc):
                raise
            return 0, 0, f"its cache could not be exercised ({type(exc).__name__})", True
        try:
            _quantize_kv_entries(entries, bits)
            settle([entries[index].state for index in targets])
        except Exception as exc:
            if is_metal_queue_dead(exc):
                raise
            return 0, 0, f"MLX cannot quantize it ({type(exc).__name__})", True
        try:
            language_model(mx.array([[0]]), cache = entries)
            settle([getattr(entry, "state", None) for entry in entries])
        except Exception as exc:
            if is_metal_queue_dead(exc):
                raise
            return 0, 0, f"it cannot attend over a quantized cache ({type(exc).__name__})", True
        # Same helper insertion uses, so the caveat matches what insertion sees.
        retainable = all(_kv_entry_nbytes(entries[index]) is not None for index in targets)
        return len(targets), skipped, None, retainable
    finally:
        _restore_mlx_rng_key(rng_key)


# Bare mx.synchronize() waits on the default stream, which generation does not use
_GENERATION_STREAM_MODULES = (
    "mlx_lm.generate",
    "mlx_vlm.generate",
    "mlx_vlm.generate.dispatch",
    "mlx_vlm.generate.ar",
    "mlx_vlm.speculative.common",
)


def _drain_generation_streams(mx):
    """Best effort: a stream raises when synchronized off its creating thread, so the drain is skipped."""

    synchronize = getattr(mx, "synchronize", None)
    if not callable(synchronize):
        return
    drained = []
    for name in _GENERATION_STREAM_MODULES:
        try:
            stream = getattr(sys.modules.get(name), "generation_stream", None)
            # 0.6.x aliases one object across every mlx_vlm.generate name.
            if stream is None or any(stream is seen for seen in drained):
                continue
            drained.append(stream)
            synchronize(stream)
        except Exception as error:
            logger.debug("MLX stream drain skipped for %s: %s", name, error)
    try:
        synchronize()
    except Exception as error:
        logger.debug("MLX default stream drain skipped: %s", error)


def _kv_quant_eligibility(
    model,
    is_vlm,
    bits = MLX_KV_BITS_CHOICES[0],
):
    """``(verdict, reason, retainable)`` for this model, verdict in full/partial/none/refused."""
    try:
        entries = _make_kv_entries(model, is_vlm)
    except Exception as exc:
        logger.warning("MLX KV quantization eligibility probe failed: %s", exc)
        return "none", "this model's KV cache layout could not be inspected", True

    if not entries:
        return "none", "this model builds no KV cache to quantize", True

    total = len(entries)
    windowed = sum(_kv_entry_windowed(entry) for entry in entries)
    language_model = getattr(model, "language_model", model) if is_vlm else model
    converted, skipped, failure, retainable = _kv_quant_probe(language_model, entries, bits)
    # Released only here, once the probe's own locals are gone, or the pages stay in the allocator.
    import mlx.core as mx

    entries.clear()
    del entries
    _drain_generation_streams(mx)
    mx.clear_cache()
    if failure is not None:
        return "refused", f"this model's KV cache cannot be quantized: {failure}", True
    if not converted:
        if skipped < total:
            reason = (
                "it has a single full-attention layer, and the last full-attention "
                "layer stays unquantized"
            )
        else:
            reason = "this model's KV cache layout cannot be quantized"
        return "none", reason, True
    if not skipped:
        return "full", "", retainable
    if windowed == skipped:
        reason = (
            "its sliding-window layers keep a fixed-size cache, so only the "
            "full-attention layers are quantized"
        )
    else:
        reason = "only some of this model's layers use a quantizable KV cache"
    return "partial", reason, retainable


def _make_kv_entries(
    model,
    is_vlm,
    max_kv_size = None,
):
    language_model = getattr(model, "language_model", model) if is_vlm else model
    if is_vlm:
        from mlx_vlm.models import cache as vlm_cache
        return vlm_cache.make_prompt_cache(language_model, max_kv_size = max_kv_size)
    from mlx_lm.models import cache as lm_cache

    return lm_cache.make_prompt_cache(language_model, max_kv_size = max_kv_size)


# Override replaces, never creates: renderers pick their target by whether a template exists
_TEMPLATE_NOT_CAPTURED = object()
MLX_TEMPLATE_NO_TARGET = (
    "this model builds its prompt without a chat template, and supplying one "
    "would drop the markers that place its images and audio"
)
MLX_TEMPLATE_CALLABLE = (
    "the installed mlx-lm supplies this model's template as code rather than text, "
    "and prefers it over any override"
)
MLX_TEMPLATE_NAMED_SET = (
    "this model ships a set of named templates rather than one, and replacing the "
    "set with a single template would lose its tool-calling variant"
)
MLX_TEMPLATE_RENDER_FAILED = "it could not render a conversation: {error}"
MLX_TEMPLATE_NOT_SETTABLE = "this model's template cannot be replaced: {error}"
MLX_TEMPLATE_DROPS_IMAGE = (
    "it does not mark where images go, so this model could not accept image input with it"
)
MLX_TEMPLATE_DROPS_AUDIO = (
    "it does not mark where audio goes, so this model could not accept audio input with it"
)


def _template_install_targets(tokenizer, processor):
    """Objects whose chat_template would be read at render time. The processor and the tokenizer can
    be the same object when a processor exposes no nested tokenizer, so the result is
    de-duplicated by identity."""
    seen, targets = [], []
    for candidate in (processor, tokenizer):
        if candidate is None:
            continue
        if any(candidate is other for other in seen):
            continue
        seen.append(candidate)
        targets.append(candidate)
    return targets


def _template_render_targets(tokenizer, processor):
    """Objects a render selector would actually read the template from. Deferred to
    chat_render_target rather than restated (#7066): it also requires the processor to be able to
    render, and mlx-vlm's get_chat_template applies the same rule for audio."""
    from core.inference.chat_template_helpers import chat_render_target

    target = chat_render_target(processor, tokenizer)
    return [target] if target is not None else []


def _usable_template(value):
    """Whether a chat_template value is one this override can replace."""
    return isinstance(value, str) and value.strip() != ""


def _native_template_source(tokenizer, processor):
    """The render target's template, so the editor's model default is the one generation really uses."""
    for target in _template_render_targets(tokenizer, processor):
        if _usable_template(getattr(target, "chat_template", None)):
            return target
    return tokenizer


def _template_override_status(override, tokenizer, processor):
    """Resolve a requested override against this model, without applying it. Returns the targets to
    install onto and a status dict; an empty target list with a reason means the override cannot
    be honored."""
    status = {
        "requested": override,
        "applied": None,
        "reason": None,
        # Kept so a later check (the audio marker) can put the model back.
        "restore": [],
    }
    if not override or not override.strip():
        return [], status
    try:
        candidates = _template_install_targets(tokenizer, processor)
        existing = [(c, getattr(c, "chat_template", None)) for c in candidates]
        rendering = _template_render_targets(tokenizer, processor)
    except Exception as exc:
        status["reason"] = MLX_TEMPLATE_NOT_SETTABLE.format(error = exc)
        return [], status
    # Check only rendering objects; an unread object's unreplaceable template cannot block the override.
    blocked = [(c, getattr(c, "chat_template", None)) for c in rendering]
    if any(getattr(c, "_chat_template", None) is not None for c in rendering):
        # apply_chat_template prefers a callable template over the attribute, so an assignment would be inert.
        status["reason"] = MLX_TEMPLATE_CALLABLE
        return [], status
    if any(isinstance(value, (dict, list)) for _, value in blocked):
        status["reason"] = MLX_TEMPLATE_NAMED_SET
        return [], status
    if not any(_usable_template(value) for _, value in blocked):
        # Text models need one; with a processor, creating one bypasses mlx-vlm's marker fallback
        if processor is not None or not blocked:
            status["reason"] = MLX_TEMPLATE_NO_TARGET
            return [], status
        return [c for c, _ in blocked], status
    # Install on every object holding a replaceable string, so both selectors keep choosing what they chose before.
    return [c for c, value in existing if _usable_template(value)], status


def _audio_marker_survives(processor, model):
    """Whether the installed template still marks where audio goes."""
    if processor is None:
        return True
    args = (processor, model, _AUDIO_PROBE_MESSAGES, 0)
    try:
        marked = _render_registered_vlm_prompt(*args, num_audios = 1)
        return bool(marked) and marked != _render_registered_vlm_prompt(*args, num_audios = 0)
    except BaseException:
        # A load must never fail on a probe, matching _classify_mlx_audio_type.
        return False


def _image_placeholder(tokenizer, processor):
    """The model's own image placeholder, when it names one."""
    for source in (processor, tokenizer):
        token = getattr(source, "image_token", None)
        if isinstance(token, str) and token:
            return token
    return None


_IMAGE_TOKEN_ID_KEYS = ("image_token_index", "image_token_id", "boi_token_index")


def _declared_image_tokens(
    tokenizer,
    processor,
    model = None,
):
    """Config token ids too: granite-vision names its token only by id, on a bare tokenizer."""
    named = []

    def add(token):
        if isinstance(token, str) and token and token not in named:
            named.append(token)

    for source in (processor, tokenizer):
        for attr in ("image_token", "boi_token"):
            add(getattr(source, attr, None))
    config, _model_type = _mlx_vlm_model_config(model) if model is not None else (None, None)
    for key in _IMAGE_TOKEN_ID_KEYS:
        token_id = config.get(key) if isinstance(config, dict) else getattr(config, key, None)
        for source in (getattr(processor, "tokenizer", None), tokenizer, processor):
            try:
                add(source.convert_ids_to_tokens(token_id))
            except BaseException:
                continue
            break
    return named


def _scaling_image_marker(
    tokenizer,
    processor,
    model = None,
):
    """Per-image marker text, or None; a load must never fail on this probe, so diff renders not names"""
    from core.inference.chat_template_helpers import apply_chat_template_for_generation

    targets = _template_render_targets(tokenizer, processor)
    if not targets:
        return None
    try:
        one, two = (
            apply_chat_template_for_generation(targets[0], probe)
            for probe in _MULTI_IMAGE_PROBE_MESSAGES
        )
        if not isinstance(one, str) or not isinstance(two, str):
            return None
        if _MULTI_IMAGE_PROBE_TEXT not in one or _MULTI_IMAGE_PROBE_TEXT not in two:
            return None
        opcodes = difflib.SequenceMatcher(None, one, two, autojunk = False).get_opcodes()
        inserted = [two[j1:j2] for tag, _i1, _i2, j1, j2 in opcodes if tag == "insert"]
        if len(inserted) != 1 or any(tag in ("replace", "delete") for tag, *_rest in opcodes):
            return None
        marker = inserted[0]
        if one.count(marker) != 1 or two.count(marker) != 2:
            return None
        named = _declared_image_tokens(tokenizer, processor, model)
        if any(token in marker for token in named):
            return marker
    except BaseException:
        return None
    return None


def _model_type_takes_several_images(model):
    """Studio renders the model's own template, so mlx-vlm's deny-list has to be read here."""
    try:
        from mlx_vlm.prompt_utils import SINGLE_IMAGE_ONLY_MODELS
    except BaseException:
        return False
    _config, model_type = _mlx_vlm_model_config(model)
    return bool(model_type) and model_type not in SINGLE_IMAGE_ONLY_MODELS


def _image_marker_survives(
    tokenizer,
    processor,
    placeholder = None,
):
    """Fails when an image renders the same as none, emits the content object, or drops the placeholder."""
    from core.inference.chat_template_helpers import apply_chat_template_for_generation

    targets = _template_render_targets(tokenizer, processor)
    if not targets:
        return True
    target = targets[0]
    try:
        marked = apply_chat_template_for_generation(target, _IMAGE_PROBE_MESSAGES)
        if not marked or marked == apply_chat_template_for_generation(target, _TEXT_PROBE_MESSAGES):
            return False
        if _vlm_prompt_issue(marked, _IMAGE_PROBE_MESSAGES):
            return False
        return placeholder is None or placeholder in marked
    except BaseException:
        # A load must never fail on a probe, matching _audio_marker_survives.
        return False


def _revoke_override_dropping(status, survives, reason):
    """Undo an installed override that stopped marking where media goes. Capability was classified
    against the native template, so an override that no longer renders the marker would leave the
    model advertising an input it can no longer place."""
    if not status["applied"] or survives():
        return status
    _restore_templates(status["restore"])
    status["applied"] = None
    status["restore"] = []
    status["reason"] = reason
    return status


def _revoke_override_that_drops_audio(status, processor, model):
    return _revoke_override_dropping(
        status,
        lambda: _audio_marker_survives(processor, model),
        MLX_TEMPLATE_DROPS_AUDIO,
    )


def _revoke_override_that_drops_image(
    status,
    tokenizer,
    processor,
    placeholder = None,
):
    return _revoke_override_dropping(
        status,
        lambda: _image_marker_survives(tokenizer, processor, placeholder),
        MLX_TEMPLATE_DROPS_IMAGE,
    )


def _restore_templates(installed):
    """Put back the templates an install replaced, newest first."""
    for target, value in reversed(installed):
        target.chat_template = value


def _install_template_override(override, tokenizer, processor, probe):
    """Install a chat template override, or report why it was not honored. ``probe`` renders a short
    conversation with whatever is installed; a template that cannot render would otherwise raise
    on every generation instead of once here, which is how a hand-edited template usually fails."""
    targets, status = _template_override_status(override, tokenizer, processor)
    if not targets:
        return status
    # Restore only what was assigned: a refused target would also refuse the restore
    installed = []
    template = MLX_TEMPLATE_RENDER_FAILED
    try:
        for target in targets:
            original = target.chat_template
            template = MLX_TEMPLATE_NOT_SETTABLE
            target.chat_template = override
            template = MLX_TEMPLATE_RENDER_FAILED
            installed.append((target, original))
        probe()
    except Exception as exc:
        _restore_templates(installed)
        status["reason"] = template.format(error = exc)
        return status
    status["applied"] = override
    status["restore"] = installed
    return status


def _kv_entry_is_bounded(entry, window):
    """Only a declared cap within the window counts; an undeclared cache grows without bound."""
    cap = getattr(entry, "max_size", None) or getattr(entry, "chunk_size", None)
    if not cap or cap > window:
        return False
    # Combined stdout+stderr, or None when unknown (exec, timeout, nonzero exit); None is never evidence a feature is absent, so callers stay conservative.
    return cap > getattr(entry, "keep", 0)


def mlx_memory_budget(*, retains_history: bool = True) -> Optional[int]:
    """Bytes a load's own footprint may occupy here, or None where Metal cannot say."""
    import mlx.core as mx

    if not mx.metal.is_available():
        return None
    recommended = mx.device_info().get("max_recommended_working_set_size")
    if not recommended or recommended <= 0:
        return None
    budget = int(recommended * MLX_MEMORY_LIMIT_FRACTION)
    if retains_history:
        budget -= _prompt_cache_max_bytes(recommended / 1e9)
    return budget if budget > 0 else None


def _snapshot_dir(model, model_name: str) -> Optional[str]:
    """Where the files this load actually opened are, or None if none are local."""
    resolved = getattr(model, "_src_path", None)
    if resolved and os.path.isdir(str(resolved)):
        return str(resolved)
    if os.path.isdir(model_name):
        return model_name
    from utils.utils import hf_cache_snapshot_dir

    snapshot = hf_cache_snapshot_dir(model_name)
    return str(snapshot) if snapshot is not None else None


def mlx_fit_to_memory(
    model_dir,
    ceiling,
    *,
    load_in_4bit: bool,
    retains_history: bool,
    kv_bits = None,
    **route,
) -> Optional[int]:
    """The context a load of *model_dir* is held to, priced at the KV width its cache takes.
    ``vision=True`` in *route* prices the mlx-vlm route a speculative text load takes."""
    budget = mlx_memory_budget(retains_history = retains_history)
    if budget is None or not ceiling or not model_dir:
        return None
    with mlx_rng_preserved():
        try:
            from core.inference.mlx_memory import mlx_fit_context

            fitted = mlx_fit_context(
                model_dir,
                budget_bytes = budget,
                max_ctx = int(ceiling),
                kv_bits = kv_bits,
                load_in_4bit = load_in_4bit,
                **route,
            )
            logger.debug("MLX fit for %s: %s tokens under %.1f GB", model_dir, fitted, budget / 1e9)
            return fitted
        except Exception as exc:
            logger.debug("MLX context fit unavailable for %s: %s", model_dir, exc)
            return None


def _fitted_context(
    model,
    model_name: str,
    ceiling,
    *,
    load_in_4bit: bool,
    retains_history: bool,
    kv_bits = None,
    is_vlm = False,
    eligibility = None,
    **route,
):
    """The fit for a resident model and the eligibility verdict it was priced under."""
    try:
        model_dir = _snapshot_dir(model, model_name)
    except Exception as exc:
        logger.debug("MLX snapshot unavailable for %s: %s", model_name, exc)
        return None, None
    verdict = eligibility
    # TurboQuant is converted by the estimator's own probe, as mlx-vlm does.
    if verdict is None and isinstance(kv_bits, int):
        verdict = _kv_quant_eligibility(model, is_vlm, kv_bits)
    fitted = mlx_fit_to_memory(
        model_dir,
        ceiling,
        load_in_4bit = load_in_4bit,
        retains_history = retains_history,
        kv_bits = kv_bits if verdict is None or verdict[0] in ("full", "partial") else None,
        **route,
    )
    return fitted, verdict


def _fit_rank(outcome, context) -> tuple:
    # An unpriceable baseline outranks every fit, so nothing is costless against it.
    return {"no_fit": 0, "fitted": 1, "fits": 2}.get(outcome, 3), context or 0


def mlx_drafter_fit(
    model_dir,
    ceiling,
    pinned,
    source,
    *,
    load_in_4bit: bool,
    costless = False,
    offload = None,
    prefill_chunk = None,
    baseline = None,
    reserve = True,
    kv_bits = None,
):
    """``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:
    only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit
    at least the context the load fits without it, on the *baseline* route (``mlx_offload_route``)
    when the offload made room for it. ``attaches`` is None when the fit could not be priced.
    ``reserve=False`` prices against MLX's own cap, spending the prompt cache's share."""
    budget = mlx_memory_budget(retains_history = reserve and mlx_vlm_snapshot_store_available())
    ceiling = pinned or ceiling
    if budget is None or not ceiling or not model_dir:
        return None, None
    with mlx_rng_preserved():
        try:
            from core.inference.mlx_memory import mlx_fit_outcome

            if kv_bits is not None and mlx_kv_quant_is_refused(model_dir, kv_bits):
                kv_bits = None
            fit = functools.partial(
                mlx_fit_outcome,
                model_dir,
                budget_bytes = budget,
                max_ctx = int(ceiling),
                load_in_4bit = load_in_4bit,
                vision = True,
                offload = offload,
                prefill_chunk = prefill_chunk,
                kv_bits = kv_bits,
            )
            outcome, fitted = fit(drafter = (source.path, source.builtin))
            if outcome == "unsizable":
                return None, fitted
            if costless and _fit_rank(outcome, fitted) < _fit_rank(*fit(**(baseline or {}))):
                return False, fitted
        except Exception as exc:
            logger.info("MLX drafter %s not priced: %s", source.path, exc)
            return None, None
    return outcome == "fits" or (outcome == "fitted" and not pinned), fitted


def mlx_prompt_cache_room(
    model_dir,
    n_ctx,
    source,
    *,
    load_in_4bit: bool,
    offload = None,
    prefill_chunk = None,
    kv_bits = None,
):
    """Bytes MLX's own cap leaves the prompt cache beside *source*'s drafter at *n_ctx*; None where unsized."""
    budget = mlx_memory_budget(retains_history = False)
    if budget is None or not n_ctx or not model_dir:
        return None
    with mlx_rng_preserved():
        try:
            from core.inference.mlx_memory import mlx_headroom
            return mlx_headroom(
                model_dir,
                budget_bytes = budget,
                n_ctx = int(n_ctx),
                load_in_4bit = load_in_4bit,
                vision = True,
                drafter = (source.path, source.builtin),
                offload = offload,
                prefill_chunk = prefill_chunk,
                kv_bits = kv_bits,
            )
        except Exception as exc:
            logger.info("MLX prompt cache room beside %s not priced: %s", source.path, exc)
            return None


def _kv_window_enforced(model, is_vlm, window):
    """Whether a cache built at *window* is bounded in every layer, None where none could be built
    to judge: both runtimes defer to a model's own make_cache and ignore max_kv_size there."""
    try:
        entries = _make_kv_entries(model, is_vlm, max_kv_size = window)
        # Inside the guard: load_model calls this unguarded, so a raise would fail the load.
        flattened = list(_flatten_kv_entries(entries))
        return bool(flattened) and all(_kv_entry_is_bounded(entry, window) for entry in flattened)
    except Exception as exc:
        logger.debug("MLX context limit probe failed: %s", exc)
        return None


def _asking_the_architecture(model_dir, question):
    """Put *question* to the tower the sizing would price, built weightless from the checkpoint."""
    with mlx_rng_preserved():
        try:
            import mlx.core as mx

            from core.inference.mlx_memory import (
                _PROBE_SHORT,
                _probe_models,
                _runtime_dtype,
                _snapshot_config,
                weightless_forward,
            )

            config = _snapshot_config(model_dir)
            if config is None:
                return None
            with weightless_forward():
                for build, make_prompt_cache, _ in _probe_models(config, _runtime_dtype()):
                    model = build()
                    try:
                        model(
                            mx.zeros((1, _PROBE_SHORT), dtype = mx.int32),
                            cache = make_prompt_cache(model),
                        )
                    except Exception:
                        continue
                    return question(model, make_prompt_cache)
        except Exception as exc:
            logger.debug("MLX architecture probe unavailable for %s: %s", model_dir, exc)
        return None


# Slider steps resend the width, and each answer is a weightless build.
_kv_refusal_cache: dict = {}
_kv_refusal_lock = threading.Lock()


def mlx_kv_quant_is_refused(model_dir, bits) -> bool:
    """Whether a load would keep this checkpoint's cache at full width despite *bits*, by the
    eligibility probe's own rule (nothing to convert, or a converted cache the model cannot
    attend over)."""

    def refused(model, make_prompt_cache):
        converted, _skipped, failure, _retainable = _kv_quant_probe(
            model, make_prompt_cache(model), bits, evaluate = False
        )
        return failure is not None or not converted

    try:
        from core.inference.mlx_memory import _checkpoint_fingerprint
        key = (os.path.realpath(model_dir), bits, _checkpoint_fingerprint(model_dir))
    except Exception:
        return _asking_the_architecture(model_dir, refused) is True
    with _kv_refusal_lock:
        if key in _kv_refusal_cache:
            return _kv_refusal_cache[key]
    answer = _asking_the_architecture(model_dir, refused) is True
    with _kv_refusal_lock:
        _kv_refusal_cache[key] = answer
        while len(_kv_refusal_cache) > 32:
            _kv_refusal_cache.pop(next(iter(_kv_refusal_cache)))
    return answer


def _turboquant_reserved_positions(count, bits):
    """Entry positions mlx-vlm reserves, learned from a control cache rather than a fixed number."""

    from mlx_vlm.generate.common import maybe_quantize_kv_cache
    from mlx_vlm.models.cache import KVCache
    from mlx_vlm.turboquant import TurboQuantKVCache

    control = [KVCache() for _ in range(count)]
    maybe_quantize_kv_cache(control, 0, MLX_KV_GROUP_SIZE, bits, kv_quant_scheme = "turboquant")
    return {
        index for index, entry in enumerate(control) if not isinstance(entry, TurboQuantKVCache)
    }


def _turboquant_status(
    model,
    bits,
    refusal = "",
):
    import mlx.core as mx

    status = dict(requested_kv_bits = bits, kv_bits = None, eligibility = "none", reason = "", note = "")
    if refusal:
        status["eligibility"] = "refused"
        status["reason"] = refusal
        logger.info("MLX TurboQuant not applied: %s", refusal)
        return status

    try:
        from mlx_vlm.generate.common import maybe_quantize_kv_cache
    except ImportError:
        # TurboQuant predates generate.common (mlx-vlm 0.6.0).
        status["eligibility"] = "refused"
        status["reason"] = "TurboQuant requires a newer mlx-vlm. Update the MLX packages and retry."
        return status
    from mlx_vlm.models.cache import make_prompt_cache
    from mlx_vlm.turboquant import TurboQuantKVCache

    language_model = getattr(model, "language_model", model)
    try:
        entries = make_prompt_cache(language_model)
    except Exception as exc:
        if is_metal_queue_dead(exc):
            raise
        logger.warning("MLX TurboQuant eligibility probe failed: %s", exc)
        status["reason"] = "this model's KV cache layout could not be inspected"
        return status
    if not entries:
        status["reason"] = "this model builds no KV cache to quantize"
        return status

    rng_key = _mlx_rng_key_words()
    try:
        reserved = _turboquant_reserved_positions(len(entries), bits)
        maybe_quantize_kv_cache(entries, 0, MLX_KV_GROUP_SIZE, bits, kv_quant_scheme = "turboquant")
        converted = declined = 0
        for index, entry in enumerate(entries):
            leaves = list(_flatten_kv_entries([entry]))
            quantized = sum(isinstance(leaf, TurboQuantKVCache) for leaf in leaves)
            converted += quantized
            if index not in reserved:
                declined += len(leaves) - quantized
        if not converted:
            status["reason"] = (
                "This model's cache keeps its native format; no layers can use TurboQuant."
            )
            return status
        for _ in range(2):
            result = language_model(mx.array([[0]]), cache = entries)
            mx.eval(getattr(result, "logits", result), [entry.state for entry in entries])
        status["kv_bits"] = bits
        status["eligibility"] = "partial" if declined else "full"
        if declined:
            status["note"] = "Some layers keep their native cache format."
        return status
    except Exception as exc:
        if is_metal_queue_dead(exc):
            raise
        logger.warning("MLX TurboQuant eligibility probe failed: %s", exc)
        status["eligibility"] = "refused"
        status["reason"] = f"this model's KV cache cannot use TurboQuant ({type(exc).__name__})"
        return status
    finally:
        _restore_mlx_rng_key(rng_key)
        # After the probe's own locals are gone, or the pages stay in the allocator.
        entries.clear()
        del entries
        _drain_generation_streams(mx)
        mx.clear_cache()


def _kv_quant_status(
    requested_bits,
    model,
    is_vlm,
    eligibility = None,
):
    """Resolve a requested bit width against this model into a status dict."""
    status = {
        "requested_kv_bits": requested_bits,
        "kv_bits": None,
        "eligibility": None,
        "reason": "",
        "note": "",
    }
    if requested_bits is None:
        return status
    verdict, reason, retainable = eligibility or _kv_quant_eligibility(
        model, is_vlm, requested_bits
    )
    status["eligibility"] = verdict
    status["reason"] = reason
    if verdict in ("full", "partial"):
        status["kv_bits"] = requested_bits
        status["note"] = "" if retainable else MLX_KV_QUANT_NO_REUSE
    else:
        logger.info("MLX KV quantization not applied: %s", reason)
    return status


PROMPT_CACHE_MEMORY_FRACTION = 0.15
PROMPT_CACHE_FALLBACK_BYTES = 2 * 1024**3


def _mlx_prompt_cache_api():
    try:
        from mlx_lm.models.cache import (
            LRUPromptCache,
            can_trim_prompt_cache,
            make_prompt_cache,
            trim_prompt_cache,
        )
    except ImportError:
        return None
    return LRUPromptCache, make_prompt_cache, can_trim_prompt_cache, trim_prompt_cache


def _prompt_cache_max_bytes(recommended_gb = None):
    override = os.environ.get("UNSLOTH_MLX_PROMPT_CACHE_BYTES")
    if override:
        try:
            return max(int(override), 0)
        except ValueError:
            logger.warning("Ignoring non-integer UNSLOTH_MLX_PROMPT_CACHE_BYTES=%r", override)
    if recommended_gb:
        return int(recommended_gb * 1e9 * PROMPT_CACHE_MEMORY_FRACTION)
    return PROMPT_CACHE_FALLBACK_BYTES


def _flatten_kv_entries(cache):
    for entry in cache:
        nested = getattr(entry, "caches", None)
        if nested is None:
            yield entry
        else:
            yield from _flatten_kv_entries(nested)


def _kv_prefix_coverage(cache):
    """Recurrent entries carry no offset, so an attention sibling must attest the token count."""
    covered = None
    for entry in _flatten_kv_entries(cache):
        offset = getattr(entry, "offset", None)
        if offset is None:
            # Upstream trims once all entries agree, so only unrewindable counts.
            if getattr(entry, "is_trimmable", lambda: False)():
                return None
            continue
        if getattr(entry, "start_position", 0):
            return None
        window = getattr(entry, "max_size", None)
        if window is not None and offset > window:
            return None
        if covered is None:
            covered = offset
        elif covered != offset:
            return None
    return covered


class _MLXPromptCacheHistory:
    def __init__(
        self,
        max_entries,
        max_bytes,
        max_kv_size = None,
        prepare = None,
    ):
        api = _mlx_prompt_cache_api()
        if api is None:
            raise RuntimeError("mlx-lm is too old for LRUPromptCache")
        lru_cls, make, can_trim, trim = api
        self._max_kv_size = max_kv_size
        self._prepare = prepare
        self._make_prompt_cache = make
        self._can_trim = can_trim
        self._trim = trim
        self._max_bytes = max_bytes
        self._lru = lru_cls(max_size = max_entries, max_bytes = max_bytes)
        self._oversized = None
        self._newest = None

    def _take_oversized(self, key, tokens):
        held, self._oversized = self._oversized, None
        if held is None:
            return None
        held_key, held_tokens, cache, nbytes = held
        if held_key != key:
            return None
        common = next(
            (i for i, (a, b) in enumerate(zip(held_tokens, tokens)) if a != b),
            min(len(held_tokens), len(tokens)),
        )
        prefix = min(common, len(tokens) - 1)
        # A prefix within budget is the LRU's to serve, so chats under it behave as before.
        if prefix * nbytes <= self._max_bytes * len(held_tokens):
            return None
        excess = len(held_tokens) - prefix
        if excess and not (self._can_trim(cache) and self._trim(cache, excess) == excess):
            return None
        return cache, list(tokens[prefix:])

    def release_oversized(self):
        self._oversized = None

    def fetch(self, model, key, tokens):
        held = self._take_oversized(key, tokens)
        if held is not None:
            return held
        cache, rest = self._lru.fetch_nearest_cache(key, list(tokens))
        if cache is not None:
            if rest:
                return cache, list(rest)
            if self._can_trim(cache) and self._trim(cache, 1) == 1:
                return cache, list(tokens[-1:])
        if len(tokens) > 1:
            head = list(tokens[:-1])
            cache, rest = self._lru.fetch_nearest_cache(key, head)
            if cache is not None:
                covered = len(head) - len(rest)
                return cache, list(tokens[covered:])
        if self._max_kv_size is None:
            cache = self._make_prompt_cache(model)
        else:
            cache = self._make_prompt_cache(model, max_kv_size = self._max_kv_size)
        return cache if self._prepare is None else self._prepare(cache), list(tokens)

    def insert(self, key, tokens, cache):
        sizes = [_kv_entry_nbytes(entry) for entry in cache]
        if any(size is None for size in sizes):
            logger.debug("MLX prompt cache: skipping state whose size is unmeasurable")
            return
        nbytes = sum(sizes)
        covered = _kv_prefix_coverage(cache)
        if covered is None:
            logger.debug("MLX prompt cache: skipping cache with unverifiable prefix coverage")
            return
        tokens = list(tokens)
        if covered > len(tokens):
            logger.debug(
                "MLX prompt cache: cache covers %d tokens but only %d were tracked",
                covered,
                len(tokens),
            )
            return
        tokens = tokens[:covered]
        if not tokens:
            return
        self._newest = (key, tokens)
        if nbytes > self._max_bytes:
            # In the LRU it would evict itself and every other conversation.
            logger.debug(
                "MLX prompt cache: holding %.2f GB entry over the %.2f GB budget for its next turn",
                nbytes / 1e9,
                self._max_bytes / 1e9,
            )
            self._oversized = (key, tokens, cache, nbytes)
            return
        self._lru.insert_cache(key, tokens, cache)

    def resume_entries(self):
        if self._newest is None:
            return []
        key, tokens = self._newest
        held = self._oversized
        if held is not None and held[0] == key and held[1] == tokens:
            return [(key, tokens, held[2], False)]
        try:
            # The trie's own entry: fetch_nearest_cache would deep-copy it.
            return [(key, tokens, self._lru._trie.get(key, tokens).prompt_cache, False)]
        except (AttributeError, KeyError):
            return []


def _resume_save_max_bytes():
    # Mirrors core.inference.llama_cpp._SLOT_SAVE_MAX_BYTES.
    try:
        return int(os.environ.get("UNSLOTH_SLOT_SAVE_MAX_BYTES") or (10 << 30))
    except ValueError:
        return 10 << 30


_RESUME_CACHE_MODULES = ("mlx.", "mlx_lm.", "mlx_vlm.", "unsloth_zoo.mlx.")


def save_resume_cache(path, key, tokens, entries):
    """Arrays as safetensors tensors, the cache layout pickled in metadata: any cache class fits."""
    import base64
    import io
    import pickle

    import mlx.core as mx

    arrays = {}

    class _Pickler(pickle.Pickler):
        def persistent_id(self, obj):
            if isinstance(obj, mx.array):
                name = f"a{len(arrays)}"
                arrays[name] = obj
                return name
            return None

    layout = io.BytesIO()
    _Pickler(layout, protocol = pickle.HIGHEST_PROTOCOL).dump((key, list(tokens), entries))
    # Renamed into place, so a reader never sees a partial file and a removed dir fails the write.
    partial = f"{path}.partial.safetensors"
    mx.save_safetensors(
        partial, arrays, {"layout": base64.b64encode(layout.getvalue()).decode("ascii")}
    )
    os.replace(partial, path)


def load_resume_cache(path):
    """``(key, tokens, entries)`` from ``save_resume_cache``; refuses any non-MLX class."""
    import base64
    import io
    import pickle

    import mlx.core as mx

    arrays, metadata = mx.load(path, return_metadata = True)

    class _Unpickler(pickle.Unpickler):
        def persistent_load(self, name):
            return arrays[name]

        def find_class(self, module, name):
            found = None
            if module.startswith(_RESUME_CACHE_MODULES) and "." not in name:
                found = super().find_class(module, name)
            if not isinstance(found, type):
                raise pickle.UnpicklingError(f"{module}.{name} is not a cache class")
            return found

    key, tokens, entries = _Unpickler(io.BytesIO(base64.b64decode(metadata["layout"]))).load()
    # Read now: the caller deletes the file once the restore returns.
    mx.eval(list(arrays.values()))
    return key, tokens, entries


def _mlx_distributed_rank_size(group = None):
    if group is None:
        return 0, 1
    rank = int(group.rank())
    world_size = int(group.size())
    if world_size < 1:
        raise ValueError(f"Invalid MLX distributed world_size={world_size}.")
    if rank < 0 or rank >= world_size:
        raise ValueError(f"Invalid MLX distributed rank={rank} for world_size={world_size}.")
    return rank, world_size


def _mlx_distributed_backend_from_env():
    if os.environ.get("MLX_JACCL_COORDINATOR") and os.environ.get("MLX_IBV_DEVICES"):
        return "jaccl"
    return None


def _init_mlx_distributed():
    """Initialize MLX distributed state, falling back to singleton metadata."""
    import mlx.core as mx

    group = None
    rank = 0
    world_size = 1
    distributed = getattr(mx, "distributed", None)
    init = getattr(distributed, "init", None) if distributed is not None else None
    if callable(init):
        backend = _mlx_distributed_backend_from_env()
        if backend is None:
            group = init()
        else:
            try:
                group = init(backend = backend)
            except TypeError:
                group = init()
        if group is not None:
            rank, world_size = _mlx_distributed_rank_size(group)
    return group, rank, world_size


def _normalize_mlx_seed(seed):
    """Reduces seeds modulo 2**64: mx.random.key raises on negatives and anything at or above 2**64."""
    return int(seed) % (2**64)


def _make_seeded_mlx_sampler(
    seed,
    *,
    temp,
    top_p,
    min_p,
    top_k,
    min_tokens_to_keep = 1,
):
    """Uses a request-scoped key, since mx.random.seed state would leak into later unseeded requests."""
    import mlx.core as mx
    from mlx_lm.sample_utils import apply_top_p, apply_min_p, apply_top_k

    if temp == 0:
        # argmax draws no randomness; seeding it would be meaningless, not wrong.
        return lambda logprobs: mx.argmax(logprobs, axis = -1)

    stages = []
    if 0 < top_p < 1.0:
        stages.append(lambda x: apply_top_p(x, top_p))
    if min_p != 0.0:
        stages.append(lambda x: apply_min_p(x, min_p, min_tokens_to_keep))
    if top_k > 0:
        stages.append(lambda x: apply_top_k(x, top_k))

    state = {"key": mx.random.key(_normalize_mlx_seed(seed))}

    def _sampler(logprobs):
        for stage in stages:
            logprobs = stage(logprobs)
        state["key"], subkey = mx.random.split(state["key"])
        return mx.random.categorical(logprobs * (1 / temp), key = subkey)

    return _sampler


def _penalized_tokens(tokens, state, window):
    """The tokens a penalty scores: the whole completion, or with ``window`` the last ``window``
    tokens of prompt and completion, as llama-server's ``repeat_last_n`` scores them."""
    if window is not None:
        return tokens[-window:]
    if state["prompt_len"] is None:
        # First call is prompt-only; latch its length.
        state["prompt_len"] = int(tokens.shape[0])
        return tokens[:0]
    return tokens[state["prompt_len"] :]


def _make_mlx_presence_penalty_processor(penalty: float, window = None):
    """Presence penalty as an mlx_lm/mlx_vlm logits processor, matching the safetensors path.
    generate_step calls processors as ``fn(tokens, logits)`` with ``tokens`` the full running
    sequence; the first call is prompt-only, so latch that length and penalize only after it."""
    state = {"prompt_len": None}

    def _processor(tokens, logits):
        generated = _penalized_tokens(tokens, state, window)
        if generated.size == 0:
            return logits
        import mlx.core as mx

        vocab = logits.shape[-1]
        # Bound ids to [0, vocab): MLX has no bounds checks (UB on OOB); strays go to scratch slot vocab
        valid = (generated >= 0) & (generated < vocab)
        safe = mx.where(valid, generated, vocab).astype(mx.int32)
        # (vocab + 1)-wide mask: duplicates idempotent, scratch column dropped
        mask = mx.zeros((vocab + 1,), dtype = logits.dtype)
        mask[safe] = penalty
        logits = logits - mask[:vocab]
        return logits

    _processor.history_only = True
    return _processor


def _row_logits_processors(processors):
    """A reply's processors, each starting at the prompt's last token."""
    if not processors:
        return []

    def _shown(processor):
        state = {"offset": None}

        def _narrowed(tokens, logits):
            if state["offset"] is None:
                state["offset"] = max(int(tokens.shape[0]) - 1, 0)
            return processor(tokens[state["offset"] :], logits)

        return _narrowed

    return [_shown(processor) for processor in processors]


def _make_mlx_frequency_penalty_processor(penalty: float, window = None):
    """Frequency penalty as an mlx_lm/mlx_vlm logits processor. Identical to the presence processor
    except the scatter *accumulates*, so a token repeated N times in the completion is charged N
    x penalty. It counts occurrences and scales once, in float32: accumulating the penalty itself
    in a float16 logits dtype rounds on every repeat, which drifts by tens of logits over a long
    run (1000 repeats at 0.3 lands on -274.25, not -300)."""
    state = {"prompt_len": None}

    def _processor(tokens, logits):
        generated = _penalized_tokens(tokens, state, window)
        if generated.size == 0:
            return logits
        import mlx.core as mx

        vocab = logits.shape[-1]
        valid = (generated >= 0) & (generated < vocab)
        safe = mx.where(valid, generated, vocab).astype(mx.int32)
        counts = mx.zeros((vocab + 1,), dtype = mx.float32).at[safe].add(1.0)
        return logits - (penalty * counts[:vocab]).astype(logits.dtype)

    _processor.history_only = True
    return _processor


def _make_mlx_logit_bias_processor(logit_bias: dict):
    """Routes out-of-range ids to a scratch slot, since MLX does no bounds checking on logit indices."""
    state = {"safe": None, "values": None, "vocab": None}

    def _processor(tokens, logits):
        import mlx.core as mx

        vocab = logits.shape[-1]
        if state["vocab"] != vocab:
            pairs = [(int(t), float(v)) for t, v in logit_bias.items()]
            state["safe"] = mx.array(
                [t if 0 <= t < vocab else vocab for t, _ in pairs], dtype = mx.int32
            )
            state["values"] = mx.array([v for _, v in pairs], dtype = mx.float32)
            state["vocab"] = vocab
        mask = mx.zeros((vocab + 1,), dtype = mx.float32).at[state["safe"]].add(state["values"])
        return logits + mask[:vocab].astype(logits.dtype)

    _processor.history_only = True
    return _processor


_DIFFUSION_RESPONSE_FORMAT = (
    "response_format is not supported on this model: it generates by "
    "diffusion rather than one token at a time, so no grammar can "
    "constrain the next token. Load an autoregressive model to use "
    "guided decoding."
)


def _vlm_generation_is_diffusion(model) -> bool:
    try:
        from mlx_vlm.generate.diffusion import is_diffusion_model
    except Exception:
        return False
    try:
        return bool(is_diffusion_model(model))
    except Exception:
        return False


def _build_grammar_constraint(
    response_format,
    tokenizer,
    prompt,
    *,
    reasoning_markers = None,
    tools = None,
    reasoning_is_extracted = False,
    reply_keeps_special_tokens = False,
):
    if response_format is None or response_format == {"type": "text"}:
        return None
    from core.inference.grammar_constraint import build_constraint

    constraint = build_constraint(
        response_format,
        tokenizer,
        prompt,
        reasoning_markers = reasoning_markers,
        tools = tools,
        reasoning_is_extracted = reasoning_is_extracted,
        reply_keeps_special_tokens = reply_keeps_special_tokens,
    )
    if constraint is not None:
        logger.info("Guided decoding active: response_format=%s", response_format.get("type"))
    return constraint


def _raw_prompt_constraint(grammar, tokenizer):
    if grammar is None:
        return None
    from core.inference.grammar_constraint import GrammarConstraint

    # Raw replies decode with skip_special_tokens, so a special id spelling a literal would vanish.
    return GrammarConstraint(grammar, tokenizer, strips_special_tokens = True)


class _PrefillStopped(Exception):
    pass


def _stream_until_cancelled(stream_generate, cancel_event, *args, **kwargs):
    def check(*_):
        if cancel_event is not None and cancel_event.is_set():
            raise _PrefillStopped

    try:
        yield from stream_generate(*args, prompt_progress_callback = check, **kwargs)
    except _PrefillStopped:
        return


@contextmanager
def _slots_until_cancelled(model, cancel_event):
    """mlx-vlm's ``generate_audio`` takes no stop hook; LFM3 samples through ``stream_slots``, so end that loop."""
    slots = getattr(model, "stream_slots", None)
    if cancel_event is None or slots is None:
        yield
        return

    def until_cancelled(*args, **kwargs):
        for slot in slots(*args, **kwargs):
            if cancel_event.is_set():
                return
            yield slot

    model.stream_slots = until_cancelled
    try:
        yield
    finally:
        del model.stream_slots


class LogprobsUnsupported(ValueError):
    """``n_probs`` asked of a model that does not sample one token per step."""

    public = True
    openai_param = "logprobs"


def _make_mlx_raw_logprob_recorder():
    """A first logits processor queueing each step's log-softmax of the model's own logits, and
    the queue. llama-server reports ``n_probs`` from the logits before any sampler, so a token
    forced by ``logit_bias`` keeps its real logprob; mlx_lm and mlx_vlm compute theirs after the
    processors. Neither runtime calls processors during prefill, so entry i is generated token i."""
    from collections import deque

    import mlx.core as mx

    recorded = deque()

    def _processor(tokens, logits):
        raw = logits.astype(mx.float32)
        recorded.append(raw - mx.logsumexp(raw, axis = -1, keepdims = True))
        return logits

    return _processor, recorded


def _token_bytes_text(tokenizer, token_id):
    kw = dict(skip_special_tokens = False, clean_up_tokenization_spaces = False)
    # SentencePiece decoders strip a sequence's leading space, so "▁the" alone decodes to
    # "the"; after a copy of itself it keeps the " the" llama-server reports.
    alone = tokenizer.decode([token_id], **kw)
    twice = tokenizer.decode([token_id, token_id], **kw)
    text = twice[len(alone) :] if twice.startswith(alone) else alone
    return text, list(text.encode("utf-8"))


def _mlx_token_logprob(raw, token_id, n_probs, tokenizer) -> dict:
    """One llama-server ``logprobs.content`` entry: the sampled token's logprob and the
    ``n_probs`` most likely tokens, all read from the unprocessed logits."""
    import mlx.core as mx

    raw = raw.reshape(-1)
    n = min(int(n_probs), raw.shape[0])
    top = mx.argpartition(-raw, kth = n - 1)[:n]
    values = raw[top]
    order = mx.argsort(-values)
    # llama-server writes the lowest float for -inf, which JSON cannot carry.
    floor = -3.4028234663852886e38
    entry_ids, entry_values = top[order].tolist(), values[order].tolist()
    text, data = _token_bytes_text(tokenizer, int(token_id))
    top_logprobs = []
    for tid, value in zip(entry_ids, entry_values):
        tid_text, tid_bytes = _token_bytes_text(tokenizer, int(tid))
        top_logprobs.append(
            {"id": int(tid), "token": tid_text, "bytes": tid_bytes, "logprob": max(value, floor)}
        )
    return {
        "id": int(token_id),
        "token": text,
        "bytes": data,
        "logprob": max(raw[int(token_id)].item(), floor),
        "top_logprobs": top_logprobs,
    }


def _with_prompt_history(processor, prompt, window):
    """*processor* also scoring the prompt tokens the runtime withholds: mlx_lm and mlx_vlm hand
    processors only what is left of the prompt after prefill and prompt-cache reuse."""
    import mlx.core as mx

    head = None

    def _processor(tokens, logits):
        nonlocal head
        if head is None:
            seen = int(tokens.shape[0])
            head = mx.array(list(prompt)[: max(len(prompt) - seen, 0)][-window:], dtype = mx.int32)
        return processor(mx.concatenate([head, tokens.astype(mx.int32)]), logits)

    # Still a function of the token history alone, so a speculative draft keeps running.
    _processor.history_only = getattr(processor, "history_only", False)
    return _processor


def _mlx_sampling_processors(
    *,
    repetition_penalty = None,
    presence_penalty: float = 0.0,
    frequency_penalty: float = 0.0,
    logit_bias = None,
    grammar_constraint = None,
    penalty_window = None,
    penalty_prompt = None,
):
    """Logits processors for the sampling knobs, or ``None`` when all are inert. Bias runs before
    the penalties, matching llama-server's sampler order. mlx_lm supplies only the repetition
    penalty here: its presence and frequency processors window the last 20 tokens *including the
    prompt*, while the penalties below score the whole completion and exclude it, so using them
    would make the same request sample differently depending on the backend. ``penalty_window``
    is llama-server's ``repeat_last_n``: one window for all three penalties, ``0`` disabling them.
    ``penalty_prompt`` is the full prompt, so the window reaches the part the runtime withholds."""
    processors = []
    if grammar_constraint is not None:
        from core.inference.grammar_constraint import make_grammar_logits_processor
        processors.append(make_grammar_logits_processor(grammar_constraint))
    if logit_bias:
        processors.append(_make_mlx_logit_bias_processor(logit_bias))
    if penalty_window == 0:
        return processors or None
    penalties = []
    if repetition_penalty is not None and float(repetition_penalty) not in (0.0, 1.0):
        from mlx_lm.sample_utils import make_logits_processors

        window = {} if penalty_window is None else {"repetition_context_size": penalty_window}
        repetition = make_logits_processors(repetition_penalty = float(repetition_penalty), **window)
        for processor in repetition:
            processor.history_only = True
        penalties.extend(repetition)
    if presence_penalty:
        penalties.append(
            _make_mlx_presence_penalty_processor(float(presence_penalty), penalty_window)
        )
    if frequency_penalty:
        penalties.append(
            _make_mlx_frequency_penalty_processor(float(frequency_penalty), penalty_window)
        )
    if penalty_window is not None and penalty_prompt:
        penalties = [
            _with_prompt_history(penalty, penalty_prompt, penalty_window) for penalty in penalties
        ]
    processors.extend(penalties)
    return processors or None


# Families whose template emits the markers (mlx-vlm 0.6.0-0.6.8 list): tokenization must not add them; laguna left out on purpose, unlike the current helper
_VLM_INLINE_SPECIAL_TOKEN_FAMILIES = ("gemma3", "gemma3n", "gemma4", "gemma4_unified")
# First mlx-vlm whose gemma-4 mask overlay skips a resumed suffix's non-square mask (#1445).
_VLM_MEDIA_BLOCK_MIN_VERSION = "0.6.4"


def _vlm_add_special_tokens(model_type, processor):
    try:
        from mlx_vlm.utils import should_add_special_tokens
        return should_add_special_tokens(model_type, processor)
    except Exception:
        # The rule those releases inline, which Studio's runtime gate still accepts.
        return (
            getattr(processor, "chat_template", None) is None
            if model_type in _VLM_INLINE_SPECIAL_TOKEN_FAMILIES
            else True
        )


def _vlm_prepared_inputs(model, processor, prompt, images, **media):
    """The processor's inputs for *prompt* and *images*, as mlx-vlm's ``stream_generate`` prepares them."""
    from mlx_vlm.utils import prepare_inputs

    config = getattr(model, "config", None)
    read = config.get if isinstance(config, dict) else lambda attr: getattr(config, attr, None)
    return prepare_inputs(
        processor,
        images = images,
        prompts = prompt,
        image_token_index = read("image_token_index"),
        add_special_tokens = _vlm_add_special_tokens(read("model_type"), processor),
        **media,
    )


def _vlm_generate_kwargs(inputs):
    """Prepared inputs as ``stream_generate`` kwargs, which then skips its own preparation."""
    inputs = dict(inputs)
    return {
        "input_ids": inputs.pop("input_ids"),
        "pixel_values": inputs.pop("pixel_values", None),
        "mask": inputs.pop("attention_mask", None),
        **inputs,
    }


class _VLMMediaBlock:
    """A prompt's rows through its last vision token, prefilled by Studio: models whose vision
    tokens attend to each other prefill in one forward that lands off the grid, and this is that
    forward cut where the grid can continue."""

    def __init__(self, model, processor, prompt, images, make_cache):
        self._model = model
        self._make_cache = make_cache
        self._inputs = _vlm_prepared_inputs(model, processor, prompt, images)

    def generate_kwargs(self):
        return _vlm_generate_kwargs(self._inputs)

    def rows(self, token_ids):
        ids = self._inputs["input_ids"]
        if ids.shape[0] != 1 or ids[0].tolist() != list(token_ids):
            return 0
        types = self._inputs.get("mm_token_type_ids", self._inputs.get("token_type_ids"))
        if types is None:
            return 0
        visual = [index for index, kind in enumerate(types[0].tolist()) if kind in (1, 2)]
        return visual[-1] + 1 if visual else 0

    def prefill(self, forward, rows):
        import mlx.core as mx

        inputs = self._inputs
        extra = {
            key: value
            for key, value in inputs.items()
            if key not in ("input_ids", "pixel_values", "attention_mask")
        }
        features = self._model.get_input_embeddings(
            input_ids = inputs["input_ids"], pixel_values = inputs.get("pixel_values"), **extra
        )
        kwargs = {
            key: extra[key][:, :rows]
            for key in ("mm_token_type_ids", "token_type_ids")
            if key in extra
        }
        if features.per_layer_inputs is not None:
            kwargs["per_layer_inputs"] = features.per_layer_inputs[:, :rows]
        entries = self._make_cache()
        forward.unrecorded(
            inputs = None, inputs_embeds = features.inputs_embeds[:, :rows], cache = entries, **kwargs
        )
        mx.eval([entry.state for entry in entries])
        return entries


class _RowStream:
    def __init__(
        self,
        *,
        prefix = "",
        sequences = (),
        normalizer = None,
        decode = None,
    ):
        self._prefix = prefix
        self._sequences = sequences
        self._normalizer = normalizer
        self._decode = decode
        self.token_ids = []
        self.sampled = ""
        self.stopped = False
        self._released = 0
        self._cut = 0
        self._output = prefix

    def feed(
        self,
        token = None,
        text = "",
    ):
        if token is not None:
            self.token_ids.append(token)
        if self._decode is not None:
            self.sampled = self._decode(self.token_ids)
        else:
            self.sampled += text or ""
        if self._sequences:
            self._cut, self.stopped = _mlx_stop_cut(self.sampled, self._sequences)
        else:
            self._cut = len(self.sampled)
        if self._normalizer is not None:
            return self._normalize(self.sampled[self._released : self._cut])
        if not self._sequences:
            # Re-decoded snapshots hold back an unfinished character, as in _generate_text.
            held = self.sampled.rstrip("\ufffd") if self._decode is not None else self.sampled
            return [self._prefix + held]
        if self._decode is not None:
            return []
        if self._cut > self._released:
            self._released = self._cut
            return [self._prefix + self.sampled[: self._cut]]
        return []

    def flush(self, text):
        """Text a decoder held until the reply ended; ``settle`` delivers it for normalized or re-decoded rows."""
        if not self._sequences or (self._normalizer is None and self._decode is None):
            # feed releases up to a stop it finds here, which settle no longer would once stopped.
            return self.feed(text = text)
        self.sampled += text
        self._cut, self.stopped = _mlx_stop_cut(self.sampled, self._sequences)
        return []

    def settle(self):
        if not self._sequences:
            if self._decode is not None and self.sampled.endswith("\ufffd"):
                return [self._prefix + self.sampled]
            return []
        if not self.stopped:
            self._cut = len(self.sampled)
        if self._normalizer is not None:
            return self._normalize(self.sampled[self._released : self._cut])
        if self._decode is not None:
            settled = self._prefix + self.sampled[: self._cut]
            return [settled] if settled != self._prefix else []
        if not self.stopped and self._released < len(self.sampled):
            return [self._prefix + self.sampled]
        return []

    def drain(self, *, cancelled):
        if self._normalizer is None:
            return []
        drained = cancelled and not self.stopped
        tail = self._normalizer.drain() if drained else self._normalizer.finish()
        return self._append(tail)

    def _normalize(self, raw):
        self._released = self._cut
        return self._append(self._normalizer.feed(raw))

    def _append(self, delta):
        if not delta:
            return []
        self._output += delta
        return [self._output]


def _messages_with_system_prompt(messages, system_prompt):
    full = [{"role": "system", "content": system_prompt}] if system_prompt else []
    full.extend(messages)
    return full


def _engine_fields(engine, name):
    return getattr(getattr(engine, name, None), "__dataclass_fields__", ())


def _asks_for_row_processors(request):
    return (
        _mlx_sampling_processors(
            repetition_penalty = request.get("repetition_penalty", 1.0),
            presence_penalty = request.get("presence_penalty", 0.0),
            frequency_penalty = request.get("frequency_penalty", 0.0),
            logit_bias = request.get("logit_bias"),
        )
        is not None
    )


def _row_processor_gap(engine, requests):
    """Why these replies' penalties cannot travel into a batch, or None."""
    if not any(_asks_for_row_processors(request) for request in requests):
        return None
    unavailable = getattr(engine, "row_logits_processors_unavailable_reason", None)
    return unavailable() if callable(unavailable) else None


def _constrained(request):
    return request.get("response_format") not in (None, {"type": "text"}) or (
        request.get("grammar") is not None
    )


def _request_batch_gap(request):
    """What a reply asks for that only the one-at-a-time decode serves, or None."""
    if request.get("images") or request.get("images_base64"):
        return "a reply replays tool images"
    if request.get("n_probs"):
        return "a reply reports token probabilities"
    return None


def _carries_video(request):
    return bool(request.get("videos") or request.get("videos_base64"))


def _batch_clips(stack, plans, processor):
    """Each row's clips as files mlx-vlm reads, removed when ``stack`` closes.

    Rows with the same clips share their files, so unsloth-zoo batches them as one group, and that
    group decodes its clips together, so its rows share one decode budget. Rows whose clips are too
    large to share it get files of their own, so each decodes alone with the whole budget.
    """
    files, shared = [], {}
    stack.callback(lambda: [_discard_video_clip(path) for path in files])

    def write(clip):
        files.append(_write_video_clip(clip))
        return files[-1]

    rows = Counter(tuple(plan.videos or ()) for plan in plans)
    clips = []
    for plan in plans:
        videos = tuple(plan.videos or ())
        paths = [
            shared[clip] if clip in shared else shared.setdefault(clip, write(clip))
            for clip in videos
        ]
        if not paths:
            clips.append({})
            continue
        try:
            rate = _video_sampling_fps(paths, processor, rows[videos])
        except _ClipsTooLarge:
            paths = [write(clip) for clip in videos]
            rate = _video_sampling_fps(paths, processor)
        clips.append({"videos": paths, "fps": rate})
    return clips


def _batched_vision_engine_gap(engine):
    def fields(name):
        return _engine_fields(engine, name)

    if not hasattr(engine, "stream_batch"):
        return "the installed unsloth-zoo has no batched vision generation"
    if "seed" not in fields("SamplingParams"):
        return "the installed unsloth-zoo cannot seed a batched reply"
    if "prompt_token_count" not in fields("GenerationResult"):
        return "the installed unsloth-zoo does not report what a batched prompt cost"
    if "logits_processors" not in fields("GenerationRequest"):
        return "the installed unsloth-zoo cannot carry a penalty into a batched reply"
    return None


class _TextRowPlan:
    __slots__ = (
        "prompt",
        "think_prefix",
        "stream",
        "sampler",
        "processors",
        "max_tokens",
        "streamed",
    )

    def __init__(
        self,
        *,
        prompt,
        think_prefix,
        stream,
        sampler,
        processors,
        max_tokens,
        streamed = None,
    ):
        self.prompt = prompt
        self.think_prefix = think_prefix
        self.stream = stream
        self.streamed = streamed
        self.sampler = sampler
        self.processors = processors
        self.max_tokens = max_tokens


class _TextBatchRow:
    __slots__ = (
        "handle",
        "plan",
        "uid",
        "cache_key",
        "all_tokens",
        "prompt_len",
        "cached",
        "generated",
        "reason",
        "processing_at",
        "ready_at",
        "failure",
    )

    def __init__(self, *, handle, plan, cache_key, all_tokens, prompt_len, cached):
        self.handle = handle
        self.plan = plan
        self.uid = None
        self.cache_key = cache_key
        self.all_tokens = all_tokens
        self.prompt_len = prompt_len
        self.cached = cached
        self.generated = 0
        self.reason = None
        self.processing_at = None
        self.ready_at = None
        self.failure = None


def _failing_alone(row, processors):
    """A row's processors, whose failure is kept on the row rather than raised through the batch."""

    def guarded(processor):
        def _processor(tokens, logits):
            if row.failure is None:
                try:
                    return processor(tokens, logits)
                except Exception as exc:
                    row.failure = exc
            return logits

        _processor.history_only = getattr(processor, "history_only", False)
        if hasattr(processor, "settle"):

            def _settle(tokens):
                if row.failure is None:
                    try:
                        processor.settle(tokens)
                    except Exception as exc:
                        row.failure = exc

            _processor.settle = _settle
        return _processor

    return [guarded(processor) for processor in processors]


def _streamed_tail(row):
    """What a row decoded from ids still holds when it ends, released as the single path's finish."""
    stream = row.plan.stream
    if row.plan.streamed is None or stream.stopped:
        return []
    tail = stream.flush(row.plan.streamed.finish(stream.sampled))
    if stream.stopped:
        row.reason = "stop"
    return tail


def _row_kwargs(request):
    """The sampling and render options one batched reply is planned with."""
    return dict(
        temperature = request.get("temperature", 0.7),
        top_p = request.get("top_p", 0.9),
        top_k = request.get("top_k", 40),
        min_p = request.get("min_p", 0.0),
        max_new_tokens = request.get("max_new_tokens", 256),
        repetition_penalty = request.get("repetition_penalty", 1.0),
        tools = request.get("tools"),
        enable_thinking = request.get("enable_thinking"),
        reasoning_effort = request.get("reasoning_effort"),
        preserve_thinking = request.get("preserve_thinking"),
        continue_final_message = request.get("continue_final_message", False),
        presence_penalty = request.get("presence_penalty", 0.0),
        seed = request.get("seed"),
        frequency_penalty = request.get("frequency_penalty", 0.0),
        logit_bias = request.get("logit_bias"),
        stop = request.get("stop"),
        prompt_ids = request.get("prompt_ids"),
        penalty_window = request.get("penalty_window"),
    )


class _TextBatchSession:
    """mlx-lm's batch with the reply set left open."""

    def __init__(
        self,
        backend,
        *,
        width,
        adapter_state = None,
        owns_model = False,
    ):
        from mlx_lm.generate import BatchGenerator

        self.backend = backend
        self._adapter_state = adapter_state
        self._rows = {}
        self._by_uid = {}
        self._settled = {}
        self._held = ExitStack()
        try:
            if owns_model:
                self._held.enter_context(backend._generation_lock)
                self._held.enter_context(
                    _temporary_mlx_adapter_state(backend._model, adapter_state)
                )
            self._held.enter_context(backend._prefill_decision_scopes())
            self._held.enter_context(backend._generation_scopes("batch"))
            self.generator = BatchGenerator(
                backend._model,
                stop_tokens = [
                    [token] for token in _mlx_stop_token_ids(backend._tokenizer, backend._model)
                ],
                completion_batch_size = width,
                prefill_batch_size = width,
                **backend._prefill_step_kwargs(BatchGenerator),
            )
        except BaseException:
            self._held.close()
            raise

    @property
    def rows_in_flight(self) -> int:
        return len(self._rows)

    @property
    def handles(self):
        return list(self._rows)

    @property
    def ending(self):
        return []

    def take_stats(self, handle):
        """The usage a retired reply settled with, once."""
        return self._settled.pop(handle, None)

    def admit(self, request, handle):
        """Take one reply in, answering with the text preceding its first token."""
        backend = self.backend
        image = request.get("image") if backend._reads_vision_input() else None
        messages = request.get("messages") or []
        row_kwargs = _row_kwargs(request)
        plan = backend._plan_text_row(
            _messages_with_system_prompt(messages, request.get("system_prompt", ""))
            if image is None
            else backend._conversation_to_render(messages, request.get("system_prompt", ""), image),
            tool_protocol_active = request.get("tool_protocol_active"),
            response_format = request.get("response_format"),
            reasoning_is_extracted = request.get("reasoning_is_extracted", False),
            grammar = request.get("grammar"),
            **row_kwargs,
        )
        began = time.perf_counter()
        if image is None:
            prefix, remainder, cache, cache_key, all_tokens = backend._batch_prompt_row(
                plan.prompt,
                self._adapter_state,
            )
        else:
            from unsloth_zoo.mlx.generate import prefill_image_prompt

            # mlx-lm batches token ids only: the image prefills here, and stays out of the token-keyed cache.
            cache = backend._make_row_cache()
            all_tokens = prefill_image_prompt(
                backend._model, backend._tokenizer, plan.prompt, [image], cache
            )
            prefix, remainder, cache_key = all_tokens[:-1], all_tokens[-1:], None
            if row_kwargs["max_new_tokens"] is None:
                plan.max_tokens = backend._unset_generation_budget(plan.prompt, all_tokens)
            else:
                backend._prompt_within_window(plan.prompt, all_tokens)
        row = _TextBatchRow(
            handle = handle,
            plan = plan,
            cache_key = cache_key,
            all_tokens = all_tokens,
            prompt_len = len(remainder) if image is None else len(all_tokens),
            cached = len(prefix) if image is None else 0,
        )
        if image is not None:
            row.processing_at = began
        uids = self.generator.insert(
            [remainder],
            max_tokens = [plan.max_tokens],
            caches = [cache],
            all_tokens = [prefix],
            samplers = [plan.sampler],
            logits_processors = [_failing_alone(row, _row_logits_processors(plan.processors))],
        )
        if len(uids) != 1:
            self.generator.remove(list(uids))
            raise RuntimeError(
                f"mlx-lm admitted {len(uids)} rows for one reply; a batch cannot "
                "tell whose tokens are whose."
            )
        row.uid = uids[0]
        self._rows[handle] = row
        self._by_uid[row.uid] = row
        logger.info(
            "Admitted reply %s to the batch: prompt=%d, cached=%d, max_tokens=%d, model=%s",
            handle,
            row.prompt_len,
            row.cached,
            plan.max_tokens,
            type(backend._model).__name__,
        )
        return plan.think_prefix

    def step(self, waiting = None):
        """Report what the batch produced, retiring every reply that ended."""
        began = time.perf_counter()
        prompt_events, events = self.generator.next()
        now = time.perf_counter() if (events or prompt_events) else None
        for event in prompt_events:
            row = self._by_uid.get(event.uid)
            if row is not None and row.processing_at is None:
                row.processing_at = began
        # A processor runs a step ahead, so the single path raises before yielding this token;
        # reported first, so a sibling ending in this step cannot answer the request as done.
        for row in [row for row in self._rows.values() if row.failure is not None]:
            self.generator.remove([row.uid])
            self._rows.pop(row.handle, None)
            self._by_uid.pop(row.uid, None)
            yield row.handle, row.failure
        for event in events:
            row = self._by_uid.get(event.uid)
            if row is None:
                continue
            if row.processing_at is None:
                row.processing_at = began
            if row.ready_at is None:
                row.ready_at = now
            yield from self._feed(row, event)
        self._release_held_cache_beside_live_rows()

    def _feed(self, row, event):
        stream = row.plan.stream
        # A stop id is held, not written: the end-of-reply flush decides whether it closes markup.
        streamed = row.plan.streamed
        text = "" if streamed is None else streamed.feed_token(int(event.token))
        row.generated += 1
        yield from (
            (row.handle, snapshot) for snapshot in stream.feed(token = event.token, text = text)
        )
        if not (stream.stopped or event.finish_reason is not None):
            return
        row.reason = "stop" if stream.stopped else event.finish_reason
        banked = self.generator.remove(
            [row.uid],
            return_prompt_caches = event.finish_reason is None,
        )
        self.backend._bank_row_cache(
            row.cache_key,
            event,
            banked.get(row.uid),
            row.all_tokens,
            stream,
        )
        yield from self._retire(row, cancelled = False)

    def withdraw(self, handles):
        rows = [self._rows[handle] for handle in handles if handle in self._rows]
        if not rows:
            return
        taken = self.generator.remove([row.uid for row in rows], return_prompt_caches = True)
        for row in rows:
            row.reason = "stop"
            self.backend._bank_row_cache(
                row.cache_key,
                None,
                taken.get(row.uid),
                row.all_tokens,
                row.plan.stream,
            )
            yield from self._retire(row, cancelled = True)
        self._release_held_cache_beside_live_rows()

    def _release_held_cache_beside_live_rows(self):
        # Decided once every row that ended together is retired: holding it beside a row still
        # decoding would raise peak memory.
        history = self.backend._prompt_cache_history
        if self._rows and history is not None:
            history.release_oversized()

    def _retire(self, row, *, cancelled):
        self._rows.pop(row.handle, None)
        self._by_uid.pop(row.uid, None)
        stream = row.plan.stream
        yield from ((row.handle, snap) for snap in _streamed_tail(row))
        yield from ((row.handle, snap) for snap in stream.settle())
        yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))
        ready_at = row.ready_at or time.perf_counter()
        decoded = time.perf_counter() - ready_at
        prefill_s = ready_at - (row.processing_at or ready_at)
        self._settled[row.handle] = _build_generation_stats(
            row.prompt_len,
            (row.prompt_len / prefill_s) if prefill_s > 0 else 0.0,
            row.generated,
            row.generated / max(decoded, 1e-9),
            row.cached,
            finish_reason = row.reason,
        )
        yield row.handle, None

    def close(self):
        try:
            try:
                if self._by_uid:
                    self.generator.remove(list(self._by_uid))
            finally:
                self.generator.close()
        finally:
            self._rows.clear()
            self._by_uid.clear()
            self._settled.clear()
            self._held.close()


class _VLMRowPlan:
    __slots__ = (
        "prompt",
        "images",
        "videos",
        "think_prefix",
        "stream",
        "kwargs",
        "sampling",
        "max_tokens",
        "processors",
        "streamed",
    )

    def __init__(
        self,
        *,
        prompt,
        images,
        think_prefix,
        stream,
        kwargs,
        sampling,
        max_tokens,
        processors = None,
        streamed = None,
        videos = None,
    ):
        self.prompt = prompt
        self.images = images
        self.videos = videos
        self.think_prefix = think_prefix
        self.stream = stream
        self.kwargs = kwargs
        self.sampling = sampling
        self.max_tokens = max_tokens
        self.processors = processors
        self.streamed = streamed


class _VisionBatchRow:
    __slots__ = (
        "handle",
        "plan",
        "row",
        "generated",
        "reason",
        "prompt_tokens",
        "cached_tokens",
        "admitted_at",
        "ready_at",
        "cancelled",
        "drafts",
        "failure",
    )

    def __init__(
        self,
        *,
        handle,
        plan,
        row = None,
    ):
        self.handle = handle
        self.plan = plan
        self.row = row
        self.generated = 0
        self.reason = None
        self.prompt_tokens = 0
        self.cached_tokens = 0
        self.admitted_at = time.perf_counter()
        self.ready_at = None
        self.cancelled = False
        self.drafts = None
        self.failure = None


def _vision_batch_reports_ids(engine, speculative):
    """Whether vision rows can be decoded from their sampled ids, as the one-at-a-time decode does."""
    return "token_ids" in _engine_fields(engine, "GenerationEvent") and (
        speculative is None or "drafts" in _engine_fields(engine, "GenerationRequest")
    )


def _row_prompt_cache_gap():
    try:
        from unsloth_zoo.mlx.generate import row_prompt_cache_unavailable_reason
    except ImportError:
        return "the installed unsloth-zoo cannot resume a batched row"
    return row_prompt_cache_unavailable_reason()


def _row_retire_gap():
    try:
        from unsloth_zoo.mlx.generate import row_retire_unavailable_reason
    except ImportError:
        return "the installed unsloth-zoo cannot hand a batched row's cache back"
    return row_retire_unavailable_reason()


def _speculative_quantized_row_gap():
    try:
        from unsloth_zoo.mlx.generate import speculative_quantized_rows_unavailable_reason
    except ImportError:
        return "the installed unsloth-zoo cannot draft a batch on a quantized KV cache"
    return speculative_quantized_rows_unavailable_reason()


def _row_quantized_cache_gap():
    try:
        from unsloth_zoo.mlx.generate import row_quantized_prompt_cache_unavailable_reason
    except ImportError:
        return "the installed unsloth-zoo cannot batch a quantized KV cache"
    return row_quantized_prompt_cache_unavailable_reason()


class _VisionBatchSession:
    def __init__(
        self,
        backend,
        *,
        width,
        adapter_state = None,
        owns_model = False,
    ):
        from unsloth_zoo.mlx import generate as engine
        from unsloth_zoo.mlx.generate import BatchStream, GenerationDefaults

        self.backend = backend
        self._adapter_state = adapter_state
        self._resumes_rows = _row_prompt_cache_gap() is None
        self._rows = {}
        self._leaving = set()
        self._by_row = {}
        self._settled = {}
        self._held = ExitStack()
        try:
            if owns_model:
                self._held.enter_context(backend._generation_lock)
                self._held.enter_context(
                    _temporary_mlx_adapter_state(backend._model, adapter_state)
                )
            self._held.enter_context(backend._prefill_decision_scopes())
            self._speculative = getattr(backend, "_speculative_draft", None)
            self._decodes_ids = _vision_batch_reports_ids(engine, self._speculative)
            self.stream = BatchStream(
                backend._model,
                backend._processor,
                defaults = GenerationDefaults(
                    prefill_batch_size = 1,
                    completion_batch_size = width,
                    # Rows quantize their own caches where the single path's decode does.
                    **backend._kv_runtime_quant_kwargs(),
                    **backend._prefill_step_kwargs(GenerationDefaults, grid = True),
                ),
                **({} if self._speculative is None else {"speculative": self._speculative}),
            )
        except BaseException:
            self._held.close()
            raise
        self._retires_rows = self._resumes_rows and _row_retire_gap() is None

    @property
    def rows_in_flight(self) -> int:
        return len(self._rows)

    @property
    def handles(self):
        return list(self._rows)

    @property
    def ending(self):
        return [handle for handle, row in self._rows.items() if row.cancelled]

    def take_stats(self, handle):
        return self._settled.pop(handle, None)

    def admit(self, request, handle):
        from unsloth_zoo.mlx.generate import (
            BatchRowRefused,
            GenerationRequest,
            SamplingParams,
        )

        backend = self.backend
        plan = backend._plan_vlm_request(request, decodes_ids = self._decodes_ids)
        row = _VisionBatchRow(handle = handle, plan = plan)
        alone = self._alone(handle) if self._retires_rows else None
        # A clip's row resumes nothing: token ids do not vary with the clip.
        state = (
            backend._vlm_batch_row_cache(self._adapter_state, plan.images, alone = alone)
            if self._resumes_rows and not plan.videos
            else None
        )
        if state is None and backend._kv_quant_bits() is not None:
            # mlx-vlm's own batch cache would be float.
            raise RowRefused("this row cannot carry the load's quantized KV cache")
        if state is None and (plan.images or plan.videos):
            backend._release_vlm_snapshots()
        elif state is None:
            backend._release_held_vlm_snapshots()
        resume = {} if state is None else {"prompt_cache_state": state}
        clips = _batch_clips(self._held, [plan], backend._processor)[0] if plan.videos else {}
        admitted_at = time.perf_counter()
        try:
            row_number = self.stream.add(
                GenerationRequest(
                    **backend._vlm_batch_prompt(plan.prompt),
                    image = plan.images[0] if plan.images else None,
                    max_tokens = plan.max_tokens,
                    sampling = SamplingParams(**plan.sampling),
                    logits_processors = plan.processors and _failing_alone(row, plan.processors),
                    **clips,
                    **resume,
                )
            )
        except BatchRowRefused as refusal:
            raise RowRefused(str(refusal)) from refusal
        row.row = row_number
        # A speculative batch prefills the row inside add(), so its clock starts before it.
        row.admitted_at = admitted_at if self._speculative is not None else time.perf_counter()
        self._rows[handle] = row
        self._by_row[row_number] = row
        logger.info(
            "Admitted vision reply %s to the batch: images=%d, max_tokens=%d, model=%s",
            handle,
            len(plan.images or ()),
            plan.max_tokens,
            type(backend._model).__name__,
        )
        return plan.think_prefix

    def _alone(self, handle):
        """No other row decodes beside ``handle``'s; rows ending this step count until it ends,
        as the speculative stream frees their caches only then."""
        rows, leaving = self._rows, self._leaving
        return lambda: all(other == handle for other in (*rows, *leaving))

    def step(self, waiting = None):
        now, stopped = None, {}
        # Only a speculative step spans many tokens, reported as they are decoded; only its unsloth-zoo has the call.
        # The batch cannot change mid-step, so a row that stopped itself ends the step and leaves after it.
        failing = {}
        events = (
            self.stream.iter_step(
                lambda: bool(stopped or failing) or (waiting is not None and waiting())
            )
            if self._speculative is not None
            else self.stream.step()
        )
        # As in _TextBatchSession.step: reported before the step's tokens, which it never yields.
        yield from self._fail([row for row in self._rows.values() if row.failure is not None])
        for event in events:
            row = self._by_row.get(event.index)
            if row is None:
                continue
            if row.failure is not None:
                # Only a drafted step decodes as it is read, so a row can fail mid-step and leave after it.
                failing[row.handle] = row
                continue
            if now is None:
                now = time.perf_counter()
            if row.ready_at is None:
                row.ready_at = now
            streamed = row.plan.streamed
            text = (
                event.delta
                if streamed is None
                else "".join(streamed.feed_token(int(token)) for token in event.token_ids)
            )
            if text and not row.cancelled:
                yield from ((row.handle, snapshot) for snapshot in row.plan.stream.feed(text = text))
                if row.plan.stream.stopped:
                    row.cancelled = True
                    row.reason = "stop"
            if event.result is not None:
                result = event.result
                if not row.cancelled:
                    row.reason = result.finish_reason
                self._record(row, result, len(result.token_ids) + (result.finish_reason == "stop"))
                self._leaving.add(row.handle)
                yield from self._retire(row, cancelled = row.cancelled)
            elif row.cancelled:
                stopped[row.handle] = row
        self._leaving.clear()
        yield from self._fail(list(failing.values()))
        for row in stopped.values():
            if row.handle in self._rows:
                yield from self._retake(row)

    def _fail(self, rows):
        for row in rows:
            self.stream.withdraw(row.row)
            self._rows.pop(row.handle, None)
            self._by_row.pop(row.row, None)
            yield row.handle, row.failure

    def withdraw(self, handles):
        for handle in list(handles):
            row = self._rows.get(handle)
            if row is None:
                continue
            row.cancelled = True
            row.reason = "stop"
            yield from self._retake(row)

    def _retake(self, row):
        managed = self.stream.withdraw(row.row)
        if managed is not None:
            self._record(row, managed, len(managed.token_ids))
            yield from self._retire(row, cancelled = True)

    def _record(self, row, result, generated):
        row.prompt_tokens, row.generated = result.prompt_token_count, generated
        row.cached_tokens = getattr(result, "cached_token_count", 0)
        if self._speculative is not None:
            row.drafts = (result.draft_tokens, result.accepted_draft_tokens)

    def _retire(self, row, *, cancelled):
        self._rows.pop(row.handle, None)
        self._by_row.pop(row.row, None)
        stream = row.plan.stream
        yield from ((row.handle, snap) for snap in _streamed_tail(row))
        yield from ((row.handle, snap) for snap in stream.settle())
        yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))
        ready_at = row.ready_at or time.perf_counter()
        decoded = time.perf_counter() - ready_at
        prefill_s = ready_at - row.admitted_at
        prefilled = row.prompt_tokens - row.cached_tokens
        self._settled[row.handle] = _build_generation_stats(
            prefilled,
            (prefilled / prefill_s) if prefill_s > 0 else 0.0,
            row.generated,
            row.generated / max(decoded, 1e-9),
            cached_n = row.cached_tokens,
            finish_reason = row.reason or "stop",
            drafts = row.drafts,
        )
        yield row.handle, None

    def close(self):
        try:
            self.stream.close()
        finally:
            self._rows.clear()
            self._by_row.clear()
            self._settled.clear()
            self._held.close()


class MLXInferenceBackend:
    def __init__(self):
        self.models = {}
        self.active_model_name = None
        self.loading_models = set()
        self.loaded_local_models = []
        self.device = "mlx"
        self._generation_lock = threading.Lock()
        # usage, timings and terminal reason of the latest generation, shipped on gen_done.
        self.last_generation_stats = None
        self.last_batch_generation_stats = []

        self._model = None
        self._model_fusion = ExitStack()
        self._tokenizer = None
        self._processor = None
        self._is_vlm = False
        self._reads_vision = None
        self._turboquant_refusal = ""
        self._speculative_draft = None
        self._prefill_step = None
        self._prefill_layer_step = None
        self._multi_image_marker = None
        self._config = {}
        self._distributed_group = None
        self._distributed_rank = 0
        self._distributed_world_size = 1

        # Recorded for unload to release pinned memory back to the OS.
        self._memory_limits_applied = {}

        # Load-time knobs read by all generation paths; bound early so a failed load still leaves dicts.
        self._kv_quant = _kv_quant_status(None, None, False)
        self._int8_prefill = _int8_prefill_status(False, None)
        self._ane_prefill = _ane_prefill_status(False, None)
        self._fast_verify = _fast_verify_status(False, None)
        self._kv_cache_window = None
        self._kv_context_budget = None
        self._served_context = None
        self._resume_load_args = None
        self._resume_files = None
        self._template_override = _template_override_status(None, None, None)[1]

        self._prompt_cache_history = None
        self._prompt_cache_unavailable = False
        self._prompt_cache_room = None
        self._vlm_snapshot_store = None
        self._vlm_snapshot_store_unavailable = False
        self._vlm_is_diffusion_model = None

    def _prompt_cache(self):
        if self._prompt_cache_history is not None or self._prompt_cache_unavailable:
            return self._prompt_cache_history
        max_bytes = self._prompt_cache_bytes()
        if max_bytes <= 0:
            self._prompt_cache_unavailable = True
            logger.info("MLX prompt cache disabled by budget")
            return None
        try:
            self._prompt_cache_history = _MLXPromptCacheHistory(
                PROMPT_CACHE_ENTRIES,
                max_bytes,
                self._kv_cache_window,
                prepare = self._prepare_kv_entries,
            )
        except Exception as exc:
            self._prompt_cache_unavailable = True
            logger.info("MLX prompt cache unavailable (%s); prefilling every request", exc)
            return None
        logger.info(
            "MLX prompt cache: %d entries, %.2f GB budget",
            PROMPT_CACHE_ENTRIES,
            max_bytes / 1e9,
        )
        return self._prompt_cache_history

    def _prompt_cache_bytes(self):
        max_bytes = _prompt_cache_max_bytes(self._memory_limits_applied.get("recommended_gb"))
        room = getattr(self, "_prompt_cache_room", None)
        return max_bytes if room is None else min(max_bytes, room)

    def _clear_prompt_cache(self):
        self._prompt_cache_history = None
        self._prompt_cache_unavailable = False
        self._prompt_cache_room = None
        self._vlm_snapshot_store = None
        self._vlm_snapshot_store_unavailable = False
        self._vlm_is_diffusion_model = None

    def _resume_weight_files(self):
        from pathlib import Path

        name = self.active_model_name
        files = []
        for root in sorted(
            {d for d in (_snapshot_dir(self._model, name), _snapshot_dir(None, name)) if d}
        ):
            weights = [*Path(root).glob("*.safetensors"), *Path(root).glob("*.json")]
            for path in sorted(weights):
                st = path.stat()
                files.append((str(path), st.st_size, st.st_mtime_ns))
        return tuple(files)

    def _resume_fingerprint(self):
        """What a restore must match: the weights this load read and every KV-affecting setting."""
        import mlx.core as mx

        files = self._resume_files
        if files is None:
            files = self._resume_weight_files()
        versions = [mx.__version__]
        for module in ("mlx_lm", "mlx_vlm"):
            versions.append(getattr(sys.modules.get(module), "__version__", None))
        return (
            self.active_model_name,
            files,
            tuple(versions),
            self._is_vlm,
            self._kv_quant_bits(),
            bool(getattr(self, "_turboquant", False)),
            self._kv_cache_window,
            bool(self._int8_prefill.get("active")),
            bool(self._ane_prefill.get("active")),
            bool(self._fast_verify.get("active")),
            self._prefill_step,
            self._resume_load_args,
        )

    def _resume_stores(self):
        return (("text", self._prompt_cache_history), ("vlm", self._vlm_snapshot_store))

    def save_prompt_cache_for_resume(self, directory):
        """Saves each store's newest turn into ``directory``; the manifest, or None."""
        if not self.active_model_name or self._distributed_world_size > 1:
            return None
        # The KV was computed from the weights read at load; files replaced since would mislabel it.
        if self._resume_files is False or (
            self._resume_files is not None and self._resume_files != self._resume_weight_files()
        ):
            return None
        # Never blocks: the worker loop advancing a batch is the thread that would release it.
        if not self._generation_lock.acquire(blocking = False):
            return None
        try:
            kept = [
                (store_name, store.resume_entries() if store is not None else ())
                for store_name, store in self._resume_stores()
            ]
            # llama-slots' bounds: a cap, and 1 GiB of disk left after it.
            total = sum(cache_entries_nbytes(item[2]) for _, items in kept for item in items)
            if total > _resume_save_max_bytes() or shutil.disk_usage(directory).free < total + (
                1 << 30
            ):
                logger.debug("MLX prompt cache: %d bytes do not fit the save budget", total)
                return None
            slots = []
            for store_name, items in kept:
                for index, (key, tokens, entries, replay) in enumerate(items):
                    filename = f"{store_name}-{index}.safetensors"
                    save_resume_cache(os.path.join(directory, filename), key, tokens, entries)
                    slots.append({"filename": filename, "store": store_name, "replay": replay})
            if not slots:
                return None
            return {
                "engine": "mlx",
                "dir": directory,
                "fingerprint": self._resume_fingerprint(),
                "slots": slots,
            }
        finally:
            self._generation_lock.release()

    def restore_prompt_cache_for_resume(self, manifest):
        """Restores a manifest saved under this exact load; the entry count."""
        if not self.active_model_name or manifest.get("engine") != "mlx":
            return 0
        if manifest.get("fingerprint") != self._resume_fingerprint():
            logger.info("MLX prompt cache: the saved context belongs to another load; not restored")
            return 0
        restored = 0
        with self._generation_lock:
            for slot in manifest.get("slots") or ():
                if slot.get("store") == "text":
                    store = self._prompt_cache()
                elif slot.get("store") == "vlm":
                    store = self._vlm_prompt_cache_store()
                else:
                    store = None
                if store is None:
                    continue
                key, tokens, entries = load_resume_cache(
                    os.path.join(manifest["dir"], slot["filename"])
                )
                if slot["store"] == "text":
                    store.insert(key, tokens, entries)
                else:
                    store.store(key, tokens, entries, replay = bool(slot.get("replay")))
                restored += 1
        return restored

    def _release_vlm_snapshots(self):
        store = getattr(self, "_vlm_snapshot_store", None)
        if store is not None:
            store.clear()

    def _release_held_vlm_snapshots(self):
        """Before a request that looks nothing up, which they would sit beside."""
        store = getattr(self, "_vlm_snapshot_store", None)
        if store is not None:
            store.release_held()

    def _vlm_prompt_cache_store(self):
        if self._vlm_snapshot_store is not None or self._vlm_snapshot_store_unavailable:
            return self._vlm_snapshot_store
        max_bytes = self._prompt_cache_bytes()
        if max_bytes <= 0:
            self._vlm_snapshot_store_unavailable = True
            logger.info("MLX VLM prompt cache disabled by budget")
            return None
        # Needs mRoPE priming, reuse reporting and diffusion routing, all added with cached_tokens
        try:
            from mlx_vlm.generate import GenerationResult
            from mlx_vlm.generate.diffusion import is_diffusion_model
            if not hasattr(GenerationResult, "cached_tokens"):
                raise ImportError("GenerationResult has no cached_tokens")
        except ImportError as exc:
            self._vlm_snapshot_store_unavailable = True
            logger.info(
                "MLX VLM prompt cache needs a newer mlx-vlm (%s); prefilling every request", exc
            )
            return None
        self._vlm_is_diffusion_model = is_diffusion_model
        self._vlm_snapshot_store = VLMPromptSnapshotStore(max_bytes)
        logger.info("MLX VLM prompt cache: %.2f GB budget", max_bytes / 1e9)
        return self._vlm_snapshot_store

    @staticmethod
    def _vlm_media_token_ids(config):
        ids = set()
        for attr in ("image_token_id", "image_token_index", "video_token_id", "video_token_index"):
            value = config.get(attr) if isinstance(config, dict) else getattr(config, attr, None)
            if value is not None:
                ids.add(int(value))
        return ids

    @staticmethod
    def _vlm_image_digest(images):
        digest = hashlib.sha256()
        for image in images:
            # What the tower sees: RGB, so palette indices and alpha decide nothing.
            rgb = image.convert("RGB")
            digest.update(f"{rgb.size}:{image.getexif().get(0x0112, 1)}:".encode())
            digest.update(rgb.tobytes())
        return digest.hexdigest()[:16]

    @staticmethod
    def _vlm_bidirectional_vision(model):
        config = getattr(getattr(model, "language_model", model), "config", None)
        return getattr(config, "use_bidirectional_attention", None) == "vision"

    def _vlm_prompt_cache_session(
        self,
        adapter_state,
        images = None,
        prompt = None,
        has_video = False,
    ):
        store = self._vlm_prompt_cache_store()
        if store is None or self._vlm_is_diffusion_model(self._model):
            # Diffusion generation never consults the prefix hook.
            return None
        if has_video:
            # Keyed by token ids the clip does not vary: a later clip would resume the first's mRoPE grid
            return None
        media_ids = self._vlm_media_token_ids(getattr(self._model, "config", None))
        if images and not media_ids:
            # Neither side can tell where the image's rows end without them.
            return None
        try:
            language_model, key, make_cache = self._vlm_snapshot_scope(adapter_state, images)
            block = self._vlm_media_block(prompt, images, make_cache)
            return VLMPromptCacheSession(
                store,
                key,
                language_model,
                make_cache,
                media_token_ids = media_ids,
                releases_unserved = bool(images),
                media_block = block,
                policy_hosts = (self._model, language_model) if block is not None else (),
                step = vlm_prefill_step(),
                window = self._session_window(),
            )
        except Exception as exc:
            # A layout that cannot be built once cannot be built later.
            self._vlm_snapshot_store = None
            self._vlm_snapshot_store_unavailable = True
            logger.info("MLX VLM prompt cache unavailable (%s); prefilling every request", exc)
            return None

    def _vlm_snapshot_scope(self, adapter_state, images):
        """The language model, store key and cache factory every path resumes and banks under."""
        from mlx_vlm.models.cache import make_prompt_cache

        language_model = getattr(self._model, "language_model", self._model)
        window = self._kv_cache_window
        # Base-vs-LoRA compare must not serve one side's KV to the other.
        key = f"{self.active_model_name}|{adapter_state!r}"
        if images:
            key += f"|{self._vlm_image_digest(images)}"
        make_cache = lambda: self._prepare_kv_entries(
            make_prompt_cache(language_model, max_kv_size = window)
        )
        return language_model, key, make_cache

    def _vlm_batch_row_cache(
        self,
        adapter_state,
        images = None,
        alone = None,
    ):
        """A batched row's ``prompt_cache_state``, or None while the store is off on an unquantized
        load. Otherwise every row gets one, so all rows' caches share the factory's layout."""
        store = self._vlm_prompt_cache_store()
        if (store is None and self._kv_quant_bits() is None) or _vlm_generation_is_diffusion(
            self._model
        ):
            return None
        media_ids = self._vlm_media_token_ids(getattr(self._model, "config", None))
        try:
            _language_model, key, make_cache = self._vlm_snapshot_scope(adapter_state, images)
        except Exception as exc:
            logger.info("MLX VLM prompt cache unavailable for a batched row (%s)", exc)
            return None
        if store is None:
            pass
        elif images and (not media_ids or self._vlm_bidirectional_vision(self._model)):
            # Unplaceable, or prefilled by the single path as a media block: nothing serves it.
            store.clear()
            store = None
        elif images:
            # Before the vision pass, which zoo runs before the row can look anything up.
            store.retain_key(key)
        return VLMBatchRowCache(
            store,
            key,
            make_cache,
            media_ids,
            releases_unserved = bool(images),
            step = vlm_prefill_step(),
            alone = alone,
        )

    @staticmethod
    def _vlm_media_block_available():
        try:
            import mlx_vlm
            from packaging.version import Version
            return Version(mlx_vlm.__version__) >= Version(_VLM_MEDIA_BLOCK_MIN_VERSION)
        except Exception:
            return False

    def _vlm_media_block(self, prompt, images, make_cache):
        if not images or prompt is None or not self._vlm_bidirectional_vision(self._model):
            return None
        if not self._vlm_media_block_available():
            return None
        try:
            return _VLMMediaBlock(self._model, self._processor, prompt, images, make_cache)
        except Exception as exc:
            # This request prefills the image as before; the store stays.
            logger.info("MLX VLM prompt cache: media block not prepared (%s)", exc)
            return None

    def _prepare_prompt_cache(self, prompt, adapter_state):
        history = self._prompt_cache()
        if history is None:
            return prompt, None, None, None, 0
        try:
            tokens = list(prompt) if isinstance(prompt, list) else self._encode_prompt(prompt)
            if not tokens:
                return prompt, None, None, None, 0
            key = f"{self.active_model_name}|{adapter_state!r}"
            cache, rest = history.fetch(self._model, key, tokens)
        except Exception as exc:
            logger.debug("MLX prompt cache lookup failed: %s", exc)
            return prompt, None, None, None, 0
        return rest, cache, key, tokens, len(tokens) - len(rest)

    def _unset_generation_budget(
        self,
        prompt,
        prompt_tokens = None,
        images = None,
        audio = None,
        videos = None,
    ):
        """Free context for an unset limit, or the default if the prompt cannot be counted."""
        prompt_n = self._prompt_within_window(prompt, prompt_tokens, images, audio, videos)
        if prompt_n is None:
            return UNSET_GENERATION_BUDGET
        return generation_budget_for_window(self._served_context, prompt_n, None)

    def _prompt_within_window(
        self,
        prompt,
        prompt_tokens = None,
        images = None,
        audio = None,
        videos = None,
    ):
        """The prompt's length, refused when it leaves no room in the served context; None if uncountable."""
        try:
            prompt_n = (
                len(prompt_tokens)
                if prompt_tokens is not None
                else self._count_prompt_tokens(prompt, images, audio, videos)
            )
        except Exception as exc:
            logger.debug("MLX prompt count failed: %s", exc)
            return None
        window = self._served_context
        # Admitted, a bounded cache would rotate the prompt's start away during prefill.
        if window and prompt_n >= int(window):
            raise context_refusal.ContextBudgetExceeded(prompt_n + 1, int(window))
        return prompt_n

    def _generation_limit(
        self,
        prompt,
        max_new_tokens,
        prompt_tokens = None,
        images = None,
        audio = None,
        videos = None,
        cap = None,
    ):
        if prompt_tokens is None and isinstance(prompt, list):
            prompt_tokens = prompt
        budget = getattr(self, "_kv_context_budget", None)
        if not budget:
            # No budget: size from the placeholder count as before, never preparing the media.
            if max_new_tokens is None:
                max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)
                if cap is not None:
                    max_new_tokens = min(max_new_tokens, cap)
            else:
                self._prompt_within_window(prompt, prompt_tokens)
            return max_new_tokens
        # Max Tokens == window is the UI's "no cap"; admitting it as-is would refuse every prompt.
        if max_new_tokens is not None and int(max_new_tokens) == int(budget):
            max_new_tokens = None
        if prompt_tokens is None:
            try:
                prompt_tokens = range(self._count_prompt_tokens(prompt, images, audio, videos))
            except Exception as exc:
                logger.debug("MLX prompt count for the context budget failed: %s", exc)
        if max_new_tokens is None:
            max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)
            if cap is not None:
                max_new_tokens = min(max_new_tokens, cap)
        if prompt_tokens is not None:
            # Refuse only a prompt with no room; a longer ask is cut at the budget like llama-server, since callers size from estimates
            self._check_context_budget(prompt, 1, prompt_tokens)
            max_new_tokens = max(1, min(int(max_new_tokens), int(budget) - len(prompt_tokens)))
        return max_new_tokens

    def _prefill_decision_scopes(self):
        """Outermost scopes of a generation, whose Int8 and Neural Engine Prefill decisions every Zoo
        scope nested in them inherits. Each is pinned off when inactive, so Zoo's full-precision NAX
        routes still run and no scope reads either env switch."""
        with ExitStack() as stack:  # a scope that fails to enter unwinds the ones before it
            for status, zoo, name in (
                (getattr(self, "_int8_prefill", None), _int8_prefill_zoo, "nax_quantized_linear"),
                (getattr(self, "_ane_prefill", None), _ane_prefill_zoo, "ane_prefill_mlp"),
            ):
                scope = (status or {}).get("scope")
                if scope is not None:
                    stack.enter_context(scope(self._model, True))
                elif zoo() is not None:
                    stack.enter_context(_mlx_optional_fusion(name, self._model, False))
            return stack.pop_all()

    def _generation_scopes(self, *call):
        """Zoo's scopes for a generation of this load, under its Int8 Prefill decision. Nested in
        _prefill_decision_scopes, which stays outermost."""
        active = (getattr(self, "_int8_prefill", None) or {}).get("active", False)
        return _mlx_inference_scopes(self._model, call, active, self._fast_verify_decision())

    def _fast_verify_decision(self):
        """This load's settled Fast Verify, or None under a Zoo that takes no such argument."""
        if _fast_verify_zoo() is None:
            return None
        return (getattr(self, "_fast_verify", None) or {}).get("active", False)

    def _kv_quant_bits(self):
        return (getattr(self, "_kv_quant", None) or {}).get("kv_bits")

    def _prepare_kv_entries(self, entries):
        bits = self._kv_quant_bits()
        # TurboQuant converts in mlx-vlm: mx.quantize cannot express its fractional widths.
        if bits is None or getattr(self, "_turboquant", False):
            return entries
        return _quantize_kv_entries(entries, bits)

    def _session_window(self):
        # TurboQuant converts the cache between forwards and a KV window rotates it per forward,
        # so neither can take in-window snapshots.
        step = getattr(self, "_prefill_step", None)
        if (
            step
            and getattr(self, "_prefill_layer_step", None)
            and getattr(self, "_kv_cache_window", None) is None
            and not self._kv_runtime_quant_kwargs()
        ):
            return step
        return vlm_prefill_step()

    def _prefill_step_kwargs(
        self,
        target = None,
        grid = False,
    ):
        step = getattr(self, "_prefill_step", None)
        if step is not None and grid:
            step = vlm_prefill_step()
        if step is None or not (target is None or _takes_prefill_step(target)):
            return {}
        return {"prefill_step_size": step}

    def _kv_runtime_quant_kwargs(self):
        """TurboQuant's width for mlx-vlm's converter; start 0, since its default leaves 5000 tokens unquantized."""
        bits = self._kv_quant_bits()
        if bits is None or not getattr(self, "_turboquant", False):
            return {}
        return {"kv_bits": bits, "kv_quant_scheme": "turboquant", "quantized_kv_start": 0}

    def _kv_quant_generate_kwargs(self):
        """A pre-quantized cache, not kv_bits: both runtimes would convert a rotating entry."""
        if self._kv_quant_bits() is None or getattr(self, "_turboquant", False):
            return {}
        return {
            "prompt_cache": self._prepare_kv_entries(_make_kv_entries(self._model, self._is_vlm))
        }

    def _check_context_budget(
        self,
        prompt,
        max_new_tokens = 0,
        prompt_tokens = None,
        images = None,
        audio = None,
        videos = None,
    ):
        """Refuse a request over the budget; an uncountable prompt is admitted."""
        budget = getattr(self, "_kv_context_budget", None)
        if not budget:
            return
        try:
            prompt_n = (
                len(prompt_tokens)
                if prompt_tokens is not None
                else self._count_prompt_tokens(prompt, images, audio, videos)
            )
        except Exception as exc:
            logger.debug("MLX prompt count for the context budget failed: %s", exc)
            return
        needed = prompt_n + max(0, int(max_new_tokens or 0))
        if needed > budget:
            raise context_refusal.ContextBudgetExceeded(needed, int(budget))

    def _kv_window_generate_kwargs(self):
        """The cache bound for a generation that builds its own cache, empty when unset. Both
        runtimes read it only when no prompt_cache is passed, so this covers the request that
        reaches generation without one rather than duplicating the bound."""
        window = getattr(self, "_kv_cache_window", None)
        return {} if window is None else {"max_kv_size": window}

    def _encode_prompt(self, prompt):
        """The tokens a generation sends for this prompt. A template that already emitted the BOS
        would otherwise get a second one, shifting every cached-prefix comparison and overstating
        a counted prompt by one."""
        bos = getattr(self._tokenizer, "bos_token", None)
        return list(
            self._tokenizer.encode(
                prompt, add_special_tokens = bos is None or not prompt.startswith(bos)
            )
        )

    def _reads_vision_input(self):
        """Whether this model reads images, which mlx-vlm serving it no longer implies."""
        reads = getattr(self, "_reads_vision", None)
        if reads is None:
            return bool(getattr(self, "_is_vlm", False))
        return bool(reads)

    def _count_media_prompt_tokens(
        self,
        prompt,
        images = None,
        audio = None,
        videos = None,
    ):
        """Prompt length as the processor expands media (as mlx-vlm's server does), else None."""
        if not (images or audio or videos):
            return None
        try:
            from mlx_vlm.utils import prepare_inputs

            config = getattr(self._model, "config", None)
            read = (
                config.get if isinstance(config, dict) else lambda attr: getattr(config, attr, None)
            )
            kwargs = {}
            if images:
                kwargs["images"] = images
            if audio:
                kwargs["audio"] = audio
            if videos:
                # Same fps as generation, not the processor's default rate.
                kwargs["videos"] = videos
                fps = _video_sampling_fps(videos, self._processor)
                if fps is not None:
                    kwargs["fps"] = fps
            inputs = prepare_inputs(
                self._processor,
                prompts = prompt,
                image_token_index = read("image_token_index"),
                add_special_tokens = _vlm_add_special_tokens(read("model_type"), self._processor),
                **kwargs,
            )
            ids = inputs["input_ids"]
            return int(getattr(ids, "size", 0)) or len(ids)
        except Exception as exc:
            logger.debug("MLX media prompt count failed: %s", exc)
            return None

    def _count_prompt_tokens(
        self,
        prompt,
        images = None,
        audio = None,
        videos = None,
    ):
        """Prompt length as generation tokenizes it; vision models follow mlx_vlm's marker rule."""
        if not self._reads_vision_input():
            return len(self._encode_prompt(prompt))
        expanded = self._count_media_prompt_tokens(prompt, images, audio, videos)
        if expanded is not None:
            return expanded
        model_type = getattr(getattr(self._model, "config", None), "model_type", None)
        add_special = _vlm_add_special_tokens(model_type, self._processor)
        return len(self._tokenizer.encode(prompt, add_special_tokens = add_special))

    def _configure_memory_limits(self):
        """Apply Metal memory caps before loading a model. memory_limit = 85% of recommended
        working-set; wired_limit = min(recommended, memory_limit). Recorded so unload can lower
        wired_limit back to release pinned RAM."""
        import mlx.core as mx

        if not mx.metal.is_available():
            return
        info = mx.device_info()
        rec_bytes = info.get("max_recommended_working_set_size")
        if not rec_bytes or rec_bytes <= 0:
            return
        rec_gb = rec_bytes / 1e9
        memory_limit_gb = rec_gb * MLX_MEMORY_LIMIT_FRACTION
        wired_limit_gb = min(rec_gb, memory_limit_gb)
        mx.set_memory_limit(int(memory_limit_gb * 1e9))
        mx.set_wired_limit(int(wired_limit_gb * 1e9))
        self._memory_limits_applied = {
            "memory_limit_gb": memory_limit_gb,
            "wired_limit_gb": wired_limit_gb,
            "recommended_gb": rec_gb,
        }
        logger.info(
            "MLX memory caps: memory_limit=%.2f GB, wired_limit=%.2f GB",
            memory_limit_gb,
            wired_limit_gb,
        )

    def _resolve_context_lengths(self, model, max_seq_length):
        """Clamps served context to MAX_REQUESTABLE_CONTEXT, as some models declare far larger windows; served length sizes the KV cache, native is unclamped"""
        native = mlx_native_context_length(model)
        served = runtime_context_length(model, max_seq_length) or native
        if served:
            served = min(int(served), MAX_REQUESTABLE_CONTEXT)
            logger.info("MLX context: served=%s native=%s", served, native)
        return served, native, native

    def _kv_cache_window_enforceable(self, served):
        """Whether a cache built for this model would really cap at *served*. A probe that could not
        run counts as not enforceable: claiming a bound nobody confirmed is what makes the
        setting look like it works."""
        if not served or served <= 0:
            return False
        enforced = _kv_window_enforced(self._model, self._is_vlm, int(served))
        if enforced is False:
            logger.warning(
                "MLX context limit of %d tokens is not enforced: this model builds at "
                "least one cache layer that declares no limit at that length.",
                int(served),
            )
        elif enforced is None:
            logger.warning(
                "MLX context limit of %d tokens is not enforced: no cache could be built "
                "to confirm it.",
                int(served),
            )
        return enforced

    def _resolve_kv_policy(
        self,
        is_vlm,
        kv_bits,
        max_seq_length,
        served,
        fitted = False,
        eligibility = None,
        turboquant = None,
    ):
        """With kv_bits, only an enforceable pin or fitted window becomes a budget, since a rotating cache cannot be quantized"""
        pinned = _positive_int(max_seq_length) is not None or fitted
        # Tri-state: True bounded, False confirmed unbounded, None unjudgeable; only True installs a bound, so clients can tell the other two apart
        confirmed = self._kv_cache_window_enforceable(served)
        enforceable = confirmed is True
        if getattr(self, "_turboquant", False) and kv_bits is not None:
            quant = turboquant or _turboquant_status(
                self._model, kv_bits, getattr(self, "_turboquant_refusal", "")
            )
        else:
            quant = _kv_quant_status(
                _normalize_mlx_kv_bits(kv_bits),
                self._model,
                is_vlm,
                eligibility,
            )
        if quant["kv_bits"] is None and enforceable:
            logger.info("MLX KV cache limited to %d tokens", int(served))
            return quant, int(served), True, None
        # No bound installed: False wherever the probe answered, None where it could not.
        budget = int(served) if quant["kv_bits"] is not None and pinned and enforceable else None
        if budget:
            logger.info("MLX context limited to %d tokens per request, not by the cache", budget)
        return quant, None, (None if confirmed is None else False), budget

    def load_model(
        self,
        config,
        max_seq_length = 2048,
        load_in_4bit = True,
        hf_token = None,
        trust_remote_code = False,
        gpu_ids = None,
        dtype = None,
        parallel_mode = None,
        distributed_group = None,
        kv_quant = None,
        chat_template_override = None,
        int8_prefill = False,
        ane_prefill = False,
        fast_verify = False,
        speculative_type = None,
        spec_draft_n_max = None,
        spec_draft_model = None,
        spec_drafters_allowed = None,
        offload = (None, None),
    ) -> bool:
        import mlx.core as mx

        from core.inference import mlx_speculative

        # Keep the token so the native-template fallback can fetch a gated model's repo template during generation.
        self._hf_token = hf_token
        self._resume_load_args = (bool(load_in_4bit), repr(dtype), repr(tuple(offload or ())))
        model_name = config.identifier if hasattr(config, "identifier") else str(config)
        is_vision = getattr(config, "is_vision", False)
        is_lora = getattr(config, "is_lora", False)
        # Route by the config the loader reads.
        route_config = getattr(config, "config_dir", None) or (
            getattr(config, "base_model", None) if is_lora else model_name
        )
        zoo_multimodal = not is_vision and _zoo_declares_multimodal(route_config, hf_token)
        vlm_only_text = not (is_vision or zoo_multimodal) and _zoo_builds_text_on_mlx_vlm(
            route_config, hf_token
        )
        kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)
        self._turboquant = turboquant
        if self._turboquant:
            try:
                from mlx_vlm.turboquant import TurboQuantKVCache
            except ImportError as exc:
                raise RuntimeError(
                    "TurboQuant requires a newer mlx-vlm. Update the MLX packages and retry."
                ) from exc
        distributed_rank, distributed_size = _mlx_distributed_rank_size(distributed_group)
        is_distributed = distributed_group is not None and distributed_size > 1
        self._turboquant_refusal = (
            _turboquant_refusal(is_vision = is_vision, is_distributed = is_distributed, is_lora = is_lora)
            if self._turboquant
            else ""
        )
        spec_mode = mlx_speculative.mlx_spec_mode(speculative_type)
        spec_reason = None
        speculates = mlx_speculative.speculates_on_route(
            spec_mode, bool(is_vision), spec_draft_model
        )
        if speculates:
            spec_reason = mlx_speculative.speculation_refusal(
                distributed = is_distributed,
                # Speculation runs on mlx-vlm and takes an adapter only on a vision or zoo-multimodal load.
                lora = is_lora and not (is_vision or zoo_multimodal),
            )
            speculates = spec_reason is None
            if (
                spec_mode == "auto" and not spec_draft_model
            ):  # not a request, so no refusal to report
                spec_reason = None
        self._speculative_draft = None
        # mlx-lm cannot batch a quantized cache; mlx-vlm, which also ships text architectures, can.
        batches_quantized_text = (
            kv_bits is not None
            and not (is_distributed or is_lora)
            and _row_quantized_cache_gap() is None
        )
        # Speculation runs on mlx-vlm models and caches.
        use_vlm = (
            is_vision
            or zoo_multimodal
            or vlm_only_text
            or speculates
            or (self._turboquant and not self._turboquant_refusal)
            or batches_quantized_text
        )
        self._distributed_group = distributed_group
        self._distributed_rank = distributed_rank
        self._distributed_world_size = distributed_size

        # GGUF is served by llama-server; here means route detection flaked, so fail loudly
        if getattr(config, "is_gguf", False):
            raise RuntimeError(
                f"MLXInferenceBackend cannot load GGUF model '{model_name}': "
                f"GGUF models must be served by llama-server in the parent "
                f"process. The /api/inference/load route should have "
                f"detected this repo as GGUF before dispatching to the MLX "
                f"orchestrator -- this fallback indicates a transient HF "
                f"Hub failure during initial detection. Retry the request."
            )

        if hf_token:
            import os
            os.environ["HF_TOKEN"] = hf_token
        self._configure_memory_limits()

        logger.info(
            "Loading %s via %s (is_lora=%s, distributed=%s, rank=%s/%s, mode=%s)",
            model_name,
            "mlx-vlm" if use_vlm else "mlx-lm",
            is_lora,
            is_distributed,
            distributed_rank,
            distributed_size,
            parallel_mode,
        )
        if is_distributed and parallel_mode not in ("pipeline", "tensor"):
            raise ValueError(
                "Unsloth: distributed MLX inference requires parallel_mode='pipeline' "
                "or parallel_mode='tensor'."
            )
        if is_distributed and is_lora:
            raise ValueError(
                "Unsloth: distributed MLX inference for LoRA adapter repos "
                "is not supported yet. Merge/export the adapter into an MLX model "
                "before distributed inference."
            )

        try:
            from unsloth_zoo.mlx.loader import FastMLXModel
        except ImportError as e:
            raise ImportError(
                "Unsloth: MLX inference requires unsloth-zoo with the MLX modules "
                "(unsloth_zoo.mlx.loader). Reinstall via install.sh on Apple Silicon."
            ) from e

        load_kwargs = {
            "max_seq_length": max_seq_length,
            "dtype": dtype,
            "load_in_4bit": load_in_4bit,
            "token": hf_token,
            "trust_remote_code": trust_remote_code,
            "text_only": not use_vlm,
            "quantize_embeddings": not is_lora,
        }
        if is_distributed:
            if parallel_mode == "pipeline":
                load_kwargs["pipeline_group"] = distributed_group
            else:
                load_kwargs["tensor_group"] = distributed_group
        precision_folder = (
            None if getattr(config, "is_gguf", False) else getattr(config, "gguf_variant", None)
        )
        load_source = model_name
        if precision_folder:
            from hub.utils.mlx_variants import download_precision_folder
            load_source = download_precision_folder(model_name, precision_folder, hf_token)

        knobs = offload

        # Auto is planned only where the load is priced: not distributed, an adapter or an explicit dtype.
        def offload_request():
            if is_distributed or is_lora or dtype is not None:
                return _mlx_offload_request(knobs, None)
            priced = dict(
                pinned = _positive_int(max_seq_length),
                load_in_4bit = load_in_4bit,
                retains_history = not use_vlm or mlx_vlm_snapshot_store_available(),
                kv_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits),
                vision = True if speculates and not is_vision else None,
            )

            def plan(model_dir, wanted):
                drafter = speculates and mlx_speculative.offload_drafter(
                    spec_mode,
                    spec_draft_model,
                    model_dir = model_dir,
                    target_name = model_name,
                    allowed = spec_drafters_allowed,
                )
                chosen, alone = mlx_resolve_drafter_offload(
                    model_dir, wanted, drafter or None, auto = spec_mode == "auto", **priced
                )
                # A repeat call sees the fractions and plans no step: keep the first baseline.
                if "auto" in wanted:
                    offload_baseline.update(route = alone and mlx_offload_route(alone))
                return chosen

            return _mlx_offload_request(knobs, plan)

        offload_baseline = {"route": None}

        offload = offload_request()
        load_kwargs.update(offload["kwargs"])

        # Freed before the replacement weights are allocated, for headroom.
        self._model_fusion.close()
        self._clear_prompt_cache()
        model = None
        try:
            model, tokenizer_or_processor = _load_offloading(
                FastMLXModel.from_pretrained, load_source, load_kwargs, offload
            )
        except Exception as exc:
            # A vlm_only_text model has no mlx-lm route to fall back to.
            if is_vision or vlm_only_text or not use_vlm or is_metal_queue_dead(exc):
                raise
            logger.warning(
                "Load of %s through mlx-vlm for %s failed (%s); serving through mlx-lm",
                model_name,
                "speculative decoding" if speculates else "its KV quantization",
                exc,
            )
        _eligibility = None
        if model is not None and batches_quantized_text and not (is_vision or self._turboquant):
            _eligibility = _kv_quant_eligibility(model, True, _normalize_mlx_kv_bits(kv_bits))
            verdict, reason, _retainable = _eligibility
            if verdict not in ("full", "partial") and not vlm_only_text:
                # Unquantized, the load gains nothing from mlx-vlm's batch; a vlm_only_text one stays on it.
                logger.info(
                    "MLX KV quantization not applied to %s (%s); serving through mlx-lm",
                    model_name,
                    reason,
                )
                model = tokenizer_or_processor = _eligibility = None
        # unsloth-zoo builds an image checkpoint mlx-vlm lacks through mlx-lm: the text paths
        # serve it, reading its images only through an unsloth-zoo that prefills them.
        served_by_mlx_lm = use_vlm and getattr(model, "_is_vlm_model", None) is False
        if served_by_mlx_lm and not _mlx_lm_reads_images(model):
            is_vision = False
        if model is None or served_by_mlx_lm:
            if speculates:
                speculates = False
                spec_reason = (
                    None
                    if spec_mode == "auto" and not spec_draft_model
                    else mlx_speculative.RUNTIME_ERROR
                )
            if self._turboquant:
                self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD
            use_vlm = False
        if model is None:
            import gc

            # Outside the except: its traceback keeps the failed load's weights alive.
            gc.collect()
            _drain_generation_streams(mx)
            mx.clear_cache()
            load_kwargs["text_only"] = True
            # Auto planned on the mlx-vlm route, and decide() consumed its knobs: plan this route afresh.
            offload = offload_request()
            load_kwargs.update(offload["kwargs"])
            model, tokenizer_or_processor = _load_offloading(
                FastMLXModel.from_pretrained, load_source, load_kwargs, offload
            )

        zoo_loaded_multimodal = zoo_multimodal and not served_by_mlx_lm
        if zoo_loaded_multimodal:
            is_vision = getattr(model, "_unsloth_supports_images", None) is True
        self._reads_vision = bool(is_vision)
        if use_vlm:
            processor = tokenizer_or_processor
            if is_lora:
                processor = load_mlx_adapter_processor(processor, model_name, hf_token)
            _stream_spm_tokens_eagerly(processor)
            self._model = model
            self._processor = processor
            self._tokenizer = getattr(processor, "tokenizer", processor)
            self._is_vlm = True
        else:
            tokenizer = tokenizer_or_processor
            if is_lora:
                tokenizer = load_mlx_adapter_tokenizer(tokenizer, model_name, hf_token)
            self._model = model
            self._tokenizer = tokenizer
            self._processor = None
            self._is_vlm = False

        _config_audio_type = getattr(config, "audio_type", None)
        if served_by_mlx_lm and _config_audio_type == "audio_vlm":
            # Omni audio input rides mlx-vlm's processor, which an mlx-lm load does not have.
            _config_audio_type = None
        _audio_type = _classify_mlx_audio_type(
            model,
            self._processor,
            is_vision or zoo_loaded_multimodal,
            config_audio_type = _config_audio_type,
        )
        _served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(
            self._model, max_seq_length
        )
        _offload = _mlx_offload_status(offload, self._model)
        _offloaded = (_offload["ple"]["applied"], _offload["experts"]["applied"])
        _offloaded = {"offload": _offloaded} if any(_offloaded) else {}
        self._prefill_step = offload["prefill_step"] if _offload["experts"]["applied"] else None
        self._prefill_layer_step = _prefill_layer_step(self._model, self._prefill_step)
        if self._prefill_step:
            _offloaded["prefill_chunk"] = self._prefill_step
        _priceable = not (is_distributed or is_lora or dtype is not None)
        spec_kind = _drafter_ctx = None
        _stretched = []
        _vlm_route = speculates and not is_vision
        if speculates and _vlm_generation_is_diffusion(self._model):
            # Neither the single path nor the resident batch may decode a diffusion model speculatively.
            speculates = False
            spec_reason = (
                None
                if spec_mode == "auto" and not spec_draft_model
                else mlx_speculative.RUNTIME_ERROR
            )
        self._fast_verify = _fast_verify_status(fast_verify, self._model)
        if speculates:
            model_dir = _snapshot_dir(self._model, model_name)
            resolution = mlx_speculative.resolve_speculation(
                spec_mode,
                spec_draft_model,
                model_dir = model_dir,
                target_name = model_name,
                allowed = spec_drafters_allowed,
            )

            if not self._turboquant and kv_bits is not None and _eligibility is None:
                _eligibility = _kv_quant_eligibility(
                    self._model, use_vlm, _normalize_mlx_kv_bits(kv_bits)
                )
            _drafter_bits = (
                _normalize_mlx_kv_bits(kv_bits)
                if not self._turboquant and _eligibility and _eligibility[0] in ("full", "partial")
                else None
            )

            def _fits(source, reserve = True):
                if not _priceable:
                    return None, None
                return mlx_drafter_fit(
                    model_dir,
                    _served_ctx,
                    _positive_int(max_seq_length),
                    source,
                    load_in_4bit = load_in_4bit,
                    costless = spec_mode == "auto",
                    baseline = offload_baseline["route"],
                    reserve = reserve,
                    kv_bits = _drafter_bits,
                    **_offloaded,
                )

            def _stretch(source):
                ok, context = _fits(source, reserve = False)
                if ok is not False:
                    _stretched.append(source)
                return ok, context

            (
                self._speculative_draft,
                spec_kind,
                spec_reason,
                _drafter_ctx,
            ) = mlx_speculative.build_draft(
                self._model,
                resolution,
                fits = _fits,
                stretch = _stretch,
                draft_n_max = _positive_int(spec_draft_n_max),
                fast_verify = self._fast_verify_decision(),
            )
            logger.info(
                "MLX speculative decoding: %s (mode %s, reason %s)",
                spec_kind or "off",
                spec_mode,
                spec_reason,
            )
        self._fast_verify = _settle_fast_verify(
            self._fast_verify, self._model, getattr(self._speculative_draft, "drafter", None)
        )
        _requested_bits = _normalize_mlx_kv_bits(kv_bits)
        _turboquant = None
        if self._turboquant:
            from core.inference.mlx_memory import MlxTurboQuant

            if kv_bits is not None:
                _turboquant = _turboquant_status(self._model, kv_bits, self._turboquant_refusal)
            _requested_bits = (
                MlxTurboQuant(kv_bits)
                if _turboquant is not None and _turboquant["kv_bits"] is not None
                else None
            )
            # Only mlx-vlm's caches convert, so a text load is priced on the route it takes.
            _vlm_route = _vlm_route or (_requested_bits is not None and not is_vision)
        _fitted_ctx, _eligibility = (
            (_drafter_ctx, _eligibility)
            if _drafter_ctx is not None
            else (None, _eligibility)
            if not _priceable or _positive_int(max_seq_length) is not None
            else _fitted_context(
                self._model,
                model_name,
                _served_ctx,
                load_in_4bit = load_in_4bit,
                retains_history = not use_vlm or mlx_vlm_snapshot_store_available(),
                kv_bits = _requested_bits,
                is_vlm = use_vlm,
                eligibility = _eligibility,
                **_offloaded,
                **({"vision": True} if _vlm_route else {}),
            )
        )
        if _fitted_ctx:
            logger.info("MLX context fitted to %d tokens from %d", _fitted_ctx, _served_ctx or 0)
            _served_ctx = _fitted_ctx
        self._served_context = _served_ctx
        # A stretched drafter spent the prompt cache's share: cap the cache to what is left beside it.
        if (
            _priceable
            and _stretched
            and getattr(self._speculative_draft, "drafter", None) is not None
        ):
            self._prompt_cache_room = mlx_prompt_cache_room(
                model_dir,
                _served_ctx,
                _stretched[-1],
                load_in_4bit = load_in_4bit,
                kv_bits = _drafter_bits,
                **_offloaded,
            )
            if self._prompt_cache_room is not None:
                logger.info(
                    "MLX prompt cache holds at most %.2f GB beside the chosen drafter",
                    self._prompt_cache_room / 1e9,
                )
        # Classified now so a cache the model cannot attend over is refused here, not on a token.
        (
            self._kv_quant,
            self._kv_cache_window,
            _ctx_enforced,
            self._kv_context_budget,
        ) = self._resolve_kv_policy(
            use_vlm,
            kv_bits,
            max_seq_length,
            _served_ctx,
            fitted = _fitted_ctx is not None,
            eligibility = _eligibility,
            turboquant = _turboquant,
        )
        if self._kv_quant["kv_bits"] is not None:
            logger.info(
                "MLX KV cache quantization: %s-bit (%s eligibility)",
                self._kv_quant["kv_bits"],
                self._kv_quant["eligibility"],
            )
        self._int8_prefill = _int8_prefill_status(int8_prefill, self._model)
        self._ane_prefill = _ane_prefill_status(ane_prefill, self._model)

        # Captured before install so the shipped template is kept, from the render target, not the nested tokenizer, which would overwrite the processor template
        native_source = _native_template_source(self._tokenizer, self._processor)
        native_template = getattr(native_source, "chat_template", None)
        native_marks_audio = bool(
            chat_template_override
            and is_audio_input_type(_audio_type)
            and _audio_marker_survives(self._processor, self._model)
        )
        image_placeholder = _image_placeholder(self._tokenizer, self._processor)
        self._template_override = _install_template_override(
            chat_template_override,
            self._tokenizer,
            self._processor,
            lambda: self._render_template_probe(use_vlm),
        )
        if native_marks_audio:
            _revoke_override_that_drops_audio(self._template_override, self._processor, self._model)
        # Unconditional for vision, not gated on the native template: an override rendering plain text silently drops the image
        if is_vision:
            _revoke_override_that_drops_image(
                self._template_override, self._tokenizer, self._processor, image_placeholder
            )
        # After the override settles, so the probe measures the template generation will use.
        self._multi_image_marker = (
            _scaling_image_marker(self._tokenizer, self._processor, self._model)
            if is_vision and _model_type_takes_several_images(self._model)
            else None
        )
        # Release after the media checks: the pairs reference tokenizer and processor and would outlive the unload
        self._template_override["restore"] = []
        if self._template_override["reason"]:
            logger.info(
                "MLX chat template override not applied: %s",
                self._template_override["reason"],
            )

        self.active_model_name = model_name
        try:
            self._resume_files = self._resume_weight_files()
        except OSError:
            self._resume_files = False
        self.models[model_name] = {
            # Per-model token for the native-template fallback (matches transformers).
            "hf_token": hf_token,
            "gguf_variant": precision_folder,
            "precision_folder": precision_folder,
            # Per-model trust_remote_code reused by the native-template reload (matches transformers).
            "trust_remote_code": trust_remote_code,
            "model": self._model,
            "tokenizer": self._tokenizer,
            "processor": self._processor,
            "is_vision": is_vision,
            "is_lora": getattr(config, "is_lora", False),
            # For a LoRA adapter the native chat template lives on the base model.
            "base_model": getattr(config, "base_model", None)
            if getattr(config, "is_lora", False)
            else None,
            # Mirrors utils.models.model_config semantics (is_audio == TTS).
            "is_audio": _audio_type is not None and _audio_type != "audio_vlm",
            "audio_type": _audio_type,
            "has_audio_input": is_audio_input_type(_audio_type),
            "speaks": _audio_type == "audio_vlm"
            and getattr(self._model, "joint_audio_text_generation", False) is True,
            "has_video_input": is_vision and _mlx_reads_video(self._processor),
            "context_length": _served_ctx,
            # Matches llama.cpp requested_n_ctx; served window alone cannot tell if a size was asked for.
            "requested_context_length": _positive_int(max_seq_length) or 0,
            "native_context_length": _native_ctx,
            "max_context_length": _max_ctx,
            # Whether the window bounds the cache: True enforced, False unbounded, None unjudged.
            "context_length_enforced": _ctx_enforced,
            "context_length_fitted": _fitted_ctx,
            "context_unbounded_when_batched": self._vision_batch_is_available(),
            "mlx_context_budget": self._kv_context_budget,
            "mlx_kv_bits": self._kv_quant["kv_bits"],
            "mlx_kv_bits_requested": self._kv_quant["requested_kv_bits"],
            "mlx_kv_quant": encode_mlx_kv_quant(self._kv_quant["kv_bits"], self._turboquant),
            "mlx_kv_quant_requested": encode_mlx_kv_quant(
                self._kv_quant["requested_kv_bits"], self._turboquant
            ),
            "mlx_kv_quant_eligibility": self._kv_quant["eligibility"],
            "mlx_kv_quant_reason": self._kv_quant["reason"],
            "mlx_kv_quant_note": self._kv_quant["note"],
            "mlx_int8_prefill": self._int8_prefill["active"],
            "mlx_int8_prefill_requested": self._int8_prefill["requested"],
            "mlx_int8_prefill_reason": self._int8_prefill["reason"],
            "mlx_ane_prefill": self._ane_prefill["active"],
            "mlx_ane_prefill_requested": self._ane_prefill["requested"],
            "mlx_ane_prefill_reason": self._ane_prefill["reason"],
            "mlx_fast_verify": self._fast_verify["active"],
            "mlx_fast_verify_requested": self._fast_verify["requested"],
            "mlx_fast_verify_reason": self._fast_verify["reason"],
            "mlx_offload": _offload,
            "chat_template_override_requested": self._template_override["requested"],
            "chat_template_override_reason": self._template_override["reason"],
            "speculative_type": spec_mode,
            "spec_draft_n_max": _positive_int(spec_draft_n_max),
            "spec_draft_model": spec_draft_model or None,
            "spec_drafter_kind": spec_kind,
            "spec_fallback_reason": spec_reason,
        }
        # Capture chat_template_info for the worker IPC reply and route capability classification.
        self._populate_chat_template_info(model_name, native_template)

        # The worker owns fixed base weights until unload; adapter requests stay scoped.
        if not is_lora:
            self._model_fusion.enter_context(_mlx_optional_fusion("fused_moe_gate_up", model))
        # Every rank of a distributed load must run each forward together.
        if not is_distributed:
            with mlx_rng_preserved(), self._prefill_decision_scopes():
                _mlx_warm_inference(
                    model, self._int8_prefill["active"], self._fast_verify_decision()
                )
            with mlx_rng_preserved():
                self._warm_resident_batch()
            if self._ane_prefill["active"]:
                self._ane_prefill = _ane_prefill_status(True, model)
                self.models[model_name].update(
                    mlx_ane_prefill = self._ane_prefill["active"],
                    mlx_ane_prefill_reason = self._ane_prefill["reason"],
                )
        logger.info("Model %s loaded successfully", model_name)
        return True

    def narrow_context(self, max_seq_length):
        """Serve a smaller context from the held weights; None where a fresh load could decide differently."""
        entry = self.models.get(self.active_model_name)
        pinned = _positive_int(max_seq_length)
        if entry is None or pinned is None or not self._served_context:
            return None
        offload = entry.get("mlx_offload") or {}
        if (
            any(part.get("applied") for part in offload.values() if isinstance(part, dict))
            or self._kv_quant["requested_kv_bits"] is not None
            or self._turboquant
            or self._prompt_cache_room is not None
            or entry.get("spec_fallback_reason")
        ):
            return None
        try:
            if not self._resume_files or self._resume_files != self._resume_weight_files():
                return None
        except OSError:
            return None
        # The served length reads this attribute before the argument.
        previous = self._model.max_seq_length
        self._model.max_seq_length = pinned
        served = self._resolve_context_lengths(self._model, pinned)[0]
        if not served or served > self._served_context:
            self._model.max_seq_length = previous
            return None
        self._clear_prompt_cache()
        self._served_context = served
        self._kv_quant, self._kv_cache_window, enforced, self._kv_context_budget = (
            self._resolve_kv_policy(self._is_vlm, None, pinned, served)
        )
        fields = {
            "context_length": served,
            "requested_context_length": pinned,
            "context_length_enforced": enforced,
            "context_length_fitted": None,
            "mlx_context_budget": self._kv_context_budget,
        }
        entry.update(fields)
        return fields

    def _warm_resident_batch(self):
        """Warm the request batch Zoo's warm-up misses: twice greedy (Qwen4 scopes build kernels only after a reply), then sampled."""
        if self.resident_unavailable_reason({}) is not None:
            return
        messages = [{"role": "user", "content": "Count from one to ten."}]
        # Cold replies would teach the drafter's controller that drafting does not pay.
        draft = self._speculative_draft
        controller = getattr(draft, "controller", None)
        if controller is not None:
            draft.controller = copy.deepcopy(controller)
        try:
            for temperature in (0.0, 0.0, 0.7):
                session = self.open_resident_batch(width = 1)
                try:
                    session.admit(
                        {"messages": messages, "temperature": temperature, "max_new_tokens": 16},
                        "warm",
                    )
                    while session.rows_in_flight:
                        for _, snapshot in session.step():
                            if isinstance(snapshot, BaseException):
                                raise snapshot
                finally:
                    session.close()
        except Exception as error:
            if is_metal_queue_dead(error):
                raise
            logger.warning(
                "MLX batch warm-up failed, so the first request runs it: %s: %s",
                type(error).__name__,
                error,
            )
        finally:
            if controller is not None:
                draft.controller = controller
            room = self._prompt_cache_room
            self._clear_prompt_cache()
            self._prompt_cache_room = room

    def _render_template_probe(self, is_vision: bool) -> str:
        """Renders through the target generation uses, since the recovery renderer returns None off-list."""
        from core.inference.chat_template_helpers import (
            apply_chat_template_for_generation,
            chat_render_target,
        )

        messages = [{"role": "user", "content": "hi"}]
        target = (
            chat_render_target(self._processor)
            if is_vision and self._processor is not None
            else self._tokenizer
        )
        rendered = apply_chat_template_for_generation(target, messages)
        if not rendered or not rendered.strip():
            raise ValueError("the template produced an empty prompt")
        return rendered

    def _populate_chat_template_info(
        self,
        model_name: str,
        native_template = _TEMPLATE_NOT_CAPTURED,
    ) -> None:
        """Records the shipped template, not an override, since capability classification reads this one."""
        entry = self.models.get(model_name)
        if not entry:
            return
        tok = entry.get("tokenizer")
        if tok is None:
            proc = entry.get("processor")
            tok = getattr(proc, "tokenizer", None) if proc else None
        info = {
            "has_template": False,
            "template": None,
            "format_type": "generic",
            "special_tokens": {},
            "template_name": None,
            # The body _generate_vlm renders an image turn with, not the tokenizer body.
            "processor_template": None,
            "renders_image": False,
            "accepts_multiple_images": False,
        }
        from core.inference.chat_template_helpers import (
            chat_render_target as _chat_render_target,
        )

        _proc = entry.get("processor")
        # A processor template alone does not mean the render selects it (#7066).
        _proc_tpl = (
            getattr(_proc, "chat_template", None) if _chat_render_target(_proc) is _proc else None
        )
        if isinstance(_proc_tpl, (str, dict, list, tuple)) and _proc_tpl:
            info["processor_template"] = _proc_tpl
        # An image model served through mlx-lm marks images with its tokenizer's template.
        info["renders_image"] = self._reads_vision_input() and (
            _proc is not None or not getattr(self, "_is_vlm", False)
        )
        info["accepts_multiple_images"] = info["renders_image"] and bool(
            getattr(self, "_multi_image_marker", None)
        )
        try:
            tpl = (
                getattr(tok, "chat_template", None)
                if native_template is _TEMPLATE_NOT_CAPTURED
                else native_template
            )
            if tpl:
                info["has_template"] = True
                info["template"] = tpl
                lower = tpl.lower()
                if "start_header_id" in lower and "end_header_id" in lower:
                    info["format_type"] = "llama3"
                elif "[inst]" in lower and "[/inst]" in lower:
                    info["format_type"] = "mistral"
                elif "<|im_start|>" in lower and "<|im_end|>" in lower:
                    info["format_type"] = "chatml"
                else:
                    info["format_type"] = "custom"
                special = {}
                for attr in ("bos_token", "eos_token", "pad_token"):
                    val = getattr(tok, attr, None)
                    if val:
                        special[attr] = val
                info["special_tokens"] = special
        except Exception as exc:
            logger.warning("MLX chat_template_info capture failed: %s", exc)
        entry["chat_template_info"] = info

    def unload_model(self, model_name: str) -> bool:
        import mlx.core as mx
        import gc

        self._model_fusion.close()
        if self._ane_prefill["active"] and self._model is not None:
            # The Neural Engine worker can hold its last request's program past the model; a failed release must not
            # stop the unload.
            try:
                _ane_prefill_zoo().release_ane_prefill(self._model)
            except Exception as exc:
                logger.warning("MLX Neural Engine prefill release failed: %s", exc)
            self._ane_prefill = _ane_prefill_status(False, None)
        self._fast_verify = _fast_verify_status(False, None)
        if model_name in self.models:
            del self.models[model_name]
        self._speculative_draft = None
        self._prefill_step = None
        self._prefill_layer_step = None
        self._model = None
        self._tokenizer = None
        self._processor = None
        self._distributed_group = None
        self._distributed_rank = 0
        self._distributed_world_size = 1
        if self.active_model_name == model_name:
            self.active_model_name = None
        self._clear_prompt_cache()
        gc.collect()
        _drain_generation_streams(mx)
        mx.clear_cache()

        if mx.metal.is_available() and self._memory_limits_applied and not self.models:
            try:
                mx.set_wired_limit(0)
                logger.info("MLX wired_limit released back to OS on unload")
            except Exception as e:
                logger.warning("Failed to release wired_limit: %s", e)
            self._memory_limits_applied = {}
        logger.info("Model %s unloaded", model_name)
        return True

    def _render_text_prompt(
        self,
        messages,
        *,
        tools = None,
        enable_thinking = None,
        reasoning_effort = None,
        preserve_thinking = None,
        continue_final_message = False,
    ):
        """Render the prompt a text generation sends, with its template metadata. Shared with
        counting, so a count cannot price a prompt the model never sees."""
        from core.inference.chat_template_helpers import (
            apply_chat_template_for_generation,
            render_with_native_template_fallback,
        )

        prompt = apply_chat_template_for_generation(
            self._tokenizer,
            messages,
            tools = tools,
            enable_thinking = enable_thinking,
            reasoning_effort = reasoning_effort,
            preserve_thinking = preserve_thinking,
            continue_final_message = continue_final_message,
        )
        if prompt is None:
            raise RuntimeError("apply_chat_template returned None — tokenizer may be incompatible")

        # Transformers parity: fall back to the native template if the template dropped tools
        model_info = self.models.get(self.active_model_name, {})
        return render_with_native_template_fallback(
            formatted_prompt = prompt,
            tokenizer = self._tokenizer,
            model_info = model_info,
            active_model_name = self.active_model_name,
            messages = messages,
            tools = tools,
            enable_thinking = enable_thinking,
            reasoning_effort = reasoning_effort,
            preserve_thinking = preserve_thinking,
            continue_final_message = continue_final_message,
            hf_token = model_info.get("hf_token"),
            return_metadata = True,
        )

    @staticmethod
    def _with_system_prompt(messages, system_prompt):
        """The conversation a request carrying this system prompt turns into. Shared with
        generation, which takes the system prompt beside the messages rather than inside them."""
        full_messages = []
        if system_prompt:
            full_messages.append({"role": "system", "content": system_prompt})
        full_messages.extend(messages)
        return full_messages

    def _conversation_to_render(
        self,
        messages,
        system_prompt,
        image,
        video = 0,
    ):
        """The turns this request renders, whichever way its reply is decoded."""
        if (self._is_vlm or self._reads_vision_input()) and (image is not None or video):
            from core.inference.chat_template_helpers import (
                chat_render_target as _chat_render_target,
            )
            full_messages = messages_with_attached_image(
                messages,
                system_prompt = system_prompt,
                structured_content = (
                    self._processor is not None
                    and _chat_render_target(self._processor) is self._processor
                ),
                image = int(image is not None),
                video = video,
            )
        else:
            full_messages = self._with_system_prompt(messages, system_prompt)
        return full_messages

    def count_chat_tokens(
        self,
        messages,
        system_prompt = "",
        *,
        tools = None,
        enable_thinking = None,
        reasoning_effort = None,
        preserve_thinking = None,
        images = None,
    ) -> int:
        """Prompt tokens this model would receive for these messages. Renders and tokenizes exactly
        as generation does."""
        if self._model is None:
            raise RuntimeError("No model loaded")
        images = list(images or ())
        full_messages = self._conversation_to_render(messages, system_prompt, images or None)
        if len(images) > 1 and not self._multi_image_marker:
            # Generation serves the newest picture alone.
            full_messages = list(full_messages)
            trim_image_turns(full_messages, images, limit = 1, keep = (len(images) - 1,))

        if self._is_vlm:
            # One image per marker, else the structured-item check raises.
            prompt, _target, _markers = self._render_vlm_prompt(
                full_messages,
                images or None,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
            )
            return self._count_prompt_tokens(prompt, images)

        render_result = self._render_text_prompt(
            full_messages,
            tools = tools,
            enable_thinking = enable_thinking,
            reasoning_effort = reasoning_effort,
            preserve_thinking = preserve_thinking,
        )
        if images and self._reads_vision_input():
            from unsloth_zoo.mlx.loader import _mlx_lm_image_inputs

            # The ids generation embeds, without running the vision tower.
            expanded = _mlx_lm_image_inputs(
                self._tokenizer,
                self._tokenizer._unsloth_image_processor,
                images,
                render_result.prompt,
            )
            return len(expanded["input_ids"][0])
        return self._count_prompt_tokens(render_result.prompt)

    def generate_chat_response(
        self,
        messages,
        system_prompt = "",
        image = None,
        images = None,
        image_ordinal = None,
        temperature = 0.7,
        top_p = 0.9,
        top_k = 40,
        min_p = 0.0,
        max_new_tokens = 256,
        repetition_penalty = 1.0,
        cancel_event = None,
        # Reasoning / tool kwargs, rendered via apply_chat_template_for_generation (transformers parity).
        tools = None,
        enable_thinking = None,
        reasoning_effort = None,
        preserve_thinking = None,
        continue_final_message = False,
        presence_penalty = 0.0,
        seed = None,
        frequency_penalty = 0.0,
        logit_bias = None,
        stop = None,
        response_format = None,
        reasoning_is_extracted = False,
        _adapter_state = None,
        # Unrestricted mode uses an EMPTY tools list, so bool(tools) cannot detect the protocol
        tool_protocol_active = None,
        videos = None,
    ) -> Generator[str, None, None]:
        if self._model is None:
            raise RuntimeError("No model loaded")
        if videos:
            if not _mlx_vlm_decodes_video():
                raise RuntimeError("The installed mlx-vlm does not read video.")
            if not self._reads_vision_input():
                raise RuntimeError("The loaded model does not read video.")

        # Reset so a failed run cannot surface stale stats.
        self.last_generation_stats = None

        full_messages = self._with_system_prompt(messages, system_prompt)

        # History first, the attachment last: MLX binds pixels to markers in order.
        attached = list(images or []) + ([image] if image is not None else [])
        # Shared with the transformers vision path so both render the same turns (#10092).
        if self._reads_vision_input() and (attached or videos):
            # Processor templates want part lists, the tokenizer fallback wants strings.
            from core.inference.chat_template_helpers import (
                chat_render_target as _chat_render_target,
            )

            _renders_via_processor = (
                self._processor is not None
                and _chat_render_target(self._processor) is self._processor
            )
            full_messages = messages_with_attached_image(
                messages,
                system_prompt = system_prompt,
                structured_content = _renders_via_processor,
                image = len(attached),
                video = len(videos or ()),
            )
            # That helper skips convos with markers, but replayed MCP picture markers belong to other pictures, so the render would be one short
            _prior_markers = image_marker_parts(full_messages)
            full_messages = top_up_image_markers(
                full_messages, len(attached), ordinal = image_ordinal
            )
            if image is not None:
                # Read off the conversation: the attachment's turn can precede a
                # replayed picture, and pixels bind to markers in document order.
                attached = pixels_in_marker_order(
                    full_messages, _prior_markers, list(images or []), image
                )
            if len(attached) > 1 and not self._multi_image_marker:
                # The template marks one image: serve the attachment, else the newest picture
                keep = next(
                    (i for i, one in enumerate(attached) if one is image), len(attached) - 1
                )
                logger.info(
                    "Serving 1 of %d images: '%s' marks one image per request.",
                    len(attached),
                    self.active_model_name,
                )
                full_messages = list(full_messages)
                trim_image_turns(full_messages, attached, limit = 1, keep = (keep,))

        if self._is_vlm:
            stream = self._generate_vlm(
                full_messages,
                attached if self._reads_vision_input() else None,
                temperature,
                top_p,
                top_k,
                min_p,
                max_new_tokens,
                repetition_penalty,
                cancel_event,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
                continue_final_message = continue_final_message,
                presence_penalty = presence_penalty,
                seed = seed,
                frequency_penalty = frequency_penalty,
                logit_bias = logit_bias,
                _adapter_state = _adapter_state,
                stop = stop,
                tool_protocol_active = tool_protocol_active,
                response_format = response_format,
                reasoning_is_extracted = reasoning_is_extracted,
                videos = videos,
            )
        else:
            stream = self._generate_text(
                full_messages,
                temperature,
                top_p,
                top_k,
                min_p,
                max_new_tokens,
                repetition_penalty,
                cancel_event,
                images = attached if self._reads_vision_input() else None,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
                continue_final_message = continue_final_message,
                presence_penalty = presence_penalty,
                seed = seed,
                frequency_penalty = frequency_penalty,
                logit_bias = logit_bias,
                _adapter_state = _adapter_state,
                stop = stop,
                tool_protocol_active = tool_protocol_active,
                response_format = response_format,
                reasoning_is_extracted = reasoning_is_extracted,
            )
        yield from stream

    def _vocab_size(self):
        """The model's token-id range, as llama-server's ``n_vocab`` bounds a prompt's ids."""
        config = getattr(self._model, "config", None)
        for owner in (
            getattr(self._model, "args", None),
            getattr(config, "text_config", None),
            config,
        ):
            size = getattr(owner, "vocab_size", None)
            if isinstance(size, int) and size > 0:
                return size
        return len(getattr(self._tokenizer, "_tokenizer", self._tokenizer))

    def resolve_completion_prompt(self, prompt) -> dict:
        """Token ids and echo text for one llama-server completion prompt, tokenized as its
        ``tokenize_mixed`` does: specials parsed, only a leading string gets BOS."""
        if self._model is None:
            raise RuntimeError("No model loaded")
        if isinstance(prompt, dict):
            prompt = prompt.get("prompt_string")
        pieces = [prompt] if isinstance(prompt, str) else prompt
        if not isinstance(pieces, list):
            raise ValueError("Invalid prompt.")
        vocab = self._vocab_size()
        ids, text, run = [], [], []
        for index, piece in enumerate(pieces + [""]):
            if isinstance(piece, int) and not isinstance(piece, bool):
                if not 0 <= piece < vocab:
                    raise ValueError("Prompt contains invalid tokens.")
                run.append(piece)
                continue
            if not isinstance(piece, str):
                raise ValueError("Invalid prompt.")
            if run:
                text.append(
                    self._tokenizer.decode(
                        run, skip_special_tokens = False, clean_up_tokenization_spaces = False
                    )
                )
                ids.extend(run)
                run = []
            if index < len(pieces):
                ids.extend(self._tokenizer.encode(piece, add_special_tokens = index == 0))
                text.append(piece)
        if not ids:
            raise ValueError("The prompt is empty.")
        return {"ids": ids, "text": "".join(text), "n_vocab": vocab}

    def generate_completion(
        self,
        prompt_ids,
        temperature = 0.7,
        top_p = 0.9,
        top_k = 40,
        min_p = 0.0,
        max_new_tokens = 256,
        repetition_penalty = 1.0,
        cancel_event = None,
        presence_penalty = 0.0,
        seed = None,
        frequency_penalty = 0.0,
        logit_bias = None,
        stop = None,
        penalty_window = None,
        grammar = None,
        n_probs = 0,
    ) -> Generator[str, None, None]:
        """Cumulative text sampled after ``prompt_ids`` with no chat template.
        ``penalty_window`` is llama-server's ``repeat_last_n``; ``n_probs`` > 0 adds each
        token's ``logprobs.content`` entry to the stats as ``token_logprobs``."""
        if self._model is None:
            raise RuntimeError("No model loaded")
        self.last_generation_stats = None
        sampling = dict(
            presence_penalty = presence_penalty,
            seed = seed,
            frequency_penalty = frequency_penalty,
            logit_bias = logit_bias,
            stop = stop,
            prompt_ids = list(prompt_ids),
            penalty_window = penalty_window,
            grammar = grammar,
        )
        if n_probs:
            sampling["n_probs"] = int(n_probs)
        args = (temperature, top_p, top_k, min_p, max_new_tokens, repetition_penalty, cancel_event)
        if self._is_vlm:
            yield from self._generate_vlm(None, None, *args, **sampling)
        else:
            yield from self._generate_text(None, *args, **sampling)

    def _mark_stopped(self):
        """Record that a text sequence ended the turn, not the runtime."""
        if isinstance(self.last_generation_stats, dict):
            self.last_generation_stats["finish_reason"] = "stop"

    def _generate_text(
        self,
        messages,
        temperature,
        top_p,
        top_k,
        min_p,
        max_new_tokens,
        repetition_penalty,
        cancel_event,
        *,
        images = None,
        tools = None,
        enable_thinking = None,
        reasoning_effort = None,
        preserve_thinking = None,
        continue_final_message = False,
        presence_penalty = 0.0,
        seed = None,
        frequency_penalty = 0.0,
        logit_bias = None,
        _adapter_state = None,
        tool_protocol_active = None,
        stop = None,
        response_format = None,
        reasoning_is_extracted = False,
        prompt_ids = None,
        penalty_window = None,
        grammar = None,
        n_probs = 0,
    ):
        from mlx_lm import stream_generate
        from mlx_lm.sample_utils import make_sampler

        if prompt_ids is not None:
            # A raw completion prompt: no template, so no prefill or reasoning channel.
            prompt, reasoning_channel_markers, think_prefix = list(prompt_ids), None, ""
            preserve_native_channels = False
            constraint = _raw_prompt_constraint(grammar, self._tokenizer)
        else:
            render_result = self._render_text_prompt(
                messages,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
                continue_final_message = continue_final_message,
            )
            prompt = render_result.prompt
            reasoning_channel_markers = render_result.reasoning_channel_markers

            preserve_native_channels = reasoning_channel_markers is not None
            # An open <think> prefilled by the template lives in the prompt, not the generated tokens; re-emit it so the
            # frontend renders the block.
            # Matches native_token_decoder below: when it runs </think> survives, so re-emit the opener.
            think_close_survives = (
                bool(tools) or tool_protocol_active or reasoning_channel_markers is not None
            ) and decoder_preserves_token(
                self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)
            )
            think_prefix = _think_prefix(
                prompt,
                getattr(self._tokenizer, "all_special_tokens", None),
                messages,
                continue_final_message,
                preserves_think_close = think_close_survives,
            )
            constraint = _build_grammar_constraint(
                response_format,
                self._tokenizer,
                prompt,
                reasoning_markers = reasoning_channel_markers,
                reasoning_is_extracted = reasoning_is_extracted,
                # Must match the prefill's answer, or a closer it re-emits would be refused.
                reply_keeps_special_tokens = preserve_native_channels or think_close_survives,
            )
            if constraint is not None and not constraint.allows_reasoning:
                think_prefix = ""
                preserve_native_channels = False
        # Not the request flag: a later tool-loop pass keeps it but renders an ordinary post-tool prompt.
        _resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))
        if seed is None:
            sampler = make_sampler(
                temp = temperature,
                top_p = top_p,
                top_k = int(top_k or 0),
                min_p = float(min_p or 0.0),
                min_tokens_to_keep = 1,
            )
        else:
            sampler = _make_seeded_mlx_sampler(
                seed,
                temp = temperature,
                top_p = top_p,
                top_k = int(top_k or 0),
                min_p = float(min_p or 0.0),
            )
        logits_processors = _mlx_sampling_processors(
            repetition_penalty = repetition_penalty,
            presence_penalty = presence_penalty,
            frequency_penalty = frequency_penalty,
            logit_bias = logit_bias,
            grammar_constraint = constraint,
            penalty_window = penalty_window,
            penalty_prompt = prompt_ids,
        )
        recorded = token_logprobs = None
        if n_probs:
            recorder, recorded = _make_mlx_raw_logprob_recorder()
            logits_processors = [recorder, *(logits_processors or ())]
            token_logprobs = []

        native_token_decoder = (
            NativeToolTokenDecoder(
                self._tokenizer,
                preserved_tokens = reasoning_control_tokens(reasoning_channel_markers),
            )
            if tools or preserve_native_channels or tool_protocol_active
            else None
        )
        if constraint is not None:
            constraint.decoded_dropping(
                native_token_decoder.dropped_ids() if native_token_decoder is not None else None
            )
        # Consulted per token on the reasoning path below, so resolved once here.
        stop_token_ids = (
            _mlx_stop_token_ids(self._tokenizer, self._model)
            if native_token_decoder is not None
            else ()
        )
        streamed_text = (
            _MlxStreamedText(
                _mlx_stream_detokenizer(self._tokenizer),
                native_token_decoder,
                self._tokenizer,
                stop_token_ids,
            )
            if preserve_native_channels
            else None
        )
        token_ids = []
        if preserve_native_channels:
            reply_text = None
        elif native_token_decoder is not None:
            reply_text = _ReplyText(
                decoder = native_token_decoder, stop_ids = stop_token_ids, prefix = think_prefix
            )
        else:
            reply_text = _ReplyText(
                self._tokenizer,
                dict(
                    skip_special_tokens = True,
                    # Cleanup would rewrite " ." inside a grammar-approved string,
                    # and llama-server returns a raw completion's pieces as is.
                    **(
                        {"clean_up_tokenization_spaces": False}
                        if constraint is not None or prompt_ids is not None
                        else {}
                    ),
                ),
            )
        normalizer = (
            make_reasoning_normalizer(
                reasoning_channel_markers,
                in_reasoning = prompt_opens_reasoning_channel(
                    prompt, reasoning_channel_markers, _resumed_partial
                ),
            )
            if preserve_native_channels
            else None
        )
        # Match sampled text, ahead of prefill and think rewriting: delivered markup would end turns the model never wrote, and never find a requested native marker
        sequences = _mlx_stop_sequences(stop)
        stopped = False
        sampled = ""
        released = 0
        # MLX diffs cumulative snapshots: keep the prefilled <think> prefix in native-protocol snapshots.
        normalized_output = think_prefix
        with (
            self._generation_lock,
            _temporary_mlx_adapter_state(self._model, _adapter_state),
            self._prefill_decision_scopes(),
            self._generation_scopes(),
        ):
            input_embeddings = None
            if images:
                from unsloth_zoo.mlx.generate import image_prompt_embeddings

                # Any two images of one size expand to the same ids, so the token-keyed cache stays out.
                prompt_tokens, input_embeddings = image_prompt_embeddings(
                    self._model, self._tokenizer, prompt, images
                )
                gen_prompt, prompt_cache, cache_key, cached_n = prompt_tokens, None, None, 0
            else:
                (
                    gen_prompt,
                    prompt_cache,
                    cache_key,
                    prompt_tokens,
                    cached_n,
                ) = self._prepare_prompt_cache(prompt, _adapter_state)
            max_new_tokens = self._generation_limit(prompt, max_new_tokens, prompt_tokens)
            logger.info(
                "Generating: prompt_len=%d, cached=%d, max_tokens=%d, model=%s, tokenizer=%s",
                len(prompt),
                cached_n,
                max_new_tokens,
                type(self._model).__name__,
                type(self._tokenizer).__name__,
            )
            final_response = None
            try:
                # Enter request-scoped model state before yielding any response.
                if think_prefix:
                    yield think_prefix
                gen_kwargs = dict(
                    prompt = gen_prompt,
                    max_tokens = max_new_tokens,
                    sampler = sampler,
                )
                gen_kwargs.update(self._kv_window_generate_kwargs())
                gen_kwargs.update(self._prefill_step_kwargs())
                if prompt_cache is not None:
                    gen_kwargs["prompt_cache"] = prompt_cache
                else:
                    gen_kwargs.update(self._kv_quant_generate_kwargs())
                if input_embeddings is not None:
                    gen_kwargs["input_embeddings"] = input_embeddings
                if logits_processors is not None:
                    gen_kwargs["logits_processors"] = logits_processors
                for response in _stream_until_cancelled(
                    stream_generate,
                    cancel_event,
                    self._model,
                    self._tokenizer,
                    **gen_kwargs,
                ):
                    final_response = response
                    token_ids.append(response.token)
                    if recorded:
                        token_logprobs.append(
                            _mlx_token_logprob(
                                recorded.popleft(), response.token, n_probs, self._tokenizer
                            )
                        )
                    if preserve_native_channels:
                        sampled += streamed_text.feed(response)
                        if sequences:
                            cut, stopped = _mlx_stop_cut(sampled, sequences)
                        else:
                            cut = len(sampled)
                        # Cut before normalizing: normalizer-written markers are this layer's own
                        delta = normalizer.feed(sampled[released:cut])
                        released = cut
                        if delta:
                            normalized_output += delta
                            yield normalized_output
                    else:
                        # Re-decoding every id rebuilds rather than extends, so an invalid byte sequence can revise
                        # characters already shown. Predates stop handling and affects plain replies too.
                        sampled = reply_text.feed(response.token)
                        if not sequences:
                            # A character whose bytes span tokens decodes to U+FFFD until its last token lands, and
                            # consumers diff snapshots by length, so yielding that would replace it for good (#8594).
                            yield think_prefix + sampled.rstrip("\ufffd")
                        else:
                            # Matched every step, delivered once at the end: revised snapshots diffed by length would splice text, and a stream cannot unsend
                            cut, stopped = _mlx_stop_cut(sampled, sequences)
                    if stopped:
                        break

                    if cancel_event and cancel_event.is_set():
                        break
                if prompt_cache is not None and prompt_tokens is not None:
                    history = self._prompt_cache_history
                    if history is not None:
                        try:
                            history.insert(cache_key, prompt_tokens + token_ids, prompt_cache)
                        except Exception as exc:
                            logger.debug("MLX prompt cache insert failed: %s", exc)
            except Exception as e:
                import traceback
                logger.error("stream_generate failed:\n%s", traceback.format_exc())
                raise
            finally:
                # Latched here so a later cancel cannot rewrite the finish reason
                if final_response is not None:
                    self.last_generation_stats = _build_generation_stats(
                        getattr(final_response, "prompt_tokens", 0),
                        getattr(final_response, "prompt_tps", 0.0),
                        getattr(final_response, "generation_tokens", 0),
                        getattr(final_response, "generation_tps", 0.0),
                        cached_n,
                        finish_reason = _mlx_finish_reason(
                            final_response,
                            _mlx_stop_token_ids(self._tokenizer, self._model),
                            getattr(final_response, "generation_tokens", 0),
                            max_new_tokens,
                        ),
                    )
                    if token_logprobs is not None:
                        self.last_generation_stats["token_logprobs"] = token_logprobs
        # The turn's settled text: delivered once for the plain path, as the tail for the native-channel one. Every
        # snapshot was matched as it arrived, so a turn no sequence ended owes all of its text, held-back partial
        # included.
        if streamed_text is not None and not stopped:
            # The flush can carry a whole sequence: it was held back unmatched, not cleared.
            sampled += streamed_text.finish(sampled)
            if sequences:
                cut, stopped = _mlx_stop_cut(sampled, sequences)
        if not sequences and streamed_text is None and sampled.endswith("\ufffd"):
            yield think_prefix + sampled
        if sequences or streamed_text is not None:
            if not stopped:
                cut = len(sampled)
            if normalizer is None:
                settled = think_prefix + sampled[:cut]
                # Settled text equal to the prefill needs no second snapshot; the prefill already went out.
                if settled != think_prefix:
                    yield settled
            else:
                delta = normalizer.feed(sampled[released:cut])
                if delta:
                    normalized_output += delta
                    yield normalized_output
        if normalizer is not None:
            # A sequence ends the turn like a stop token, so a reasoning block it cut inside is closed; only a cancelled turn drains, since more was coming
            cancelled = not stopped and cancel_event is not None and cancel_event.is_set()
            tail = normalizer.drain() if cancelled else normalizer.finish()
            if tail:
                normalized_output += tail
                yield normalized_output
        if stopped:
            self._mark_stopped()

    def _plan_text_row(
        self,
        messages,
        *,
        temperature,
        top_p,
        top_k,
        min_p,
        max_new_tokens,
        repetition_penalty,
        tools = None,
        enable_thinking = None,
        reasoning_effort = None,
        preserve_thinking = None,
        continue_final_message = False,
        presence_penalty = 0.0,
        seed = None,
        frequency_penalty = 0.0,
        logit_bias = None,
        stop = None,
        tool_protocol_active = None,
        prompt_ids = None,
        penalty_window = None,
        response_format = None,
        reasoning_is_extracted = False,
        grammar = None,
    ) -> "_TextRowPlan":
        from mlx_lm.sample_utils import make_sampler

        if prompt_ids is not None:
            prompt, reasoning_channel_markers, think_prefix = list(prompt_ids), None, ""
            _resumed_partial = False
            preserve_native_channels = False
            constraint = _raw_prompt_constraint(grammar, self._tokenizer)
        else:
            render_result = self._render_text_prompt(
                messages,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
                continue_final_message = continue_final_message,
            )
            prompt = render_result.prompt
            reasoning_channel_markers = render_result.reasoning_channel_markers
            _resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))

            preserve_native_channels = reasoning_channel_markers is not None
            think_close_survives = (
                bool(tools) or tool_protocol_active or preserve_native_channels
            ) and decoder_preserves_token(
                self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)
            )
            think_prefix = _think_prefix(
                prompt,
                getattr(self._tokenizer, "all_special_tokens", None),
                messages,
                continue_final_message,
                preserves_think_close = think_close_survives,
            )
            constraint = _build_grammar_constraint(
                response_format,
                self._tokenizer,
                prompt,
                reasoning_markers = reasoning_channel_markers,
                reasoning_is_extracted = reasoning_is_extracted,
                reply_keeps_special_tokens = preserve_native_channels or think_close_survives,
            )
            if constraint is not None and not constraint.allows_reasoning:
                think_prefix = ""
                preserve_native_channels = False
        if seed is None:
            sampler = make_sampler(
                temp = temperature,
                top_p = top_p,
                top_k = int(top_k or 0),
                min_p = float(min_p or 0.0),
                min_tokens_to_keep = 1,
            )
        else:
            sampler = _make_seeded_mlx_sampler(
                seed,
                temp = temperature,
                top_p = top_p,
                top_k = int(top_k or 0),
                min_p = float(min_p or 0.0),
            )
        logits_processors = _mlx_sampling_processors(
            repetition_penalty = repetition_penalty,
            presence_penalty = presence_penalty,
            frequency_penalty = frequency_penalty,
            logit_bias = logit_bias,
            grammar_constraint = constraint,
            penalty_window = penalty_window,
            penalty_prompt = prompt_ids,
        )

        normalizer = (
            make_reasoning_normalizer(
                reasoning_channel_markers,
                in_reasoning = prompt_opens_reasoning_channel(
                    prompt, reasoning_channel_markers, _resumed_partial
                ),
            )
            if preserve_native_channels
            else None
        )
        # As _generate_text decodes the reply, so a batched turn shows the same native controls.
        native_token_decoder = (
            NativeToolTokenDecoder(
                self._tokenizer,
                preserved_tokens = reasoning_control_tokens(reasoning_channel_markers),
            )
            if tools or preserve_native_channels or tool_protocol_active
            else None
        )
        if constraint is not None:
            constraint.decoded_dropping(
                native_token_decoder.dropped_ids() if native_token_decoder is not None else None
            )
        stop_token_ids = (
            _mlx_stop_token_ids(self._tokenizer, self._model)
            if native_token_decoder is not None
            else ()
        )
        streamed = (
            _MlxStreamedText(
                _mlx_stream_detokenizer(self._tokenizer),
                native_token_decoder,
                self._tokenizer,
                stop_token_ids,
            )
            if preserve_native_channels
            else None
        )
        _require_row_detokenizer(streamed)
        if streamed is not None:
            decode = None
        elif native_token_decoder is not None:
            decode = _ReplyText(
                decoder = native_token_decoder, stop_ids = stop_token_ids, prefix = think_prefix
            )
        else:
            decode = _ReplyText(
                self._tokenizer,
                dict(
                    skip_special_tokens = True,
                    **(
                        {"clean_up_tokenization_spaces": False}
                        if constraint is not None or prompt_ids is not None
                        else {}
                    ),
                ),
            )
        if max_new_tokens is None:
            max_tokens = self._unset_generation_budget(prompt, prompt_ids)
        else:
            self._prompt_within_window(prompt, prompt_ids)
            max_tokens = max_new_tokens
        row = _RowStream(
            prefix = think_prefix,
            sequences = _mlx_stop_sequences(stop),
            normalizer = normalizer,
            decode = decode,
        )
        return _TextRowPlan(
            prompt = prompt,
            think_prefix = think_prefix,
            stream = row,
            streamed = streamed,
            sampler = sampler,
            processors = logits_processors,
            max_tokens = max_tokens,
        )

    def _render_vlm_prompt(
        self,
        messages,
        images,
        *,
        videos = None,
        tools = None,
        enable_thinking = None,
        reasoning_effort = None,
        preserve_thinking = None,
        continue_final_message = False,
    ):
        """Render the prompt a vision generation sends, and the target that rendered it. Shared with
        counting, as _render_text_prompt is for text models, so a count cannot price a prompt the
        model never sees. A text-only conversation passes images=None."""
        from core.inference.chat_template_helpers import (
            apply_chat_template_for_generation,
            chat_render_target,
        )

        # Same render target as the healing catalog the route builds (#7066)
        chat_target = (
            _video_render_target(self._processor) if videos else chat_render_target(self._processor)
        )

        attached_images = 0 if images is None else len(images)
        structured_images = sum(
            _count_vlm_images(message.get("content"))
            for message in messages
            if isinstance(message, dict)
        )
        if structured_images != attached_images:
            raise RuntimeError(
                f"VLM conversation contains {structured_images} structured image "
                f"item(s) for {attached_images} attached image(s)."
            )
        attached_videos = 0 if videos is None else len(videos)
        structured_videos = sum(
            _count_vlm_videos(message.get("content"))
            for message in messages
            if isinstance(message, dict)
        )
        if structured_videos != attached_videos:
            raise RuntimeError(
                f"VLM conversation contains {structured_videos} structured video "
                f"item(s) for {attached_videos} attached video(s)."
            )
        prompt = None
        has_tool_history = _vlm_messages_have_tool_history(messages)
        prompt_error = None
        try:
            prompt = apply_chat_template_for_generation(
                chat_target,
                messages,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
                continue_final_message = continue_final_message,
            )
        except Exception as exc:
            if images is None or has_tool_history:
                raise
            prompt_error = exc
        prompt_issue = (
            _vlm_prompt_issue(prompt, messages) if prompt_error is None else "a rendering error"
        )
        if prompt_issue and has_tool_history:
            raise RuntimeError(
                f"VLM chat template returned {prompt_issue} and cannot be recovered "
                "without dropping tool-call history."
            ) from prompt_error
        if prompt_issue and attached_videos:
            # mlx-vlm's registered renderers take image and audio counts only.
            raise RuntimeError(
                f"VLM chat template returned {prompt_issue} and cannot be recovered "
                "for a video turn."
            ) from prompt_error

        if images is not None and prompt_issue:
            if tools or any(
                value is not None
                for value in (enable_thinking, reasoning_effort, preserve_thinking)
            ):
                if prompt_error is not None:
                    raise prompt_error
                raise RuntimeError(
                    f"VLM chat template returned {prompt_issue} and cannot be recovered "
                    "without dropping requested tools or reasoning controls."
                )
            try:
                recovered_prompt = _render_registered_vlm_prompt(
                    self._processor,
                    self._model,
                    messages,
                    len(images),
                    continue_final_message = continue_final_message,
                )
            except Exception as recovery_error:
                if prompt_error is not None:
                    raise prompt_error
                raise RuntimeError(
                    f"VLM chat template returned {prompt_issue}; model-aware "
                    f"recovery failed: {recovery_error}"
                ) from recovery_error
            if recovered_prompt is None:
                if prompt_error is not None:
                    raise prompt_error
                raise RuntimeError(
                    f"VLM chat template returned {prompt_issue}, and no registered "
                    "MLX VLM renderer was available for this model."
                )
            recovered_issue = _vlm_prompt_issue(recovered_prompt, messages)
            if recovered_issue:
                if prompt_error is not None:
                    raise prompt_error
                raise RuntimeError(
                    f"Model-aware VLM rendering returned {recovered_issue} for "
                    f"{attached_images} attached image(s)."
                )
            prompt = recovered_prompt
        elif prompt_issue:
            raise RuntimeError(f"VLM chat template returned {prompt_issue}.") from prompt_error
        # mlx-vlm binds pixel_values[i] to the i-th image token: miscounted markers answer wrong.
        if attached_images > 1 and self._multi_image_marker:
            marked = prompt.count(self._multi_image_marker)
            if marked != attached_images:
                raise RuntimeError(
                    f"VLM chat template marked {marked} image(s) for {attached_images} attached."
                )

        markers = detect_reasoning_channel_markers(chat_target, tools = tools)
        if not self._reads_vision_input():
            # No images means no image tokens to place, so skipping this would drop the tool schema.
            from core.inference.chat_template_helpers import (
                render_with_native_template_fallback,
            )

            model_info = self.models.get(self.active_model_name, {})
            recovered = render_with_native_template_fallback(
                formatted_prompt = prompt,
                tokenizer = self._tokenizer,
                model_info = model_info,
                active_model_name = self.active_model_name,
                messages = messages,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
                continue_final_message = continue_final_message,
                hf_token = model_info.get("hf_token"),
                return_metadata = True,
            )
            prompt, markers = recovered.prompt, recovered.reasoning_channel_markers
        return prompt, chat_target, markers

    def _media_generation_limit(self, prompt, max_new_tokens, prompt_ids, images, videos):
        # Counted from a temp file (base64 is unreadable to the processor), discarded here.
        counted_clips = []
        try:
            if getattr(self, "_kv_context_budget", None):
                for clip in videos or ():
                    counted_clips.append(_write_video_clip(clip))
            return self._generation_limit(
                prompt,
                max_new_tokens,
                prompt_ids,
                images = images,
                videos = counted_clips or None,
                cap = UNSET_GENERATION_BUDGET if images or videos else None,
            )
        finally:
            for counted_clip in counted_clips:
                _discard_video_clip(counted_clip)

    def _plan_vlm_request(
        self,
        request,
        *,
        decodes_ids = False,
    ) -> "_VLMRowPlan":
        image = request.get("image")
        videos = request.get("videos") or None
        messages = request.get("messages") or []
        conversation = (
            []
            if request.get("prompt_ids") is not None
            else self._conversation_to_render(
                messages, request.get("system_prompt", ""), image, video = len(videos or ())
            )
        )
        return self._plan_vlm_row(
            conversation,
            image,
            tool_protocol_active = request.get("tool_protocol_active"),
            response_format = request.get("response_format"),
            reasoning_is_extracted = request.get("reasoning_is_extracted", False),
            decodes_ids = decodes_ids,
            videos = videos,
            **_row_kwargs(request),
        )

    def _plan_vlm_row(
        self,
        messages,
        image,
        *,
        temperature,
        top_p,
        top_k,
        min_p,
        max_new_tokens,
        repetition_penalty,
        tools = None,
        enable_thinking = None,
        reasoning_effort = None,
        preserve_thinking = None,
        continue_final_message = False,
        presence_penalty = 0.0,
        seed = None,
        frequency_penalty = 0.0,
        logit_bias = None,
        stop = None,
        prompt_ids = None,
        penalty_window = None,
        tool_protocol_active = None,
        response_format = None,
        reasoning_is_extracted = False,
        decodes_ids = False,
        videos = None,
    ) -> "_VLMRowPlan":
        """``decodes_ids``: the batch reports each row's ids, so the reply is decoded from them as
        ``_generate_vlm`` decodes it; otherwise from the batch's text, which keeps no native controls."""
        images = [image] if image is not None else None
        constraint = token_decoder = None
        if prompt_ids is not None:
            prompt, markers, think_prefix = list(prompt_ids), None, ""
        else:
            prompt, chat_target, markers = self._render_vlm_prompt(
                messages,
                images,
                videos = videos,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
                continue_final_message = continue_final_message,
            )

            native = decodes_ids and bool(tools or tool_protocol_active or markers is not None)
            think_prefix = _think_prefix(
                prompt,
                getattr(chat_target, "all_special_tokens", None),
                messages,
                continue_final_message,
                preserves_think_close = native
                and decoder_preserves_token(self._tokenizer, "</think>"),
            )
            if decodes_ids:
                constraint = _build_grammar_constraint(
                    response_format,
                    chat_target,
                    prompt,
                    reasoning_markers = markers,
                    tools = tools,
                    reasoning_is_extracted = reasoning_is_extracted,
                    reply_keeps_special_tokens = True,
                )
            if constraint is not None and not constraint.allows_reasoning:
                think_prefix = ""
            elif constraint is not None:
                from core.inference.chat_template_helpers import detect_think_prefill
                think_prefix = detect_think_prefill(prompt, preserves_think_close = True)
            if constraint is not None and _vlm_generation_is_diffusion(self._model):
                from core.inference.grammar_constraint import ResponseFormatError
                raise ResponseFormatError(_DIFFUSION_RESPONSE_FORMAT)
            if native and self._tokenizer:
                token_decoder = NativeToolTokenDecoder(
                    self._tokenizer,
                    preserved_tokens = reasoning_control_tokens(markers),
                )
        normalizer = (
            make_reasoning_normalizer(
                markers,
                in_reasoning = prompt_opens_reasoning_channel(
                    prompt,
                    markers,
                    bool(continue_final_message and trailing_assistant_text(messages)),
                ),
            )
            # The normalizer would rewrite marker text inside the document.
            if markers is not None and (constraint is None or constraint.allows_reasoning)
            else None
        )
        row = _RowStream(
            prefix = think_prefix,
            sequences = _mlx_stop_sequences(stop),
            normalizer = normalizer,
        )
        max_new_tokens = self._media_generation_limit(
            prompt, max_new_tokens, prompt_ids, images, videos
        )
        vlm_kwargs = dict(
            max_tokens = max_new_tokens,
            temperature = temperature,
            top_p = top_p,
            top_k = int(top_k or 0),
            min_p = float(min_p or 0.0),
        )
        vlm_kwargs.update(self._kv_window_generate_kwargs())
        if seed is not None:
            vlm_kwargs["sampler"] = _make_seeded_mlx_sampler(
                seed,
                temp = temperature,
                top_p = top_p,
                top_k = int(top_k or 0),
                min_p = float(min_p or 0.0),
            )
        processors = _mlx_sampling_processors(
            repetition_penalty = repetition_penalty,
            presence_penalty = presence_penalty,
            frequency_penalty = frequency_penalty,
            logit_bias = logit_bias,
            grammar_constraint = constraint,
            penalty_window = penalty_window,
            penalty_prompt = prompt_ids,
        )
        if processors:
            # Not the repetition_penalty shortcut: mlx-vlm would build it outside our narrowing
            vlm_kwargs["logits_processors"] = _row_logits_processors(processors)
        streamed = None
        if constraint is not None:
            # mlx-vlm stops on the config's ids, which some repos set apart from the tokenizer's.
            constraint.stops_on(_mlx_stop_token_ids(self._tokenizer, self._model))
            if token_decoder is not None:
                constraint.decoded_dropping(token_decoder.dropped_ids())
        if token_decoder is not None:
            streamed = _MlxStreamedText(
                _mlx_stream_detokenizer(self._processor)
                or _mlx_stream_detokenizer(self._tokenizer),
                token_decoder,
                self._tokenizer,
                _mlx_stop_token_ids(self._tokenizer, self._model),
            )
            _require_row_detokenizer(streamed)

        return _VLMRowPlan(
            prompt = prompt,
            images = images,
            videos = videos,
            think_prefix = think_prefix,
            stream = row,
            kwargs = vlm_kwargs,
            sampling = {
                "temperature": vlm_kwargs["temperature"],
                "top_p": vlm_kwargs["top_p"],
                "top_k": max(vlm_kwargs["top_k"], 0),
                "min_p": vlm_kwargs["min_p"],
                "seed": seed,
            },
            max_tokens = vlm_kwargs["max_tokens"],
            processors = processors,
            streamed = streamed,
        )

    def _generate_vlm(
        self,
        messages,
        images,
        temperature,
        top_p,
        top_k,
        min_p,
        max_new_tokens,
        repetition_penalty,
        cancel_event,
        *,
        tools = None,
        enable_thinking = None,
        reasoning_effort = None,
        preserve_thinking = None,
        continue_final_message = False,
        presence_penalty = 0.0,
        seed = None,
        frequency_penalty = 0.0,
        logit_bias = None,
        _adapter_state = None,
        tool_protocol_active = None,
        stop = None,
        videos = None,
        response_format = None,
        reasoning_is_extracted = False,
        prompt_ids = None,
        penalty_window = None,
        grammar = None,
        n_probs = 0,
    ):
        from mlx_vlm import stream_generate as vlm_stream

        from core.inference.chat_template_helpers import detect_think_prefill

        images = list(images) if images else None
        if prompt_ids is not None:
            prompt, chat_target, vlm_reasoning_markers, prefill = "", None, None, ""
        else:
            prompt, chat_target, vlm_reasoning_markers = self._render_vlm_prompt(
                messages,
                images,
                videos = videos or None,
                tools = tools,
                enable_thinking = enable_thinking,
                reasoning_effort = reasoning_effort,
                preserve_thinking = preserve_thinking,
                continue_final_message = continue_final_message,
            )
            # Detected once: the decoder keeps the delimiters the normalizer below consumes.
            prefill = _think_prefix(
                prompt,
                getattr(chat_target, "all_special_tokens", None),
                messages,
                continue_final_message,
                # The same activation the decoder below uses: in unrestricted mode ``tools`` is
                # empty while the protocol is live, so ``bool(tools)`` said the closer would be
                # stripped, the opener was suppressed, and the stream ran on to an orphan
                # ``</think>``. Mirrors the text path.
                preserves_think_close = (
                    bool(tools) or tool_protocol_active or vlm_reasoning_markers is not None
                )
                and decoder_preserves_token(self._tokenizer, "</think>"),
            )
        vlm_continued = bool(continue_final_message and trailing_assistant_text(messages))
        # Matched on the sampled text, for the reason _generate_text gives.
        sequences = _mlx_stop_sequences(stop)
        stopped = False
        max_new_tokens = self._media_generation_limit(
            prompt, max_new_tokens, prompt_ids, images, videos
        )
        logger.info(
            "VLM generating: prompt_len=%d, images=%d, videos=%d",
            len(prompt),
            len(images or ()),
            len(videos or ()),
        )
        # generate_step expects temperature=; temp= is silently ignored (stuck at greedy)
        vlm_kwargs = dict(
            max_tokens = max_new_tokens,
            temperature = temperature,
            top_p = top_p,
            top_k = int(top_k or 0),
            min_p = float(min_p or 0.0),
        )
        vlm_kwargs.update(self._kv_window_generate_kwargs())
        if seed is not None:
            # generate_step builds its sampler only if sampler is None, so a seeded one must carry the full temp/top_p/min_p/top_k chain or they go silently off
            vlm_kwargs["sampler"] = _make_seeded_mlx_sampler(
                seed,
                temp = temperature,
                top_p = top_p,
                top_k = int(top_k or 0),
                min_p = float(min_p or 0.0),
            )
        _rep_active = repetition_penalty is not None and float(repetition_penalty) not in (
            0.0,
            1.0,
        )
        constraint = (
            _raw_prompt_constraint(grammar, self._tokenizer)
            if prompt_ids is not None
            else _build_grammar_constraint(
                response_format,
                chat_target,
                prompt,
                reasoning_markers = vlm_reasoning_markers,
                tools = tools,
                reasoning_is_extracted = reasoning_is_extracted,
                reply_keeps_special_tokens = True,
            )
        )
        document_only = constraint is not None and not constraint.allows_reasoning
        if document_only:
            prefill = ""
        elif constraint is not None:
            # The grammar lets a kept </think> close the block: re-emit the opener or it all reads as content.
            prefill = detect_think_prefill(prompt, preserves_think_close = True)
        if constraint is not None and _vlm_generation_is_diffusion(self._model):
            from core.inference.grammar_constraint import ResponseFormatError
            raise ResponseFormatError(_DIFFUSION_RESPONSE_FORMAT)
        recorded = token_logprobs = None
        if n_probs:
            if _vlm_generation_is_diffusion(self._model):
                raise LogprobsUnsupported(
                    "logprobs is not supported on this model: it generates by diffusion rather "
                    "than one token at a time."
                )
            recorder, recorded = _make_mlx_raw_logprob_recorder()
            token_logprobs = []
        draft = self._speculative_draft
        if (
            presence_penalty
            or frequency_penalty
            or logit_bias
            or constraint is not None
            or (_rep_active and draft is not None)
            or penalty_window is not None
            or recorded is not None
        ):
            # These need custom processors: pass the full list (repetition + the rest) instead of the
            # repetition_penalty shortcut so all apply.
            vlm_kwargs["logits_processors"] = _mlx_sampling_processors(
                repetition_penalty = repetition_penalty,
                presence_penalty = presence_penalty,
                frequency_penalty = frequency_penalty,
                logit_bias = logit_bias,
                grammar_constraint = constraint,
                penalty_window = penalty_window,
                penalty_prompt = prompt_ids,
            )
            if recorded is not None:
                # Not history_only, so no draft runs: its verify pass would record rejected tokens.
                vlm_kwargs["logits_processors"] = [
                    recorder,
                    *(vlm_kwargs["logits_processors"] or ()),
                ]
        elif _rep_active:
            vlm_kwargs["repetition_penalty"] = float(repetition_penalty)

        # Keep native tool controls and reasoning delimiters, as the text path does, so the parser can tell a wrapped call from markerless prose
        vlm_token_decoder = (
            NativeToolTokenDecoder(
                self._tokenizer,
                preserved_tokens = reasoning_control_tokens(vlm_reasoning_markers),
            )
            if (tools or tool_protocol_active or vlm_reasoning_markers is not None)
            and self._tokenizer
            else None
        )
        if constraint is not None:
            # mlx-vlm stops on the config's ids, which some repos set apart from the tokenizer's.
            constraint.stops_on(_mlx_stop_token_ids(self._tokenizer, self._model))
        if constraint is not None and vlm_token_decoder is not None:
            constraint.decoded_dropping(vlm_token_decoder.dropped_ids())
        # Runtime EOS may be an allowlisted control and would trail each answer
        vlm_stop_ids = (
            _mlx_stop_token_ids(self._tokenizer, self._model)
            if vlm_token_decoder is not None
            else ()
        )
        vlm_detokenizer = (
            _mlx_stream_detokenizer(self._processor) or _mlx_stream_detokenizer(self._tokenizer)
            if vlm_token_decoder is not None
            else None
        )
        vlm_streamed_text = (
            _MlxStreamedText(vlm_detokenizer, vlm_token_decoder, self._tokenizer, vlm_stop_ids)
            if vlm_token_decoder is not None
            else None
        )

        if prompt_ids is not None:
            import mlx.core as mx
            vlm_kwargs["input_ids"] = mx.array([list(prompt_ids)])
            # As the text path and llama-server: control tokens are not reply text.
            vlm_kwargs["skip_special_tokens"] = True
        elif not self._reads_vision_input():
            # mlx-vlm adds special tokens by family, doubling a BOS the template already emitted.
            import mlx.core as mx
            vlm_kwargs["input_ids"] = mx.array([self._encode_prompt(prompt)])
        session = self._vlm_prompt_cache_session(
            _adapter_state, images, prompt, has_video = bool(videos)
        )
        vlm_kwargs.update(self._kv_runtime_quant_kwargs())
        if session is not None:
            vlm_kwargs["prompt_cache"] = session.cache
            vlm_kwargs["prompt_cache_state"] = session
            if session.media_block is not None:
                vlm_kwargs.update(session.media_block.generate_kwargs())
            # Reused and unreused turns must prefill on the same grid to match.
            vlm_kwargs["prefill_step_size"] = session.window
        else:
            vlm_kwargs.update(self._kv_quant_generate_kwargs())
            vlm_kwargs.update(self._prefill_step_kwargs())
        session_scope = session if session is not None else nullcontext()
        processors = vlm_kwargs.get("logits_processors") or ()
        if (
            draft is None
            or not all(
                getattr(processor, "history_only", False) or hasattr(processor, "settle")
                for processor in processors
            )
            or _vlm_generation_is_diffusion(self._model)
        ):
            draft = None
        elif "input_ids" not in vlm_kwargs and not videos:
            vlm_kwargs.update(
                _vlm_generate_kwargs(
                    _vlm_prepared_inputs(self._model, self._processor, prompt, images)
                )
            )

        def _stream_vlm_snapshots():
            nonlocal stopped
            sampled = ""
            released = 0
            # Hold the lock and adapter state for the whole stream (Base-vs-LoRA compare)
            with (
                self._generation_lock,
                _temporary_mlx_adapter_state(self._model, _adapter_state),
                self._prefill_decision_scopes(),
                ExitStack() as generation_scope,
                session_scope,
            ):
                if session is None and (images or videos):
                    # Free headroom for the vision pass under the lock; a video turn resumes no snapshot, so retained ones are pure occupancy
                    self._release_vlm_snapshots()
                elif session is None:
                    self._release_held_vlm_snapshots()
                generation_scope.enter_context(
                    self._generation_scopes("vision", *(() if draft is None else ("speculative",)))
                )
                final_response = None
                clip_paths = []
                try:
                    if videos:
                        for clip in videos:
                            clip_paths.append(_write_video_clip(clip))
                        clip_media = {"video": clip_paths}
                        frame_rate = _video_sampling_fps(clip_paths, self._processor)
                        if frame_rate is not None:
                            clip_media["fps"] = frame_rate
                        if draft is None or "input_ids" in vlm_kwargs:
                            vlm_kwargs.update(clip_media)
                        else:
                            clip_media["videos"] = clip_media.pop("video")
                            vlm_kwargs.update(
                                _vlm_generate_kwargs(
                                    _vlm_prepared_inputs(
                                        self._model, self._processor, prompt, images, **clip_media
                                    )
                                )
                            )
                    if draft is not None:
                        from unsloth_zoo.mlx.generate import SamplingParams
                        draft.prepare(
                            vlm_kwargs["input_ids"][0].tolist(),
                            SamplingParams(
                                temperature = temperature,
                                top_p = top_p,
                                top_k = max(int(top_k or 0), 0),
                                min_p = float(min_p or 0.0),
                                seed = seed,
                            ),
                            processors,
                        )
                        vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)
                    # Emit prefilled <think> first; inside the adapter so unsupported requests raise before any output.
                    if prefill:
                        yield prefill
                    with closing(
                        _iter_vlm_responses(
                            vlm_stream(
                                self._model,
                                self._processor,
                                prompt,
                                images,
                                **vlm_kwargs,
                            )
                        )
                    ) as responses:
                        raw_ids = [] if prompt_ids is not None else None
                        raw_text = (
                            _ReplyText(
                                self._tokenizer,
                                dict(skip_special_tokens = True, clean_up_tokenization_spaces = False),
                            )
                            if raw_ids is not None
                            else None
                        )
                        for response in responses:
                            final_response = response
                            raw_id = (
                                getattr(response, "token", None) if raw_ids is not None else None
                            )
                            filled = getattr(response, "generation_tokens", None)
                            if raw_id is not None and (
                                filled is None or int(filled) == len(raw_ids) + 1
                            ):
                                # The streaming detokenizer trims the first leading space; decode the ids.
                                raw_ids.append(int(raw_id))
                                if recorded:
                                    token_logprobs.append(
                                        _mlx_token_logprob(
                                            recorded.popleft(),
                                            int(raw_id),
                                            n_probs,
                                            self._tokenizer,
                                        )
                                    )
                                decoded = raw_text.feed(raw_ids[-1]).rstrip("\ufffd")
                                token_text = (
                                    decoded[len(sampled) :] if decoded.startswith(sampled) else ""
                                )
                            elif (
                                raw_ids is not None
                                and filled is not None
                                and int(filled) <= len(raw_ids)
                            ):
                                token_text = ""
                            elif vlm_streamed_text is not None:
                                token_text = vlm_streamed_text.feed(response)
                            else:
                                # A block naming several ids (a denoised step): the runtime's text stands.
                                raw_ids = None
                                token_text = (
                                    response.text if hasattr(response, "text") else str(response)
                                )
                            sampled += token_text
                            if not sequences:
                                yield prefill + sampled
                            else:
                                cut, stopped = _mlx_stop_cut(sampled, sequences)
                                # These deltas only append, so the cut never moves back over text already released.
                                if cut > released:
                                    released = cut
                                    yield prefill + sampled[:cut]
                                if stopped:
                                    break
                            if cancel_event and cancel_event.is_set():
                                break
                    if vlm_streamed_text is not None and not stopped:
                        tail = vlm_streamed_text.finish(sampled)
                        if tail:
                            sampled += tail
                            if not sequences:
                                yield prefill + sampled
                            else:
                                cut, stopped = _mlx_stop_cut(sampled, sequences)
                                if cut > released:
                                    released = cut
                                    yield prefill + sampled[:cut]
                    # As in _generate_text: what was withheld is ordinary text now.
                    if sequences and not stopped and released < len(sampled):
                        yield prefill + sampled
                finally:
                    for clip_path in clip_paths:
                        _discard_video_clip(clip_path)
                    cached_n = produced_n = 0
                    produced_s = 0.0
                    if session is not None:
                        # Not from mlx-vlm's hook, which a stop or cancel never reaches.
                        try:
                            session.finish()
                        except Exception as exc:
                            logger.debug("MLX VLM prompt cache: snapshot not stored (%s)", exc)
                        # What mlx-vlm reused, less the rows produced here as prefill.
                        cached_n = int(getattr(final_response, "cached_tokens", 0) or 0)
                        produced_n, produced_s = session.produced_tokens, session.produced_seconds
                        cached_n = max(cached_n - produced_n, 0)
                    # mlx_vlm exposes the same stats fields as mlx_lm, minus a finish reason, so that one is derived.
                    if final_response is not None:
                        tokenizer = getattr(self._processor, "tokenizer", self._processor)
                        stop_ids = _mlx_stop_token_ids(tokenizer, self._model)
                        # mlx-vlm rates the full prompt against its prefill, which omits rows the session produced first.
                        prompt_n = int(getattr(final_response, "prompt_tokens", 0) or 0)
                        prefilled_n = max(prompt_n - cached_n, 0)
                        prompt_tps = float(getattr(final_response, "prompt_tps", 0.0) or 0.0)
                        if prompt_tps and (cached_n or produced_s):
                            prompt_tps = prefilled_n / (prompt_n / prompt_tps + produced_s)
                        self.last_generation_stats = _build_generation_stats(
                            prefilled_n,
                            prompt_tps,
                            getattr(final_response, "generation_tokens", 0),
                            getattr(final_response, "generation_tps", 0.0),
                            cached_n,
                            finish_reason = _mlx_finish_reason(
                                final_response,
                                stop_ids,
                                getattr(final_response, "generation_tokens", 0),
                                max_new_tokens,
                            ),
                            drafts = None
                            if draft is None
                            else (draft.draft_n, draft.draft_n_accepted),
                        )
                        if token_logprobs is not None:
                            self.last_generation_stats["token_logprobs"] = token_logprobs

        if document_only or prompt_ids is not None:
            # The normalizer would rewrite marker text inside the document.
            yield from _stream_vlm_snapshots()
        else:
            yield from normalize_reasoning_snapshots(
                _stream_vlm_snapshots(),
                chat_target,
                cancel_event,
                markers = vlm_reasoning_markers,
                tools = tools,
                prompt = prompt,
                continued = vlm_continued,
                ended = lambda: stopped,
            )
        if stopped:
            self._mark_stopped()

    def _kv_policy_batch_reason(self, resident = False):
        if self._kv_quant_bits() is None and not getattr(self, "_kv_context_budget", None):
            return None
        # Only resident vision rows carry their own cache, where the quantization lives.
        if resident and self._is_vlm:
            if self._kv_quant_bits() is not None and getattr(self, "_speculative_draft", None):
                if getattr(self, "_turboquant", False):
                    return "a speculative batch cannot run TurboQuant caches"
                return _speculative_quantized_row_gap()
            return _row_quantized_cache_gap()
        return "the load quantizes or budgets its KV cache, which a batch does not carry"

    def batch_unavailable_reason(self, requests):
        if self._model is None:
            return "no model is loaded"
        for request in requests:
            reason = _request_batch_gap(request)
            if reason is not None:
                return reason
        reason = self._video_batch_gap(requests)
        if reason is not None:
            return reason
        reason = self._kv_policy_batch_reason(
            resident = getattr(self, "_speculative_draft", None) is not None
        )
        if reason is not None:
            return reason
        if len(requests) < 2:
            return "fewer than two replies were requested"
        if self._is_vlm:
            reason = self._vlm_batch_unavailable_reason(requests)
            if reason is None and self._speculative_draft is not None:
                reason = self._vlm_resident_unavailable_reason({})
            if reason is not None:
                return reason
            if any(_mlx_stop_sequences(request.get("stop")) for request in requests):
                return "a reply asks for stop sequences"
            return None
        return self._text_batch_unavailable_reason()

    def resident_unavailable_reason(self, request):
        if self._model is None:
            return "no model is loaded"
        reason = _request_batch_gap(request)
        if reason is None and _carries_video(request):
            # Whether a clip's row fits the open batch is known only once its frames are decoded.
            reason = "a reply carries a video clip"
        if reason is not None:
            return reason
        reason = self._kv_policy_batch_reason(resident = True)
        if reason is not None:
            return reason
        if self._is_vlm:
            return self._vlm_resident_unavailable_reason(request)
        return self._text_batch_unavailable_reason()

    def _video_batch_gap(self, requests):
        if not any(map(_carries_video, requests)):
            return None
        # The one-at-a-time decode refuses these clips by name.
        if not self._is_vlm or not self._reads_vision_input() or not _mlx_vlm_decodes_video():
            return "a reply carries a video clip this load does not read"
        if self._speculative_draft is not None and self._kv_quant_bits() is not None:
            # A clip's row resumes no cache, and only a resumed row carries the quantization.
            return "a quantized speculative batch does not decode a video clip"
        try:
            from unsloth_zoo.mlx import generate as engine
        except ImportError:
            return "the installed unsloth-zoo has no MLX generation"
        if "videos" not in _engine_fields(engine, "GenerationRequest"):
            return "the installed unsloth-zoo cannot batch a video clip"
        return None

    def _text_batch_unavailable_reason(self):
        """Why the installed mlx-lm cannot batch this load's replies, or None."""
        try:
            from unsloth_zoo.mlx.generate import stream_unavailable_reason
        except ImportError:
            return "the installed unsloth-zoo cannot keep a batch open"
        return stream_unavailable_reason(self._model, self._tokenizer)

    def _vlm_resident_unavailable_reason(self, request):
        reason = self._vlm_batch_unavailable_reason([request], resident = True)
        if reason is not None:
            return reason
        try:
            from unsloth_zoo.mlx.generate import stream_unavailable_reason
        except ImportError:
            return "the installed unsloth-zoo cannot keep a vision batch open"
        return stream_unavailable_reason(self._model, self._processor)

    def open_resident_batch(
        self,
        *,
        width,
        adapter_state = None,
    ):
        reason = self.resident_unavailable_reason({})
        if reason is not None:
            raise RuntimeError(f"MLX batched generation is unavailable: {reason}")
        session = _VisionBatchSession if self._is_vlm else _TextBatchSession
        return session(
            self,
            width = width,
            adapter_state = adapter_state,
            owns_model = True,
        )

    def _vision_batch_is_available(self) -> bool:
        """Whether this load will decode vision replies together, asked without a request."""
        if not self._is_vlm:
            return False
        return (
            self.batch_unavailable_reason([{}, {}]) is None
            or self.resident_unavailable_reason({}) is None
        )

    def _vlm_batch_unavailable_reason(
        self,
        requests,
        *,
        resident = False,
    ):
        try:
            from unsloth_zoo.mlx import generate as engine
        except ImportError:
            return "the installed unsloth-zoo has no MLX generation"
        gap = _batched_vision_engine_gap(engine)
        if gap is not None:
            return gap
        gap = _row_processor_gap(engine, requests)
        if gap is not None:
            return gap
        # Only the resident session decodes rows from their ids, as the one-at-a-time decode does.
        decodes_ids = resident and _vision_batch_reports_ids(engine, self._speculative_draft)
        if not decodes_ids and any(
            request.get("tools") or request.get("tool_protocol_active") for request in requests
        ):
            return "a reply runs the tool protocol"
        if not decodes_ids and any(_constrained(request) for request in requests):
            return "a reply is constrained to a response format"
        if any(request.get("prompt_ids") is not None for request in requests):
            # The batch detokenizer keeps control tokens and trims the first leading space.
            return "a raw prompt on a vision model decodes on its own"
        if not getattr(self._model, "_is_vlm_model", False):
            return "this model was not loaded as a vision model"
        return None

    def generate_chat_batch(
        self,
        requests,
        *,
        cancel_event = None,
        _adapter_state = None,
    ):
        reason = self.batch_unavailable_reason(requests)
        if reason is not None:
            raise RuntimeError(f"MLX batched generation is unavailable: {reason}")

        self.last_batch_generation_stats = [None] * len(requests)
        self.last_generation_stats = None

        kwargs = {"cancel_event": cancel_event, "_adapter_state": _adapter_state}
        if not self._is_vlm:
            yield from self._generate_session_batch(requests, _TextBatchSession, **kwargs)
        elif self._speculative_draft is not None:  # stream_batch has no speculative mode
            yield from self._generate_session_batch(requests, _VisionBatchSession, **kwargs)
        else:
            yield from self._generate_vlm_batch(requests, **kwargs)

    def _generate_session_batch(
        self,
        requests,
        session_type,
        *,
        cancel_event = None,
        _adapter_state = None,
    ):
        def report(events):
            for row, snapshot in events:
                if isinstance(snapshot, BaseException):
                    # Every row answers one command, which fails as a single reply would.
                    raise snapshot
                if snapshot is None:
                    self.last_batch_generation_stats[row] = session.take_stats(row)
                yield row, snapshot

        with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state):
            session = session_type(
                self,
                width = len(requests),
                adapter_state = _adapter_state,
            )
            try:
                prefixes = [
                    (row, session.admit(request, row)) for row, request in enumerate(requests)
                ]
                for row, prefix in prefixes:
                    if prefix:
                        yield row, prefix
                while session.rows_in_flight:
                    yield from report(session.step(cancel_event and cancel_event.is_set))
                    if cancel_event is not None and cancel_event.is_set():
                        yield from report(session.withdraw(list(session.handles)))
            finally:
                session.close()

    def _vlm_batch_prompt(self, prompt):
        if isinstance(prompt, list):
            return {"prompt_token_ids": prompt}
        from unsloth_zoo.mlx.generate import vlm_batch_adds_special_tokens

        bos = getattr(self._tokenizer, "bos_token", None)
        if (
            not self._reads_vision_input()
            and bos
            and prompt.startswith(bos)
            and vlm_batch_adds_special_tokens(self._model, self._processor)
        ):
            # The single path encodes a text load's prompt itself, keeping only the template's BOS.
            return {"prompt": prompt[len(bos) :]}
        return {"prompt": prompt}

    def _generate_vlm_batch(
        self,
        requests,
        *,
        cancel_event = None,
        _adapter_state = None,
    ):
        import time

        from unsloth_zoo.mlx.generate import (
            GenerationDefaults,
            GenerationRequest,
            SamplingParams,
            stream_batch,
        )

        plans = [self._plan_vlm_request(request) for request in requests]
        defaults = GenerationDefaults(
            max_tokens = max(plan.max_tokens for plan in plans),
            prefill_batch_size = len(plans),
            completion_batch_size = len(plans),
            **self._prefill_step_kwargs(GenerationDefaults, grid = True),
        )

        with (
            self._generation_lock,
            _temporary_mlx_adapter_state(self._model, _adapter_state),
            self._prefill_decision_scopes(),
            ExitStack() as clips,
        ):
            batch = [
                GenerationRequest(
                    **self._vlm_batch_prompt(plan.prompt),
                    image = plan.images[0] if plan.images else None,
                    **row_clips,
                    max_tokens = plan.max_tokens,
                    sampling = SamplingParams(**plan.sampling),
                    logits_processors = plan.processors,
                )
                for plan, row_clips in zip(plans, _batch_clips(clips, plans, self._processor))
            ]
            if any(plan.images or plan.videos for plan in plans):
                self._release_vlm_snapshots()
            else:
                self._release_held_vlm_snapshots()
            logger.info(
                "Generating %d vision replies as one batch: images=%d, videos=%d, max_tokens=%s, model=%s",
                len(plans),
                sum(1 for plan in plans if plan.images),
                sum(len(plan.videos or ()) for plan in plans),
                [plan.max_tokens for plan in plans],
                type(self._model).__name__,
            )
            for row, plan in enumerate(plans):
                if plan.think_prefix:
                    yield row, plan.think_prefix
            started = time.perf_counter()
            prefilled_at: list[float | None] = [None] * len(plans)
            counted: dict[int, tuple[int, int]] = {}
            pending = set(range(len(plans)))
            events = stream_batch(self._model, self._processor, batch, defaults = defaults)

            def _retire(row, result, *, cancelled):
                stream = plans[row].stream
                yield from ((row, snap) for snap in stream.settle())
                yield from ((row, snap) for snap in stream.drain(cancelled = cancelled))
                ready_at = prefilled_at[row] or time.perf_counter()
                decoded = time.perf_counter() - ready_at
                prefill_s = ready_at - started
                seen_prompt, seen_generated = counted.get(row, (0, 0))
                prompt_n = result.prompt_token_count if result is not None else seen_prompt
                generated = (
                    len(result.token_ids) + (result.finish_reason == "stop")
                    if result is not None
                    else seen_generated
                )
                self.last_batch_generation_stats[row] = _build_generation_stats(
                    prompt_n,
                    (prompt_n / prefill_s) if prefill_s > 0 else 0.0,
                    generated,
                    generated / max(decoded, 1e-9),
                    finish_reason = (result.finish_reason if result is not None else "stop"),
                )
                yield row, None

            try:
                for event in events:
                    row = event.index
                    if row not in pending:
                        continue
                    if prefilled_at[row] is None:
                        prefilled_at[row] = time.perf_counter()
                    counted[row] = (
                        getattr(event, "prompt_tokens", 0),
                        getattr(event, "generated_tokens", 0),
                    )
                    if event.delta:
                        yield from (
                            (row, snapshot) for snapshot in plans[row].stream.feed(text = event.delta)
                        )
                    if event.result is not None:
                        pending.discard(row)
                        yield from _retire(row, event.result, cancelled = False)
                    if cancel_event is not None and cancel_event.is_set():
                        break
                if pending and not (cancel_event is not None and cancel_event.is_set()):
                    raise RuntimeError(
                        f"Batched vision generation ended with {len(pending)} of "
                        f"{len(plans)} replies unfinished."
                    )
                for row in sorted(pending):
                    yield from _retire(row, None, cancelled = True)
            finally:
                close = getattr(events, "close", None)
                if callable(close):
                    close()

    def _batch_prompt_row(self, prompt, adapter_state):
        rest, cache, key, tokens, cached_n = self._prepare_prompt_cache(prompt, adapter_state)
        if cache is None or tokens is None:
            tokens = list(prompt) if isinstance(prompt, list) else self._encode_prompt(prompt)
            rest, cached_n, key = tokens, 0, None
            cache = self._make_row_cache()
        return list(tokens[:cached_n]), list(rest), cache, key, list(tokens)

    def _make_row_cache(self):
        """A row's own cache: mlx-lm applies ``max_kv_size`` only to caches it builds."""
        from mlx_lm.models.cache import make_prompt_cache

        window = getattr(self, "_kv_cache_window", None)
        if window is None:
            return make_prompt_cache(self._model)
        return make_prompt_cache(self._model, max_kv_size = window)

    def _bank_row_cache(self, cache_key, event, withdrawn, prompt_tokens, stream):
        history = self._prompt_cache_history
        if history is None or cache_key is None:
            return
        cache = getattr(event, "prompt_cache", None)
        tokens = getattr(event, "all_tokens", None)
        if cache is None and withdrawn is not None:
            cache, tokens = withdrawn
        if cache is None:
            return
        if not tokens:
            tokens = prompt_tokens + stream.token_ids
        try:
            history.insert(cache_key, list(tokens), cache)
        except Exception as exc:
            logger.debug("MLX prompt cache insert failed: %s", exc)

    def generate_audio_input_response(
        self,
        messages,
        system_prompt,
        audio_array,
        max_new_tokens = 512,
        use_adapter = None,
        cancel_event = None,
        stop = None,
        extra_audio_arrays = None,
        speak = False,
        tools = None,
        **_sampler,
    ):
        """Audio-input chat (omni models): waveform in, incremental text deltas out (the audio route
        forwards deltas, unlike the snapshot-diffing text/vision paths). Greedy, so the worker's
        sampler kwargs go unused. ``speak`` answers in the model's own voice instead: the reply
        text, then one ``{"speech": <base64 wav>, "sample_rate": ...}`` item. ``tools`` (a list,
        empty for tool history alone) renders the catalogue and keeps ``messages`` as sent."""
        entry = self.models.get(self.active_model_name) or {}
        if entry.get("audio_type") != "audio_vlm":
            raise RuntimeError(
                "Audio input is not supported for this model on the MLX backend: "
                "no verified audio-capable tower/processor was detected at load."
            )

        # mlx-vlm silently drops all but the first clip; the route refuses this earlier.
        if extra_audio_arrays:
            raise RuntimeError("This MLX model takes one audio file per message.")

        if speak:
            yield from self._speak_audio_reply(
                messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event
            )
            return

        from mlx_vlm import stream_generate as vlm_stream

        if not system_prompt:
            system_prompt = "You are an assistant that transcribes speech accurately."

        audio_messages = messages_with_attached_image(
            alternating_turns(messages) if tools is None else messages,
            system_prompt = system_prompt,
            fallback_user_text = "Please transcribe this audio.",
            image = 0,
            audio = audio_array,
        )
        prompt = _render_registered_vlm_prompt(
            self._processor,
            self._model,
            audio_messages,
            num_images = 0,
            num_audios = 1,
            **({"tools": tools} if tools else {}),
        )
        if prompt is None:
            raise RuntimeError(
                "mlx-vlm has no registered prompt renderer for this model family; "
                "cannot build an audio prompt."
            )

        max_new_tokens = self._generation_limit(
            prompt,
            max_new_tokens,
            audio = [audio_array],
            cap = UNSET_GENERATION_BUDGET,
        )
        logger.info("MLX audio-input generating: prompt_len=%d", len(prompt))
        markers = detect_reasoning_channel_markers(self._processor)
        normalizer = make_reasoning_normalizer(markers) if markers is not None else None
        # Matched on the sampled deltas, for the reason _generate_text gives.
        sequences = _mlx_stop_sequences(stop)
        sampled = ""
        released = 0
        stopped = False
        # Hold adapter state for the whole stream so Base-vs-LoRA compare stays correct
        with (
            self._generation_lock,
            _temporary_mlx_adapter_state(self._model, use_adapter),
            self._prefill_decision_scopes(),
            ExitStack() as generation_scope,
        ):
            # As on the image path: the tower gets the headroom, under the lock.
            self._release_vlm_snapshots()
            generation_scope.enter_context(self._generation_scopes("vision", "audio"))
            final_response = None
            try:
                with closing(
                    _iter_vlm_responses(
                        vlm_stream(
                            self._model,
                            self._processor,
                            prompt,
                            audio = [audio_array],
                            max_tokens = max_new_tokens,
                            # Greedy; the knobs below are load-time state, not caller kwargs.
                            temperature = 0.0,
                            **self._kv_runtime_quant_kwargs(),
                            **self._kv_quant_generate_kwargs(),
                            **self._kv_window_generate_kwargs(),
                            **self._prefill_step_kwargs(),
                        )
                    )
                ) as responses:
                    for response in responses:
                        final_response = response
                        sampled += response.text if hasattr(response, "text") else str(response)
                        if sequences:
                            cut, stopped = _mlx_stop_cut(sampled, sequences)
                        else:
                            cut = len(sampled)
                        # Cut before normalizing; the normalizer's own markers are unmatched here, as the prefill is.
                        delta = sampled[released:cut]
                        released = cut
                        if normalizer is not None:
                            delta = normalizer.feed(delta)
                        if delta:
                            yield delta
                        if stopped:
                            break
                        if cancel_event and cancel_event.is_set():
                            break
            finally:
                # As the vision path derives it: no finish reason here, and unset reads as a natural end.
                if final_response is not None:
                    tokenizer = getattr(self._processor, "tokenizer", self._processor)
                    self.last_generation_stats = _build_generation_stats(
                        getattr(final_response, "prompt_tokens", 0),
                        getattr(final_response, "prompt_tps", 0.0),
                        getattr(final_response, "generation_tokens", 0),
                        getattr(final_response, "generation_tps", 0.0),
                        finish_reason = _mlx_finish_reason(
                            final_response,
                            _mlx_stop_token_ids(tokenizer, self._model),
                            getattr(final_response, "generation_tokens", 0),
                            max_new_tokens,
                        ),
                    )
        if sequences and not stopped:
            delta = sampled[released:]
            if normalizer is not None:
                delta = normalizer.feed(delta)
            if delta:
                yield delta
        if normalizer is not None:
            # As in _generate_text: a sequence closes the block it cut inside.
            cancelled = not stopped and cancel_event is not None and cancel_event.is_set()
            tail = normalizer.drain() if cancelled else normalizer.finish()
            if tail:
                yield tail
        if stopped:
            self._mark_stopped()

    def _speak_audio_reply(
        self,
        messages,
        system_prompt,
        audio_array,
        max_new_tokens,
        use_adapter,
        cancel_event = None,
    ):
        """One spoken question answered in interleaved text and speech, not streamed."""
        import base64

        from mlx_vlm.generate import generate_audio

        if not (self.models.get(self.active_model_name) or {}).get("speaks"):
            raise RuntimeError("The loaded model cannot answer in speech.")
        # The model card's prompt for speech chat; a user prompt still applies after it.
        system = "\n\n".join(p for p in (_SPEECH_REPLY_SYSTEM_PROMPT, system_prompt) if p)
        prompt = _render_registered_vlm_prompt(
            self._processor,
            self._model,
            messages_with_attached_image(
                alternating_turns(messages),
                system_prompt = system,
                image = 0,
                audio = audio_array,
            ),
            num_images = 0,
            num_audios = 1,
        )
        max_new_tokens = self._generation_limit(
            prompt, max_new_tokens, audio = [audio_array], cap = UNSET_GENERATION_BUDGET
        )
        with (
            self._generation_lock,
            _temporary_mlx_adapter_state(self._model, use_adapter),
            ExitStack() as generation_scope,
        ):
            self._release_vlm_snapshots()
            generation_scope.enter_context(self._generation_scopes("vision", "audio"))
            generation_scope.enter_context(_slots_until_cancelled(self._model, cancel_event))
            result = generate_audio(
                self._model,
                self._processor,
                prompt,
                audio = [audio_array],
                generation_mode = "interleaved",
                max_tokens = max_new_tokens,
            )
            cancelled = cancel_event is not None and cancel_event.is_set()
            # Nothing to encode once stopped, nor for a reply that ended before its first speech frame.
            spoke = not cancelled and result.audio is not None and result.audio.size > 0
            wav = result.to_wav_bytes() if spoke else None
        self.last_generation_stats = _build_generation_stats(
            result.prompt_tokens,
            result.prompt_tps,
            result.generation_tokens,
            result.generation_tps,
            finish_reason = result.finish_reason,
        )
        if cancelled:
            return
        if result.text:
            yield result.text
        if spoke:
            yield {
                "speech": base64.b64encode(wav).decode("ascii"),
                "sample_rate": result.sample_rate,
            }

    def generate_with_adapter_control(
        self,
        use_adapter = None,
        cancel_event = None,
        **gen_kwargs,
    ) -> Generator[str, None, None]:
        yield from self.generate_chat_response(
            cancel_event = cancel_event,
            _adapter_state = use_adapter,
            **gen_kwargs,
        )

    def reset_generation_state(self, caller_cancel_event = None):
        # caller_cancel_event: signature parity with the orchestrator; unused here.
        import mlx.core as mx
        import gc

        gc.collect()
        _drain_generation_streams(mx)
        mx.clear_cache()