Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

177 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Core inference backend."""

from __future__ import annotations

from unsloth import FastLanguageModel, FastVisionModel
from unsloth.chat_templates import get_chat_template
from transformers import BatchEncoding, TextIteratorStreamer, TextStreamer
from peft import PeftModel, PeftModelForCausalLM

import contextlib
import inspect
import json
import re
import sys
import threading
import torch
from pathlib import Path
from typing import Optional, Union, Generator, Tuple
from utils.models import ModelConfig, get_base_model_from_lora
from utils.paths import is_model_cached
from utils.transformers_dtype import dtype_kwargs
from utils.utils import format_error_message
from utils.hardware import (
    get_device,
    clear_gpu_cache,
    log_gpu_memory,
    get_device_map,
    raise_if_offloaded,
    get_visible_gpu_count,
)
from core.inference.audio_codecs import AudioCodecManager
from core.inference.context_refusal import ContextBudgetExceeded
from core.inference.runtime_context import (
    UNSET_GENERATION_BUDGET,
    generation_budget_within_context,
    runtime_context_length,
)
from core.inference.message_content import content_to_text, named_turn
from core.inference.model_ids import is_deepseek_ocr
from core.inference.lfm3_conv_fallback import disable_lfm3_triton_conv_where_unsupported
from core.inference.chat_eos import (
    chat_eos_repair,
    resolve_chat_turn_end_eos_ids_using,
)
from core.inference.chat_template_helpers import (
    alternating_turns,
    build_dac_tts_prompt,
    make_reasoning_normalizer,
    detect_reasoning_channel_markers,
    detect_think_prefill,
    messages_with_attached_image,
    neutralize_control_markup_in_messages,
    neutralize_tts_prompt_text,
    prompt_opens_reasoning_channel,
    trailing_assistant_text,
    tts_prompt_with_voice,
    tts_voice_options,
)
from core.inference.presence_penalty import _make_presence_penalty_processor
from core.inference.generation_timing import (
    GenerationTimer,
    build_generation_timings,
    with_prefill_boundary_processor,
)
from core.inference.incremental_streamer import IncrementalTextIteratorStreamer
from core.inference.native_tool_tokens import (
    NativeToolTokenDecoder,
    closes_an_open_envelope,
    decoder_preserves_token,
    reasoning_control_tokens,
    stop_token_text,
)
from core.inference.mlx_inference import (
    _mlx_stop_cut,
    _mlx_stop_sequences,
    _scaling_image_marker,
)
from io import StringIO
import structlog
from loggers import get_logger
from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory


logger = get_logger(__name__)


def _load_in_4bit_kwargs(load_in_4bit: bool) -> dict:
    # True is the loaders' default; passing it reads as an explicit request to requantize fp8 checkpoints to NF4.
    return {} if load_in_4bit else {"load_in_4bit": False}


_OFFLOAD_FIT_FRACTION = 0.9


def _loader_repo(path: str, load_in_4bit: bool) -> str:
    """The repo Unsloth's mapper will load for `path`, else `path`."""
    try:
        from unsloth.models import loader, loader_utils

        if not loader.ALLOW_BITSANDBYTES:
            load_in_4bit = False
        tables = (
            loader_utils.FLOAT_TO_INT_MAPPER,
            loader_utils.INT_TO_FLOAT_MAPPER,
            loader_utils.MAP_TO_UNSLOTH_16bit,
        )
        # Unknown names would fetch the remote mapper.
        name = path.lower()
        if name in loader_utils.BAD_MAPPINGS or not any(name in t for t in tables):
            return path
        return loader_utils.get_model_name(path, load_in_4bit = load_in_4bit) or path
    except Exception:
        return path


def _complete_local_safetensors_bytes(folder: Path) -> Optional[int]:
    """None while a shard the index names is missing (interrupted download)."""
    try:
        index = folder / "model.safetensors.index.json"
        if index.is_file():
            shards = set(json.loads(index.read_text(encoding = "utf-8"))["weight_map"].values())
            files = [folder / shard for shard in shards]
            if not all(f.is_file() for f in files):
                return None
        else:
            files = list(folder.glob("*.safetensors"))
        return sum(f.stat().st_size for f in files) or None
    except Exception:
        return None


def _checkpoint_bytes(name: str, hf_token = None) -> Optional[int]:
    folder = Path(name)
    if not folder.is_dir():
        try:
            from huggingface_hub import snapshot_download
            folder = Path(snapshot_download(name, local_files_only = True, allow_patterns = ["*.json"]))
        except Exception:
            folder = None
    if folder is not None:
        local = _complete_local_safetensors_bytes(folder)
        if local:
            return local
    if Path(name).is_dir():
        return None
    try:
        from huggingface_hub import HfApi

        info = HfApi().model_info(name, token = hf_token, files_metadata = True)
        total = sum(
            s.size or 0
            for s in info.siblings or ()
            if s.rfilename.endswith(".safetensors") and "/" not in s.rfilename
        )
        return total or None
    except Exception:
        return None


def _offload_kwargs_for_fit(
    name: str,
    device_map,
    load_in_4bit: bool,
    hf_token = None,
    mapped: bool = True,
) -> dict:
    """Gemma 4 E2B 4-bit is 8.1 GB on an 8 GB card (#13388); "auto" still loads it all onto the GPU if it fits."""
    # "sequential" over several visible cards pools them; "auto" would plan for one.
    if device_map != "sequential" or not torch.cuda.is_available():
        return {}
    try:
        if torch.cuda.device_count() != 1:
            return {}
        free = torch.cuda.mem_get_info(0)[0]
    except Exception:
        return {}
    # mapped = False: use_exact_model_name reads `name` verbatim.
    size = _checkpoint_bytes(_loader_repo(name, load_in_4bit) if mapped else name, hf_token)
    if not size or size < _OFFLOAD_FIT_FRACTION * free:
        return {}
    logger.info(
        f"Checkpoint is {size / 2**30:.2f} GiB with {free / 2**30:.2f} GiB free; loading with offload_layers='auto'"
    )
    return {"offload_layers": "auto"}


def _load_with_offload_fallback(load, **kwargs):
    """A single-GPU load the size check missed that spills to the CPU: free it, retry with offload_layers = "auto"."""
    try:
        return load(**kwargs)
    except ValueError as e:
        text = str(e)
        spilled = "does not fit in GPU memory" in text or text.startswith(
            "Some modules are dispatched on the CPU or the disk"
        )
        if (
            not spilled
            or "offload_layers" in kwargs
            or kwargs.get("device_map") != "sequential"
            or not torch.cuda.is_available()
            or torch.cuda.device_count() != 1
        ):
            raise
    # After the except block: its traceback's frames held the half-built model.
    import gc

    gc.collect()
    torch.cuda.empty_cache()
    logger.info(
        f"Model did not fit in GPU memory; retrying with offload_layers='auto' "
        f"({torch.cuda.memory_allocated() / 2**30:.2f} GiB still allocated)"
    )
    return load(offload_layers = "auto", **kwargs)


def _hf_token_for_loader(hf_token: Optional[str] | bool) -> Optional[str] | bool:
    if hf_token is False:
        return False
    return hf_token.strip() if isinstance(hf_token, str) and hf_token.strip() else None


def _exact_model_name_for_load(config: ModelConfig, load_in_4bit: bool) -> Optional[str]:
    """config.path when the mapped repo is missing but user weights exist; else cached casing or None."""
    if config.is_local or config.is_lora or not config.path:
        return None
    try:
        from unsloth.models import loader, loader_utils

        # ModelScope downloads every repo id to its own cache, so the HF cache says nothing here.
        if loader.USE_MODELSCOPE:
            return None
        # Resolve the repo the loader will fetch on this host, after its bitsandbytes fallbacks.
        if not loader.ALLOW_BITSANDBYTES:
            load_in_4bit = False
        name = config.path.lower()
        if name in loader_utils.BAD_MAPPINGS:
            return None
        table = (
            loader_utils.FLOAT_TO_INT_MAPPER if load_in_4bit else loader_utils.MAP_TO_UNSLOTH_16bit
        )
        # Avoid fetching the remote mapper for unknown names.
        if name not in table:
            return None
        target = loader_utils.get_model_name(config.path, load_in_4bit = load_in_4bit)
        if target and not loader.ALLOW_PREQUANTIZED_MODELS:
            target = loader._strip_unsloth_bnb_4bit_suffix(target)
    except Exception as e:
        logger.debug(f"Could not resolve the Unsloth mapping for {config.path}: {e}")
        return None
    if not target or target.lower() == name:
        return None
    from utils.utils import (
        active_hf_cache_loadable_snapshot,
        active_hf_cache_repo_spelling,
    )

    cached_target = active_hf_cache_repo_spelling(target)
    if cached_target == target:
        return None  # on disk under the name the mapper will ask for: nothing to do
    if cached_target is not None:
        logger.info(f"Loading cached {cached_target} instead of downloading {target} again")
        return cached_target
    snapshot = active_hf_cache_loadable_snapshot(config.path)
    if snapshot is None:
        return None
    try:
        checkpoint = json.loads((snapshot / "config.json").read_text(encoding = "utf-8"))
    except (OSError, ValueError):
        return None
    if not isinstance(checkpoint, dict) or checkpoint.get("quantization_config") is not None:
        return None
    logger.info(f"Loading cached {config.path} as named instead of downloading {target}")
    return config.path


def _decodes_by_concatenation(tokenizer) -> bool:
    """True when decode(a + b) == decode(a) + decode(b) whenever decode(a) ends on a whole character:
    a byte-level BPE (bytes joined, then lossy UTF-8) with no clean-up pass."""
    if getattr(tokenizer, "clean_up_tokenization_spaces", True) is not False:
        return False
    try:
        from tokenizers import decoders
        from transformers import PreTrainedTokenizerFast

        if not isinstance(tokenizer, PreTrainedTokenizerFast):
            return False
        # A subclass or instance that overrides decode / _decode may post-process the whole text.
        for name in ("decode", "_decode"):
            method = getattr(tokenizer, name, None)
            if getattr(method, "__func__", None) is not getattr(
                PreTrainedTokenizerFast, name, None
            ):
                return False
        return type(tokenizer.backend_tokenizer.decoder) is decoders.ByteLevel
    except Exception:
        return False


class HarmonyTextStreamer:
    """Streaming text decoder for the gpt-oss harmony channel protocol.

    gpt-oss emits multi-channel output via ``<|channel|>analysis<|message|>...``
    / ``<|channel|>final<|message|>...``. Plain skip_special_tokens streaming
    glues channel names to content. This decodes with skip_special_tokens=False
    and parses statefully: emit ``<think>`` on first analysis, stream analysis,
    emit ``</think>`` on first final, stream final. Tracking per-channel lengths
    avoids the delta-on-transformed bug where wrapping tags shift position.
    Same put/end/iterator interface as TextIteratorStreamer.
    """

    import re as _re

    _HARMONY_RE = _re.compile(
        r"<\|channel\|>(\w+)<\|message\|>(.*?)(?=<\|end\|>|<\|channel\|>|\Z)",
        _re.DOTALL,
    )

    def __init__(
        self,
        tokenizer,
        *,
        skip_prompt: bool = True,
        timeout: float = 0.2,
    ):
        import queue

        self.tokenizer = tokenizer
        self.skip_prompt = skip_prompt
        self.timeout = timeout

        self._queue: queue.Queue = queue.Queue()
        self._token_ids: list = []
        self._prompt_len: int = 0
        self._is_first_put: bool = True
        self._stop: bool = False

        self._emitted_think_open: bool = False
        self._emitted_think_close: bool = False
        self._analysis_emitted: int = 0
        self._final_emitted: int = 0

        # Incremental decode: text of the first _settled_n generated ids, ending on a whole character.
        self._concat_decode: bool = _decodes_by_concatenation(tokenizer)
        self._settled_text: str = ""
        self._settled_n: int = 0
        self._scan_raw: str = ""
        self._scan_pos: int = 0
        self._open_match = None
        self._stable_final: bool = False

    def put(self, value):
        import torch

        if isinstance(value, torch.Tensor):
            ids = value[0].tolist() if value.dim() > 1 else value.tolist()
        elif isinstance(value, (list, tuple)):
            ids = list(value)
        else:
            ids = [value]

        if self._is_first_put and self.skip_prompt:
            self._prompt_len = len(ids)
            self._token_ids = list(ids)
            self._is_first_put = False
            return

        self._token_ids.extend(ids)

        if self._concat_decode:
            # Re-decoding the whole reply each token is quadratic; only ids after the settled text are decoded.
            start = self._prompt_len + self._settled_n
            tail = self.tokenizer.decode(self._token_ids[start:], skip_special_tokens = False)
            raw = self._settled_text + tail
            if tail and not tail.endswith("\ufffd"):
                self._settled_text = raw
                self._settled_n = len(self._token_ids) - self._prompt_len
        else:
            gen_ids = self._token_ids[self._prompt_len :]
            raw = self.tokenizer.decode(gen_ids, skip_special_tokens = False)
        # A trailing U+FFFD may be a character whose bytes are still arriving; end() emits it.
        self._process_incremental(raw.rstrip("\ufffd"))

    def end(self):
        gen_ids = self._token_ids[self._prompt_len :]
        if gen_ids:
            raw = self.tokenizer.decode(gen_ids, skip_special_tokens = False)
            self._process_incremental(raw)

        if self._emitted_think_open and not self._emitted_think_close:
            self._queue.put("</think>")
            self._emitted_think_close = True

        self._stop = True
        self._queue.put(None)

    def __iter__(self):
        return self

    def __next__(self):
        from queue import Empty
        while True:
            try:
                val = self._queue.get(timeout = self.timeout)
            except Empty:
                if self._stop:
                    raise StopIteration
                raise  # propagate Empty so caller can check thread liveness
            if val is None:
                raise StopIteration
            return val

    def _process_incremental(self, raw: str) -> None:
        """Emit per-channel deltas; same output as re-matching all of ``raw``, but only the open channel is
        rescanned: a closed channel never changes, and replaying one can only close ``<think>`` (final)."""
        if not raw.startswith(self._scan_raw):
            self._scan_pos = 0
            self._open_match = None
            self._stable_final = False
            old_len = 0
        else:
            old_len = len(self._scan_raw)
        self._scan_raw = raw
        if self._stable_final and self._emitted_think_open and not self._emitted_think_close:
            self._queue.put("</think>")
            self._emitted_think_close = True

        pos = self._scan_pos
        if self._open_match is not None:
            channel, start = self._open_match
            # The open channel ran to the old end, so a terminator can only start in the new tail.
            end = len(raw)
            for marker in ("<|end|>", "<|channel|>"):
                found = raw.find(marker, max(start, old_len - len(marker) + 1))
                if 0 <= found < end:
                    end = found
            self._emit_channel(channel, raw, start, end)
            if end == len(raw):
                return
            self._close_channel(channel, end)
            pos = end

        for m in self._HARMONY_RE.finditer(raw, pos):
            channel = m.group(1).lower()
            self._emit_channel(channel, raw, m.start(2), m.end(2))
            if m.end() == len(raw):
                self._open_match = (channel, m.start(2))
            else:
                self._close_channel(channel, m.end())

    def _close_channel(self, channel: str, end: int) -> None:
        self._open_match = None
        self._scan_pos = end
        if channel in ("final", "assistant"):
            self._stable_final = True

    def _emit_channel(self, channel: str, raw: str, start: int, end: int) -> None:
        if channel == "analysis":
            if not self._emitted_think_open:
                self._queue.put("<think>")
                self._emitted_think_open = True

            if end - start > self._analysis_emitted:
                self._queue.put(raw[start + self._analysis_emitted : end])
                self._analysis_emitted = end - start

        elif channel in ("final", "assistant"):
            if self._emitted_think_open and not self._emitted_think_close:
                self._queue.put("</think>")
                self._emitted_think_close = True

            if end - start > self._final_emitted:
                self._queue.put(raw[start + self._final_emitted : end])
                self._final_emitted = end - start


class ReasoningTextIteratorStreamer(IncrementalTextIteratorStreamer):
    """TextIteratorStreamer that preserves native channel tokens until parsed."""

    def __init__(
        self,
        tokenizer,
        *,
        markers: tuple[str, ...],
        skip_prompt: bool = True,
        timeout: float = 0.2,
        cancel_event = None,
        in_reasoning: bool = False,
        **decode_kwargs,
    ):
        decode_kwargs["skip_special_tokens"] = False
        stream_tokenizer = NativeToolTokenDecoder(
            tokenizer, preserved_tokens = reasoning_control_tokens(markers)
        )
        super().__init__(
            stream_tokenizer, skip_prompt = skip_prompt, timeout = timeout, **decode_kwargs
        )
        self._normalizer = make_reasoning_normalizer(markers, in_reasoning = in_reasoning)
        self._cancel_event = cancel_event
        self._aborted = False

    def abort(self):
        """Mark generation as failed so ``end`` drains without closing."""
        self._aborted = True

    def on_finalized_text(
        self,
        text: str,
        stream_end: bool = False,
    ):
        """Queue canonical deltas, closing only on natural stream completion."""
        delta = self._normalizer.feed(text)
        if delta:
            self.text_queue.put(delta, timeout = self.timeout)

        if stream_end:
            cancelled = self._aborted or (
                self._cancel_event is not None and self._cancel_event.is_set()
            )
            tail = self._normalizer.drain() if cancelled else self._normalizer.finish()
            if tail:
                self.text_queue.put(tail, timeout = self.timeout)
            self.text_queue.put(self.stop_signal, timeout = self.timeout)


class _GenerationThreadError(RuntimeError):
    """Generation worker failures that should propagate through stream routes."""


# What clean_up_tokenization_spaces deletes a space before; a rule spans at most four chars.
_CLEANUP_TAILS = (".", "?", "!", ",", "' ", "n't", "'m", "'s", "'ve", "'re")
_CLEANUP_SPAN_CHARS = 4
# Tokens, not chars: four chars can be four tokens, plus room for tokens decoding to "".
_CLEANUP_SPAN_TOKENS = 8


def _cleanup_join_pending(text: str) -> bool:
    """Only the last few characters matter, from a space onward; a trailing space counts as pending."""
    window = text[-_CLEANUP_SPAN_CHARS:]
    for i, char in enumerate(window):
        if char != " ":
            continue
        rest = window[i + 1 :]
        if not rest or any(tail.startswith(rest) for tail in _CLEANUP_TAILS):
            return True
    return False


def _decoder_cleans_up(streamer) -> bool:
    """Skips the widened re-decode for tokenizers that default clean_up_tokenization_spaces off."""
    kwargs = getattr(streamer, "decode_kwargs", None) or {}
    if "clean_up_tokenization_spaces" in kwargs:
        return bool(kwargs["clean_up_tokenization_spaces"])
    tokenizer = getattr(streamer, "tokenizer", None)
    for _ in range(4):
        if tokenizer is None:
            break
        flag = getattr(tokenizer, "clean_up_tokenization_spaces", None)
        if flag is not None:
            return bool(flag)
        tokenizer = getattr(tokenizer, "tokenizer", None)
    return False


class _StopSequenceStreamer:
    def __init__(self, streamer, stop):
        self.streamer = streamer
        self.sequences = _mlx_stop_sequences(stop)
        self.matched = threading.Event()
        self.token_ids = []
        self.text = ""
        self.settled = ""
        self.prefix_offset = 0
        self.read_offset = 0
        self.scan_from = 0
        self.longest = max((len(s) for s in self.sequences), default = 0)
        self.released = 0
        self.cut = 0
        self.finished = False
        self.next_tokens_are_prompt = bool(getattr(streamer, "skip_prompt", False))
        self.is_harmony = isinstance(streamer, HarmonyTextStreamer)
        self.cleans_up = _decoder_cleans_up(streamer)

    def __next__(self):
        return next(self.streamer)

    def __iter__(self):
        return self

    def put(self, value):
        if not self.sequences:
            return self.streamer.put(value)
        if self.finished or self.matched.is_set():
            return
        # Matches the supported batch-one TextIteratorStreamer protocol.
        if len(value.shape) > 1:
            if value.shape[0] > 1:
                raise ValueError("TextStreamer only supports batch size 1")
            value = value[0]
        if self.next_tokens_are_prompt:
            self.next_tokens_are_prompt = False
            return
        self.token_ids.extend(value.tolist())
        self._decode_new_tokens()
        # Earlier text was already scanned; only a stop overlapping new text can match.
        start = self.scan_from
        cut, matched = _mlx_stop_cut(self.text[start:], self.sequences)
        self.cut = start + cut
        if matched:
            # This runs in the producer, before model.generate checks criteria.
            self.matched.set()
        else:
            self.scan_from = max(start, len(self.settled) - self.longest + 1)
        cut = self.cut
        if not matched and not self.is_harmony and cut > self.released:
            # Keep displayed text stable under word buffering; stop detection above must still see every token.
            if not self.streamer._is_chinese_char(ord(self.text[cut - 1])):
                cut = (
                    max(
                        self.text.rfind(" ", self.released, cut),
                        self.text.rfind("\n", self.released, cut),
                    )
                    + 1
                )
        self._publish(cut)

    def _decode(self, token_ids):
        decode_kwargs = (
            {"skip_special_tokens": False} if self.is_harmony else self.streamer.decode_kwargs
        )
        return self.streamer.tokenizer.decode(token_ids, **decode_kwargs)

    def _decode_new_tokens(self):
        # Re-decoding the whole reply each token is quadratic; decode a short window.
        prefix = self._decode(self.token_ids[self.prefix_offset : self.read_offset])
        window = self._decode(self.token_ids[self.prefix_offset :])
        if (
            self.cleans_up
            and window.startswith(prefix)
            and _cleanup_join_pending(self.settled + window[len(prefix) :])
        ):
            # A cleanup rule split across the join is invisible from the short window and would diverge
            # permanently, so re-read a few tokens behind the join.
            wider = max(0, self.read_offset - _CLEANUP_SPAN_TOKENS)
            if wider < self.prefix_offset:
                wide_prefix = self._decode(self.token_ids[wider : self.read_offset])
                # Splicing onto text the prefix does not end is worse than the divergence; use the short window.
                if self.settled.endswith(wide_prefix):
                    prefix = wide_prefix
                    window = self._decode(self.token_ids[wider:])
        if window.endswith("\ufffd"):
            # Bytes still arriving can rewrite the window (unfinished emoji shows as U+FFFD), so wait for them.
            tail = window[len(prefix) :] if window.startswith(prefix) else ""
            self.text = self.settled + tail
            return
        if not window.startswith(prefix):
            # New tokens can rewrite settled text, e.g. space cleanup; rebuild it from every token and rescan.
            self.settled = self._decode(self.token_ids)
            self.prefix_offset = max(0, len(self.token_ids) - 4)
            self.read_offset = len(self.token_ids)
            self.scan_from = 0
        elif len(window) > len(prefix):
            self.settled += window[len(prefix) :]
            self.prefix_offset = self.read_offset
            self.read_offset = len(self.token_ids)
        self.text = self.settled

    def _publish(self, cut):
        if cut <= self.released:
            return
        if self.is_harmony:
            # Harmony consumes cumulative raw text and synthesizes tags itself.
            self.streamer._process_incremental(self.text[:cut])
        else:
            # The reasoning subclass normalizes here; plain text just queues it.
            self.streamer.on_finalized_text(self.text[self.released : cut])
        self.released = cut

    def end(self):
        if not self.sequences:
            return self.streamer.end()
        if self.finished:
            return
        self.finished = True
        if not self.matched.is_set() and self.read_offset < len(self.token_ids):
            # Bytes that never resolved still belong to the reply, as a full decode shows.
            self.text = self._decode(self.token_ids)
        # A natural end releases a partial unmatched stop and unresolved bytes.
        self._publish(self.cut if self.matched.is_set() else len(self.text))
        # Caches are empty as put() feeds raw text, so end() only closes reasoning and sends the sentinel.
        self.streamer.end()

    def abort(self):
        abort = getattr(self.streamer, "abort", None)
        if abort is not None:
            abort()


def _decode_graphs():
    """Unsloth's CUDA-graph decode module, or None for an Unsloth without the Studio scope."""
    try:
        from unsloth.models import _decode_graphs as graphs
    except Exception:
        return None
    return graphs if hasattr(graphs, "studio_generation") else None


def _decode_graph_scope():
    """Graphed decode on by default for this thread's chat generate(): this worker only serves
    inference. Llama / Qwen3 on NVIDIA CUDA only; UNSLOTH_DECODE_CUDA_GRAPHS=0 still turns it off."""
    graphs = _decode_graphs()
    return graphs.studio_generation() if graphs is not None else contextlib.nullcontext()


def _release_decode_graphs(model, generation_lock) -> None:
    """Frees the model's decode graphs and decode buffers now rather than at garbage collection.
    Skipped while a generate() still holds the lock: that thread's model reference frees them."""
    graphs = _decode_graphs()
    if graphs is None or not hasattr(model, "modules"):
        return
    if generation_lock is not None and not generation_lock.acquire(blocking = False):
        return
    try:
        graphs.release_for_training(model)
    except Exception as e:
        logger.warning(f"Could not release decode graphs: {e}")
    finally:
        if generation_lock is not None:
            generation_lock.release()


_CUSTOM_GENERATE_REJECTED_KWARGS = ("logits_processor", "stopping_criteria")


class _GenerationHalted(Exception):
    pass


class _HaltingStreamer:
    def __init__(self, streamer, events):
        self.streamer = streamer
        self.events = events

    def put(self, value):
        if any(ev.is_set() for ev in self.events):
            raise _GenerationHalted
        return self.streamer.put(value)

    def end(self):
        return self.streamer.end()

    def __getattr__(self, name):
        return getattr(self.streamer, name)


def _supported_generate_kwargs(model, generation_kwargs: dict) -> dict:
    if not isinstance(getattr(model, "_generation_fields_unsupported", None), dict):
        return generation_kwargs
    kept = {k: v for k, v in generation_kwargs.items() if k not in _CUSTOM_GENERATE_REJECTED_KWARGS}
    model_type = getattr(getattr(model, "config", None), "model_type", None)
    if model_type in {"lfm3_omni", "lfm3_audio"}:
        kept["generation_mode"] = "text"
    events = [
        c.ev
        for c in (generation_kwargs.get("stopping_criteria") or ())
        if getattr(c, "ev", None) is not None
    ]
    if events and kept.get("streamer") is not None:
        kept["streamer"] = _HaltingStreamer(kept["streamer"], events)
    return kept


def _generate_supported(model, generation_kwargs: dict):
    kwargs = _supported_generate_kwargs(model, generation_kwargs)
    try:
        return model.generate(**kwargs)
    except _GenerationHalted:
        streamer = generation_kwargs.get("streamer")
        matched = getattr(streamer, "matched", None)
        prompt = generation_kwargs.get("input_ids")
        if matched is not None and matched.is_set() and prompt is not None:
            return prompt.new_tensor([prompt[0].tolist() + streamer.token_ids])
        return None


def _refuse_without_generate(model, model_name: str) -> None:
    if not callable(getattr(model, "generate", None)):
        raise ValueError(
            f"{model_name} loads as {type(model).__name__}, which cannot generate text. "
            "Unsloth Studio has no chat path for this architecture yet."
        )


def _audio_chat_auto_model(config):
    if not getattr(config, "has_audio_input", False) or getattr(config, "is_vision", False):
        return None
    if getattr(config, "audio_type", None) != "audio_vlm":
        return None
    import transformers

    return getattr(transformers, "AutoModelForMultimodalLM", None)


# Audio sampling stays with each checkpoint's generation_config: LFM3-Audio and -Omni differ.
_SPEECH_REPLY_SYSTEM_PROMPT = "Respond with interleaved text and audio."
_TTS_LOCALES = {
    "ar": "Arabic",
    "da": "Danish",
    "de": "German",
    "el": "Greek",
    "en": "American English",
    "en-us": "American English",
    "en-gb": "British English",
    "fi": "Finnish",
    "fr": "French",
    "he": "Hebrew",
    "it": "Italian",
    "ja": "Japanese",
    "ko": "Korean",
    "nl": "Dutch",
    "pl": "Polish",
    "pt": "Portuguese",
    "es": "Spanish",
    "sv": "Swedish",
    "tr": "Turkish",
    "zh": "Chinese",
}


def _speech_codec(model, processor):
    """The processor's bundled codec when the model speaks (LFM3-Audio / -Omni), else None."""
    codec = getattr(processor, "audio_tokenizer", None)
    if codec is None or not isinstance(
        getattr(getattr(model, "config", None), "interleaved_n_audio", None), int
    ):
        return None
    return codec


def _tts_system_prompt(language: Optional[str], audio_options: Optional[dict]) -> str:
    """The model card's TTS prompt; an unknown language falls back to American English."""
    lang = str(language or "").strip()
    locale = _TTS_LOCALES.get(lang.lower().replace("_", "-")) or next(
        (name for name in _TTS_LOCALES.values() if name.lower() == lang.lower()),
        "American English",
    )
    voice = str((audio_options or {}).get("voice") or "").strip().lower()
    return f"Perform TTS in {locale} using a {'male' if voice == 'male' else 'female'} voice."


class _SpeechStopped(Exception):
    pass


def _speech_stop_streamer(cancel_event):
    """LFM3's generate refuses stopping_criteria; a streamer raising on put ends it."""
    from transformers.generation.streamers import BaseStreamer

    class _StopOnCancel(BaseStreamer):
        def put(self, value):
            if cancel_event.is_set():
                raise _SpeechStopped

        def end(self):
            pass

    return _StopOnCancel()


def _processor_takes_text_first(processor) -> bool:
    """LFM3-Omni's processor is (text, audio, ..., images); the usual VLM order is (images, text)."""
    # unsloth_zoo wraps __call__ as (*args, **kwargs); read the first class that names its arguments.
    for cls in type(processor).__mro__:
        call = cls.__dict__.get("__call__")
        if call is None:
            continue
        try:
            params = list(inspect.signature(call).parameters.values())[1:]
        except (TypeError, ValueError):
            return False
        if params and params[0].kind is not inspect.Parameter.VAR_POSITIONAL:
            return params[0].name == "text"
    return False


# LlamaModel_fast_forward cuts input_ids at max_seq_length, so decoding past it crashes; FastModel's stock forward does not.
def _truncates_past_window(model) -> bool:
    return any(
        getattr(getattr(type(module), "forward", None), "__name__", "") == "LlamaModel_fast_forward"
        for module in model.modules()
    )


def _probe_prompt_bos(tokenizer, prompt):
    """(tokenizer, prompt, encoding) when the rendered prompt already starts with BOS, else None.
    Most templates render BOS, so no second one is added; zephyr and tinyllama-chat do not."""
    tok = getattr(tokenizer, "tokenizer", tokenizer)
    bos_token_id = getattr(tok, "bos_token_id", None)
    if bos_token_id is None:
        return None
    try:
        encoding = tok(prompt, add_special_tokens = False)
        ids = encoding["input_ids"]
    except Exception:
        return None
    while isinstance(ids, (list, tuple)) and ids and isinstance(ids[0], (list, tuple)):
        ids = ids[0]
    if len(ids) and ids[0] == bos_token_id:
        return tok, prompt, encoding
    return None


def _prompt_already_has_bos(tokenizer, prompt):
    return _probe_prompt_bos(tokenizer, prompt) is not None


def _tokenize_prompt(
    tokenizer,
    prompt,
    add_special_tokens,
    bos_probe = None,
):
    """``tokenizer(prompt, return_tensors="pt", ...)``, reusing a matching bos_probe encoding."""
    if not add_special_tokens and bos_probe is not None:
        probe_tok, probe_prompt, encoding = bos_probe
        ids = encoding["input_ids"] if isinstance(encoding, BatchEncoding) else None
        if (
            probe_tok is tokenizer
            and probe_prompt == prompt
            and isinstance(ids, list)
            and not (ids and isinstance(ids[0], (list, tuple)))
        ):
            return BatchEncoding(
                {key: [value] for key, value in encoding.items()},
                encoding = encoding.encodings,
                tensor_type = "pt",
            )
    return tokenizer(prompt, return_tensors = "pt", add_special_tokens = add_special_tokens)


_MAPPED_TEMPLATE_LOCK = threading.Lock()


def _mapped_template_tokenizer(model_info, tokenizer, template_name):
    with _MAPPED_TEMPLATE_LOCK:
        cached = model_info.get("_mapped_template_tokenizer")
        if cached is not None and cached[0] is tokenizer and cached[1] == template_name:
            return cached[2]
        remapped = get_chat_template(tokenizer, chat_template = template_name)
        model_info["_mapped_template_tokenizer"] = (tokenizer, template_name, remapped)
        return remapped


_MISSING = object()


class _OcrNeedsImageError(ValueError):
    public = True
    openai_param = "messages"


def _without_image_parts(messages) -> list[dict]:
    """Drops image parts; a turn left with only text collapses back to that text."""
    out = []
    for message in messages:
        content = message.get("content")
        if not isinstance(content, list):
            out.append(message)
            continue
        kept = [
            part
            for part in content
            if not (
                isinstance(part, dict) and part.get("type") in ("image", "image_url", "input_image")
            )
        ]
        if len(kept) == len(content):
            out.append(message)
            continue
        texts = [
            part.get("text", "")
            for part in kept
            if isinstance(part, dict) and part.get("type") == "text"
        ]
        out.append({**message, "content": "\n".join(texts) if len(kept) == len(texts) else kept})
    return out


_CANVAS_MASK = "\u2591"
_CANVAS_FRAME_INTERVAL_S = 0.1


def _accepts_on_frame(generate_text) -> bool:
    """An older core would swallow on_frame unused."""
    import inspect

    candidates = [generate_text]
    try:
        from unsloth.models import diffusion_samplers
        candidates.append(diffusion_samplers.generate_text)
    except ImportError:
        pass
    for fn in candidates:
        try:
            if "on_frame" in inspect.signature(fn).parameters:
                return True
        except (TypeError, ValueError):
            continue
    return False


def _render_canvas(tokenizer, tokens, mask_id) -> str:
    tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
    parts, run = [], []
    for token in tokens:
        if mask_id is not None and token == mask_id:
            if run:
                parts.append(tokenizer.decode(run, skip_special_tokens = True))
                run = []
            parts.append(_CANVAS_MASK)
        else:
            run.append(token)
    if run:
        parts.append(tokenizer.decode(run, skip_special_tokens = True))
    return "".join(parts)


def _diffusion_eos_ids(model, tokenizer, chat) -> tuple:
    """The ids ``generate_text`` cuts its reply at; empty on an older core."""
    import importlib
    try:
        samplers = importlib.import_module("unsloth.models.diffusion_samplers")
        base = samplers.unwrap_diffusion_model(model)
        ids = list(samplers._eos_ids(base, tokenizer))
        if chat:
            ids += [i for i in samplers._turn_end_ids(tokenizer) if i not in ids]
        return tuple(ids)
    except Exception:
        return ()


def _diffusion_frame_relay(
    tokenizer,
    sink,
    interval_s = None,
    stop = None,
    eos_ids = (),
):
    """Core-sampler on_frame: one frame per interval to sink; never raises into the sampler."""
    import time

    last = [float("-inf")]
    if interval_s is None:
        interval_s = _CANVAS_FRAME_INTERVAL_S

    def on_frame(frame):
        now = time.monotonic()
        # A block's last step is never throttled: later frames build on it.
        block_end = int(frame.get("step", 0)) >= int(frame.get("total", 0)) > 0
        if now - last[0] < interval_s and not block_end:
            return
        last[0] = now
        try:
            tokens = list(frame.get("tokens") or ())
            for index, token in enumerate(tokens):
                if token in eos_ids:
                    tokens = tokens[:index]
                    break
            text = _render_canvas(tokenizer, tokens, frame.get("mask_id"))
            for sequence in stop or ():
                if sequence and sequence in text:
                    text = text[: text.index(sequence)]
            total = int(frame.get("total", 0))
            sink(
                {
                    "type": "diffusion_frame",
                    "block": int(frame.get("block", 0)),
                    # Zero-based for the canvas; the sampler's block-end frame is step == total.
                    "step": max(min(int(frame.get("step", 0)), total - 1), 0),
                    "total": total,
                    "text": text,
                }
            )
        except Exception as exc:
            logger.debug("Dropped a diffusion canvas frame: %s", exc)

    return on_frame


# o200k_harmony's non-reserved special tokens (tokenizer_config.json added_tokens_decoder).
_HARMONY_PROTOCOL_TOKEN_RE = re.compile(
    r"<\|(?:start|end|message|channel|constrain|return|call|startoftext|endoftext|endofprompt)\|>"
)


class InferenceBackend:
    """Unified inference backend supporting text, vision, and LoRA models"""

    # Class attribute too, so a backend built with __new__ still answers.
    last_generation_stats = None

    def __init__(self):
        self.models = {}
        self.active_model_name = None
        self.loading_models = set()
        self.loaded_local_models = []
        from core.inference.defaults import get_default_models

        self.default_models = get_default_models()
        self.device = get_device().value
        self._audio_codec_manager = AudioCodecManager()
        self.last_generation_stats = None
        # Set by the worker for one request that asked for canvas frames.
        self.diffusion_frame_sink = None

        # Plain Lock, NOT RLock: reentrancy would let concurrent compare-mode requests race on the GPU.
        import threading

        self._generation_lock = threading.Lock()
        self._model_state_lock = threading.Lock()

        logger.info(f"InferenceBackend initialized on {self.device}")

    @staticmethod
    def _normalize_top_k(top_k: int) -> int:
        # API uses -1 to disable top-k; transformers uses 0.
        return 0 if top_k < 0 else top_k

    def _resolve_chat_eos(self, model_name: str) -> None:
        """Caches chat turn-end stop ids at load and repairs generation_config; some Qwen3.5/3.6 need it."""
        info = self.models.get(model_name) or {}
        model = info.get("model")
        container = info.get("tokenizer")
        tokenizer = getattr(container, "tokenizer", container)
        if model is None or tokenizer is None:
            return
        # VLMs keep chat_template on the processor; resolve ids on the generation tokenizer.
        template_source = container if getattr(container, "chat_template", None) else tokenizer
        try:
            turn_end_ids = resolve_chat_turn_end_eos_ids_using(template_source, tokenizer)
        except Exception as e:  # never block a load on eos resolution
            logger.warning("Chat turn-end eos resolution failed for %s: %s", model_name, e)
            return
        info["chat_turn_end_eos_ids"] = turn_end_ids

        gen = getattr(model, "generation_config", None)
        if gen is None:
            return
        repaired = chat_eos_repair(gen.eos_token_id, turn_end_ids)
        if repaired is None:
            return
        previous = gen.eos_token_id
        gen.eos_token_id = repaired
        logger.info(
            "Repaired generation_config.eos_token_id for %s: %s -> %s",
            model_name,
            previous,
            repaired,
        )

    @_invalidates_gpu_memory("transformers load")
    def load_model(
        self,
        config: ModelConfig,
        max_seq_length: int = 2048,
        dtype = None,
        load_in_4bit: bool = True,
        hf_token: Optional[str] | bool = None,
        trust_remote_code: bool = False,
        gpu_ids: Optional[list[int]] = None,
        audio_codec_path: Optional[str] = None,
    ) -> bool:
        """Keep the token so native-template fallback can fetch a gated repo template during generation."""
        self._hf_token = hf_token
        # GGUF uses max_seq_length=0 as "model default"; Unsloth crashes on it.
        if max_seq_length <= 0:
            max_seq_length = 2048

        try:
            model_name = config.identifier

            if model_name in self.models and self.models[model_name].get("model"):
                logger.info(f"Model {model_name} already loaded")
                if hf_token:
                    self.models[model_name]["hf_token"] = hf_token
                self.active_model_name = model_name
                return True

            if model_name in self.loading_models:
                logger.info(f"Model {model_name} is already being loaded")
                return False

            self.loading_models.add(model_name)
            device_map = get_device_map(gpu_ids)
            logger.info(
                f"Using device_map='{device_map}' ({get_visible_gpu_count()} GPU(s) visible)"
            )

            self.models[model_name] = {
                # Native-template fallback must use this model's own hf_token, not the one loaded last.
                "hf_token": hf_token,
                # Reuse this model trust_remote_code consent for the template reload; never run unconsented code.
                "trust_remote_code": trust_remote_code,
                "is_vision": config.is_vision,
                "is_lora": config.is_lora,
                "is_audio": config.is_audio,
                "audio_type": config.audio_type,
                "has_audio_input": config.has_audio_input,
                "model_path": config.path,
                "base_model": config.base_model if config.is_lora else None,
                "loaded_adapters": {},
                "active_adapter": None,
            }

            if config.is_audio:
                audio_type = config.audio_type
                adapter_info = " (LoRA adapter)" if config.is_lora else ""
                logger.info(f"Loading audio ({audio_type}) model{adapter_info}: {model_name}")
                log_gpu_memory(f"Before loading {model_name}")

                if audio_type == "csm":
                    from unsloth import FastModel
                    from transformers import CsmForConditionalGeneration

                    model, processor = FastModel.from_pretrained(
                        config.path,
                        auto_model = CsmForConditionalGeneration,
                        load_in_4bit = False,
                        device_map = device_map,
                        token = _hf_token_for_loader(hf_token),
                        trust_remote_code = trust_remote_code,
                    )
                    FastModel.for_inference(model)
                    self.models[model_name]["model"] = model
                    self.models[model_name]["tokenizer"] = processor
                    self.models[model_name]["processor"] = processor
                elif audio_type == "bicodec":
                    import os
                    from core.inference.audio_codecs import resolve_bicodec_repo_path
                    from unsloth import FastModel

                    if config.is_lora and config.base_model:
                        # LoRA adapter: base_model is .../Spark-TTS-0.5B/LLM;
                        # BiCodec weights live in the parent dir.
                        from utils.models.spark_tts_adapter import spark_tts_adapter_load_target

                        base_path = config.base_model
                        abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
                            base_path, hf_token = hf_token
                        )

                        logger.info(
                            f"Spark-TTS LoRA: loading adapter from {config.path}, BiCodec from {abs_repo_path}"
                        )
                        with spark_tts_adapter_load_target(
                            config.path, hf_token = _hf_token_for_loader(hf_token)
                        ) as load_target:
                            model, tokenizer = FastModel.from_pretrained(
                                load_target,
                                dtype = torch.float32,
                                # Flash Attention cannot run float32
                                attn_implementation = "sdpa",
                                load_in_4bit = False,
                                device_map = device_map,
                                token = _hf_token_for_loader(hf_token),
                                trust_remote_code = trust_remote_code,
                            )
                    elif config.is_local and os.path.isdir(config.path):
                        # A merged export has no LLM/ child or BiCodec weights; resolve those from the recorded base.
                        llm_path = os.path.join(config.path, "LLM")
                        if not os.path.isdir(llm_path):
                            llm_path = config.path
                        abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
                            config.path, hf_token = hf_token
                        )
                        logger.info(
                            f"Spark-TTS merged export: LLM from {llm_path}, BiCodec from {abs_repo_path}"
                        )
                    else:
                        abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
                            config.path, hf_token = hf_token
                        )
                        llm_path = os.path.join(abs_repo_path, "LLM")
                        logger.info(
                            f"Spark-TTS: repo at {abs_repo_path}, loading LLM from {llm_path}"
                        )

                    if not (config.is_lora and config.base_model):
                        model, tokenizer = FastModel.from_pretrained(
                            llm_path,
                            dtype = torch.float32,
                            # Flash Attention cannot run float32
                            attn_implementation = "sdpa",
                            load_in_4bit = False,
                            device_map = device_map,
                            token = _hf_token_for_loader(hf_token),
                            trust_remote_code = trust_remote_code,
                        )

                    FastModel.for_inference(model)
                    self.models[model_name]["model"] = model
                    self.models[model_name]["tokenizer"] = tokenizer
                    self.models[model_name]["model_repo_path"] = abs_repo_path
                elif audio_type == "dac":
                    # OuteTTS uses FastModel (not FastLanguageModel)
                    from unsloth import FastModel

                    model, tokenizer = FastModel.from_pretrained(
                        config.path,
                        max_seq_length = max_seq_length,
                        load_in_4bit = False,
                        device_map = device_map,
                        token = _hf_token_for_loader(hf_token),
                        trust_remote_code = trust_remote_code,
                    )
                    FastModel.for_inference(model)
                    self.models[model_name]["model"] = model
                    self.models[model_name]["tokenizer"] = tokenizer
                elif audio_type == "whisper":
                    from unsloth import FastModel
                    from transformers import WhisperForConditionalGeneration

                    model, tokenizer = FastModel.from_pretrained(
                        config.path,
                        auto_model = WhisperForConditionalGeneration,
                        whisper_language = "English",
                        whisper_task = "transcribe",
                        load_in_4bit = False,
                        device_map = device_map,
                        token = _hf_token_for_loader(hf_token),
                        trust_remote_code = trust_remote_code,
                    )
                    FastModel.for_inference(model)
                    model.eval()

                    from transformers import pipeline as hf_pipeline

                    whisper_pipe = hf_pipeline(
                        "automatic-speech-recognition",
                        model = model,
                        tokenizer = tokenizer.tokenizer,
                        feature_extractor = tokenizer.feature_extractor,
                        processor = tokenizer,
                        return_language = True,
                        **dtype_kwargs(torch.float16),
                    )
                    self.models[model_name]["model"] = model
                    self.models[model_name]["tokenizer"] = tokenizer
                    self.models[model_name]["whisper_pipeline"] = whisper_pipe
                else:
                    # SNAC (Orpheus) uses FastLanguageModel
                    model, tokenizer = FastLanguageModel.from_pretrained(
                        model_name = config.path,
                        max_seq_length = max_seq_length,
                        load_in_4bit = False,
                        device_map = device_map,
                        token = _hf_token_for_loader(hf_token),
                        trust_remote_code = trust_remote_code,
                    )
                    FastLanguageModel.for_inference(model)
                    self.models[model_name]["model"] = model
                    self.models[model_name]["tokenizer"] = tokenizer
                    voice_options = tts_voice_options(audio_type)
                    if voice_options:
                        self.models[model_name]["audio_options"] = voice_options

                # Load an external codec for TTS audio; whisper (ASR) and audio_vlm (audio input) need none.
                if audio_type not in ("whisper", "audio_vlm"):
                    model_repo_path = audio_codec_path or self.models[model_name].get(
                        "model_repo_path"
                    )
                    self._audio_codec_manager.load_codec(
                        audio_type, self.device, model_repo_path = model_repo_path
                    )

                raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")
                disable_lfm3_triton_conv_where_unsupported(
                    self.models[model_name]["model"], torch, logger
                )
                self.models[model_name]["context_length"] = runtime_context_length(
                    self.models[model_name].get("model"),
                    max_seq_length,
                )

                self.active_model_name = model_name
                self.loading_models.discard(model_name)
                logger.info(f"Successfully loaded audio model: {model_name}")
                log_gpu_memory(f"After loading {model_name}")
                return True

            model_type = "vision" if config.is_vision else "text"
            adapter_info = " (LoRA adapter)" if self.models[model_name]["is_lora"] else ""
            logger.info(f"Loading {model_type} model{adapter_info}: {model_name}")
            log_gpu_memory(f"Before loading {model_name}")

            exact_model_name = _exact_model_name_for_load(config, load_in_4bit)
            use_exact_model_name = exact_model_name is not None
            load_path = exact_model_name or config.path
            from utils.models.diffusion_lm import (
                diffusion_lm_profile_for_model,
                loaded_diffusion_profile,
            )

            diffusion_lm_profile = diffusion_lm_profile_for_model(
                load_path, hf_token = _hf_token_for_loader(hf_token)
            )
            if not diffusion_lm_profile and config.is_lora and config.base_model:
                # A Hub LoRA has no config.json: classify its base before the vision branch claims DiffusionGemma.
                diffusion_lm_profile = diffusion_lm_profile_for_model(
                    config.base_model, hf_token = _hf_token_for_loader(hf_token)
                )
            # A LoRA loads its base model.
            offload_kwargs = _offload_kwargs_for_fit(
                config.base_model if config.is_lora and config.base_model else load_path,
                device_map,
                load_in_4bit,
                _hf_token_for_loader(hf_token),
                mapped = not use_exact_model_name,
            )
            if diffusion_lm_profile:
                # Text-diffusion LMs (LLaDA, Dream, DiffusionGemma, ...) decode with their own sampler,
                # never left-to-right generate(); DiffusionGemma's vision tower is not used here.
                from unsloth import FastModel

                model, tokenizer = FastModel.from_pretrained(
                    model_name = load_path,
                    max_seq_length = max_seq_length,
                    dtype = dtype,
                    **_load_in_4bit_kwargs(load_in_4bit),
                    device_map = device_map,
                    token = _hf_token_for_loader(hf_token),
                    trust_remote_code = trust_remote_code,
                    use_exact_model_name = use_exact_model_name,
                )
                FastModel.for_inference(model)
                loaded_profile = loaded_diffusion_profile(model)
                if not loaded_profile:
                    # As training does: an autoregressive load must never reach the diffusion sampler.
                    raise RuntimeError(
                        f"'{load_path}' is a text-diffusion model ({diffusion_lm_profile}), but the installed "
                        "Unsloth loaded it as an ordinary model. Update with `unsloth studio update`."
                    )
                self.models[model_name]["diffusion_lm_profile"] = loaded_profile
                self.models[model_name]["is_vision"] = False
                self.models[model_name]["model"] = model
                self.models[model_name]["tokenizer"] = tokenizer
                logger.info(
                    f"Loaded text-diffusion model for chat (profile: "
                    f"{self.models[model_name]['diffusion_lm_profile']})"
                )

            elif config.is_vision:
                model, processor = _load_with_offload_fallback(
                    FastVisionModel.from_pretrained,
                    model_name = load_path,
                    max_seq_length = max_seq_length,
                    dtype = dtype,
                    **_load_in_4bit_kwargs(load_in_4bit),
                    device_map = device_map,
                    token = _hf_token_for_loader(hf_token),
                    trust_remote_code = trust_remote_code,
                    use_exact_model_name = use_exact_model_name,
                    **offload_kwargs,
                )

                FastVisionModel.for_inference(model)

                # FastVisionModel may return a raw tokenizer (e.g. Gemma-3); load the real processor.
                from transformers import ProcessorMixin

                if not (
                    isinstance(processor, ProcessorMixin) or hasattr(processor, "image_processor")
                ):
                    # LoRA adapters: use base model. Local merged exports: read base from export_metadata.json.
                    processor_source = config.base_model if config.is_lora else config.identifier
                    if not config.is_lora and config.is_local:
                        _meta_path = Path(config.path) / "export_metadata.json"
                        try:
                            if _meta_path.exists():
                                _meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig"))
                                if _meta.get("base_model"):
                                    processor_source = _meta["base_model"]
                        except Exception:
                            pass
                    logger.warning(
                        f"FastVisionModel returned {type(processor).__name__} (no image_processor) "
                        f"for '{model_name}' — loading proper processor from '{processor_source}'"
                    )
                    from transformers import AutoProcessor

                    processor = AutoProcessor.from_pretrained(
                        processor_source,
                        token = _hf_token_for_loader(hf_token),
                        trust_remote_code = trust_remote_code,
                    )
                    logger.info(f"Loaded {type(processor).__name__} from {processor_source}")

                self.models[model_name]["model"] = model
                self.models[model_name]["tokenizer"] = processor
                self.models[model_name]["processor"] = processor

            elif _audio_chat_auto_model(config) is not None:
                # Its text load builds the bare backbone; the multimodal class carries generate().
                from unsloth import FastModel

                model, processor = _load_with_offload_fallback(
                    FastModel.from_pretrained,
                    model_name = load_path,
                    auto_model = _audio_chat_auto_model(config),
                    max_seq_length = max_seq_length,
                    dtype = dtype,
                    **_load_in_4bit_kwargs(load_in_4bit),
                    device_map = device_map,
                    token = _hf_token_for_loader(hf_token),
                    trust_remote_code = trust_remote_code,
                    use_exact_model_name = use_exact_model_name,
                    **offload_kwargs,
                )
                FastModel.for_inference(model)
                if not hasattr(processor, "feature_extractor"):
                    from transformers import AutoProcessor
                    processor = AutoProcessor.from_pretrained(
                        config.base_model if config.is_lora else load_path,
                        token = _hf_token_for_loader(hf_token),
                        trust_remote_code = trust_remote_code,
                    )
                _refuse_without_generate(model, model_name)
                self.models[model_name]["model"] = model
                self.models[model_name]["tokenizer"] = processor
                self.models[model_name]["processor"] = processor

            else:
                model, tokenizer = _load_with_offload_fallback(
                    FastLanguageModel.from_pretrained,
                    model_name = load_path,
                    max_seq_length = max_seq_length,
                    dtype = dtype,
                    **_load_in_4bit_kwargs(load_in_4bit),
                    device_map = device_map,
                    token = _hf_token_for_loader(hf_token),
                    trust_remote_code = trust_remote_code,
                    use_exact_model_name = use_exact_model_name,
                    **offload_kwargs,
                )

                FastLanguageModel.for_inference(model)

                self.models[model_name]["model"] = model
                self.models[model_name]["tokenizer"] = tokenizer
                # A Hub LoRA has no config.json for the probe above, but core still loads a diffusion base
                # through its diffusion path: decode it with its sampler, never left-to-right.
                loaded_profile = loaded_diffusion_profile(model)
                if loaded_profile:
                    self.models[model_name]["diffusion_lm_profile"] = loaded_profile
                    logger.info(f"Loaded text-diffusion model for chat (profile: {loaded_profile})")
                else:
                    _refuse_without_generate(model, model_name)

            if self.models[model_name].get("audio_type") == "audio_vlm":
                self.models[model_name]["speaks"] = (
                    _speech_codec(
                        self.models[model_name].get("model"),
                        self.models[model_name].get("processor"),
                    )
                    is not None
                )

            raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")
            disable_lfm3_triton_conv_where_unsupported(
                self.models[model_name]["model"], torch, logger
            )
            self.models[model_name]["context_length"] = runtime_context_length(
                self.models[model_name].get("model"),
                max_seq_length,
            )

            self._resolve_chat_eos(model_name)
            self._load_chat_template_info(model_name)

            self.active_model_name = model_name
            self.loading_models.discard(model_name)

            logger.info(f"Successfully loaded model: {model_name}")
            log_gpu_memory(f"After loading {model_name}")
            return True

        except Exception as e:
            logger.error(f"Failed to load model: {e}")
            error_msg = format_error_message(e, config.identifier)

            if model_name in self.models:
                del self.models[model_name]
            self.loading_models.discard(model_name)

            raise Exception(error_msg)

    @_invalidates_gpu_memory("transformers unload")
    def unload_model(self, model_name: str) -> bool:
        """Remove a model from the registry and clear GPU memory."""
        if model_name in self.models:
            try:
                if self.models[model_name].get("is_audio"):
                    self._audio_codec_manager.unload()

                logger.info(f"Unloading model '{model_name}' from memory.")
                model = self.models[model_name].get("model")
                del self.models[model_name]
                _release_decode_graphs(model, getattr(self, "_generation_lock", None))
                model = None

                if self.active_model_name == model_name:
                    self.active_model_name = None

                clear_gpu_cache()

                # On spawn platforms keep trainer files so concurrent dataset.map() workers can import them.
                import sys as _sys
                from utils.cache_cleanup import clear_unsloth_compiled_cache

                _preserve = ["Unsloth*Trainer.py"] if _sys.platform in ("win32", "darwin") else None
                clear_unsloth_compiled_cache(preserve_patterns = _preserve)

                logger.info(f"Model '{model_name}' successfully unloaded.")
                return True
            except Exception as e:
                logger.error(f"Error while unloading model '{model_name}': {e}")
                return False
        else:
            logger.warning(
                f"Attempted to unload model '{model_name}', but it was not found in the registry."
            )
            return True

    def revert_to_base_model(self, base_model_name: str) -> bool:
        """Revert the model to its pristine base state by unloading and deleting all adapter
        configurations."""
        if base_model_name not in self.models:
            return False

        model = self.models[base_model_name].get("model")

        try:
            if isinstance(model, (PeftModel, PeftModelForCausalLM)):
                logger.info(f"Unloading LoRA adapters from '{base_model_name}'...")
                unwrapped_base_model = model.unload()
                self.models[base_model_name]["model"] = unwrapped_base_model
                model = unwrapped_base_model

            # A leftover peft_config causes "multiple adapters" warnings on the next from_pretrained().
            if hasattr(model, "peft_config"):
                del model.peft_config

            logger.info(f"Model '{base_model_name}' reverted to clean base state.")
            return True

        except Exception as e:
            logger.error(f"Failed to revert model to base state: {e}")
            import traceback

            logger.error(traceback.format_exc())
            return False

    def load_for_eval(
        self,
        lora_path: str,
        max_seq_length: int = 2048,
        dtype = None,
        load_in_4bit: bool = True,
        hf_token: Optional[str] = None,
        gpu_ids: Optional[list[int]] = None,
    ) -> Tuple[bool, Optional[str], Optional[str]]:
        """Ensure the base model and the given adapter are loaded. Idempotent."""
        try:
            from utils.models import ModelConfig

            lora_config = ModelConfig.from_lora_path(lora_path, hf_token)
            if not lora_config:
                return False, None, None

            base_model_name = lora_config.base_model

            if base_model_name not in self.models or not self.models[base_model_name].get("model"):
                logger.info(f"Base model '{base_model_name}' not loaded, loading now.")
                base_config = ModelConfig.from_ui_selection(base_model_name, None, is_lora = False)
                if not self.load_model(
                    base_config,
                    max_seq_length,
                    dtype,
                    load_in_4bit,
                    hf_token,
                    gpu_ids = gpu_ids,
                ):
                    return False, None, None

            self.active_model_name = base_model_name

            adapter_name = lora_path.split("/")[-1].replace(".", "_")

            # load_adapter only reads from disk if the model does not already have this adapter.
            adapter_success = self.load_adapter(
                base_model_name = base_model_name,
                adapter_path = lora_path,
                adapter_name = adapter_name,
            )
            if not adapter_success:
                return False, base_model_name, None

            return True, base_model_name, adapter_name

        except Exception as e:
            logger.error(f"Error during load_for_eval: {e}")
            import traceback

            logger.error(traceback.format_exc())
            return False, None, None

    def load_adapter(self, base_model_name: str, adapter_path: str, adapter_name: str) -> bool:
        """Load an adapter onto the model only if not already attached."""
        model = self.models[base_model_name].get("model")

        # Most reliable check: adapter name already in the model's config.
        if hasattr(model, "peft_config") and adapter_name in model.peft_config:
            logger.info(
                f"Adapter '{adapter_name}' is already attached to the model. Skipping load."
            )
            return True

        try:
            logger.info(
                f"Loading new adapter '{adapter_name}' from '{adapter_path}' onto {base_model_name}"
            )
            model.load_adapter(adapter_path, adapter_name = adapter_name)

            if "loaded_adapters" not in self.models[base_model_name]:
                self.models[base_model_name]["loaded_adapters"] = {}
            self.models[base_model_name]["loaded_adapters"][adapter_name] = adapter_path

            total_adapters = len(getattr(model, "peft_config", {}))
            logger.info(
                f"Adapter '{adapter_name}' loaded successfully. (Total unique adapters on model: {total_adapters})"
            )
            return True
        except Exception as e:
            logger.error(f"Failed to load adapter '{adapter_name}': {e}")
            return False

    def set_active_adapter(self, base_model_name: str, adapter_name: str) -> bool:
        model = self.models[base_model_name].get("model")
        try:
            logger.info(f"Setting active adapter to: '{adapter_name}'")
            model.set_adapter(adapter_name)
            self.models[base_model_name]["active_adapter"] = adapter_name
            return True
        except Exception as e:
            logger.error(f"Failed to set active adapter to '{adapter_name}': {e}")
            return False

    def _apply_adapter_state(self, use_adapter: Optional[Union[bool, str]]) -> None:
        """Must hold _generation_lock. use_adapter: None keeps, False base, True current, str named adapter."""
        if use_adapter is None:
            return

        base = self.active_model_name
        if not base or base not in self.models:
            return

        model_info = self.models[base]
        model = model_info.get("model")
        if model is None:
            return

        if use_adapter is False:
            if isinstance(model, (PeftModel, PeftModelForCausalLM)):
                logger.info(
                    f"Compare mode: disabling adapters on '{base}' for base model generation"
                )
                model.base_model.disable_adapter_layers()
            else:
                logger.info(f"Compare mode: model '{base}' is not a PeftModel, already base")

        elif use_adapter is True:
            if isinstance(model, (PeftModel, PeftModelForCausalLM)):
                logger.info(f"Compare mode: enabling adapters on '{base}' for LoRA generation")
                model.base_model.enable_adapter_layers()
            else:
                logger.warning("use_adapter=true but model is not a PeftModel")

        elif isinstance(use_adapter, str):
            if isinstance(model, (PeftModel, PeftModelForCausalLM)):
                logger.info(f"Compare mode: enabling adapter '{use_adapter}' on '{base}'")
                model.base_model.enable_adapter_layers()
                self.set_active_adapter(base, use_adapter)
            else:
                logger.warning(f"use_adapter='{use_adapter}' but model is not a PeftModel")

    @contextlib.contextmanager
    def _adapter_scope(self, use_adapter: Optional[Union[bool, str]]):
        """Must hold _generation_lock. A base-model (False) request re-enables the LoRA on exit: plain requests keep the adapter state."""
        try:
            self._apply_adapter_state(use_adapter)
            yield
        finally:
            model = (self.models.get(self.active_model_name) or {}).get("model")
            if use_adapter is False and isinstance(model, PeftModel):
                model.base_model.enable_adapter_layers()

    def generate_with_adapter_control(
        self,
        use_adapter: Optional[Union[bool, str]] = None,
        cancel_event = None,
        **gen_kwargs,
    ) -> Generator[str, None, None]:
        """Toggle and generate are serialized by _generation_lock so SSE handlers on one loop cannot
        race."""
        yield from self._generate_chat_response_inner(
            cancel_event = cancel_event, _adapter_state = use_adapter, **gen_kwargs
        )

    def generate_chat_completion_with_tools(
        self,
        messages: list,
        tools: list,
        system_prompt: str = "",
        temperature: float = 0.7,
        top_p: float = 0.9,
        top_k: int = 40,
        min_p: float = 0.0,
        max_new_tokens: Optional[int] = 2048,
        repetition_penalty: float = 1.0,
        cancel_event = None,
        enable_thinking: Optional[bool] = None,
        reasoning_effort: Optional[str] = None,
        preserve_thinking: Optional[bool] = None,
        continue_final_message: bool = False,
        max_tool_iterations: int = 25,
        auto_heal_tool_calls: bool = True,
        nudge_tool_calls: Optional[bool] = None,
        tool_call_timeout: int = 300,
        session_id: Optional[str] = None,
        thread_id: Optional[str] = None,
        rag_scope: Optional[dict] = None,
        website_policy: Optional[dict] = None,
        presence_penalty: float = 0.0,
        reasoning_prefilled: bool = False,
        deduplicate_tool_calls: bool = True,
    ):
        """Yields the same event-dict protocol as the GGUF path so one route helper streams both
        backends."""
        from core.inference.safetensors_agentic import run_safetensors_tool_loop
        from core.inference.tools import execute_tool

        # Latest turn's usage only; cleared every turn so a failed turn leaves it empty, not stale.
        _turn_stats: dict = {}

        def _single_turn(
            conv: list,
            *,
            active_tools: Optional[list[dict]] = None,
            tool_protocol_active: Optional[bool] = None,
        ):
            # Conv has the system message; active_tools lets the loop drop one-shot tools like render_html.
            turn_tools = active_tools if active_tools is not None else tools
            _turn_stats["stats"] = None
            try:
                yield from self._generate_chat_response_inner(
                    messages = conv,
                    system_prompt = "",
                    temperature = temperature,
                    top_p = top_p,
                    top_k = top_k,
                    min_p = min_p,
                    max_new_tokens = max_new_tokens,
                    repetition_penalty = repetition_penalty,
                    cancel_event = cancel_event,
                    tools = turn_tools,
                    enable_thinking = enable_thinking,
                    reasoning_effort = reasoning_effort,
                    preserve_thinking = preserve_thinking,
                    # Self-limiting: the conversation ends on a tool result, so later turns render as ordinary turns.
                    continue_final_message = continue_final_message,
                    presence_penalty = presence_penalty,
                    tool_protocol_active = tool_protocol_active,
                )
            finally:
                _turn_stats["stats"] = self.last_generation_stats

        initial = list(messages)
        if system_prompt:
            initial = [{"role": "system", "content": system_prompt}] + initial

        # Same profile as the renderer; catalog must be safe under every template this turn may use.
        from core.inference.chat_template_helpers import (
            mapped_chat_template,
            markup_for_tokenizer,
            renderable_tool_catalog,
        )

        _model_info = self.models.get(self.active_model_name) or {}
        # Resolved BEFORE the profile: the mapper installs its template during the render.
        _mapped_tpl = mapped_chat_template(_model_info, self.active_model_name)

        yield from run_safetensors_tool_loop(
            markup = markup_for_tokenizer(_model_info.get("tokenizer"), tools, _mapped_tpl),
            renderable_tools = renderable_tool_catalog(
                tools,
                _model_info.get("tokenizer"),
                _model_info,
                active_model_name = self.active_model_name,
                template = _mapped_tpl,
            ),
            single_turn = _single_turn,
            messages = initial,
            tools = tools,
            execute_tool = execute_tool,
            cancel_event = cancel_event,
            auto_heal_tool_calls = auto_heal_tool_calls,
            deduplicate_tool_calls = deduplicate_tool_calls,
            nudge_tool_calls = nudge_tool_calls,
            max_tool_iterations = max_tool_iterations,
            tool_call_timeout = tool_call_timeout,
            session_id = session_id,
            thread_id = thread_id,
            rag_scope = rag_scope,
            website_policy = website_policy,
            reasoning_prefilled = reasoning_prefilled,
            continue_final_message = continue_final_message,
            # So a conversation search can be sized against what this model can hold.
            context_length = _model_info.get("context_length"),
            max_tokens = max_new_tokens,
            generation_stats_holder = _turn_stats,
        )

    def generate_chat_response(
        self,
        messages: list,
        system_prompt: str,
        image = None,
        images: Optional[list] = None,
        image_ordinal: Optional[int] = None,
        temperature: float = 0.7,
        top_p: float = 0.9,
        top_k: int = 40,
        min_p: float = 0.0,
        max_new_tokens: Optional[int] = 256,
        repetition_penalty: float = 1.0,
        cancel_event = None,
        tools: Optional[list] = None,
        enable_thinking: Optional[bool] = None,
        reasoning_effort: Optional[str] = None,
        preserve_thinking: Optional[bool] = None,
        continue_final_message: bool = False,
        presence_penalty: float = 0.0,
        tool_protocol_active: Optional[bool] = None,
        stop: Optional[list] = None,
    ) -> Generator[str, None, None]:
        """Forwards tools and reasoning flags into apply_chat_template for templates that understand
        them."""
        yield from self._generate_chat_response_inner(
            messages = messages,
            system_prompt = system_prompt,
            image = image,
            images = images,
            image_ordinal = image_ordinal,
            temperature = temperature,
            top_p = top_p,
            top_k = top_k,
            min_p = min_p,
            max_new_tokens = max_new_tokens,
            repetition_penalty = repetition_penalty,
            cancel_event = cancel_event,
            tools = tools,
            enable_thinking = enable_thinking,
            reasoning_effort = reasoning_effort,
            preserve_thinking = preserve_thinking,
            continue_final_message = continue_final_message,
            tool_protocol_active = tool_protocol_active,
            presence_penalty = presence_penalty,
            stop = stop,
        )

    def _generate_chat_response_inner(
        self,
        messages: list,
        system_prompt: str = "",
        image = None,
        images: Optional[list] = None,
        image_ordinal: Optional[int] = None,
        temperature: float = 0.7,
        top_p: float = 0.9,
        top_k: int = 40,
        min_p: float = 0.0,
        max_new_tokens: Optional[int] = 256,
        repetition_penalty: float = 1.0,
        cancel_event = None,
        _adapter_state = None,
        tools: Optional[list] = None,
        enable_thinking: Optional[bool] = None,
        reasoning_effort: Optional[str] = None,
        preserve_thinking: Optional[bool] = None,
        continue_final_message: bool = False,
        presence_penalty: float = 0.0,
        tool_protocol_active: Optional[bool] = None,
        stop: Optional[list] = None,
    ) -> Generator[str, None, None]:
        """Keeps native tool tokens per tool_protocol_active: unrestricted mode sends an empty tools
        list."""
        if not self.active_model_name:
            raise RuntimeError("No active model")

        model_info = self.models[self.active_model_name]
        if model_info.get("diffusion_lm_profile"):
            template_messages = (
                [{"role": "system", "content": system_prompt}] + messages
                if system_prompt
                else messages
            )
            yield from self._generate_diffusion_lm(
                model_info,
                messages = _without_image_parts(template_messages),
                temperature = temperature,
                max_new_tokens = max_new_tokens,
                cancel_event = cancel_event,
                _adapter_state = _adapter_state,
                stop = stop,
            )
            return
        is_vision = model_info.get("is_vision", False)
        tokenizer = model_info.get("tokenizer") or model_info.get("processor")
        tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
        top_k = self._normalize_top_k(top_k)

        if is_deepseek_ocr(model_info.get("model")):
            yield from self._generate_deepseek_ocr_response(
                messages, image, images, max_new_tokens, cancel_event, _adapter_state, stop
            )
            return

        if is_vision and (image or images):
            # FastVisionModel may return a raw tokenizer, not a ProcessorMixin (e.g. Gemma-3), so check first.
            from transformers import ProcessorMixin

            processor = model_info.get("processor")
            has_image_processing = processor is not None and (
                isinstance(processor, ProcessorMixin) or hasattr(processor, "image_processor")
            )
            if has_image_processing:
                yield from self._generate_vision_response(
                    messages,
                    system_prompt,
                    image,
                    temperature,
                    top_p,
                    top_k,
                    min_p,
                    max_new_tokens,
                    repetition_penalty,
                    cancel_event = cancel_event,
                    _adapter_state = _adapter_state,
                    presence_penalty = presence_penalty,
                    continue_final_message = continue_final_message,
                    tools = tools,
                    images = images,
                    image_ordinal = image_ordinal,
                    enable_thinking = enable_thinking,
                    reasoning_effort = reasoning_effort,
                    preserve_thinking = preserve_thinking,
                    tool_protocol_active = tool_protocol_active,
                    stop = stop,
                )
                return
            else:
                logger.warning(
                    f"Model '{self.active_model_name}' is marked as vision but its processor "
                    f"({type(processor).__name__}) has no image_processor — "
                    f"falling back to text-only generation (image will be ignored)."
                )
                # Strip image placeholders: a text template rejects them or renders tokens with no pixels.
                messages = _without_image_parts(messages)

        # Text path: messages are already in ChatML format from eval.py.

        try:
            from utils.datasets import (
                MODEL_TO_TEMPLATE_MAPPER,
                get_tokenizer_chat_template,
            )
            model_name_lower = self.active_model_name.lower()

            if model_name_lower in MODEL_TO_TEMPLATE_MAPPER:
                template_name = MODEL_TO_TEMPLATE_MAPPER[model_name_lower]
                logger.info(
                    f"Applying chat template '{template_name}' for {self.active_model_name}"
                )

                tokenizer = _mapped_template_tokenizer(model_info, tokenizer, template_name)
                # The mapper installs the template only at generate time: UNION into the load-time cache.
                # Take marker strings from the mapped template but resolve ids on the original tokenizer.
                try:
                    _gen_tok = model_info.get("tokenizer") or tokenizer
                    refreshed = resolve_chat_turn_end_eos_ids_using(
                        getattr(tokenizer, "tokenizer", tokenizer),
                        getattr(_gen_tok, "tokenizer", _gen_tok),
                    )
                    existing = model_info.get("chat_turn_end_eos_ids") or []
                    model_info["chat_turn_end_eos_ids"] = sorted(set(existing) | set(refreshed))
                except Exception as e:
                    logger.warning(f"Could not refresh chat turn-end eos after template: {e}")
            else:
                logger.info(
                    f"No registered Unsloth template for {self.active_model_name}, using tokenizer default"
                )
        except Exception as e:
            logger.warning(f"Could not apply get_chat_template: {e}")

        if system_prompt:
            template_messages = [{"role": "system", "content": system_prompt}] + messages
        else:
            template_messages = messages
        reasoning_channel_markers_resolved = False
        add_special_tokens = True
        bos_probe = None
        try:
            if not (hasattr(tokenizer, "chat_template") and tokenizer.chat_template):
                raise ValueError(
                    f"Model '{self.active_model_name}' has no chat_template set in its "
                    f"tokenizer_config.json. This is usually a problem with the model's "
                    f"HuggingFace repository — it is missing a 'chat_template' key. "
                    f"Please use a model that includes a chat template, or manually set "
                    f"one via tokenizer.chat_template before inference."
                )
            reasoning_channel_markers = None
            from core.inference.chat_template_helpers import (
                messages_have_tool_history,
                render_native_template,
                render_with_native_template_fallback,
            )

            try:
                formatted_prompt = self._apply_chat_template_for_generation(
                    tokenizer,
                    template_messages,
                    tools = tools,
                    enable_thinking = enable_thinking,
                    reasoning_effort = reasoning_effort,
                    preserve_thinking = preserve_thinking,
                    continue_final_message = continue_final_message,
                )
            except Exception:
                if not messages_have_tool_history(template_messages):
                    raise
                render_result = render_native_template(
                    model_info = model_info,
                    active_model_name = self.active_model_name,
                    messages = template_messages,
                    tools = tools,
                    enable_thinking = enable_thinking,
                    reasoning_effort = reasoning_effort,
                    preserve_thinking = preserve_thinking,
                    continue_final_message = continue_final_message,
                    apply_fn = self._apply_chat_template_for_generation,
                    hf_token = model_info.get("hf_token"),
                    return_metadata = True,
                )
                if render_result is None:
                    raise
            else:
                # use the native template shared with MLX when an override ignores requested tools.
                render_result = render_with_native_template_fallback(
                    formatted_prompt = formatted_prompt,
                    tokenizer = tokenizer,
                    model_info = model_info,
                    active_model_name = self.active_model_name,
                    messages = template_messages,
                    tools = tools,
                    enable_thinking = enable_thinking,
                    reasoning_effort = reasoning_effort,
                    preserve_thinking = preserve_thinking,
                    continue_final_message = continue_final_message,
                    apply_fn = self._apply_chat_template_for_generation,
                    hf_token = model_info.get("hf_token"),
                    return_metadata = True,
                )
            formatted_prompt = render_result.prompt
            reasoning_channel_markers = render_result.reasoning_channel_markers
            reasoning_channel_markers_resolved = True
            # suppress the tokenizer's BOS only when the template already emitted one.
            bos_probe = _probe_prompt_bos(tokenizer, formatted_prompt)
            add_special_tokens = bos_probe is None

            logger.debug(f"Formatted prompt: {formatted_prompt[:200]}...")
        except Exception as e:
            logger.error(f"Error applying chat template: {e}")
            formatted_prompt = self.format_chat_prompt(
                messages, system_prompt, continue_final_message = continue_final_message
            )
            reasoning_channel_markers = None
            reasoning_channel_markers_resolved = True

        yield from self.generate_stream(
            formatted_prompt,
            temperature,
            top_p,
            top_k,
            min_p,
            max_new_tokens,
            repetition_penalty,
            cancel_event = cancel_event,
            _adapter_state = _adapter_state,
            presence_penalty = presence_penalty,
            reasoning_channel_markers = reasoning_channel_markers,
            reasoning_channel_markers_resolved = reasoning_channel_markers_resolved,
            continued = bool(continue_final_message and trailing_assistant_text(template_messages)),
            preserve_tool_tokens = bool(tools)
            if tool_protocol_active is None
            else tool_protocol_active,
            add_special_tokens = add_special_tokens,
            stop = stop,
            _bos_probe = bos_probe,
        )

    def _generate_deepseek_ocr_response(
        self,
        messages,
        image,
        images,
        max_new_tokens,
        cancel_event,
        _adapter_state,
        stop = None,
    ) -> Generator[str, None, None]:
        """DeepSeek-OCR ships no processor or chat template: its own infer() tiles the image."""
        import tempfile

        from core.inference.chat_template_helpers import last_user_text

        self.last_generation_stats = None
        picture = image if image is not None else next(iter(images or ()), None)
        if picture is None:
            raise _OcrNeedsImageError(
                "DeepSeek-OCR reads text from an image: attach one to your message."
            )
        model_info = self.models[self.active_model_name]
        tokenizer = model_info.get("tokenizer") or model_info.get("processor")
        prompt = "<image>\n" + (last_user_text(messages) or "Free OCR.")
        infer = model_info["model"].infer
        # infer() hardcodes max_new_tokens=8192 and no stopping criteria: bound the generate() it calls.
        owner = infer.__self__
        previous = owner.__dict__.get("generate", _MISSING)
        generate = owner.generate
        stopping_criteria = self._cancel_stopping_criteria(cancel_event)

        def bounded_generate(*args, **kwargs):
            if max_new_tokens:
                kwargs["max_new_tokens"] = min(
                    kwargs.get("max_new_tokens", max_new_tokens), max_new_tokens
                )
            if stopping_criteria is not None:
                kwargs["stopping_criteria"] = stopping_criteria
            return generate(*args, **kwargs)

        with (
            tempfile.TemporaryDirectory() as tmp,
            self._generation_lock,
            self._adapter_scope(_adapter_state),
        ):
            path = str(Path(tmp) / "image.png")
            picture.convert("RGB").save(path)
            owner.generate = bounded_generate
            try:
                # Gundam preset from each model card: 640 px tiles for v1, 768 for OCR-2.
                text = infer(
                    getattr(tokenizer, "tokenizer", tokenizer),
                    prompt = prompt,
                    image_file = path,
                    output_path = tmp,
                    base_size = 1024,
                    image_size = 640
                    if "DeepseekOCRForCausalLM" in model_info["model"].config.architectures
                    else 768,
                    crop_mode = True,
                    save_results = False,
                    eval_mode = True,
                )
            finally:
                if previous is _MISSING:
                    del owner.generate
                else:
                    owner.generate = previous
        text = text or ""
        for sequence in stop or ():
            if sequence and sequence in text:
                text = text[: text.index(sequence)]
        yield text

    def _generate_vision_response(
        self,
        messages,
        system_prompt,
        image,
        temperature,
        top_p,
        top_k,
        min_p,
        max_new_tokens,
        repetition_penalty,
        cancel_event = None,
        _adapter_state = None,
        presence_penalty: float = 0.0,
        continue_final_message: bool = False,
        tools: Optional[list] = None,
        images: Optional[list] = None,
        image_ordinal: Optional[int] = None,
        enable_thinking: Optional[bool] = None,
        reasoning_effort: Optional[str] = None,
        preserve_thinking: Optional[bool] = None,
        tool_protocol_active: Optional[bool] = None,
        stop: Optional[list] = None,
    ) -> Generator[str, None, None]:
        """Handle vision model generation with true token-by-token streaming."""
        # Reset so a failed or uncountable run cannot surface stale stats.
        self.last_generation_stats = None

        model_info = self.models[self.active_model_name]
        model = model_info["model"]
        processor = model_info["processor"]
        # FastVisionModel may return a raw tokenizer; unwrap for tokenize-only ops.
        raw_tokenizer = getattr(processor, "tokenizer", processor)

        # Scans back so a continuation ending in an assistant partial still prompts from the user turn.
        from core.inference.chat_template_helpers import (
            last_user_text,
            messages_have_tool_history,
            messages_with_attached_image,
            render_advertising_tools,
            trailing_assistant_text,
            vlm_prompt_issue,
        )
        from core.inference.mcp_images import (
            image_marker_parts,
            pixels_in_marker_order,
            top_up_image_markers,
        )

        # History first, attachment last: pixels bind to markers in order.
        attached = list(images or []) + ([image] if image is not None else [])
        user_message = last_user_text(messages)
        continue_partial = trailing_assistant_text(messages) if continue_final_message else None

        if not user_message:
            user_message = "Describe this image." if attached else "Hello"

        if attached:
            has_tool_history = messages_have_tool_history(messages)
            # tool_choice="none" and a forced unknown name also arrive with tools=None (#10092).
            folded_system = not system_prompt and any(
                isinstance(m, dict) and m.get("role") in ("system", "developer") for m in messages
            )
            # Rebuilding from the newest user text would drop the system turn and OpenAI tool history.
            vision_messages = messages_with_attached_image(
                messages,
                system_prompt = system_prompt,
                fallback_user_text = user_message,
                structured_content = True,
            )
            # Existing markers are left alone, which is wrong for replayed MCP pictures.
            _prior_markers = image_marker_parts(vision_messages)
            vision_messages = top_up_image_markers(
                vision_messages, len(attached), ordinal = image_ordinal
            )
            if image is not None:
                attached = pixels_in_marker_order(
                    vision_messages, _prior_markers, list(images or []), image
                )
            if bool(tools) or has_tool_history or folded_system:
                # The conversation the LAST render used, not the no-tools probe's (#10092).
                rendered_with: dict = {"messages": vision_messages}

                def _render_vision(catalog):
                    rendered_with["messages"] = vision_messages
                    try:
                        rendered = self._apply_chat_template_for_generation(
                            processor,
                            vision_messages,
                            tools = catalog,
                            enable_thinking = enable_thinking,
                            reasoning_effort = reasoning_effort,
                            preserve_thinking = preserve_thinking,
                            continue_final_message = bool(continue_partial),
                        )
                    except Exception as e:  # noqa: F841 -- read by the fallback below
                        without_system = [
                            m
                            for m in vision_messages
                            if not (isinstance(m, dict) and m.get("role") == "system")
                        ]
                        # Only where an unsupported system role is the last explanation.
                        if (
                            has_tool_history
                            or rendered_with.get("system_ok")
                            or len(without_system) == len(vision_messages)
                        ):
                            raise
                        logger.warning(
                            f"Vision processor for '{self.active_model_name}' may not support "
                            f"system messages; retrying without. Original error: {e}"
                        )
                        rendered = self._apply_chat_template_for_generation(
                            processor,
                            without_system,
                            tools = catalog,
                            enable_thinking = enable_thinking,
                            reasoning_effort = reasoning_effort,
                            preserve_thinking = preserve_thinking,
                            continue_final_message = bool(continue_partial),
                        )
                        rendered_with["messages"] = without_system
                        return rendered
                    else:
                        # A render that kept the system turn proves the role is supported.
                        if any(
                            isinstance(m, dict) and m.get("role") == "system"
                            for m in vision_messages
                        ):
                            rendered_with["system_ok"] = True
                        return rendered

                input_text, tools_advertised = render_advertising_tools(_render_vision, tools)
                vision_messages = rendered_with["messages"]

                prompt_issue = vlm_prompt_issue(input_text, vision_messages)
                if prompt_issue and has_tool_history:
                    raise RuntimeError(
                        f"Vision chat template returned {prompt_issue} and cannot be recovered "
                        "without dropping tool-call history."
                    )
                if prompt_issue:
                    raise RuntimeError(f"Vision chat template returned {prompt_issue}.")

                if tools and not tools_advertised:
                    # Served anyway: refusing turns an answerable question into an error.
                    logger.warning(
                        "Vision chat template for '%s' rendered the same prompt with and "
                        "without the requested tools; serving the turn without the catalog.",
                        self.active_model_name,
                    )
            else:

                def _render_plain_vision(msgs):
                    return self._apply_chat_template_for_generation(
                        processor,
                        msgs,
                        enable_thinking = enable_thinking,
                        reasoning_effort = reasoning_effort,
                        preserve_thinking = preserve_thinking,
                        continue_final_message = bool(continue_partial),
                    )

                try:
                    input_text = _render_plain_vision(vision_messages)
                except Exception as e:
                    # Safe here: no catalog and no tool history to hide a failure behind.
                    if system_prompt:
                        logger.warning(
                            f"Vision processor for '{self.active_model_name}' may not support "
                            f"system messages; retrying without. Original error: {e}"
                        )
                        vision_messages = [m for m in vision_messages if m.get("role") != "system"]
                        input_text = _render_plain_vision(vision_messages)
                    else:
                        raise
            _images = attached[0] if len(attached) == 1 else attached
            if _processor_takes_text_first(processor):
                inputs = processor(
                    text = input_text,
                    images = _images,
                    add_special_tokens = False,
                    return_tensors = "pt",
                ).to(model.device)
            else:
                inputs = processor(
                    _images,
                    input_text,
                    add_special_tokens = False,
                    return_tensors = "pt",
                ).to(model.device)
            prompt_text = input_text
        else:
            formatted_prompt = self.format_chat_prompt(
                messages, system_prompt, continue_final_message = continue_final_message
            )
            bos_probe = _probe_prompt_bos(raw_tokenizer, formatted_prompt)
            inputs = _tokenize_prompt(
                raw_tokenizer, formatted_prompt, bos_probe is None, bos_probe
            ).to(model.device)
            prompt_text = formatted_prompt

        try:
            # Re-emit open <think> prefill swallowed by skip_prompt; image turns carry tools too, so it applies.
            _preserve_tool_tokens = (
                bool(tools) if tool_protocol_active is None else tool_protocol_active
            )
            think_prefix = detect_think_prefill(
                prompt_text,
                getattr(raw_tokenizer, "all_special_tokens", None),
                # Ask the decoder: with no usable all_special_ids it skips specials and drops the closer.
                preserves_think_close = _preserve_tool_tokens
                and decoder_preserves_token(raw_tokenizer, "</think>"),
            )
            import threading

            streamer = self._make_text_streamer(
                raw_tokenizer,
                protocol_source = processor,
                # The text-only fallback did not use the processor template; pass tools to match the render.
                reasoning_channel_markers = detect_reasoning_channel_markers(processor, tools = tools)
                # Any attached image: no markers here would leak native reasoning as visible answer text.
                if attached
                else None,
                reasoning_channel_markers_resolved = True,
                prompt = prompt_text,
                continued = bool(continue_partial),
                skip_prompt = True,
                timeout = 0.2,
                cancel_event = cancel_event,
                use_harmony = self._is_gpt_oss_model(),
                preserve_tool_tokens = _preserve_tool_tokens,
            )

            _vision_input_ids = inputs.get("input_ids") if hasattr(inputs, "get") else None
            prompt_len = int(_vision_input_ids.shape[1]) if _vision_input_ids is not None else None
            max_new_tokens = generation_budget_within_context(
                model, prompt_len or 0, max_new_tokens
            )

            generation_kwargs = dict(
                **inputs,
                streamer = streamer,
                max_new_tokens = max_new_tokens,
                use_cache = True,
                do_sample = temperature > 0,
                temperature = temperature,
                top_p = top_p,
                top_k = top_k,
                min_p = min_p,
            )
            # Presence penalty (GGUF parity) for VLM chat.
            _pp = (
                _make_presence_penalty_processor(presence_penalty, prompt_len)
                if _vision_input_ids is not None
                else None
            )
            if repetition_penalty != 1.0 and _vision_input_ids is not None:
                from transformers import LogitsProcessorList, RepetitionPenaltyLogitsProcessor

                # Prompt ids skipped: mllama's <|image|> id lies past the LM head.
                try:
                    _rp = RepetitionPenaltyLogitsProcessor(
                        repetition_penalty, prompt_ignore_length = prompt_len
                    )
                except TypeError:
                    # prompt_ignore_length landed in transformers 4.52; the floor is 4.51.3.
                    class _PromptSkippingRepetitionPenalty(RepetitionPenaltyLogitsProcessor):
                        def __call__(self, input_ids, scores):
                            return super().__call__(input_ids[:, prompt_len:], scores)

                    _rp = _PromptSkippingRepetitionPenalty(repetition_penalty)
                _pp = LogitsProcessorList([_rp, *(_pp or [])])
            timer = GenerationTimer()
            generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)
            stop_streamer = _StopSequenceStreamer(streamer, stop)
            if stop_streamer.sequences:
                generation_kwargs["streamer"] = stop_streamer
            stopping_criteria = self._cancel_stopping_criteria(cancel_event, stop_streamer.matched)
            if stopping_criteria is not None:
                generation_kwargs["stopping_criteria"] = stopping_criteria
            active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs)

            err: dict[str, str] = {}
            gen_outputs: dict = {}

            def generate_fn():
                with self._generation_lock, self._adapter_scope(_adapter_state):
                    try:
                        # Started inside the lock so a queued request's wait is not billed as prefill.
                        timer.start()
                        # Only the returned sequences carry an exact generated-token count; see generate_stream.
                        sequences = _generate_supported(model, generation_kwargs)
                        if sequences is not None:
                            gen_outputs["sequences"] = sequences
                    except Exception as e:
                        err["msg"] = str(e)
                        stop_streamer.abort()
                        logger.error(f"Vision generation error in thread: {e}")
                    finally:
                        timer.finish()
                        try:
                            stop_streamer.end()
                        except Exception:
                            pass

            thread = threading.Thread(target = generate_fn)
            thread.start()

            output = think_prefix
            # Emit prefilled <think> before the first token so the block renders during slow prompt prefill.
            if think_prefix:
                yield think_prefix
            from queue import Empty
            import time

            generation_complete = False
            cancel_deadline = None
            try:
                while True:
                    if cancel_event is not None and cancel_event.is_set():
                        if cancel_deadline is None:
                            cancel_deadline = time.monotonic() + 10
                        elif time.monotonic() >= cancel_deadline:
                            break
                    try:
                        new_token = next(stop_streamer)
                    except StopIteration:
                        generation_complete = True
                        break
                    except Empty:
                        if not thread.is_alive():
                            generation_complete = True
                            output = yield from self._drain_streamer_tail(
                                stop_streamer, output, active_stop_token_ids
                            )
                            break
                        if cancel_deadline is not None:
                            remaining = cancel_deadline - time.monotonic()
                            if remaining <= 0:
                                break
                            thread.join(timeout = remaining)
                            if thread.is_alive():
                                break
                            generation_complete = True
                            output = yield from self._drain_streamer_tail(
                                stop_streamer, output, active_stop_token_ids
                            )
                            break
                        continue
                    if new_token:
                        output, cleaned = self._append_stream_delta(
                            output, new_token, active_stop_token_ids
                        )
                        yield cleaned
            finally:
                if cancel_event is not None and not generation_complete:
                    cancel_event.set()
                join_timeout = 10
                if cancel_deadline is not None:
                    join_timeout = max(0, cancel_deadline - time.monotonic())
                thread.join(timeout = join_timeout)
                if thread.is_alive():
                    logger.warning(
                        "Vision generation thread did not exit after cancel/join timeout"
                    )

            if "sequences" in gen_outputs:
                self._record_generation_stats(
                    prompt_tokens = prompt_len,
                    completion_tokens = self._generated_token_count(
                        model, gen_outputs["sequences"], prompt_len
                    ),
                    max_new_tokens = max_new_tokens,
                    ended_on_stop_token = stop_streamer.matched.is_set()
                    or self._ended_on_stop_token(gen_outputs["sequences"], active_stop_token_ids),
                    cancelled = not generation_complete
                    or (cancel_event is not None and cancel_event.is_set()),
                    timer = timer,
                )

            if err.get("msg"):
                raise _GenerationThreadError(err["msg"])

        except _GenerationThreadError:
            raise
        except Exception as e:
            logger.error(f"Vision generation error: {e}")
            raise

    def generate_audio_input_response(
        self,
        messages,
        system_prompt,
        audio_array,
        temperature,
        top_p,
        top_k,
        min_p,
        max_new_tokens,
        repetition_penalty,
        use_adapter: Optional[Union[bool, str]] = None,
        cancel_event = None,
        extra_audio_arrays: Optional[list] = None,
        tools: Optional[list] = None,
        speak: bool = False,
    ) -> Generator[str, None, None]:
        """Audio goes into the chat messages via processor.apply_chat_template (Gemma 3n pattern).
        ``tools`` (a list, empty for tool history alone) renders the catalogue and keeps
        ``messages`` as sent. ``speak`` answers in the model's own voice, as the MLX backend
        does: the reply text, then one ``{"speech": <base64 wav>, "sample_rate"}`` item."""
        import threading
        import numpy as np

        # Reset so a failed or uncountable run cannot surface stale stats.
        self.last_generation_stats = None

        if speak:
            if extra_audio_arrays:
                raise RuntimeError("A spoken reply answers one audio clip.")
            yield from self._speak_audio_reply(
                messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event
            )
            return

        model_info = self.models[self.active_model_name]
        model = model_info["model"]
        processor = model_info.get("processor") or model_info.get("tokenizer")
        raw_tokenizer = getattr(processor, "tokenizer", processor)

        if not system_prompt:
            system_prompt = "You are an assistant that transcribes speech accurately."

        # Gemma 3n format: audio goes INTO apply_chat_template, one item per clip in order.
        audio_messages = messages_with_attached_image(
            alternating_turns(messages) if tools is None else messages,
            system_prompt = system_prompt,
            fallback_user_text = "Please transcribe this audio.",
            structured_content = True,
            image = 0,
            audio = audio_array,
            extra_audio = extra_audio_arrays or (),
        )

        # Direct processor render, so neutralize here with this processor's own profile (#7066).
        from core.inference.chat_template_helpers import neutralize_for_render

        audio_messages, tools, _ = neutralize_for_render(processor, audio_messages, tools)

        inputs = processor.apply_chat_template(
            audio_messages,
            add_generation_prompt = True,
            tokenize = True,
            return_dict = True,
            return_tensors = "pt",
            truncation = False,
            **({"tools": tools} if tools else {}),
        ).to(model.device)

        try:
            from queue import Empty

            # As the text path: a rendered catalogue keeps native tool-call tokens for the healer.
            streamer = IncrementalTextIteratorStreamer(
                NativeToolTokenDecoder(raw_tokenizer) if tools else raw_tokenizer,
                skip_prompt = True,
                skip_special_tokens = not tools,
                timeout = 0.2,
            )

            _audio_input_ids = inputs.get("input_ids") if hasattr(inputs, "get") else None
            prompt_len = int(_audio_input_ids.shape[1]) if _audio_input_ids is not None else None
            max_new_tokens = generation_budget_within_context(
                model, prompt_len or 0, max_new_tokens
            )

            # Notebook uses do_sample=False (greedy) for ASR accuracy
            generation_kwargs = dict(
                **inputs,
                streamer = streamer,
                max_new_tokens = max_new_tokens,
                use_cache = True,
                do_sample = False,
            )

            timer = GenerationTimer()
            generation_kwargs["logits_processor"] = with_prefill_boundary_processor(None, timer)
            stopping_criteria = self._cancel_stopping_criteria(cancel_event)
            if stopping_criteria is not None:
                generation_kwargs["stopping_criteria"] = stopping_criteria
            active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs)

            err: dict[str, str] = {}
            gen_outputs: dict = {}

            def generate_fn():
                with self._generation_lock, self._adapter_scope(use_adapter):
                    try:
                        # Started after the adapter swap so only model.generate() is timed.
                        timer.start()
                        # Only the returned sequences carry an exact generated-token count; see generate_stream.
                        sequences = _generate_supported(model, generation_kwargs)
                        if sequences is not None:
                            gen_outputs["sequences"] = sequences
                    except Exception as e:
                        err["msg"] = str(e)
                        logger.error(f"Audio input generation error in thread: {e}")
                    finally:
                        timer.finish()
                        try:
                            streamer.end()
                        except Exception:
                            pass

            thread = threading.Thread(target = generate_fn)
            thread.start()

            output = ""
            # The finally always sets cancel_event, so this flag tells a user stop from a normal end.
            generation_complete = False
            try:
                while True:
                    if cancel_event is not None and cancel_event.is_set():
                        break
                    try:
                        new_token = next(streamer)
                    except StopIteration:
                        generation_complete = True
                        break
                    except Empty:
                        if not thread.is_alive():
                            generation_complete = True
                            break
                        continue
                    if new_token:
                        output += new_token
                        yield new_token
            finally:
                if cancel_event is not None:
                    cancel_event.set()
                thread.join(timeout = 10)
                if thread.is_alive():
                    logger.warning(
                        "Audio input generation thread did not exit after cancel/join timeout"
                    )

            if "sequences" in gen_outputs:
                self._record_generation_stats(
                    prompt_tokens = prompt_len,
                    completion_tokens = self._generated_token_count(
                        model, gen_outputs["sequences"], prompt_len
                    ),
                    max_new_tokens = max_new_tokens,
                    ended_on_stop_token = self._ended_on_stop_token(
                        gen_outputs["sequences"], active_stop_token_ids
                    ),
                    cancelled = not generation_complete,
                    timer = timer,
                )

            if err.get("msg"):
                raise _GenerationThreadError(err["msg"])

        except _GenerationThreadError:
            raise
        except Exception as e:
            logger.error(f"Audio input generation error: {e}")
            raise

    def _speak_audio_reply(
        self, messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event
    ):
        import base64

        system = "\n\n".join(p for p in (_SPEECH_REPLY_SYSTEM_PROMPT, system_prompt) if p)
        conversation = messages_with_attached_image(
            alternating_turns(messages),
            system_prompt = system,
            structured_content = True,
            image = 0,
            audio = audio_array,
        )
        result = self._generate_speech(
            conversation, "<|mixed_start|>", max_new_tokens, use_adapter, cancel_event
        )
        if result is None:
            return
        text, wav, sample_rate = result
        if text:
            yield text
        if wav:
            yield {"speech": base64.b64encode(wav).decode("ascii"), "sample_rate": sample_rate}

    def _speak_text(self, text, max_new_tokens, use_adapter, cancel_event, language, audio_options):
        """The loaded LFM3 model reads ``text`` aloud in its TTS mode: ``(wav_bytes, sample_rate)``."""
        from core.inference.audio_errors import AudioGenerationCancelledError

        conversation = [
            {"role": "system", "content": _tts_system_prompt(language, audio_options)},
            {"role": "user", "content": text},
        ]
        result = self._generate_speech(
            conversation, "<|audio_start|>", max_new_tokens, use_adapter, cancel_event
        )
        if result is None:
            raise AudioGenerationCancelledError("Audio generation cancelled")
        _, wav, sample_rate = result
        if not wav:
            raise RuntimeError("The model returned no speech.")
        return wav, sample_rate

    def _generate_speech(self, conversation, prefill, max_new_tokens, use_adapter, cancel_event):
        """Render ``conversation`` with the assistant turn opened by ``prefill`` and generate.
        Returns ``(text, wav_bytes or None, sample_rate)``, or None once stopped."""
        from core.inference.audio_codecs import _numpy_to_wav_bytes
        from core.inference.chat_template_helpers import neutralize_for_render

        model_info = self.models[self.active_model_name]
        model = model_info["model"]
        processor = model_info.get("processor") or model_info.get("tokenizer")
        codec = _speech_codec(model, processor)
        if codec is None:
            raise RuntimeError("The loaded model cannot answer in speech.")
        # Neutralize before the prefill, which is itself control markup (#7066).
        conversation, _, _ = neutralize_for_render(processor, conversation, None)
        conversation = [*conversation, {"role": "assistant", "content": prefill}]
        inputs = processor.apply_chat_template(
            conversation,
            continue_final_message = True,
            tokenize = True,
            return_dict = True,
            return_tensors = "pt",
            truncation = False,
        ).to(model.device)
        prompt_len = int(inputs["input_ids"].shape[1])
        budget = generation_budget_within_context(model, prompt_len, max_new_tokens)
        # Unset: the free window, capped as the MLX path caps it.
        max_new_tokens = budget if max_new_tokens else min(budget, UNSET_GENERATION_BUDGET)
        kwargs = dict(
            **inputs,
            max_new_tokens = max_new_tokens,
            do_sample = False,
            return_dict_in_generate = True,
        )
        if cancel_event is not None:
            stopper = _speech_stop_streamer(cancel_event)
            kwargs.update(streamer = stopper, audio_streamer = stopper)
        timer = GenerationTimer()
        with self._generation_lock, self._adapter_scope(use_adapter):
            if cancel_event is not None and cancel_event.is_set():
                return None
            timer.start()
            try:
                out = model.generate(**kwargs)
            except _SpeechStopped:
                return None
            finally:
                timer.finish()
            lengths = getattr(out, "audio_lengths", None)
            frames = int(lengths[0]) if lengths is not None else 0
            wav, sample_rate = None, int(getattr(codec.config, "sampling_rate", None) or 24000)
            if frames > 0:
                device = next(model.parameters()).device
                if next(codec.parameters()).device != device:
                    codec.to(device = device)
                with torch.inference_mode():
                    waveform = codec.decode(out.audio_codes[:1, :, :frames]).audio_values[0]
                wav = _numpy_to_wav_bytes(waveform.float().cpu().numpy(), sample_rate)
        if cancel_event is not None and cancel_event.is_set():
            return None
        raw_tokenizer = getattr(processor, "tokenizer", processor)
        text = raw_tokenizer.decode(out.sequences[0, prompt_len:], skip_special_tokens = True)
        self._record_generation_stats(
            prompt_tokens = prompt_len,
            completion_tokens = int(out.sequences.shape[1]) - prompt_len,
            max_new_tokens = max_new_tokens,
            timer = timer,
        )
        return text.strip(), wav, sample_rate

    def generate_whisper_response(
        self,
        audio_array,
        use_adapter: Optional[Union[bool, str]] = None,
        cancel_event = None,
        extra_audio_arrays: Optional[list] = None,
    ) -> Generator[str, None, None]:
        """Whisper ASR over an audio array; yields text from the pipeline built at model load."""
        # No token counts here; clear so gen_done does not ship a previous turn's stats.
        self.last_generation_stats = None

        model_info = self.models[self.active_model_name]
        whisper_pipe = model_info.get("whisper_pipeline")
        if not whisper_pipe:
            yield "Error: Whisper pipeline not initialized"
            return

        clips = [audio_array, *(extra_audio_arrays or [])]
        try:
            for index, clip in enumerate(clips):
                if cancel_event is not None and cancel_event.is_set():
                    return
                with self._generation_lock, self._adapter_scope(use_adapter):
                    result = whisper_pipe({"raw": clip, "sampling_rate": 16000})

                text = result.get("text", "") if isinstance(result, dict) else str(result)
                if text:
                    yield f"\n\n{text}" if index else text
        except Exception as e:
            logger.error(f"Whisper ASR error: {e}")
            yield f"Error: {str(e)}"

    def _is_gpt_oss_model(self, model_name: str = None) -> bool:
        """Whether the given (or active) model uses the gpt-oss harmony protocol."""
        from utils.datasets import is_gpt_oss_model_name
        return is_gpt_oss_model_name(model_name or self.active_model_name or "")

    def _make_text_streamer(
        self,
        tokenizer,
        *,
        protocol_source = None,
        reasoning_channel_markers = None,
        reasoning_channel_markers_resolved: bool = False,
        prompt: Optional[str] = None,
        continued: bool = False,
        skip_prompt: bool = True,
        timeout: float = 0.2,
        cancel_event = None,
        use_harmony: bool = False,
        preserve_tool_tokens: bool = False,
    ):
        """Create the streamer matching this model's native response protocol."""
        if use_harmony:
            try:
                return HarmonyTextStreamer(
                    tokenizer,
                    skip_prompt = skip_prompt,
                    timeout = timeout,
                )
            except Exception as e:
                logger.warning(f"HarmonyTextStreamer init failed, falling back: {e}")
                return IncrementalTextIteratorStreamer(
                    tokenizer,
                    skip_prompt = skip_prompt,
                    skip_special_tokens = True,
                    timeout = timeout,
                )

        markers = (
            reasoning_channel_markers
            if reasoning_channel_markers_resolved
            else reasoning_channel_markers
            or detect_reasoning_channel_markers(protocol_source or tokenizer)
        )
        if markers is not None:
            return ReasoningTextIteratorStreamer(
                tokenizer,
                markers = markers,
                skip_prompt = skip_prompt,
                timeout = timeout,
                cancel_event = cancel_event,
                in_reasoning = prompt_opens_reasoning_channel(prompt, markers, continued),
            )
        stream_tokenizer = NativeToolTokenDecoder(tokenizer) if preserve_tool_tokens else tokenizer
        return IncrementalTextIteratorStreamer(
            stream_tokenizer,
            skip_prompt = skip_prompt,
            skip_special_tokens = not preserve_tool_tokens,
            timeout = timeout,
        )

    def _append_stream_delta(
        self,
        output: str,
        new_token: str,
        stop_token_ids = None,
    ):
        """Append a streamer delta and apply response-boundary cleanup."""
        output += new_token
        return output, self._clean_generated_text(output, stop_token_ids = stop_token_ids)

    def _drain_streamer_tail(
        self,
        streamer,
        output: str,
        stop_token_ids = None,
    ):
        """Drain queued streamer text after the producer exits."""
        while True:
            try:
                new_token = next(streamer)
            except StopIteration:
                return output
            except Exception:
                return output
            if new_token:
                output, cleaned = self._append_stream_delta(
                    output, new_token, stop_token_ids = stop_token_ids
                )
                yield cleaned

    def _generate_diffusion_lm(
        self,
        model_info: dict,
        prompt: Optional[str] = None,
        messages: Optional[list] = None,
        temperature: float = 0.0,
        max_new_tokens: Optional[int] = 256,
        cancel_event = None,
        _adapter_state = None,
        stop: Optional[list] = None,
    ) -> Generator[str, None, None]:
        """A text-diffusion LM fills a whole block at once, so the answer arrives as one completion: there is
        no left-to-right token stream to forward. Holds the generation lock until the sampler returns, and
        the sampler polls ``cancel_event`` between denoising steps."""
        # No token stats come back from the sampler; a previous request's must not be reported for this one.
        self.last_generation_stats = None
        from utils.models.diffusion_lm import require_diffusion_api

        if messages:
            # Same choke point as the left-to-right renderer: role / turn markup inside message text is
            # neutralised before the sampler renders the chat template.
            from core.inference.chat_template_helpers import neutralize_for_render
            tokenizer = model_info["tokenizer"]
            messages, _, _ = neutralize_for_render(
                getattr(tokenizer, "tokenizer", tokenizer), messages, None
            )
        generate_text = getattr(require_diffusion_api("FastDiffusionModel"), "generate_text", None)
        if generate_text is None:
            raise RuntimeError(
                "Unsloth: chatting with a text-diffusion model needs a newer Unsloth "
                "(FastDiffusionModel.generate_text). Update with `unsloth studio update`."
            )
        try:
            from unsloth.models.diffusion_samplers import (
                DiffusionGenerationCancelled as cancelled_error,
            )
        except ImportError:  # an older core: no cancellation error to catch
            cancelled_error = ()
        frame_kwargs = {}
        sink = getattr(self, "diffusion_frame_sink", None)
        if sink is not None and _accepts_on_frame(generate_text):
            frame_kwargs["on_frame"] = _diffusion_frame_relay(
                model_info["tokenizer"],
                sink,
                stop = stop,
                eos_ids = _diffusion_eos_ids(
                    model_info["model"], model_info["tokenizer"], messages is not None
                ),
            )
        with self._generation_lock, self._adapter_scope(_adapter_state):
            try:
                text = generate_text(
                    model_info["model"],
                    model_info["tokenizer"],
                    prompt = prompt,
                    messages = messages,
                    max_new_tokens = int(max_new_tokens or 256),
                    temperature = float(temperature or 0.0),
                    stop_callback = (lambda: cancel_event is not None and cancel_event.is_set()),
                    **frame_kwargs,
                )
            except cancelled_error:
                return
        text = text or ""
        for sequence in stop or ():
            if sequence and sequence in text:
                text = text[: text.index(sequence)]
        if text and not (cancel_event is not None and cancel_event.is_set()):
            yield text

    def generate_stream(
        self,
        prompt: str,
        temperature: float = 0.7,
        top_p: float = 0.9,
        top_k: int = 40,
        min_p: float = 0.0,
        max_new_tokens: Optional[int] = 256,
        repetition_penalty: float = 1.0,
        cancel_event = None,
        _adapter_state = None,
        presence_penalty: float = 0.0,
        reasoning_channel_markers = None,
        reasoning_channel_markers_resolved: bool = False,
        continued: bool = False,
        preserve_tool_tokens: bool = False,
        add_special_tokens: bool = True,
        stop: Optional[list] = None,
        _bos_probe = None,
    ) -> Generator[str, None, None]:
        """Rendered chat prompts pass add_special_tokens=False; raw prompts keep the tokenizer's BOS
        default."""
        if not self.active_model_name:
            raise RuntimeError("No active model")

        # Reset so a failed or uncountable run cannot surface stale stats.
        self.last_generation_stats = None

        model_info = self.models[self.active_model_name]
        if model_info.get("diffusion_lm_profile"):
            yield from self._generate_diffusion_lm(
                model_info,
                prompt = prompt,
                temperature = temperature,
                max_new_tokens = max_new_tokens,
                cancel_event = cancel_event,
                _adapter_state = _adapter_state,
                stop = stop,
            )
            return
        model = model_info["model"]
        # For VLMs the stored tokenizer is the processor; unwrap so skip_prompt works.
        tokenizer = model_info["tokenizer"]
        tokenizer = getattr(tokenizer, "tokenizer", tokenizer)

        try:
            inputs = _tokenize_prompt(tokenizer, prompt, add_special_tokens, _bos_probe).to(
                model.device
            )

            import threading

            # skip_prompt swallows a template-prefilled <think>; re-emit it. gpt-oss emits its own tags.
            think_prefix = (
                ""
                if self._is_gpt_oss_model()
                else detect_think_prefill(
                    prompt,
                    getattr(tokenizer, "all_special_tokens", None),
                    # Both streamers keep </think> through NativeToolTokenDecoder, so the opener must be re-emitted.
                    preserves_think_close = (
                        preserve_tool_tokens or reasoning_channel_markers is not None
                    )
                    and decoder_preserves_token(tokenizer, "</think>"),
                )
            )

            streamer = self._make_text_streamer(
                tokenizer,
                protocol_source = model_info.get("tokenizer"),
                reasoning_channel_markers = reasoning_channel_markers,
                reasoning_channel_markers_resolved = reasoning_channel_markers_resolved,
                prompt = prompt,
                continued = continued,
                skip_prompt = True,
                timeout = 0.2,
                cancel_event = cancel_event,
                use_harmony = self._is_gpt_oss_model(),
                preserve_tool_tokens = preserve_tool_tokens,
            )

            prompt_len = int(inputs["input_ids"].shape[1])
            window = runtime_context_length(model)
            if window and prompt_len > window:
                if "truncates_past_window" not in model_info:
                    model_info["truncates_past_window"] = _truncates_past_window(model)
                if model_info["truncates_past_window"]:
                    raise ContextBudgetExceeded(prompt_len, window)
            max_new_tokens = generation_budget_within_context(model, prompt_len, max_new_tokens)

            generation_kwargs = dict(
                **inputs,
                streamer = streamer,
                max_new_tokens = max_new_tokens,
                temperature = temperature,
                top_p = top_p,
                top_k = top_k,
                min_p = min_p,
                repetition_penalty = repetition_penalty,
                do_sample = temperature > 0,
                # Resolved once at load (chat_template-derived turn-end tokens).
                eos_token_id = model_info.get("chat_turn_end_eos_ids") or tokenizer.eos_token_id,
                pad_token_id = tokenizer.eos_token_id
                if tokenizer.pad_token_id is None
                else tokenizer.pad_token_id,
            )
            active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs)
            # Presence penalty (GGUF parity); prompt_len excludes prompt tokens.
            _pp = _make_presence_penalty_processor(presence_penalty, prompt_len)
            timer = GenerationTimer()
            generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)
            stop_streamer = _StopSequenceStreamer(streamer, stop)
            if stop_streamer.sequences:
                generation_kwargs["streamer"] = stop_streamer
            stopping_criteria = self._cancel_stopping_criteria(cancel_event, stop_streamer.matched)
            if stopping_criteria is not None:
                generation_kwargs["stopping_criteria"] = stopping_criteria

            def generate_fn():
                with self._generation_lock, self._adapter_scope(_adapter_state):
                    try:
                        # Started after the adapter swap so only model.generate() is timed.
                        timer.start()
                        # Only the returned sequences carry an exact token count.
                        with _decode_graph_scope():
                            sequences = _generate_supported(model, generation_kwargs)
                            if sequences is not None:
                                gen_outputs["sequences"] = sequences
                    except Exception as e:
                        err["msg"] = str(e)
                        stop_streamer.abort()
                        logger.error(f"Generation error: {e}")
                    finally:
                        timer.finish()
                        try:
                            stop_streamer.end()
                        except Exception:
                            pass

            err: dict[str, str] = {}
            gen_outputs: dict = {}
            thread = threading.Thread(target = generate_fn)
            thread.start()

            output = think_prefix
            # Emit prefilled <think> before the first token so the block renders during slow prompt prefill.
            if think_prefix:
                yield think_prefix
            from queue import Empty
            import time

            generation_complete = False
            cancel_deadline = None
            try:
                while True:
                    if cancel_event is not None and cancel_event.is_set():
                        if cancel_deadline is None:
                            cancel_deadline = time.monotonic() + 10
                        elif time.monotonic() >= cancel_deadline:
                            break
                    try:
                        new_token = next(stop_streamer)
                    except StopIteration:
                        generation_complete = True
                        break
                    except Empty:
                        if not thread.is_alive():
                            generation_complete = True
                            output = yield from self._drain_streamer_tail(
                                stop_streamer, output, active_stop_token_ids
                            )
                            break
                        if cancel_deadline is not None:
                            remaining = cancel_deadline - time.monotonic()
                            if remaining <= 0:
                                break
                            thread.join(timeout = remaining)
                            if thread.is_alive():
                                break
                            generation_complete = True
                            output = yield from self._drain_streamer_tail(
                                stop_streamer, output, active_stop_token_ids
                            )
                            break
                        continue
                    if new_token:
                        output, cleaned = self._append_stream_delta(
                            output, new_token, active_stop_token_ids
                        )
                        yield cleaned
            finally:
                # Set only on early exit: it is a shared mp.Event and would cancel the next serialized request.
                if cancel_event is not None and not generation_complete:
                    cancel_event.set()
                join_timeout = 10
                if cancel_deadline is not None:
                    join_timeout = max(0, cancel_deadline - time.monotonic())
                thread.join(timeout = join_timeout)
                if thread.is_alive():
                    logger.warning("Generation thread did not exit after cancel/join timeout")

            if "sequences" in gen_outputs:
                self._record_generation_stats(
                    prompt_tokens = prompt_len,
                    completion_tokens = self._generated_token_count(
                        model, gen_outputs["sequences"], prompt_len
                    ),
                    max_new_tokens = max_new_tokens,
                    ended_on_stop_token = stop_streamer.matched.is_set()
                    or self._ended_on_stop_token(gen_outputs["sequences"], active_stop_token_ids),
                    cancelled = not generation_complete
                    or (cancel_event is not None and cancel_event.is_set()),
                    timer = timer,
                )

            if err.get("msg"):
                raise _GenerationThreadError(err["msg"])

        except _GenerationThreadError:
            raise
        except Exception as e:
            logger.error(f"Error during generation: {e}")
            raise

    def generate_audio_response(
        self,
        text: str,
        temperature: float = 0.6,
        top_p: float = 0.95,
        top_k: int = 50,
        min_p: float = 0.0,
        max_new_tokens: int = 2048,
        repetition_penalty: float = 1.0,
        use_adapter: Optional[Union[bool, str]] = None,
        cancel_event = None,
        instructions: Optional[str] = None,
        language: Optional[str] = None,
        seed: Optional[int] = None,
        audio_options: Optional[dict] = None,
    ) -> Tuple[bytes, int]:
        """Args unused: reserved for native audio models; codec TTS has no instructions, language or seed.
        A chat model that speaks (LFM3) reads the text aloud in its TTS mode, in ``language``."""
        del instructions, seed
        if not self.active_model_name:
            raise RuntimeError("No active model")

        model_info = self.models[self.active_model_name]
        audio_type = model_info.get("audio_type")
        model = model_info["model"]
        tokenizer = model_info.get("tokenizer")

        if not audio_type:
            raise RuntimeError(f"Model {self.active_model_name} is not an audio model")
        if audio_type == "audio_vlm" and model_info.get("speaks"):
            return self._speak_text(
                text, max_new_tokens, use_adapter, cancel_event, language, audio_options
            )

        top_k = self._normalize_top_k(top_k)
        # All four codecs concatenate prompts, not templates; this is their one choke point.
        text = tts_prompt_with_voice(
            neutralize_tts_prompt_text(text, audio_type), audio_type, audio_options
        )

        if cancel_event is not None and cancel_event.is_set():
            raise RuntimeError("Audio generation cancelled")
        with self._generation_lock, self._adapter_scope(use_adapter):
            if cancel_event is not None and cancel_event.is_set():
                raise RuntimeError("Audio generation cancelled")
            stopping_criteria = self._cancel_stopping_criteria(cancel_event)
            self.last_generation_stats = None

            if audio_type == "snac":
                result = self._generate_snac(
                    model,
                    tokenizer,
                    text,
                    temperature,
                    top_p,
                    max_new_tokens,
                    repetition_penalty,
                    stopping_criteria = stopping_criteria,
                    cancel_event = cancel_event,
                )
            elif audio_type == "csm":
                processor = model_info.get("processor", tokenizer)
                result = self._generate_csm(
                    model,
                    processor,
                    text,
                    max_new_tokens,
                    stopping_criteria = stopping_criteria,
                    cancel_event = cancel_event,
                )
            elif audio_type == "bicodec":
                result = self._generate_bicodec(
                    model,
                    tokenizer,
                    text,
                    temperature,
                    top_k,
                    max_new_tokens,
                    stopping_criteria = stopping_criteria,
                    cancel_event = cancel_event,
                )
            elif audio_type == "dac":
                result = self._generate_dac(
                    model,
                    tokenizer,
                    text,
                    temperature,
                    top_k,
                    top_p,
                    min_p,
                    max_new_tokens,
                    repetition_penalty,
                    stopping_criteria = stopping_criteria,
                    cancel_event = cancel_event,
                )
            else:
                raise RuntimeError(f"Unknown audio_type: {audio_type}")
            if cancel_event is not None and cancel_event.is_set():
                raise RuntimeError("Audio generation cancelled")
            return result

    def _generate_snac(
        self,
        model,
        tokenizer,
        text,
        temperature,
        top_p,
        max_new_tokens,
        repetition_penalty,
        *,
        stopping_criteria = None,
        cancel_event = None,
    ):
        """Generate audio using SNAC codec (Orpheus)."""
        device = model.device
        start_token = torch.tensor([[128259]], device = device)  # START_OF_HUMAN
        end_tokens = torch.tensor([[128009, 128260]], device = device)  # EOT, END_OF_HUMAN
        text_ids = tokenizer(text, return_tensors = "pt").input_ids.to(device)
        input_ids = torch.cat([start_token, text_ids, end_tokens], dim = 1)
        attention_mask = torch.ones_like(input_ids)

        generated = model.generate(
            input_ids = input_ids,
            attention_mask = attention_mask,
            max_new_tokens = max_new_tokens,
            do_sample = True,
            temperature = temperature,
            top_p = top_p,
            repetition_penalty = repetition_penalty,
            eos_token_id = 128258,  # END_OF_SPEECH
            use_cache = True,
            **({"stopping_criteria": stopping_criteria} if stopping_criteria is not None else {}),
        )
        if cancel_event is not None and cancel_event.is_set():
            raise RuntimeError("Audio generation cancelled")
        self._record_generation_stats(
            prompt_tokens = input_ids.shape[1],
            completion_tokens = self._generated_token_count(model, generated, input_ids.shape[1]),
            max_new_tokens = max_new_tokens,
            ended_on_stop_token = self._ended_on_stop_token(generated, 128258),
        )
        return self._audio_codec_manager.decode_snac(generated, str(device))

    def _generate_csm(
        self,
        model,
        processor,
        text,
        max_new_tokens,
        *,
        stopping_criteria = None,
        cancel_event = None,
    ):
        """Generate audio using CSM (Sesame)."""
        speaker_id = 0
        inputs = processor(
            f"[{speaker_id}]{text}", add_special_tokens = True, return_tensors = "pt"
        ).to(model.device)
        audio_values = model.generate(
            **inputs,
            max_new_tokens = max_new_tokens,
            output_audio = True,
            **({"stopping_criteria": stopping_criteria} if stopping_criteria is not None else {}),
        )
        if cancel_event is not None and cancel_event.is_set():
            raise RuntimeError("Audio generation cancelled")
        return self._audio_codec_manager.decode_csm(audio_values)

    def _generate_bicodec(
        self,
        model,
        tokenizer,
        text,
        temperature,
        top_k,
        max_new_tokens,
        *,
        stopping_criteria = None,
        cancel_event = None,
    ):
        """Generate audio using BiCodec (Spark-TTS)."""
        prompt = "<|task_tts|><|start_content|>" + text + "<|end_content|><|start_global_token|>"
        inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
        generated = model.generate(
            **inputs,
            max_new_tokens = max_new_tokens,
            do_sample = True,
            temperature = temperature,
            top_k = top_k,
            eos_token_id = tokenizer.eos_token_id,
            pad_token_id = tokenizer.pad_token_id,
            **({"stopping_criteria": stopping_criteria} if stopping_criteria is not None else {}),
        )
        if cancel_event is not None and cancel_event.is_set():
            raise RuntimeError("Audio generation cancelled")
        prompt_len = inputs.input_ids.shape[1]
        self._record_generation_stats(
            prompt_tokens = prompt_len,
            completion_tokens = self._generated_token_count(model, generated, prompt_len),
            max_new_tokens = max_new_tokens,
            ended_on_stop_token = self._ended_on_stop_token(generated, tokenizer.eos_token_id),
        )
        new_tokens = generated[:, prompt_len:]
        decoded_text = tokenizer.batch_decode(new_tokens, skip_special_tokens = False)[0]
        return self._audio_codec_manager.decode_bicodec(decoded_text, str(model.device))

    def _generate_dac(
        self,
        model,
        tokenizer,
        text,
        temperature,
        top_k,
        top_p,
        min_p,
        max_new_tokens,
        repetition_penalty,
        *,
        stopping_criteria = None,
        cancel_event = None,
    ):
        """Generate audio using DAC (OuteTTS). Follows Oute_TTS_(1B).ipynb exactly."""
        # 64-token repetition window, same as the OuteTTS notebook.
        self._patch_repetition_penalty_processor()

        prompt = build_dac_tts_prompt(text)

        with torch.inference_mode():
            # Autocast device from the model, not the backend: a CPU-fallback DAC may sit on a GPU host.
            device_type = (
                model.device.type
                if hasattr(model.device, "type")
                else str(model.device).split(":", 1)[0]
            )
            # Clamp to autocast-supported backends (e.g. "meta" under offloaded loading raises).
            if device_type not in ("cuda", "xpu", "mps", "cpu"):
                device_type = "cpu"
            # CPU/XPU autocast accept only bf16/fp16; skip autocast for fp32 models.
            autocast_dtype_supported = model.dtype in (torch.bfloat16, torch.float16)
            if device_type in ("cpu", "xpu") and not autocast_dtype_supported:
                autocast_ctx = contextlib.nullcontext()
            else:
                autocast_ctx = torch.amp.autocast(device_type, dtype = model.dtype)
            with autocast_ctx:
                inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
                generated = model.generate(
                    **inputs,
                    temperature = temperature,
                    top_k = top_k,
                    top_p = top_p,
                    min_p = min_p,
                    repetition_penalty = repetition_penalty,
                    max_new_tokens = max_new_tokens,
                    **(
                        {"stopping_criteria": stopping_criteria}
                        if stopping_criteria is not None
                        else {}
                    ),
                )
        if cancel_event is not None and cancel_event.is_set():
            raise RuntimeError("Audio generation cancelled")
        self._record_generation_stats(
            prompt_tokens = inputs.input_ids.shape[1],
            completion_tokens = self._generated_token_count(
                model, generated, inputs.input_ids.shape[1]
            ),
            max_new_tokens = max_new_tokens,
            ended_on_stop_token = self._ended_on_stop_token(
                generated, self._generation_stop_token_ids(model, {})
            ),
        )
        decoded_text = tokenizer.batch_decode(generated, skip_special_tokens = False)[0]
        return self._audio_codec_manager.decode_dac(decoded_text, str(model.device))

    _repetition_penalty_patched = False

    @classmethod
    def _patch_repetition_penalty_processor(cls):
        """Monkey-patch transformers' RepetitionPenaltyLogitsProcessor with the OuteTTS notebook's
        64-token sliding-window variant. Applied once per process."""
        if cls._repetition_penalty_patched:
            return
        cls._repetition_penalty_patched = True

        from transformers import LogitsProcessor
        import transformers.generation.utils as generation_utils

        class RepetitionPenaltyLogitsProcessorPatch(LogitsProcessor):
            def __init__(self, penalty: float):
                self.penalty_last_n = 64
                if not isinstance(penalty, float) or penalty <= 0:
                    raise ValueError(f"`penalty` has to be a positive float, but is {penalty}")
                self.penalty = penalty

            @torch.no_grad()
            def __call__(
                self, input_ids: torch.LongTensor, scores: torch.FloatTensor
            ) -> torch.FloatTensor:
                if self.penalty_last_n == 0 or self.penalty == 1.0:
                    return scores
                batch_size, seq_len = input_ids.shape
                vocab_size = scores.shape[-1]
                for b in range(batch_size):
                    start_index = max(0, seq_len - self.penalty_last_n)
                    window_indices = input_ids[b, start_index:]
                    if window_indices.numel() == 0:
                        continue
                    for token_id in set(window_indices.tolist()):
                        if token_id >= vocab_size:
                            continue
                        logit = scores[b, token_id]
                        scores[b, token_id] = (
                            logit * self.penalty if logit <= 0 else logit / self.penalty
                        )
                return scores

        generation_utils.RepetitionPenaltyLogitsProcessor = RepetitionPenaltyLogitsProcessorPatch
        logger.info("Patched RepetitionPenaltyLogitsProcessor with 64-token window for OuteTTS")

    def _apply_chat_template_for_generation(
        self,
        tokenizer,
        messages: list,
        *,
        tools: Optional[list] = None,
        enable_thinking: Optional[bool] = None,
        reasoning_effort: Optional[str] = None,
        preserve_thinking: Optional[bool] = None,
        continue_final_message: bool = False,
    ) -> str:
        """Peels kwargs the template does not understand; delegates to a dependency-light helper for
        testing."""
        from core.inference.chat_template_helpers import (
            apply_chat_template_for_generation,
        )
        return apply_chat_template_for_generation(
            tokenizer,
            messages,
            tools = tools,
            enable_thinking = enable_thinking,
            reasoning_effort = reasoning_effort,
            preserve_thinking = preserve_thinking,
            continue_final_message = continue_final_message,
        )

    def format_chat_prompt(
        self,
        messages: list,
        system_prompt: str = None,
        continue_final_message: bool = False,
    ) -> str:
        if not self.active_model_name or self.active_model_name not in self.models:
            logger.error("No active model available")
            return ""

        if self.models[self.active_model_name].get("tokenizer") is None:
            logger.error("Tokenizer not loaded for active model")
            return ""

        chat_template_info = self.models[self.active_model_name].get("chat_template_info", {})
        tokenizer = self.models[self.active_model_name]["tokenizer"]
        tokenizer = getattr(tokenizer, "tokenizer", tokenizer)

        chat_messages = []

        if system_prompt:
            chat_messages.append({"role": "system", "content": system_prompt})

        last_role = "system" if system_prompt else None

        for msg in messages:
            role = msg.get("role", "")
            content = content_to_text(msg.get("content", ""))
            reasoning_content = msg.get("reasoning_content")
            has_reasoning_content = (
                role == "assistant"
                and isinstance(reasoning_content, str)
                and bool(reasoning_content.strip())
            )

            if role in ["system", "user", "assistant"] and (
                content.strip() or has_reasoning_content
            ):
                if role == last_role:
                    logger.debug(f"Skipping consecutive {role} message to maintain alternation")
                    continue

                if role == "user":
                    chat_messages.append(named_turn({"role": role, "content": content}, msg))
                    last_role = role
                elif role == "assistant":
                    assistant_message = named_turn({"role": role, "content": content}, msg)
                    if has_reasoning_content:
                        assistant_message["reasoning_content"] = reasoning_content
                    chat_messages.append(assistant_message)
                    last_role = role
                elif role == "system":
                    continue

        # A continuation resumes that turn, so dropping it would restart the answer.
        _continuing = continue_final_message and bool(
            chat_messages and chat_messages[-1]["role"] == "assistant"
        )
        if not _continuing and chat_messages and chat_messages[-1]["role"] == "assistant":
            logger.debug("Removing final assistant message to ensure proper alternation")
            chat_messages.pop()

        # Direct tokenizer render bypasses the choke point, so neutralize here too (#7066).
        from core.inference.chat_template_helpers import (
            markup_for_tokenizer,
            render_prompt_with_boundary,
        )

        chat_messages = neutralize_control_markup_in_messages(
            chat_messages, None, markup_for_tokenizer(tokenizer)
        )

        logger.info(f"Sending {len(chat_messages)} messages to tokenizer:")
        for i, msg in enumerate(chat_messages):
            logger.debug(f"  {i}: {msg['role']} - {msg['content'][:50]}...")

        try:
            formatted_prompt = render_prompt_with_boundary(
                tokenizer, chat_messages, continue_final_message = _continuing
            )
            logger.info(f"Successfully applied tokenizer's native chat template")
            return formatted_prompt
        except Exception as e:
            error_msg = str(e).lower()
            if "chat_template is not set" in error_msg or "no template argument" in error_msg:
                logger.info(
                    f"Base model detected - no built-in chat template available, using fallback formatting"
                )
            else:
                logger.warning(f"Failed to apply tokenizer chat template: {e}")
            logger.debug(
                f"""Failed with messages: {[f"{m['role']}: {m['content'][:30]}..." for m in chat_messages]}"""
            )

        # Manual formatters close the assistant turn, so render without the partial and append it.
        partial = chat_messages[-1]["content"] if _continuing else None
        manual_messages = chat_messages[:-1] if _continuing else chat_messages

        if chat_template_info.get("has_template", False):
            logger.info("Falling back to manual template formatting based on detected patterns")
            template_type = chat_template_info.get("format_type", "generic")
            manual_prompt = self._format_chat_manual(
                manual_messages,
                template_type,
                chat_template_info.get("special_tokens", {}),
            )
            logger.debug(f"Manual template result: {manual_prompt[:200]}...")
        else:
            logger.info("Using generic chat formatting for base model")
            manual_prompt = self._format_generic_template(manual_messages, {})
        return f"{manual_prompt}{partial}" if partial else manual_prompt

    def _format_chat_manual(self, messages: list, template_type: str, special_tokens: dict) -> str:
        """Manual chat-formatting fallback when the tokenizer template fails."""
        if template_type == "llama3":
            return self._format_llama3_template(messages, special_tokens)
        elif template_type == "mistral":
            return self._format_mistral_template(messages, special_tokens)
        elif template_type == "chatml":
            return self._format_chatml_template(messages, special_tokens)
        elif template_type == "alpaca":
            return self._format_alpaca_template(messages, special_tokens)
        else:
            return self._format_generic_template(messages, special_tokens)

    def _format_llama3_template(self, messages: list, special_tokens: dict) -> str:
        bos_token = special_tokens.get("bos_token", "<|begin_of_text|>")
        formatted = bos_token

        for msg in messages:
            role = msg["role"]
            content = content_to_text(msg["content"])
            formatted += f"<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>"

        formatted += "<|start_header_id|>assistant<|end_header_id|>\n\n"
        return formatted

    def _format_mistral_template(self, messages: list, special_tokens: dict) -> str:
        bos_token = special_tokens.get("bos_token", "<s>")
        formatted = bos_token

        system_msg = None
        conversation = []

        for msg in messages:
            if msg["role"] == "system":
                system_msg = content_to_text(msg["content"])
            else:
                conversation.append(msg)

        i = 0
        while i < len(conversation):
            if conversation[i]["role"] == "user":
                user_content = content_to_text(conversation[i]["content"])

                if system_msg and i == 0:
                    user_content = f"{system_msg}\n\n{user_content}"

                formatted += f"[INST] {user_content} [/INST]"

                if i + 1 < len(conversation) and conversation[i + 1]["role"] == "assistant":
                    formatted += f" {content_to_text(conversation[i + 1]['content'])}</s>"
                    i += 2
                else:
                    formatted += " "
                    break
            else:
                i += 1

        return formatted

    def _format_chatml_template(self, messages: list, special_tokens: dict) -> str:
        formatted = ""

        for msg in messages:
            role = msg["role"]
            content = content_to_text(msg["content"])
            formatted += f"<|im_start|>{role}\n{content}<|im_end|>\n"

        formatted += "<|im_start|>assistant\n"
        return formatted

    def _format_alpaca_template(self, messages: list, special_tokens: dict) -> str:
        formatted = ""
        system_msg = None

        for msg in messages:
            content = content_to_text(msg["content"])
            if msg["role"] == "system":
                system_msg = content
            elif msg["role"] == "user":
                if system_msg:
                    formatted += f"### Instruction:\n{system_msg}\n\n### Input:\n{content}\n\n### Response:\n"
                    system_msg = None
                else:
                    formatted += f"### Human:\n{content}\n\n### Assistant:\n"
            elif msg["role"] == "assistant":
                formatted += f"{content}\n\n"

        return formatted

    def _format_generic_template(self, messages: list, special_tokens: dict) -> str:
        formatted = ""

        for msg in messages:
            role = msg["role"].title()
            content = content_to_text(msg["content"])
            formatted += f"{role}: {content}\n"

        formatted += "Assistant: "
        return formatted

    def check_vision_model_compatibility(self) -> bool:
        current_model = self.get_current_model()
        if current_model and current_model in self.models:
            return self.models[current_model].get("is_vision", False)
        return False

    def _reset_model_generation_state(self, model_name: str):
        """Reset generation state for a specific model to prevent contamination."""
        if model_name not in self.models:
            return

        model = self.models[model_name].get("model")
        if not model:
            return

        try:
            if hasattr(model, "past_key_values"):
                model.past_key_values = None
            if hasattr(model, "generation_config"):
                if hasattr(model.generation_config, "past_key_values"):
                    model.generation_config.past_key_values = None

            logger.debug(f"Reset generation state for model: {model_name}")
        except Exception as e:
            logger.warning(f"Could not fully reset model state for {model_name}: {e}")

    def reset_generation_state(self, caller_cancel_event = None):
        """caller_cancel_event is accepted for signature parity only; this never cancels a live
        generation."""
        try:
            for model_name in self.models.keys():
                self._reset_model_generation_state(model_name)

            clear_gpu_cache()
            logger.debug("Cleared GPU cache")

            import gc

            gc.collect()
            logger.info("Performed comprehensive generation state reset")

        except Exception as e:
            logger.warning(f"Could not fully reset generation state: {e}")

    def resize_image(
        self,
        img,
        max_size: int = 800,
    ):
        """Resize image while maintaining aspect ratio if either dimension exceeds max_size"""
        if img is None:
            return None
        if img.size[0] > max_size or img.size[1] > max_size:
            from PIL import Image

            ratio = min(max_size / img.size[0], max_size / img.size[1])
            new_size = (int(img.size[0] * ratio), int(img.size[1] * ratio))
            return img.resize(new_size, Image.Resampling.LANCZOS)
        return img

    def _generation_stop_token_ids(self, model, generation_kwargs: dict):
        """Return the stop-token ids active for a ``generate`` call."""
        if "eos_token_id" in generation_kwargs:
            return generation_kwargs.get("eos_token_id")
        generation_config = getattr(model, "generation_config", None)
        eos_token_id = getattr(generation_config, "eos_token_id", None)
        if eos_token_id is not None:
            return eos_token_id
        config = getattr(model, "config", None)
        return getattr(config, "eos_token_id", None)

    def _generated_token_count(self, model, outputs, prompt_len) -> Optional[int]:
        """Returns None for unrecognised outputs, so the caller never reports a count it did not measure."""
        sequences = getattr(outputs, "sequences", outputs)
        shape = getattr(sequences, "shape", None)
        if shape is None or len(shape) < 2:
            return None
        total = int(shape[-1])
        if getattr(getattr(model, "config", None), "is_encoder_decoder", False):
            return max(total - 1, 0)
        if prompt_len is None or total < int(prompt_len):
            return None
        return total - int(prompt_len)

    def _ended_on_stop_token(self, outputs, stop_token_ids) -> bool:
        """Whether the final token is a stop token rather than one cut off by the cap. A response
        landing exactly on ``max_new_tokens`` ended naturally; only one that did NOT emit a stop
        token ran out of budget."""
        sequences = getattr(outputs, "sequences", outputs)
        if isinstance(stop_token_ids, int):
            stop_token_ids = (stop_token_ids,)
        try:
            last_id = int(sequences[0, -1])
            return last_id in {int(token_id) for token_id in (stop_token_ids or ())}
        except Exception:
            return False

    def _record_generation_stats(
        self,
        *,
        prompt_tokens,
        completion_tokens,
        max_new_tokens,
        ended_on_stop_token: bool = False,
        cancelled: bool = False,
        timer = None,
    ) -> None:
        """Cancelled runs never report finish_reason length: they stopped by request, not at the
        token cap."""
        if completion_tokens is None:
            return
        completion_tokens = int(completion_tokens)
        prompt_tokens = int(prompt_tokens or 0)
        stats = {
            "usage": {
                "prompt_tokens": prompt_tokens,
                "completion_tokens": completion_tokens,
                "total_tokens": prompt_tokens + completion_tokens,
            },
            "truncated": (
                not cancelled
                and not ended_on_stop_token
                and bool(max_new_tokens)
                and completion_tokens >= int(max_new_tokens)
            ),
        }
        timings = build_generation_timings(
            prompt_n = prompt_tokens,
            predicted_n = completion_tokens,
            prompt_ms = timer.prompt_ms if timer is not None else None,
            predicted_ms = timer.predicted_ms if timer is not None else None,
        )
        if timings is not None:
            stats["timings"] = timings
        self.last_generation_stats = stats

    def _cancel_stopping_criteria(self, *events):
        """Build a Transformers stopping criteria list for user cancellation."""
        events = [ev for ev in events if ev is not None]
        if not events:
            return None
        from transformers.generation.stopping_criteria import (
            StoppingCriteria,
            StoppingCriteriaList,
        )

        class _CancelCriteria(StoppingCriteria):
            def __init__(self, ev):
                self.ev = ev

            def __call__(self, input_ids, scores, **kwargs):
                return self.ev.is_set()

        return StoppingCriteriaList([_CancelCriteria(ev) for ev in events])

    def _clean_generated_text(
        self,
        text: str,
        *,
        stop_token_ids = None,
    ) -> str:
        """Strip leaked response-boundary tokens after streaming."""
        if self._is_gpt_oss_model():
            # Harmony vocabulary only: a literal <|im_end|> in an answer is content.
            text = _HARMONY_PROTOCOL_TOKEN_RE.sub("", text)
            return text.strip()

        tokenizer = self.models.get(self.active_model_name, {}).get("tokenizer")
        tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
        if tokenizer:
            if stop_token_ids is None:
                stop_token_ids = self.models.get(self.active_model_name, {}).get(
                    "chat_turn_end_eos_ids"
                )
            if isinstance(stop_token_ids, int):
                stop_token_ids = (stop_token_ids,)
            for token_id in stop_token_ids or ():
                token = stop_token_text(tokenizer, token_id)
                if isinstance(token, str) and token and text.endswith(token):
                    if closes_an_open_envelope(text, token):
                        # A native closer that is also the stop token still needs its opener present.
                        continue
                    text = text[: -len(token)]
                elif (
                    isinstance(token, str)
                    and token
                    and text.endswith("</think>")
                    and text[: -len("</think>")].endswith(token)
                ):
                    text = text[: -len("</think>") - len(token)] + "</think>"
        return text.strip()

    def _load_chat_template_info(self, model_name: str):
        if model_name not in self.models or not self.models[model_name].get("tokenizer"):
            return

        tokenizer = self.models[model_name]["tokenizer"]
        chat_template_info = {
            "has_template": False,
            "template": None,
            "format_type": "generic",
            "special_tokens": {},
            "template_name": None,
            # Image turns render through the processor template, usually a different file (#10092).
            "processor_template": None,
            # Distinct from processor_template: a template-less processor still places the image.
            "renders_image": False,
        }
        processor = self.models[model_name].get("processor")
        # Same check as the image-ignoring fallback, since FastVisionModel may return a raw tokenizer.
        try:
            from transformers import ProcessorMixin
            processes_images = processor is not None and (
                isinstance(processor, ProcessorMixin) or hasattr(processor, "image_processor")
            )
        except Exception:
            processes_images = processor is not None and hasattr(processor, "image_processor")
        chat_template_info["renders_image"] = bool(processes_images)
        # Not mlx-vlm's SINGLE_IMAGE_ONLY_MODELS: that list is its prompt builder's limit, while the
        # transformers processors for llava_next and mllama take one image per marker.
        chat_template_info["accepts_multiple_images"] = bool(processes_images) and bool(
            _scaling_image_marker(tokenizer, processor, self.models[model_name].get("model"))
        )
        if processes_images:
            processor_template = getattr(processor, "chat_template", None)
            # Narrowing the named-template list form away disables the image-turn override.
            if isinstance(processor_template, (str, dict, list, tuple)) and processor_template:
                chat_template_info["processor_template"] = processor_template

        try:
            from utils.datasets import MODEL_TO_TEMPLATE_MAPPER
            model_name_lower = model_name.lower()
            if model_name_lower in MODEL_TO_TEMPLATE_MAPPER:
                chat_template_info["template_name"] = MODEL_TO_TEMPLATE_MAPPER[model_name_lower]
                logger.info(
                    f"Detected template '{chat_template_info['template_name']}' for {model_name} from mapper"
                )
            else:
                # Partial match (for variants like model_name-bnb-4bit)
                for key in MODEL_TO_TEMPLATE_MAPPER:
                    if key in model_name_lower or model_name_lower in key:
                        chat_template_info["template_name"] = MODEL_TO_TEMPLATE_MAPPER[key]
                        logger.info(
                            f"Detected template '{chat_template_info['template_name']}' for {model_name} (partial match)"
                        )
                        break
        except Exception as e:
            logger.warning(f"Could not detect template from mapper for {model_name}: {e}")

        try:
            if hasattr(tokenizer, "chat_template") and tokenizer.chat_template:
                chat_template_info["has_template"] = True
                chat_template_info["template"] = tokenizer.chat_template

                template_str = tokenizer.chat_template.lower()

                if "start_header_id" in template_str and "end_header_id" in template_str:
                    chat_template_info["format_type"] = "llama3"
                elif "[inst]" in template_str and "[/inst]" in template_str:
                    chat_template_info["format_type"] = "mistral"
                elif "<|im_start|>" in template_str and "<|im_end|>" in template_str:
                    chat_template_info["format_type"] = "chatml"
                elif "### instruction:" in template_str or "### human:" in template_str:
                    chat_template_info["format_type"] = "alpaca"
                else:
                    chat_template_info["format_type"] = "custom"

                logger.info(
                    f"Loaded chat template for {model_name} (detected as {chat_template_info['format_type']} format)"
                )
                logger.debug(f"Template preview: {tokenizer.chat_template[:200]}...")

                special_tokens = {}
                if hasattr(tokenizer, "bos_token") and tokenizer.bos_token:
                    special_tokens["bos_token"] = tokenizer.bos_token
                if hasattr(tokenizer, "eos_token") and tokenizer.eos_token:
                    special_tokens["eos_token"] = tokenizer.eos_token
                if hasattr(tokenizer, "pad_token") and tokenizer.pad_token:
                    special_tokens["pad_token"] = tokenizer.pad_token

                chat_template_info["special_tokens"] = special_tokens

            else:
                logger.info(f"No chat template found for {model_name}, will use generic formatting")

        except Exception as e:
            logger.error(f"Error loading chat template info for {model_name}: {e}")

        self.models[model_name]["chat_template_info"] = chat_template_info

        if chat_template_info["has_template"]:
            logger.info(
                f"Chat template loaded for {model_name}: {chat_template_info['format_type']} format"
            )
        else:
            logger.info(f"No built-in chat template for {model_name}, will use generic formatting")

    def get_current_model(self) -> Optional[str]:
        """Currently active model name."""
        return self.active_model_name

    def is_model_loading(self) -> bool:
        return len(self.loading_models) > 0

    def get_loading_model(self) -> Optional[str]:
        return next(iter(self.loading_models)) if self.loading_models else None

    def load_model_simple(
        self,
        model_path: str,
        hf_token: Optional[str] = None,
        max_seq_length: int = 2048,
        load_in_4bit: bool = True,
    ) -> bool:
        """Simple model-loading wrapper for the chat interface: takes a string path and builds the
        ModelConfig internally."""
        try:
            config = ModelConfig.from_ui_selection(
                model_path,
                lora_path = None,  # No LoRA for chat
                is_lora = False,
            )

            return self.load_model(
                config = config,
                max_seq_length = max_seq_length,
                dtype = None,
                load_in_4bit = load_in_4bit,
                hf_token = hf_token,
            )

        except Exception as e:
            logger.error(f"Error in load_model_simple: {e}")
            return False


inference_backend = InferenceBackend()


def get_inference_backend() -> InferenceBackend:
    return inference_backend