Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Core inference backend."""
from __future__ import annotations
from unsloth import FastLanguageModel, FastVisionModel
from unsloth.chat_templates import get_chat_template
from transformers import BatchEncoding, TextIteratorStreamer, TextStreamer
from peft import PeftModel, PeftModelForCausalLM
import contextlib
import inspect
import json
import re
import sys
import threading
import torch
from pathlib import Path
from typing import Optional, Union, Generator, Tuple
from utils.models import ModelConfig, get_base_model_from_lora
from utils.paths import is_model_cached
from utils.transformers_dtype import dtype_kwargs
from utils.utils import format_error_message
from utils.hardware import (
get_device,
clear_gpu_cache,
log_gpu_memory,
get_device_map,
raise_if_offloaded,
get_visible_gpu_count,
)
from core.inference.audio_codecs import AudioCodecManager
from core.inference.context_refusal import ContextBudgetExceeded
from core.inference.runtime_context import (
UNSET_GENERATION_BUDGET,
generation_budget_within_context,
runtime_context_length,
)
from core.inference.message_content import content_to_text, named_turn
from core.inference.model_ids import is_deepseek_ocr
from core.inference.lfm3_conv_fallback import disable_lfm3_triton_conv_where_unsupported
from core.inference.chat_eos import (
chat_eos_repair,
resolve_chat_turn_end_eos_ids_using,
)
from core.inference.chat_template_helpers import (
alternating_turns,
build_dac_tts_prompt,
make_reasoning_normalizer,
detect_reasoning_channel_markers,
detect_think_prefill,
messages_with_attached_image,
neutralize_control_markup_in_messages,
neutralize_tts_prompt_text,
prompt_opens_reasoning_channel,
trailing_assistant_text,
tts_prompt_with_voice,
tts_voice_options,
)
from core.inference.presence_penalty import _make_presence_penalty_processor
from core.inference.generation_timing import (
GenerationTimer,
build_generation_timings,
with_prefill_boundary_processor,
)
from core.inference.incremental_streamer import IncrementalTextIteratorStreamer
from core.inference.native_tool_tokens import (
NativeToolTokenDecoder,
closes_an_open_envelope,
decoder_preserves_token,
reasoning_control_tokens,
stop_token_text,
)
from core.inference.mlx_inference import (
_mlx_stop_cut,
_mlx_stop_sequences,
_scaling_image_marker,
)
from io import StringIO
import structlog
from loggers import get_logger
from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory
logger = get_logger(__name__)
def _load_in_4bit_kwargs(load_in_4bit: bool) -> dict:
# True is the loaders' default; passing it reads as an explicit request to requantize fp8 checkpoints to NF4.
return {} if load_in_4bit else {"load_in_4bit": False}
_OFFLOAD_FIT_FRACTION = 0.9
def _loader_repo(path: str, load_in_4bit: bool) -> str:
"""The repo Unsloth's mapper will load for `path`, else `path`."""
try:
from unsloth.models import loader, loader_utils
if not loader.ALLOW_BITSANDBYTES:
load_in_4bit = False
tables = (
loader_utils.FLOAT_TO_INT_MAPPER,
loader_utils.INT_TO_FLOAT_MAPPER,
loader_utils.MAP_TO_UNSLOTH_16bit,
)
# Unknown names would fetch the remote mapper.
name = path.lower()
if name in loader_utils.BAD_MAPPINGS or not any(name in t for t in tables):
return path
return loader_utils.get_model_name(path, load_in_4bit = load_in_4bit) or path
except Exception:
return path
def _complete_local_safetensors_bytes(folder: Path) -> Optional[int]:
"""None while a shard the index names is missing (interrupted download)."""
try:
index = folder / "model.safetensors.index.json"
if index.is_file():
shards = set(json.loads(index.read_text(encoding = "utf-8"))["weight_map"].values())
files = [folder / shard for shard in shards]
if not all(f.is_file() for f in files):
return None
else:
files = list(folder.glob("*.safetensors"))
return sum(f.stat().st_size for f in files) or None
except Exception:
return None
def _checkpoint_bytes(name: str, hf_token = None) -> Optional[int]:
folder = Path(name)
if not folder.is_dir():
try:
from huggingface_hub import snapshot_download
folder = Path(snapshot_download(name, local_files_only = True, allow_patterns = ["*.json"]))
except Exception:
folder = None
if folder is not None:
local = _complete_local_safetensors_bytes(folder)
if local:
return local
if Path(name).is_dir():
return None
try:
from huggingface_hub import HfApi
info = HfApi().model_info(name, token = hf_token, files_metadata = True)
total = sum(
s.size or 0
for s in info.siblings or ()
if s.rfilename.endswith(".safetensors") and "/" not in s.rfilename
)
return total or None
except Exception:
return None
def _offload_kwargs_for_fit(
name: str,
device_map,
load_in_4bit: bool,
hf_token = None,
mapped: bool = True,
) -> dict:
"""Gemma 4 E2B 4-bit is 8.1 GB on an 8 GB card (#13388); "auto" still loads it all onto the GPU if it fits."""
# "sequential" over several visible cards pools them; "auto" would plan for one.
if device_map != "sequential" or not torch.cuda.is_available():
return {}
try:
if torch.cuda.device_count() != 1:
return {}
free = torch.cuda.mem_get_info(0)[0]
except Exception:
return {}
# mapped = False: use_exact_model_name reads `name` verbatim.
size = _checkpoint_bytes(_loader_repo(name, load_in_4bit) if mapped else name, hf_token)
if not size or size < _OFFLOAD_FIT_FRACTION * free:
return {}
logger.info(
f"Checkpoint is {size / 2**30:.2f} GiB with {free / 2**30:.2f} GiB free; loading with offload_layers='auto'"
)
return {"offload_layers": "auto"}
def _load_with_offload_fallback(load, **kwargs):
"""A single-GPU load the size check missed that spills to the CPU: free it, retry with offload_layers = "auto"."""
try:
return load(**kwargs)
except ValueError as e:
text = str(e)
spilled = "does not fit in GPU memory" in text or text.startswith(
"Some modules are dispatched on the CPU or the disk"
)
if (
not spilled
or "offload_layers" in kwargs
or kwargs.get("device_map") != "sequential"
or not torch.cuda.is_available()
or torch.cuda.device_count() != 1
):
raise
# After the except block: its traceback's frames held the half-built model.
import gc
gc.collect()
torch.cuda.empty_cache()
logger.info(
f"Model did not fit in GPU memory; retrying with offload_layers='auto' "
f"({torch.cuda.memory_allocated() / 2**30:.2f} GiB still allocated)"
)
return load(offload_layers = "auto", **kwargs)
def _hf_token_for_loader(hf_token: Optional[str] | bool) -> Optional[str] | bool:
if hf_token is False:
return False
return hf_token.strip() if isinstance(hf_token, str) and hf_token.strip() else None
def _exact_model_name_for_load(config: ModelConfig, load_in_4bit: bool) -> Optional[str]:
"""config.path when the mapped repo is missing but user weights exist; else cached casing or None."""
if config.is_local or config.is_lora or not config.path:
return None
try:
from unsloth.models import loader, loader_utils
# ModelScope downloads every repo id to its own cache, so the HF cache says nothing here.
if loader.USE_MODELSCOPE:
return None
# Resolve the repo the loader will fetch on this host, after its bitsandbytes fallbacks.
if not loader.ALLOW_BITSANDBYTES:
load_in_4bit = False
name = config.path.lower()
if name in loader_utils.BAD_MAPPINGS:
return None
table = (
loader_utils.FLOAT_TO_INT_MAPPER if load_in_4bit else loader_utils.MAP_TO_UNSLOTH_16bit
)
# Avoid fetching the remote mapper for unknown names.
if name not in table:
return None
target = loader_utils.get_model_name(config.path, load_in_4bit = load_in_4bit)
if target and not loader.ALLOW_PREQUANTIZED_MODELS:
target = loader._strip_unsloth_bnb_4bit_suffix(target)
except Exception as e:
logger.debug(f"Could not resolve the Unsloth mapping for {config.path}: {e}")
return None
if not target or target.lower() == name:
return None
from utils.utils import (
active_hf_cache_loadable_snapshot,
active_hf_cache_repo_spelling,
)
cached_target = active_hf_cache_repo_spelling(target)
if cached_target == target:
return None # on disk under the name the mapper will ask for: nothing to do
if cached_target is not None:
logger.info(f"Loading cached {cached_target} instead of downloading {target} again")
return cached_target
snapshot = active_hf_cache_loadable_snapshot(config.path)
if snapshot is None:
return None
try:
checkpoint = json.loads((snapshot / "config.json").read_text(encoding = "utf-8"))
except (OSError, ValueError):
return None
if not isinstance(checkpoint, dict) or checkpoint.get("quantization_config") is not None:
return None
logger.info(f"Loading cached {config.path} as named instead of downloading {target}")
return config.path
def _decodes_by_concatenation(tokenizer) -> bool:
"""True when decode(a + b) == decode(a) + decode(b) whenever decode(a) ends on a whole character:
a byte-level BPE (bytes joined, then lossy UTF-8) with no clean-up pass."""
if getattr(tokenizer, "clean_up_tokenization_spaces", True) is not False:
return False
try:
from tokenizers import decoders
from transformers import PreTrainedTokenizerFast
if not isinstance(tokenizer, PreTrainedTokenizerFast):
return False
# A subclass or instance that overrides decode / _decode may post-process the whole text.
for name in ("decode", "_decode"):
method = getattr(tokenizer, name, None)
if getattr(method, "__func__", None) is not getattr(
PreTrainedTokenizerFast, name, None
):
return False
return type(tokenizer.backend_tokenizer.decoder) is decoders.ByteLevel
except Exception:
return False
class HarmonyTextStreamer:
"""Streaming text decoder for the gpt-oss harmony channel protocol.
gpt-oss emits multi-channel output via ``<|channel|>analysis<|message|>...``
/ ``<|channel|>final<|message|>...``. Plain skip_special_tokens streaming
glues channel names to content. This decodes with skip_special_tokens=False
and parses statefully: emit ``<think>`` on first analysis, stream analysis,
emit ``</think>`` on first final, stream final. Tracking per-channel lengths
avoids the delta-on-transformed bug where wrapping tags shift position.
Same put/end/iterator interface as TextIteratorStreamer.
"""
import re as _re
_HARMONY_RE = _re.compile(
r"<\|channel\|>(\w+)<\|message\|>(.*?)(?=<\|end\|>|<\|channel\|>|\Z)",
_re.DOTALL,
)
def __init__(
self,
tokenizer,
*,
skip_prompt: bool = True,
timeout: float = 0.2,
):
import queue
self.tokenizer = tokenizer
self.skip_prompt = skip_prompt
self.timeout = timeout
self._queue: queue.Queue = queue.Queue()
self._token_ids: list = []
self._prompt_len: int = 0
self._is_first_put: bool = True
self._stop: bool = False
self._emitted_think_open: bool = False
self._emitted_think_close: bool = False
self._analysis_emitted: int = 0
self._final_emitted: int = 0
# Incremental decode: text of the first _settled_n generated ids, ending on a whole character.
self._concat_decode: bool = _decodes_by_concatenation(tokenizer)
self._settled_text: str = ""
self._settled_n: int = 0
self._scan_raw: str = ""
self._scan_pos: int = 0
self._open_match = None
self._stable_final: bool = False
def put(self, value):
import torch
if isinstance(value, torch.Tensor):
ids = value[0].tolist() if value.dim() > 1 else value.tolist()
elif isinstance(value, (list, tuple)):
ids = list(value)
else:
ids = [value]
if self._is_first_put and self.skip_prompt:
self._prompt_len = len(ids)
self._token_ids = list(ids)
self._is_first_put = False
return
self._token_ids.extend(ids)
if self._concat_decode:
# Re-decoding the whole reply each token is quadratic; only ids after the settled text are decoded.
start = self._prompt_len + self._settled_n
tail = self.tokenizer.decode(self._token_ids[start:], skip_special_tokens = False)
raw = self._settled_text + tail
if tail and not tail.endswith("\ufffd"):
self._settled_text = raw
self._settled_n = len(self._token_ids) - self._prompt_len
else:
gen_ids = self._token_ids[self._prompt_len :]
raw = self.tokenizer.decode(gen_ids, skip_special_tokens = False)
# A trailing U+FFFD may be a character whose bytes are still arriving; end() emits it.
self._process_incremental(raw.rstrip("\ufffd"))
def end(self):
gen_ids = self._token_ids[self._prompt_len :]
if gen_ids:
raw = self.tokenizer.decode(gen_ids, skip_special_tokens = False)
self._process_incremental(raw)
if self._emitted_think_open and not self._emitted_think_close:
self._queue.put("</think>")
self._emitted_think_close = True
self._stop = True
self._queue.put(None)
def __iter__(self):
return self
def __next__(self):
from queue import Empty
while True:
try:
val = self._queue.get(timeout = self.timeout)
except Empty:
if self._stop:
raise StopIteration
raise # propagate Empty so caller can check thread liveness
if val is None:
raise StopIteration
return val
def _process_incremental(self, raw: str) -> None:
"""Emit per-channel deltas; same output as re-matching all of ``raw``, but only the open channel is
rescanned: a closed channel never changes, and replaying one can only close ``<think>`` (final)."""
if not raw.startswith(self._scan_raw):
self._scan_pos = 0
self._open_match = None
self._stable_final = False
old_len = 0
else:
old_len = len(self._scan_raw)
self._scan_raw = raw
if self._stable_final and self._emitted_think_open and not self._emitted_think_close:
self._queue.put("</think>")
self._emitted_think_close = True
pos = self._scan_pos
if self._open_match is not None:
channel, start = self._open_match
# The open channel ran to the old end, so a terminator can only start in the new tail.
end = len(raw)
for marker in ("<|end|>", "<|channel|>"):
found = raw.find(marker, max(start, old_len - len(marker) + 1))
if 0 <= found < end:
end = found
self._emit_channel(channel, raw, start, end)
if end == len(raw):
return
self._close_channel(channel, end)
pos = end
for m in self._HARMONY_RE.finditer(raw, pos):
channel = m.group(1).lower()
self._emit_channel(channel, raw, m.start(2), m.end(2))
if m.end() == len(raw):
self._open_match = (channel, m.start(2))
else:
self._close_channel(channel, m.end())
def _close_channel(self, channel: str, end: int) -> None:
self._open_match = None
self._scan_pos = end
if channel in ("final", "assistant"):
self._stable_final = True
def _emit_channel(self, channel: str, raw: str, start: int, end: int) -> None:
if channel == "analysis":
if not self._emitted_think_open:
self._queue.put("<think>")
self._emitted_think_open = True
if end - start > self._analysis_emitted:
self._queue.put(raw[start + self._analysis_emitted : end])
self._analysis_emitted = end - start
elif channel in ("final", "assistant"):
if self._emitted_think_open and not self._emitted_think_close:
self._queue.put("</think>")
self._emitted_think_close = True
if end - start > self._final_emitted:
self._queue.put(raw[start + self._final_emitted : end])
self._final_emitted = end - start
class ReasoningTextIteratorStreamer(IncrementalTextIteratorStreamer):
"""TextIteratorStreamer that preserves native channel tokens until parsed."""
def __init__(
self,
tokenizer,
*,
markers: tuple[str, ...],
skip_prompt: bool = True,
timeout: float = 0.2,
cancel_event = None,
in_reasoning: bool = False,
**decode_kwargs,
):
decode_kwargs["skip_special_tokens"] = False
stream_tokenizer = NativeToolTokenDecoder(
tokenizer, preserved_tokens = reasoning_control_tokens(markers)
)
super().__init__(
stream_tokenizer, skip_prompt = skip_prompt, timeout = timeout, **decode_kwargs
)
self._normalizer = make_reasoning_normalizer(markers, in_reasoning = in_reasoning)
self._cancel_event = cancel_event
self._aborted = False
def abort(self):
"""Mark generation as failed so ``end`` drains without closing."""
self._aborted = True
def on_finalized_text(
self,
text: str,
stream_end: bool = False,
):
"""Queue canonical deltas, closing only on natural stream completion."""
delta = self._normalizer.feed(text)
if delta:
self.text_queue.put(delta, timeout = self.timeout)
if stream_end:
cancelled = self._aborted or (
self._cancel_event is not None and self._cancel_event.is_set()
)
tail = self._normalizer.drain() if cancelled else self._normalizer.finish()
if tail:
self.text_queue.put(tail, timeout = self.timeout)
self.text_queue.put(self.stop_signal, timeout = self.timeout)
class _GenerationThreadError(RuntimeError):
"""Generation worker failures that should propagate through stream routes."""
# What clean_up_tokenization_spaces deletes a space before; a rule spans at most four chars.
_CLEANUP_TAILS = (".", "?", "!", ",", "' ", "n't", "'m", "'s", "'ve", "'re")
_CLEANUP_SPAN_CHARS = 4
# Tokens, not chars: four chars can be four tokens, plus room for tokens decoding to "".
_CLEANUP_SPAN_TOKENS = 8
def _cleanup_join_pending(text: str) -> bool:
"""Only the last few characters matter, from a space onward; a trailing space counts as pending."""
window = text[-_CLEANUP_SPAN_CHARS:]
for i, char in enumerate(window):
if char != " ":
continue
rest = window[i + 1 :]
if not rest or any(tail.startswith(rest) for tail in _CLEANUP_TAILS):
return True
return False
def _decoder_cleans_up(streamer) -> bool:
"""Skips the widened re-decode for tokenizers that default clean_up_tokenization_spaces off."""
kwargs = getattr(streamer, "decode_kwargs", None) or {}
if "clean_up_tokenization_spaces" in kwargs:
return bool(kwargs["clean_up_tokenization_spaces"])
tokenizer = getattr(streamer, "tokenizer", None)
for _ in range(4):
if tokenizer is None:
break
flag = getattr(tokenizer, "clean_up_tokenization_spaces", None)
if flag is not None:
return bool(flag)
tokenizer = getattr(tokenizer, "tokenizer", None)
return False
class _StopSequenceStreamer:
def __init__(self, streamer, stop):
self.streamer = streamer
self.sequences = _mlx_stop_sequences(stop)
self.matched = threading.Event()
self.token_ids = []
self.text = ""
self.settled = ""
self.prefix_offset = 0
self.read_offset = 0
self.scan_from = 0
self.longest = max((len(s) for s in self.sequences), default = 0)
self.released = 0
self.cut = 0
self.finished = False
self.next_tokens_are_prompt = bool(getattr(streamer, "skip_prompt", False))
self.is_harmony = isinstance(streamer, HarmonyTextStreamer)
self.cleans_up = _decoder_cleans_up(streamer)
def __next__(self):
return next(self.streamer)
def __iter__(self):
return self
def put(self, value):
if not self.sequences:
return self.streamer.put(value)
if self.finished or self.matched.is_set():
return
# Matches the supported batch-one TextIteratorStreamer protocol.
if len(value.shape) > 1:
if value.shape[0] > 1:
raise ValueError("TextStreamer only supports batch size 1")
value = value[0]
if self.next_tokens_are_prompt:
self.next_tokens_are_prompt = False
return
self.token_ids.extend(value.tolist())
self._decode_new_tokens()
# Earlier text was already scanned; only a stop overlapping new text can match.
start = self.scan_from
cut, matched = _mlx_stop_cut(self.text[start:], self.sequences)
self.cut = start + cut
if matched:
# This runs in the producer, before model.generate checks criteria.
self.matched.set()
else:
self.scan_from = max(start, len(self.settled) - self.longest + 1)
cut = self.cut
if not matched and not self.is_harmony and cut > self.released:
# Keep displayed text stable under word buffering; stop detection above must still see every token.
if not self.streamer._is_chinese_char(ord(self.text[cut - 1])):
cut = (
max(
self.text.rfind(" ", self.released, cut),
self.text.rfind("\n", self.released, cut),
)
+ 1
)
self._publish(cut)
def _decode(self, token_ids):
decode_kwargs = (
{"skip_special_tokens": False} if self.is_harmony else self.streamer.decode_kwargs
)
return self.streamer.tokenizer.decode(token_ids, **decode_kwargs)
def _decode_new_tokens(self):
# Re-decoding the whole reply each token is quadratic; decode a short window.
prefix = self._decode(self.token_ids[self.prefix_offset : self.read_offset])
window = self._decode(self.token_ids[self.prefix_offset :])
if (
self.cleans_up
and window.startswith(prefix)
and _cleanup_join_pending(self.settled + window[len(prefix) :])
):
# A cleanup rule split across the join is invisible from the short window and would diverge
# permanently, so re-read a few tokens behind the join.
wider = max(0, self.read_offset - _CLEANUP_SPAN_TOKENS)
if wider < self.prefix_offset:
wide_prefix = self._decode(self.token_ids[wider : self.read_offset])
# Splicing onto text the prefix does not end is worse than the divergence; use the short window.
if self.settled.endswith(wide_prefix):
prefix = wide_prefix
window = self._decode(self.token_ids[wider:])
if window.endswith("\ufffd"):
# Bytes still arriving can rewrite the window (unfinished emoji shows as U+FFFD), so wait for them.
tail = window[len(prefix) :] if window.startswith(prefix) else ""
self.text = self.settled + tail
return
if not window.startswith(prefix):
# New tokens can rewrite settled text, e.g. space cleanup; rebuild it from every token and rescan.
self.settled = self._decode(self.token_ids)
self.prefix_offset = max(0, len(self.token_ids) - 4)
self.read_offset = len(self.token_ids)
self.scan_from = 0
elif len(window) > len(prefix):
self.settled += window[len(prefix) :]
self.prefix_offset = self.read_offset
self.read_offset = len(self.token_ids)
self.text = self.settled
def _publish(self, cut):
if cut <= self.released:
return
if self.is_harmony:
# Harmony consumes cumulative raw text and synthesizes tags itself.
self.streamer._process_incremental(self.text[:cut])
else:
# The reasoning subclass normalizes here; plain text just queues it.
self.streamer.on_finalized_text(self.text[self.released : cut])
self.released = cut
def end(self):
if not self.sequences:
return self.streamer.end()
if self.finished:
return
self.finished = True
if not self.matched.is_set() and self.read_offset < len(self.token_ids):
# Bytes that never resolved still belong to the reply, as a full decode shows.
self.text = self._decode(self.token_ids)
# A natural end releases a partial unmatched stop and unresolved bytes.
self._publish(self.cut if self.matched.is_set() else len(self.text))
# Caches are empty as put() feeds raw text, so end() only closes reasoning and sends the sentinel.
self.streamer.end()
def abort(self):
abort = getattr(self.streamer, "abort", None)
if abort is not None:
abort()
def _decode_graphs():
"""Unsloth's CUDA-graph decode module, or None for an Unsloth without the Studio scope."""
try:
from unsloth.models import _decode_graphs as graphs
except Exception:
return None
return graphs if hasattr(graphs, "studio_generation") else None
def _decode_graph_scope():
"""Graphed decode on by default for this thread's chat generate(): this worker only serves
inference. Llama / Qwen3 on NVIDIA CUDA only; UNSLOTH_DECODE_CUDA_GRAPHS=0 still turns it off."""
graphs = _decode_graphs()
return graphs.studio_generation() if graphs is not None else contextlib.nullcontext()
def _release_decode_graphs(model, generation_lock) -> None:
"""Frees the model's decode graphs and decode buffers now rather than at garbage collection.
Skipped while a generate() still holds the lock: that thread's model reference frees them."""
graphs = _decode_graphs()
if graphs is None or not hasattr(model, "modules"):
return
if generation_lock is not None and not generation_lock.acquire(blocking = False):
return
try:
graphs.release_for_training(model)
except Exception as e:
logger.warning(f"Could not release decode graphs: {e}")
finally:
if generation_lock is not None:
generation_lock.release()
_CUSTOM_GENERATE_REJECTED_KWARGS = ("logits_processor", "stopping_criteria")
class _GenerationHalted(Exception):
pass
class _HaltingStreamer:
def __init__(self, streamer, events):
self.streamer = streamer
self.events = events
def put(self, value):
if any(ev.is_set() for ev in self.events):
raise _GenerationHalted
return self.streamer.put(value)
def end(self):
return self.streamer.end()
def __getattr__(self, name):
return getattr(self.streamer, name)
def _supported_generate_kwargs(model, generation_kwargs: dict) -> dict:
if not isinstance(getattr(model, "_generation_fields_unsupported", None), dict):
return generation_kwargs
kept = {k: v for k, v in generation_kwargs.items() if k not in _CUSTOM_GENERATE_REJECTED_KWARGS}
model_type = getattr(getattr(model, "config", None), "model_type", None)
if model_type in {"lfm3_omni", "lfm3_audio"}:
kept["generation_mode"] = "text"
events = [
c.ev
for c in (generation_kwargs.get("stopping_criteria") or ())
if getattr(c, "ev", None) is not None
]
if events and kept.get("streamer") is not None:
kept["streamer"] = _HaltingStreamer(kept["streamer"], events)
return kept
def _generate_supported(model, generation_kwargs: dict):
kwargs = _supported_generate_kwargs(model, generation_kwargs)
try:
return model.generate(**kwargs)
except _GenerationHalted:
streamer = generation_kwargs.get("streamer")
matched = getattr(streamer, "matched", None)
prompt = generation_kwargs.get("input_ids")
if matched is not None and matched.is_set() and prompt is not None:
return prompt.new_tensor([prompt[0].tolist() + streamer.token_ids])
return None
def _refuse_without_generate(model, model_name: str) -> None:
if not callable(getattr(model, "generate", None)):
raise ValueError(
f"{model_name} loads as {type(model).__name__}, which cannot generate text. "
"Unsloth Studio has no chat path for this architecture yet."
)
def _audio_chat_auto_model(config):
if not getattr(config, "has_audio_input", False) or getattr(config, "is_vision", False):
return None
if getattr(config, "audio_type", None) != "audio_vlm":
return None
import transformers
return getattr(transformers, "AutoModelForMultimodalLM", None)
# Audio sampling stays with each checkpoint's generation_config: LFM3-Audio and -Omni differ.
_SPEECH_REPLY_SYSTEM_PROMPT = "Respond with interleaved text and audio."
_TTS_LOCALES = {
"ar": "Arabic",
"da": "Danish",
"de": "German",
"el": "Greek",
"en": "American English",
"en-us": "American English",
"en-gb": "British English",
"fi": "Finnish",
"fr": "French",
"he": "Hebrew",
"it": "Italian",
"ja": "Japanese",
"ko": "Korean",
"nl": "Dutch",
"pl": "Polish",
"pt": "Portuguese",
"es": "Spanish",
"sv": "Swedish",
"tr": "Turkish",
"zh": "Chinese",
}
def _speech_codec(model, processor):
"""The processor's bundled codec when the model speaks (LFM3-Audio / -Omni), else None."""
codec = getattr(processor, "audio_tokenizer", None)
if codec is None or not isinstance(
getattr(getattr(model, "config", None), "interleaved_n_audio", None), int
):
return None
return codec
def _tts_system_prompt(language: Optional[str], audio_options: Optional[dict]) -> str:
"""The model card's TTS prompt; an unknown language falls back to American English."""
lang = str(language or "").strip()
locale = _TTS_LOCALES.get(lang.lower().replace("_", "-")) or next(
(name for name in _TTS_LOCALES.values() if name.lower() == lang.lower()),
"American English",
)
voice = str((audio_options or {}).get("voice") or "").strip().lower()
return f"Perform TTS in {locale} using a {'male' if voice == 'male' else 'female'} voice."
class _SpeechStopped(Exception):
pass
def _speech_stop_streamer(cancel_event):
"""LFM3's generate refuses stopping_criteria; a streamer raising on put ends it."""
from transformers.generation.streamers import BaseStreamer
class _StopOnCancel(BaseStreamer):
def put(self, value):
if cancel_event.is_set():
raise _SpeechStopped
def end(self):
pass
return _StopOnCancel()
def _processor_takes_text_first(processor) -> bool:
"""LFM3-Omni's processor is (text, audio, ..., images); the usual VLM order is (images, text)."""
# unsloth_zoo wraps __call__ as (*args, **kwargs); read the first class that names its arguments.
for cls in type(processor).__mro__:
call = cls.__dict__.get("__call__")
if call is None:
continue
try:
params = list(inspect.signature(call).parameters.values())[1:]
except (TypeError, ValueError):
return False
if params and params[0].kind is not inspect.Parameter.VAR_POSITIONAL:
return params[0].name == "text"
return False
# LlamaModel_fast_forward cuts input_ids at max_seq_length, so decoding past it crashes; FastModel's stock forward does not.
def _truncates_past_window(model) -> bool:
return any(
getattr(getattr(type(module), "forward", None), "__name__", "") == "LlamaModel_fast_forward"
for module in model.modules()
)
def _probe_prompt_bos(tokenizer, prompt):
"""(tokenizer, prompt, encoding) when the rendered prompt already starts with BOS, else None.
Most templates render BOS, so no second one is added; zephyr and tinyllama-chat do not."""
tok = getattr(tokenizer, "tokenizer", tokenizer)
bos_token_id = getattr(tok, "bos_token_id", None)
if bos_token_id is None:
return None
try:
encoding = tok(prompt, add_special_tokens = False)
ids = encoding["input_ids"]
except Exception:
return None
while isinstance(ids, (list, tuple)) and ids and isinstance(ids[0], (list, tuple)):
ids = ids[0]
if len(ids) and ids[0] == bos_token_id:
return tok, prompt, encoding
return None
def _prompt_already_has_bos(tokenizer, prompt):
return _probe_prompt_bos(tokenizer, prompt) is not None
def _tokenize_prompt(
tokenizer,
prompt,
add_special_tokens,
bos_probe = None,
):
"""``tokenizer(prompt, return_tensors="pt", ...)``, reusing a matching bos_probe encoding."""
if not add_special_tokens and bos_probe is not None:
probe_tok, probe_prompt, encoding = bos_probe
ids = encoding["input_ids"] if isinstance(encoding, BatchEncoding) else None
if (
probe_tok is tokenizer
and probe_prompt == prompt
and isinstance(ids, list)
and not (ids and isinstance(ids[0], (list, tuple)))
):
return BatchEncoding(
{key: [value] for key, value in encoding.items()},
encoding = encoding.encodings,
tensor_type = "pt",
)
return tokenizer(prompt, return_tensors = "pt", add_special_tokens = add_special_tokens)
_MAPPED_TEMPLATE_LOCK = threading.Lock()
def _mapped_template_tokenizer(model_info, tokenizer, template_name):
with _MAPPED_TEMPLATE_LOCK:
cached = model_info.get("_mapped_template_tokenizer")
if cached is not None and cached[0] is tokenizer and cached[1] == template_name:
return cached[2]
remapped = get_chat_template(tokenizer, chat_template = template_name)
model_info["_mapped_template_tokenizer"] = (tokenizer, template_name, remapped)
return remapped
_MISSING = object()
class _OcrNeedsImageError(ValueError):
public = True
openai_param = "messages"
def _without_image_parts(messages) -> list[dict]:
"""Drops image parts; a turn left with only text collapses back to that text."""
out = []
for message in messages:
content = message.get("content")
if not isinstance(content, list):
out.append(message)
continue
kept = [
part
for part in content
if not (
isinstance(part, dict) and part.get("type") in ("image", "image_url", "input_image")
)
]
if len(kept) == len(content):
out.append(message)
continue
texts = [
part.get("text", "")
for part in kept
if isinstance(part, dict) and part.get("type") == "text"
]
out.append({**message, "content": "\n".join(texts) if len(kept) == len(texts) else kept})
return out
_CANVAS_MASK = "\u2591"
_CANVAS_FRAME_INTERVAL_S = 0.1
def _accepts_on_frame(generate_text) -> bool:
"""An older core would swallow on_frame unused."""
import inspect
candidates = [generate_text]
try:
from unsloth.models import diffusion_samplers
candidates.append(diffusion_samplers.generate_text)
except ImportError:
pass
for fn in candidates:
try:
if "on_frame" in inspect.signature(fn).parameters:
return True
except (TypeError, ValueError):
continue
return False
def _render_canvas(tokenizer, tokens, mask_id) -> str:
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
parts, run = [], []
for token in tokens:
if mask_id is not None and token == mask_id:
if run:
parts.append(tokenizer.decode(run, skip_special_tokens = True))
run = []
parts.append(_CANVAS_MASK)
else:
run.append(token)
if run:
parts.append(tokenizer.decode(run, skip_special_tokens = True))
return "".join(parts)
def _diffusion_eos_ids(model, tokenizer, chat) -> tuple:
"""The ids ``generate_text`` cuts its reply at; empty on an older core."""
import importlib
try:
samplers = importlib.import_module("unsloth.models.diffusion_samplers")
base = samplers.unwrap_diffusion_model(model)
ids = list(samplers._eos_ids(base, tokenizer))
if chat:
ids += [i for i in samplers._turn_end_ids(tokenizer) if i not in ids]
return tuple(ids)
except Exception:
return ()
def _diffusion_frame_relay(
tokenizer,
sink,
interval_s = None,
stop = None,
eos_ids = (),
):
"""Core-sampler on_frame: one frame per interval to sink; never raises into the sampler."""
import time
last = [float("-inf")]
if interval_s is None:
interval_s = _CANVAS_FRAME_INTERVAL_S
def on_frame(frame):
now = time.monotonic()
# A block's last step is never throttled: later frames build on it.
block_end = int(frame.get("step", 0)) >= int(frame.get("total", 0)) > 0
if now - last[0] < interval_s and not block_end:
return
last[0] = now
try:
tokens = list(frame.get("tokens") or ())
for index, token in enumerate(tokens):
if token in eos_ids:
tokens = tokens[:index]
break
text = _render_canvas(tokenizer, tokens, frame.get("mask_id"))
for sequence in stop or ():
if sequence and sequence in text:
text = text[: text.index(sequence)]
total = int(frame.get("total", 0))
sink(
{
"type": "diffusion_frame",
"block": int(frame.get("block", 0)),
# Zero-based for the canvas; the sampler's block-end frame is step == total.
"step": max(min(int(frame.get("step", 0)), total - 1), 0),
"total": total,
"text": text,
}
)
except Exception as exc:
logger.debug("Dropped a diffusion canvas frame: %s", exc)
return on_frame
# o200k_harmony's non-reserved special tokens (tokenizer_config.json added_tokens_decoder).
_HARMONY_PROTOCOL_TOKEN_RE = re.compile(
r"<\|(?:start|end|message|channel|constrain|return|call|startoftext|endoftext|endofprompt)\|>"
)
class InferenceBackend:
"""Unified inference backend supporting text, vision, and LoRA models"""
# Class attribute too, so a backend built with __new__ still answers.
last_generation_stats = None
def __init__(self):
self.models = {}
self.active_model_name = None
self.loading_models = set()
self.loaded_local_models = []
from core.inference.defaults import get_default_models
self.default_models = get_default_models()
self.device = get_device().value
self._audio_codec_manager = AudioCodecManager()
self.last_generation_stats = None
# Set by the worker for one request that asked for canvas frames.
self.diffusion_frame_sink = None
# Plain Lock, NOT RLock: reentrancy would let concurrent compare-mode requests race on the GPU.
import threading
self._generation_lock = threading.Lock()
self._model_state_lock = threading.Lock()
logger.info(f"InferenceBackend initialized on {self.device}")
@staticmethod
def _normalize_top_k(top_k: int) -> int:
# API uses -1 to disable top-k; transformers uses 0.
return 0 if top_k < 0 else top_k
def _resolve_chat_eos(self, model_name: str) -> None:
"""Caches chat turn-end stop ids at load and repairs generation_config; some Qwen3.5/3.6 need it."""
info = self.models.get(model_name) or {}
model = info.get("model")
container = info.get("tokenizer")
tokenizer = getattr(container, "tokenizer", container)
if model is None or tokenizer is None:
return
# VLMs keep chat_template on the processor; resolve ids on the generation tokenizer.
template_source = container if getattr(container, "chat_template", None) else tokenizer
try:
turn_end_ids = resolve_chat_turn_end_eos_ids_using(template_source, tokenizer)
except Exception as e: # never block a load on eos resolution
logger.warning("Chat turn-end eos resolution failed for %s: %s", model_name, e)
return
info["chat_turn_end_eos_ids"] = turn_end_ids
gen = getattr(model, "generation_config", None)
if gen is None:
return
repaired = chat_eos_repair(gen.eos_token_id, turn_end_ids)
if repaired is None:
return
previous = gen.eos_token_id
gen.eos_token_id = repaired
logger.info(
"Repaired generation_config.eos_token_id for %s: %s -> %s",
model_name,
previous,
repaired,
)
@_invalidates_gpu_memory("transformers load")
def load_model(
self,
config: ModelConfig,
max_seq_length: int = 2048,
dtype = None,
load_in_4bit: bool = True,
hf_token: Optional[str] | bool = None,
trust_remote_code: bool = False,
gpu_ids: Optional[list[int]] = None,
audio_codec_path: Optional[str] = None,
) -> bool:
"""Keep the token so native-template fallback can fetch a gated repo template during generation."""
self._hf_token = hf_token
# GGUF uses max_seq_length=0 as "model default"; Unsloth crashes on it.
if max_seq_length <= 0:
max_seq_length = 2048
try:
model_name = config.identifier
if model_name in self.models and self.models[model_name].get("model"):
logger.info(f"Model {model_name} already loaded")
if hf_token:
self.models[model_name]["hf_token"] = hf_token
self.active_model_name = model_name
return True
if model_name in self.loading_models:
logger.info(f"Model {model_name} is already being loaded")
return False
self.loading_models.add(model_name)
device_map = get_device_map(gpu_ids)
logger.info(
f"Using device_map='{device_map}' ({get_visible_gpu_count()} GPU(s) visible)"
)
self.models[model_name] = {
# Native-template fallback must use this model's own hf_token, not the one loaded last.
"hf_token": hf_token,
# Reuse this model trust_remote_code consent for the template reload; never run unconsented code.
"trust_remote_code": trust_remote_code,
"is_vision": config.is_vision,
"is_lora": config.is_lora,
"is_audio": config.is_audio,
"audio_type": config.audio_type,
"has_audio_input": config.has_audio_input,
"model_path": config.path,
"base_model": config.base_model if config.is_lora else None,
"loaded_adapters": {},
"active_adapter": None,
}
if config.is_audio:
audio_type = config.audio_type
adapter_info = " (LoRA adapter)" if config.is_lora else ""
logger.info(f"Loading audio ({audio_type}) model{adapter_info}: {model_name}")
log_gpu_memory(f"Before loading {model_name}")
if audio_type == "csm":
from unsloth import FastModel
from transformers import CsmForConditionalGeneration
model, processor = FastModel.from_pretrained(
config.path,
auto_model = CsmForConditionalGeneration,
load_in_4bit = False,
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastModel.for_inference(model)
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = processor
self.models[model_name]["processor"] = processor
elif audio_type == "bicodec":
import os
from core.inference.audio_codecs import resolve_bicodec_repo_path
from unsloth import FastModel
if config.is_lora and config.base_model:
# LoRA adapter: base_model is .../Spark-TTS-0.5B/LLM;
# BiCodec weights live in the parent dir.
from utils.models.spark_tts_adapter import spark_tts_adapter_load_target
base_path = config.base_model
abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
base_path, hf_token = hf_token
)
logger.info(
f"Spark-TTS LoRA: loading adapter from {config.path}, BiCodec from {abs_repo_path}"
)
with spark_tts_adapter_load_target(
config.path, hf_token = _hf_token_for_loader(hf_token)
) as load_target:
model, tokenizer = FastModel.from_pretrained(
load_target,
dtype = torch.float32,
# Flash Attention cannot run float32
attn_implementation = "sdpa",
load_in_4bit = False,
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
elif config.is_local and os.path.isdir(config.path):
# A merged export has no LLM/ child or BiCodec weights; resolve those from the recorded base.
llm_path = os.path.join(config.path, "LLM")
if not os.path.isdir(llm_path):
llm_path = config.path
abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
config.path, hf_token = hf_token
)
logger.info(
f"Spark-TTS merged export: LLM from {llm_path}, BiCodec from {abs_repo_path}"
)
else:
abs_repo_path = audio_codec_path or resolve_bicodec_repo_path(
config.path, hf_token = hf_token
)
llm_path = os.path.join(abs_repo_path, "LLM")
logger.info(
f"Spark-TTS: repo at {abs_repo_path}, loading LLM from {llm_path}"
)
if not (config.is_lora and config.base_model):
model, tokenizer = FastModel.from_pretrained(
llm_path,
dtype = torch.float32,
# Flash Attention cannot run float32
attn_implementation = "sdpa",
load_in_4bit = False,
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastModel.for_inference(model)
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = tokenizer
self.models[model_name]["model_repo_path"] = abs_repo_path
elif audio_type == "dac":
# OuteTTS uses FastModel (not FastLanguageModel)
from unsloth import FastModel
model, tokenizer = FastModel.from_pretrained(
config.path,
max_seq_length = max_seq_length,
load_in_4bit = False,
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastModel.for_inference(model)
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = tokenizer
elif audio_type == "whisper":
from unsloth import FastModel
from transformers import WhisperForConditionalGeneration
model, tokenizer = FastModel.from_pretrained(
config.path,
auto_model = WhisperForConditionalGeneration,
whisper_language = "English",
whisper_task = "transcribe",
load_in_4bit = False,
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastModel.for_inference(model)
model.eval()
from transformers import pipeline as hf_pipeline
whisper_pipe = hf_pipeline(
"automatic-speech-recognition",
model = model,
tokenizer = tokenizer.tokenizer,
feature_extractor = tokenizer.feature_extractor,
processor = tokenizer,
return_language = True,
**dtype_kwargs(torch.float16),
)
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = tokenizer
self.models[model_name]["whisper_pipeline"] = whisper_pipe
else:
# SNAC (Orpheus) uses FastLanguageModel
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = config.path,
max_seq_length = max_seq_length,
load_in_4bit = False,
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
FastLanguageModel.for_inference(model)
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = tokenizer
voice_options = tts_voice_options(audio_type)
if voice_options:
self.models[model_name]["audio_options"] = voice_options
# Load an external codec for TTS audio; whisper (ASR) and audio_vlm (audio input) need none.
if audio_type not in ("whisper", "audio_vlm"):
model_repo_path = audio_codec_path or self.models[model_name].get(
"model_repo_path"
)
self._audio_codec_manager.load_codec(
audio_type, self.device, model_repo_path = model_repo_path
)
raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")
disable_lfm3_triton_conv_where_unsupported(
self.models[model_name]["model"], torch, logger
)
self.models[model_name]["context_length"] = runtime_context_length(
self.models[model_name].get("model"),
max_seq_length,
)
self.active_model_name = model_name
self.loading_models.discard(model_name)
logger.info(f"Successfully loaded audio model: {model_name}")
log_gpu_memory(f"After loading {model_name}")
return True
model_type = "vision" if config.is_vision else "text"
adapter_info = " (LoRA adapter)" if self.models[model_name]["is_lora"] else ""
logger.info(f"Loading {model_type} model{adapter_info}: {model_name}")
log_gpu_memory(f"Before loading {model_name}")
exact_model_name = _exact_model_name_for_load(config, load_in_4bit)
use_exact_model_name = exact_model_name is not None
load_path = exact_model_name or config.path
from utils.models.diffusion_lm import (
diffusion_lm_profile_for_model,
loaded_diffusion_profile,
)
diffusion_lm_profile = diffusion_lm_profile_for_model(
load_path, hf_token = _hf_token_for_loader(hf_token)
)
if not diffusion_lm_profile and config.is_lora and config.base_model:
# A Hub LoRA has no config.json: classify its base before the vision branch claims DiffusionGemma.
diffusion_lm_profile = diffusion_lm_profile_for_model(
config.base_model, hf_token = _hf_token_for_loader(hf_token)
)
# A LoRA loads its base model.
offload_kwargs = _offload_kwargs_for_fit(
config.base_model if config.is_lora and config.base_model else load_path,
device_map,
load_in_4bit,
_hf_token_for_loader(hf_token),
mapped = not use_exact_model_name,
)
if diffusion_lm_profile:
# Text-diffusion LMs (LLaDA, Dream, DiffusionGemma, ...) decode with their own sampler,
# never left-to-right generate(); DiffusionGemma's vision tower is not used here.
from unsloth import FastModel
model, tokenizer = FastModel.from_pretrained(
model_name = load_path,
max_seq_length = max_seq_length,
dtype = dtype,
**_load_in_4bit_kwargs(load_in_4bit),
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
use_exact_model_name = use_exact_model_name,
)
FastModel.for_inference(model)
loaded_profile = loaded_diffusion_profile(model)
if not loaded_profile:
# As training does: an autoregressive load must never reach the diffusion sampler.
raise RuntimeError(
f"'{load_path}' is a text-diffusion model ({diffusion_lm_profile}), but the installed "
"Unsloth loaded it as an ordinary model. Update with `unsloth studio update`."
)
self.models[model_name]["diffusion_lm_profile"] = loaded_profile
self.models[model_name]["is_vision"] = False
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = tokenizer
logger.info(
f"Loaded text-diffusion model for chat (profile: "
f"{self.models[model_name]['diffusion_lm_profile']})"
)
elif config.is_vision:
model, processor = _load_with_offload_fallback(
FastVisionModel.from_pretrained,
model_name = load_path,
max_seq_length = max_seq_length,
dtype = dtype,
**_load_in_4bit_kwargs(load_in_4bit),
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
use_exact_model_name = use_exact_model_name,
**offload_kwargs,
)
FastVisionModel.for_inference(model)
# FastVisionModel may return a raw tokenizer (e.g. Gemma-3); load the real processor.
from transformers import ProcessorMixin
if not (
isinstance(processor, ProcessorMixin) or hasattr(processor, "image_processor")
):
# LoRA adapters: use base model. Local merged exports: read base from export_metadata.json.
processor_source = config.base_model if config.is_lora else config.identifier
if not config.is_lora and config.is_local:
_meta_path = Path(config.path) / "export_metadata.json"
try:
if _meta_path.exists():
_meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig"))
if _meta.get("base_model"):
processor_source = _meta["base_model"]
except Exception:
pass
logger.warning(
f"FastVisionModel returned {type(processor).__name__} (no image_processor) "
f"for '{model_name}' — loading proper processor from '{processor_source}'"
)
from transformers import AutoProcessor
processor = AutoProcessor.from_pretrained(
processor_source,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
logger.info(f"Loaded {type(processor).__name__} from {processor_source}")
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = processor
self.models[model_name]["processor"] = processor
elif _audio_chat_auto_model(config) is not None:
# Its text load builds the bare backbone; the multimodal class carries generate().
from unsloth import FastModel
model, processor = _load_with_offload_fallback(
FastModel.from_pretrained,
model_name = load_path,
auto_model = _audio_chat_auto_model(config),
max_seq_length = max_seq_length,
dtype = dtype,
**_load_in_4bit_kwargs(load_in_4bit),
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
use_exact_model_name = use_exact_model_name,
**offload_kwargs,
)
FastModel.for_inference(model)
if not hasattr(processor, "feature_extractor"):
from transformers import AutoProcessor
processor = AutoProcessor.from_pretrained(
config.base_model if config.is_lora else load_path,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
)
_refuse_without_generate(model, model_name)
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = processor
self.models[model_name]["processor"] = processor
else:
model, tokenizer = _load_with_offload_fallback(
FastLanguageModel.from_pretrained,
model_name = load_path,
max_seq_length = max_seq_length,
dtype = dtype,
**_load_in_4bit_kwargs(load_in_4bit),
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
use_exact_model_name = use_exact_model_name,
**offload_kwargs,
)
FastLanguageModel.for_inference(model)
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = tokenizer
# A Hub LoRA has no config.json for the probe above, but core still loads a diffusion base
# through its diffusion path: decode it with its sampler, never left-to-right.
loaded_profile = loaded_diffusion_profile(model)
if loaded_profile:
self.models[model_name]["diffusion_lm_profile"] = loaded_profile
logger.info(f"Loaded text-diffusion model for chat (profile: {loaded_profile})")
else:
_refuse_without_generate(model, model_name)
if self.models[model_name].get("audio_type") == "audio_vlm":
self.models[model_name]["speaks"] = (
_speech_codec(
self.models[model_name].get("model"),
self.models[model_name].get("processor"),
)
is not None
)
raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")
disable_lfm3_triton_conv_where_unsupported(
self.models[model_name]["model"], torch, logger
)
self.models[model_name]["context_length"] = runtime_context_length(
self.models[model_name].get("model"),
max_seq_length,
)
self._resolve_chat_eos(model_name)
self._load_chat_template_info(model_name)
self.active_model_name = model_name
self.loading_models.discard(model_name)
logger.info(f"Successfully loaded model: {model_name}")
log_gpu_memory(f"After loading {model_name}")
return True
except Exception as e:
logger.error(f"Failed to load model: {e}")
error_msg = format_error_message(e, config.identifier)
if model_name in self.models:
del self.models[model_name]
self.loading_models.discard(model_name)
raise Exception(error_msg)
@_invalidates_gpu_memory("transformers unload")
def unload_model(self, model_name: str) -> bool:
"""Remove a model from the registry and clear GPU memory."""
if model_name in self.models:
try:
if self.models[model_name].get("is_audio"):
self._audio_codec_manager.unload()
logger.info(f"Unloading model '{model_name}' from memory.")
model = self.models[model_name].get("model")
del self.models[model_name]
_release_decode_graphs(model, getattr(self, "_generation_lock", None))
model = None
if self.active_model_name == model_name:
self.active_model_name = None
clear_gpu_cache()
# On spawn platforms keep trainer files so concurrent dataset.map() workers can import them.
import sys as _sys
from utils.cache_cleanup import clear_unsloth_compiled_cache
_preserve = ["Unsloth*Trainer.py"] if _sys.platform in ("win32", "darwin") else None
clear_unsloth_compiled_cache(preserve_patterns = _preserve)
logger.info(f"Model '{model_name}' successfully unloaded.")
return True
except Exception as e:
logger.error(f"Error while unloading model '{model_name}': {e}")
return False
else:
logger.warning(
f"Attempted to unload model '{model_name}', but it was not found in the registry."
)
return True
def revert_to_base_model(self, base_model_name: str) -> bool:
"""Revert the model to its pristine base state by unloading and deleting all adapter
configurations."""
if base_model_name not in self.models:
return False
model = self.models[base_model_name].get("model")
try:
if isinstance(model, (PeftModel, PeftModelForCausalLM)):
logger.info(f"Unloading LoRA adapters from '{base_model_name}'...")
unwrapped_base_model = model.unload()
self.models[base_model_name]["model"] = unwrapped_base_model
model = unwrapped_base_model
# A leftover peft_config causes "multiple adapters" warnings on the next from_pretrained().
if hasattr(model, "peft_config"):
del model.peft_config
logger.info(f"Model '{base_model_name}' reverted to clean base state.")
return True
except Exception as e:
logger.error(f"Failed to revert model to base state: {e}")
import traceback
logger.error(traceback.format_exc())
return False
def load_for_eval(
self,
lora_path: str,
max_seq_length: int = 2048,
dtype = None,
load_in_4bit: bool = True,
hf_token: Optional[str] = None,
gpu_ids: Optional[list[int]] = None,
) -> Tuple[bool, Optional[str], Optional[str]]:
"""Ensure the base model and the given adapter are loaded. Idempotent."""
try:
from utils.models import ModelConfig
lora_config = ModelConfig.from_lora_path(lora_path, hf_token)
if not lora_config:
return False, None, None
base_model_name = lora_config.base_model
if base_model_name not in self.models or not self.models[base_model_name].get("model"):
logger.info(f"Base model '{base_model_name}' not loaded, loading now.")
base_config = ModelConfig.from_ui_selection(base_model_name, None, is_lora = False)
if not self.load_model(
base_config,
max_seq_length,
dtype,
load_in_4bit,
hf_token,
gpu_ids = gpu_ids,
):
return False, None, None
self.active_model_name = base_model_name
adapter_name = lora_path.split("/")[-1].replace(".", "_")
# load_adapter only reads from disk if the model does not already have this adapter.
adapter_success = self.load_adapter(
base_model_name = base_model_name,
adapter_path = lora_path,
adapter_name = adapter_name,
)
if not adapter_success:
return False, base_model_name, None
return True, base_model_name, adapter_name
except Exception as e:
logger.error(f"Error during load_for_eval: {e}")
import traceback
logger.error(traceback.format_exc())
return False, None, None
def load_adapter(self, base_model_name: str, adapter_path: str, adapter_name: str) -> bool:
"""Load an adapter onto the model only if not already attached."""
model = self.models[base_model_name].get("model")
# Most reliable check: adapter name already in the model's config.
if hasattr(model, "peft_config") and adapter_name in model.peft_config:
logger.info(
f"Adapter '{adapter_name}' is already attached to the model. Skipping load."
)
return True
try:
logger.info(
f"Loading new adapter '{adapter_name}' from '{adapter_path}' onto {base_model_name}"
)
model.load_adapter(adapter_path, adapter_name = adapter_name)
if "loaded_adapters" not in self.models[base_model_name]:
self.models[base_model_name]["loaded_adapters"] = {}
self.models[base_model_name]["loaded_adapters"][adapter_name] = adapter_path
total_adapters = len(getattr(model, "peft_config", {}))
logger.info(
f"Adapter '{adapter_name}' loaded successfully. (Total unique adapters on model: {total_adapters})"
)
return True
except Exception as e:
logger.error(f"Failed to load adapter '{adapter_name}': {e}")
return False
def set_active_adapter(self, base_model_name: str, adapter_name: str) -> bool:
model = self.models[base_model_name].get("model")
try:
logger.info(f"Setting active adapter to: '{adapter_name}'")
model.set_adapter(adapter_name)
self.models[base_model_name]["active_adapter"] = adapter_name
return True
except Exception as e:
logger.error(f"Failed to set active adapter to '{adapter_name}': {e}")
return False
def _apply_adapter_state(self, use_adapter: Optional[Union[bool, str]]) -> None:
"""Must hold _generation_lock. use_adapter: None keeps, False base, True current, str named adapter."""
if use_adapter is None:
return
base = self.active_model_name
if not base or base not in self.models:
return
model_info = self.models[base]
model = model_info.get("model")
if model is None:
return
if use_adapter is False:
if isinstance(model, (PeftModel, PeftModelForCausalLM)):
logger.info(
f"Compare mode: disabling adapters on '{base}' for base model generation"
)
model.base_model.disable_adapter_layers()
else:
logger.info(f"Compare mode: model '{base}' is not a PeftModel, already base")
elif use_adapter is True:
if isinstance(model, (PeftModel, PeftModelForCausalLM)):
logger.info(f"Compare mode: enabling adapters on '{base}' for LoRA generation")
model.base_model.enable_adapter_layers()
else:
logger.warning("use_adapter=true but model is not a PeftModel")
elif isinstance(use_adapter, str):
if isinstance(model, (PeftModel, PeftModelForCausalLM)):
logger.info(f"Compare mode: enabling adapter '{use_adapter}' on '{base}'")
model.base_model.enable_adapter_layers()
self.set_active_adapter(base, use_adapter)
else:
logger.warning(f"use_adapter='{use_adapter}' but model is not a PeftModel")
@contextlib.contextmanager
def _adapter_scope(self, use_adapter: Optional[Union[bool, str]]):
"""Must hold _generation_lock. A base-model (False) request re-enables the LoRA on exit: plain requests keep the adapter state."""
try:
self._apply_adapter_state(use_adapter)
yield
finally:
model = (self.models.get(self.active_model_name) or {}).get("model")
if use_adapter is False and isinstance(model, PeftModel):
model.base_model.enable_adapter_layers()
def generate_with_adapter_control(
self,
use_adapter: Optional[Union[bool, str]] = None,
cancel_event = None,
**gen_kwargs,
) -> Generator[str, None, None]:
"""Toggle and generate are serialized by _generation_lock so SSE handlers on one loop cannot
race."""
yield from self._generate_chat_response_inner(
cancel_event = cancel_event, _adapter_state = use_adapter, **gen_kwargs
)
def generate_chat_completion_with_tools(
self,
messages: list,
tools: list,
system_prompt: str = "",
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
min_p: float = 0.0,
max_new_tokens: Optional[int] = 2048,
repetition_penalty: float = 1.0,
cancel_event = None,
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
continue_final_message: bool = False,
max_tool_iterations: int = 25,
auto_heal_tool_calls: bool = True,
nudge_tool_calls: Optional[bool] = None,
tool_call_timeout: int = 300,
session_id: Optional[str] = None,
thread_id: Optional[str] = None,
rag_scope: Optional[dict] = None,
website_policy: Optional[dict] = None,
presence_penalty: float = 0.0,
reasoning_prefilled: bool = False,
deduplicate_tool_calls: bool = True,
):
"""Yields the same event-dict protocol as the GGUF path so one route helper streams both
backends."""
from core.inference.safetensors_agentic import run_safetensors_tool_loop
from core.inference.tools import execute_tool
# Latest turn's usage only; cleared every turn so a failed turn leaves it empty, not stale.
_turn_stats: dict = {}
def _single_turn(
conv: list,
*,
active_tools: Optional[list[dict]] = None,
tool_protocol_active: Optional[bool] = None,
):
# Conv has the system message; active_tools lets the loop drop one-shot tools like render_html.
turn_tools = active_tools if active_tools is not None else tools
_turn_stats["stats"] = None
try:
yield from self._generate_chat_response_inner(
messages = conv,
system_prompt = "",
temperature = temperature,
top_p = top_p,
top_k = top_k,
min_p = min_p,
max_new_tokens = max_new_tokens,
repetition_penalty = repetition_penalty,
cancel_event = cancel_event,
tools = turn_tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
# Self-limiting: the conversation ends on a tool result, so later turns render as ordinary turns.
continue_final_message = continue_final_message,
presence_penalty = presence_penalty,
tool_protocol_active = tool_protocol_active,
)
finally:
_turn_stats["stats"] = self.last_generation_stats
initial = list(messages)
if system_prompt:
initial = [{"role": "system", "content": system_prompt}] + initial
# Same profile as the renderer; catalog must be safe under every template this turn may use.
from core.inference.chat_template_helpers import (
mapped_chat_template,
markup_for_tokenizer,
renderable_tool_catalog,
)
_model_info = self.models.get(self.active_model_name) or {}
# Resolved BEFORE the profile: the mapper installs its template during the render.
_mapped_tpl = mapped_chat_template(_model_info, self.active_model_name)
yield from run_safetensors_tool_loop(
markup = markup_for_tokenizer(_model_info.get("tokenizer"), tools, _mapped_tpl),
renderable_tools = renderable_tool_catalog(
tools,
_model_info.get("tokenizer"),
_model_info,
active_model_name = self.active_model_name,
template = _mapped_tpl,
),
single_turn = _single_turn,
messages = initial,
tools = tools,
execute_tool = execute_tool,
cancel_event = cancel_event,
auto_heal_tool_calls = auto_heal_tool_calls,
deduplicate_tool_calls = deduplicate_tool_calls,
nudge_tool_calls = nudge_tool_calls,
max_tool_iterations = max_tool_iterations,
tool_call_timeout = tool_call_timeout,
session_id = session_id,
thread_id = thread_id,
rag_scope = rag_scope,
website_policy = website_policy,
reasoning_prefilled = reasoning_prefilled,
continue_final_message = continue_final_message,
# So a conversation search can be sized against what this model can hold.
context_length = _model_info.get("context_length"),
max_tokens = max_new_tokens,
generation_stats_holder = _turn_stats,
)
def generate_chat_response(
self,
messages: list,
system_prompt: str,
image = None,
images: Optional[list] = None,
image_ordinal: Optional[int] = None,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
min_p: float = 0.0,
max_new_tokens: Optional[int] = 256,
repetition_penalty: float = 1.0,
cancel_event = None,
tools: Optional[list] = None,
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
continue_final_message: bool = False,
presence_penalty: float = 0.0,
tool_protocol_active: Optional[bool] = None,
stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""Forwards tools and reasoning flags into apply_chat_template for templates that understand
them."""
yield from self._generate_chat_response_inner(
messages = messages,
system_prompt = system_prompt,
image = image,
images = images,
image_ordinal = image_ordinal,
temperature = temperature,
top_p = top_p,
top_k = top_k,
min_p = min_p,
max_new_tokens = max_new_tokens,
repetition_penalty = repetition_penalty,
cancel_event = cancel_event,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
continue_final_message = continue_final_message,
tool_protocol_active = tool_protocol_active,
presence_penalty = presence_penalty,
stop = stop,
)
def _generate_chat_response_inner(
self,
messages: list,
system_prompt: str = "",
image = None,
images: Optional[list] = None,
image_ordinal: Optional[int] = None,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
min_p: float = 0.0,
max_new_tokens: Optional[int] = 256,
repetition_penalty: float = 1.0,
cancel_event = None,
_adapter_state = None,
tools: Optional[list] = None,
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
continue_final_message: bool = False,
presence_penalty: float = 0.0,
tool_protocol_active: Optional[bool] = None,
stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""Keeps native tool tokens per tool_protocol_active: unrestricted mode sends an empty tools
list."""
if not self.active_model_name:
raise RuntimeError("No active model")
model_info = self.models[self.active_model_name]
if model_info.get("diffusion_lm_profile"):
template_messages = (
[{"role": "system", "content": system_prompt}] + messages
if system_prompt
else messages
)
yield from self._generate_diffusion_lm(
model_info,
messages = _without_image_parts(template_messages),
temperature = temperature,
max_new_tokens = max_new_tokens,
cancel_event = cancel_event,
_adapter_state = _adapter_state,
stop = stop,
)
return
is_vision = model_info.get("is_vision", False)
tokenizer = model_info.get("tokenizer") or model_info.get("processor")
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
top_k = self._normalize_top_k(top_k)
if is_deepseek_ocr(model_info.get("model")):
yield from self._generate_deepseek_ocr_response(
messages, image, images, max_new_tokens, cancel_event, _adapter_state, stop
)
return
if is_vision and (image or images):
# FastVisionModel may return a raw tokenizer, not a ProcessorMixin (e.g. Gemma-3), so check first.
from transformers import ProcessorMixin
processor = model_info.get("processor")
has_image_processing = processor is not None and (
isinstance(processor, ProcessorMixin) or hasattr(processor, "image_processor")
)
if has_image_processing:
yield from self._generate_vision_response(
messages,
system_prompt,
image,
temperature,
top_p,
top_k,
min_p,
max_new_tokens,
repetition_penalty,
cancel_event = cancel_event,
_adapter_state = _adapter_state,
presence_penalty = presence_penalty,
continue_final_message = continue_final_message,
tools = tools,
images = images,
image_ordinal = image_ordinal,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
tool_protocol_active = tool_protocol_active,
stop = stop,
)
return
else:
logger.warning(
f"Model '{self.active_model_name}' is marked as vision but its processor "
f"({type(processor).__name__}) has no image_processor — "
f"falling back to text-only generation (image will be ignored)."
)
# Strip image placeholders: a text template rejects them or renders tokens with no pixels.
messages = _without_image_parts(messages)
# Text path: messages are already in ChatML format from eval.py.
try:
from utils.datasets import (
MODEL_TO_TEMPLATE_MAPPER,
get_tokenizer_chat_template,
)
model_name_lower = self.active_model_name.lower()
if model_name_lower in MODEL_TO_TEMPLATE_MAPPER:
template_name = MODEL_TO_TEMPLATE_MAPPER[model_name_lower]
logger.info(
f"Applying chat template '{template_name}' for {self.active_model_name}"
)
tokenizer = _mapped_template_tokenizer(model_info, tokenizer, template_name)
# The mapper installs the template only at generate time: UNION into the load-time cache.
# Take marker strings from the mapped template but resolve ids on the original tokenizer.
try:
_gen_tok = model_info.get("tokenizer") or tokenizer
refreshed = resolve_chat_turn_end_eos_ids_using(
getattr(tokenizer, "tokenizer", tokenizer),
getattr(_gen_tok, "tokenizer", _gen_tok),
)
existing = model_info.get("chat_turn_end_eos_ids") or []
model_info["chat_turn_end_eos_ids"] = sorted(set(existing) | set(refreshed))
except Exception as e:
logger.warning(f"Could not refresh chat turn-end eos after template: {e}")
else:
logger.info(
f"No registered Unsloth template for {self.active_model_name}, using tokenizer default"
)
except Exception as e:
logger.warning(f"Could not apply get_chat_template: {e}")
if system_prompt:
template_messages = [{"role": "system", "content": system_prompt}] + messages
else:
template_messages = messages
reasoning_channel_markers_resolved = False
add_special_tokens = True
bos_probe = None
try:
if not (hasattr(tokenizer, "chat_template") and tokenizer.chat_template):
raise ValueError(
f"Model '{self.active_model_name}' has no chat_template set in its "
f"tokenizer_config.json. This is usually a problem with the model's "
f"HuggingFace repository — it is missing a 'chat_template' key. "
f"Please use a model that includes a chat template, or manually set "
f"one via tokenizer.chat_template before inference."
)
reasoning_channel_markers = None
from core.inference.chat_template_helpers import (
messages_have_tool_history,
render_native_template,
render_with_native_template_fallback,
)
try:
formatted_prompt = self._apply_chat_template_for_generation(
tokenizer,
template_messages,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
continue_final_message = continue_final_message,
)
except Exception:
if not messages_have_tool_history(template_messages):
raise
render_result = render_native_template(
model_info = model_info,
active_model_name = self.active_model_name,
messages = template_messages,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
continue_final_message = continue_final_message,
apply_fn = self._apply_chat_template_for_generation,
hf_token = model_info.get("hf_token"),
return_metadata = True,
)
if render_result is None:
raise
else:
# use the native template shared with MLX when an override ignores requested tools.
render_result = render_with_native_template_fallback(
formatted_prompt = formatted_prompt,
tokenizer = tokenizer,
model_info = model_info,
active_model_name = self.active_model_name,
messages = template_messages,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
continue_final_message = continue_final_message,
apply_fn = self._apply_chat_template_for_generation,
hf_token = model_info.get("hf_token"),
return_metadata = True,
)
formatted_prompt = render_result.prompt
reasoning_channel_markers = render_result.reasoning_channel_markers
reasoning_channel_markers_resolved = True
# suppress the tokenizer's BOS only when the template already emitted one.
bos_probe = _probe_prompt_bos(tokenizer, formatted_prompt)
add_special_tokens = bos_probe is None
logger.debug(f"Formatted prompt: {formatted_prompt[:200]}...")
except Exception as e:
logger.error(f"Error applying chat template: {e}")
formatted_prompt = self.format_chat_prompt(
messages, system_prompt, continue_final_message = continue_final_message
)
reasoning_channel_markers = None
reasoning_channel_markers_resolved = True
yield from self.generate_stream(
formatted_prompt,
temperature,
top_p,
top_k,
min_p,
max_new_tokens,
repetition_penalty,
cancel_event = cancel_event,
_adapter_state = _adapter_state,
presence_penalty = presence_penalty,
reasoning_channel_markers = reasoning_channel_markers,
reasoning_channel_markers_resolved = reasoning_channel_markers_resolved,
continued = bool(continue_final_message and trailing_assistant_text(template_messages)),
preserve_tool_tokens = bool(tools)
if tool_protocol_active is None
else tool_protocol_active,
add_special_tokens = add_special_tokens,
stop = stop,
_bos_probe = bos_probe,
)
def _generate_deepseek_ocr_response(
self,
messages,
image,
images,
max_new_tokens,
cancel_event,
_adapter_state,
stop = None,
) -> Generator[str, None, None]:
"""DeepSeek-OCR ships no processor or chat template: its own infer() tiles the image."""
import tempfile
from core.inference.chat_template_helpers import last_user_text
self.last_generation_stats = None
picture = image if image is not None else next(iter(images or ()), None)
if picture is None:
raise _OcrNeedsImageError(
"DeepSeek-OCR reads text from an image: attach one to your message."
)
model_info = self.models[self.active_model_name]
tokenizer = model_info.get("tokenizer") or model_info.get("processor")
prompt = "<image>\n" + (last_user_text(messages) or "Free OCR.")
infer = model_info["model"].infer
# infer() hardcodes max_new_tokens=8192 and no stopping criteria: bound the generate() it calls.
owner = infer.__self__
previous = owner.__dict__.get("generate", _MISSING)
generate = owner.generate
stopping_criteria = self._cancel_stopping_criteria(cancel_event)
def bounded_generate(*args, **kwargs):
if max_new_tokens:
kwargs["max_new_tokens"] = min(
kwargs.get("max_new_tokens", max_new_tokens), max_new_tokens
)
if stopping_criteria is not None:
kwargs["stopping_criteria"] = stopping_criteria
return generate(*args, **kwargs)
with (
tempfile.TemporaryDirectory() as tmp,
self._generation_lock,
self._adapter_scope(_adapter_state),
):
path = str(Path(tmp) / "image.png")
picture.convert("RGB").save(path)
owner.generate = bounded_generate
try:
# Gundam preset from each model card: 640 px tiles for v1, 768 for OCR-2.
text = infer(
getattr(tokenizer, "tokenizer", tokenizer),
prompt = prompt,
image_file = path,
output_path = tmp,
base_size = 1024,
image_size = 640
if "DeepseekOCRForCausalLM" in model_info["model"].config.architectures
else 768,
crop_mode = True,
save_results = False,
eval_mode = True,
)
finally:
if previous is _MISSING:
del owner.generate
else:
owner.generate = previous
text = text or ""
for sequence in stop or ():
if sequence and sequence in text:
text = text[: text.index(sequence)]
yield text
def _generate_vision_response(
self,
messages,
system_prompt,
image,
temperature,
top_p,
top_k,
min_p,
max_new_tokens,
repetition_penalty,
cancel_event = None,
_adapter_state = None,
presence_penalty: float = 0.0,
continue_final_message: bool = False,
tools: Optional[list] = None,
images: Optional[list] = None,
image_ordinal: Optional[int] = None,
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
tool_protocol_active: Optional[bool] = None,
stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""Handle vision model generation with true token-by-token streaming."""
# Reset so a failed or uncountable run cannot surface stale stats.
self.last_generation_stats = None
model_info = self.models[self.active_model_name]
model = model_info["model"]
processor = model_info["processor"]
# FastVisionModel may return a raw tokenizer; unwrap for tokenize-only ops.
raw_tokenizer = getattr(processor, "tokenizer", processor)
# Scans back so a continuation ending in an assistant partial still prompts from the user turn.
from core.inference.chat_template_helpers import (
last_user_text,
messages_have_tool_history,
messages_with_attached_image,
render_advertising_tools,
trailing_assistant_text,
vlm_prompt_issue,
)
from core.inference.mcp_images import (
image_marker_parts,
pixels_in_marker_order,
top_up_image_markers,
)
# History first, attachment last: pixels bind to markers in order.
attached = list(images or []) + ([image] if image is not None else [])
user_message = last_user_text(messages)
continue_partial = trailing_assistant_text(messages) if continue_final_message else None
if not user_message:
user_message = "Describe this image." if attached else "Hello"
if attached:
has_tool_history = messages_have_tool_history(messages)
# tool_choice="none" and a forced unknown name also arrive with tools=None (#10092).
folded_system = not system_prompt and any(
isinstance(m, dict) and m.get("role") in ("system", "developer") for m in messages
)
# Rebuilding from the newest user text would drop the system turn and OpenAI tool history.
vision_messages = messages_with_attached_image(
messages,
system_prompt = system_prompt,
fallback_user_text = user_message,
structured_content = True,
)
# Existing markers are left alone, which is wrong for replayed MCP pictures.
_prior_markers = image_marker_parts(vision_messages)
vision_messages = top_up_image_markers(
vision_messages, len(attached), ordinal = image_ordinal
)
if image is not None:
attached = pixels_in_marker_order(
vision_messages, _prior_markers, list(images or []), image
)
if bool(tools) or has_tool_history or folded_system:
# The conversation the LAST render used, not the no-tools probe's (#10092).
rendered_with: dict = {"messages": vision_messages}
def _render_vision(catalog):
rendered_with["messages"] = vision_messages
try:
rendered = self._apply_chat_template_for_generation(
processor,
vision_messages,
tools = catalog,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
continue_final_message = bool(continue_partial),
)
except Exception as e: # noqa: F841 -- read by the fallback below
without_system = [
m
for m in vision_messages
if not (isinstance(m, dict) and m.get("role") == "system")
]
# Only where an unsupported system role is the last explanation.
if (
has_tool_history
or rendered_with.get("system_ok")
or len(without_system) == len(vision_messages)
):
raise
logger.warning(
f"Vision processor for '{self.active_model_name}' may not support "
f"system messages; retrying without. Original error: {e}"
)
rendered = self._apply_chat_template_for_generation(
processor,
without_system,
tools = catalog,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
continue_final_message = bool(continue_partial),
)
rendered_with["messages"] = without_system
return rendered
else:
# A render that kept the system turn proves the role is supported.
if any(
isinstance(m, dict) and m.get("role") == "system"
for m in vision_messages
):
rendered_with["system_ok"] = True
return rendered
input_text, tools_advertised = render_advertising_tools(_render_vision, tools)
vision_messages = rendered_with["messages"]
prompt_issue = vlm_prompt_issue(input_text, vision_messages)
if prompt_issue and has_tool_history:
raise RuntimeError(
f"Vision chat template returned {prompt_issue} and cannot be recovered "
"without dropping tool-call history."
)
if prompt_issue:
raise RuntimeError(f"Vision chat template returned {prompt_issue}.")
if tools and not tools_advertised:
# Served anyway: refusing turns an answerable question into an error.
logger.warning(
"Vision chat template for '%s' rendered the same prompt with and "
"without the requested tools; serving the turn without the catalog.",
self.active_model_name,
)
else:
def _render_plain_vision(msgs):
return self._apply_chat_template_for_generation(
processor,
msgs,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
continue_final_message = bool(continue_partial),
)
try:
input_text = _render_plain_vision(vision_messages)
except Exception as e:
# Safe here: no catalog and no tool history to hide a failure behind.
if system_prompt:
logger.warning(
f"Vision processor for '{self.active_model_name}' may not support "
f"system messages; retrying without. Original error: {e}"
)
vision_messages = [m for m in vision_messages if m.get("role") != "system"]
input_text = _render_plain_vision(vision_messages)
else:
raise
_images = attached[0] if len(attached) == 1 else attached
if _processor_takes_text_first(processor):
inputs = processor(
text = input_text,
images = _images,
add_special_tokens = False,
return_tensors = "pt",
).to(model.device)
else:
inputs = processor(
_images,
input_text,
add_special_tokens = False,
return_tensors = "pt",
).to(model.device)
prompt_text = input_text
else:
formatted_prompt = self.format_chat_prompt(
messages, system_prompt, continue_final_message = continue_final_message
)
bos_probe = _probe_prompt_bos(raw_tokenizer, formatted_prompt)
inputs = _tokenize_prompt(
raw_tokenizer, formatted_prompt, bos_probe is None, bos_probe
).to(model.device)
prompt_text = formatted_prompt
try:
# Re-emit open <think> prefill swallowed by skip_prompt; image turns carry tools too, so it applies.
_preserve_tool_tokens = (
bool(tools) if tool_protocol_active is None else tool_protocol_active
)
think_prefix = detect_think_prefill(
prompt_text,
getattr(raw_tokenizer, "all_special_tokens", None),
# Ask the decoder: with no usable all_special_ids it skips specials and drops the closer.
preserves_think_close = _preserve_tool_tokens
and decoder_preserves_token(raw_tokenizer, "</think>"),
)
import threading
streamer = self._make_text_streamer(
raw_tokenizer,
protocol_source = processor,
# The text-only fallback did not use the processor template; pass tools to match the render.
reasoning_channel_markers = detect_reasoning_channel_markers(processor, tools = tools)
# Any attached image: no markers here would leak native reasoning as visible answer text.
if attached
else None,
reasoning_channel_markers_resolved = True,
prompt = prompt_text,
continued = bool(continue_partial),
skip_prompt = True,
timeout = 0.2,
cancel_event = cancel_event,
use_harmony = self._is_gpt_oss_model(),
preserve_tool_tokens = _preserve_tool_tokens,
)
_vision_input_ids = inputs.get("input_ids") if hasattr(inputs, "get") else None
prompt_len = int(_vision_input_ids.shape[1]) if _vision_input_ids is not None else None
max_new_tokens = generation_budget_within_context(
model, prompt_len or 0, max_new_tokens
)
generation_kwargs = dict(
**inputs,
streamer = streamer,
max_new_tokens = max_new_tokens,
use_cache = True,
do_sample = temperature > 0,
temperature = temperature,
top_p = top_p,
top_k = top_k,
min_p = min_p,
)
# Presence penalty (GGUF parity) for VLM chat.
_pp = (
_make_presence_penalty_processor(presence_penalty, prompt_len)
if _vision_input_ids is not None
else None
)
if repetition_penalty != 1.0 and _vision_input_ids is not None:
from transformers import LogitsProcessorList, RepetitionPenaltyLogitsProcessor
# Prompt ids skipped: mllama's <|image|> id lies past the LM head.
try:
_rp = RepetitionPenaltyLogitsProcessor(
repetition_penalty, prompt_ignore_length = prompt_len
)
except TypeError:
# prompt_ignore_length landed in transformers 4.52; the floor is 4.51.3.
class _PromptSkippingRepetitionPenalty(RepetitionPenaltyLogitsProcessor):
def __call__(self, input_ids, scores):
return super().__call__(input_ids[:, prompt_len:], scores)
_rp = _PromptSkippingRepetitionPenalty(repetition_penalty)
_pp = LogitsProcessorList([_rp, *(_pp or [])])
timer = GenerationTimer()
generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)
stop_streamer = _StopSequenceStreamer(streamer, stop)
if stop_streamer.sequences:
generation_kwargs["streamer"] = stop_streamer
stopping_criteria = self._cancel_stopping_criteria(cancel_event, stop_streamer.matched)
if stopping_criteria is not None:
generation_kwargs["stopping_criteria"] = stopping_criteria
active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs)
err: dict[str, str] = {}
gen_outputs: dict = {}
def generate_fn():
with self._generation_lock, self._adapter_scope(_adapter_state):
try:
# Started inside the lock so a queued request's wait is not billed as prefill.
timer.start()
# Only the returned sequences carry an exact generated-token count; see generate_stream.
sequences = _generate_supported(model, generation_kwargs)
if sequences is not None:
gen_outputs["sequences"] = sequences
except Exception as e:
err["msg"] = str(e)
stop_streamer.abort()
logger.error(f"Vision generation error in thread: {e}")
finally:
timer.finish()
try:
stop_streamer.end()
except Exception:
pass
thread = threading.Thread(target = generate_fn)
thread.start()
output = think_prefix
# Emit prefilled <think> before the first token so the block renders during slow prompt prefill.
if think_prefix:
yield think_prefix
from queue import Empty
import time
generation_complete = False
cancel_deadline = None
try:
while True:
if cancel_event is not None and cancel_event.is_set():
if cancel_deadline is None:
cancel_deadline = time.monotonic() + 10
elif time.monotonic() >= cancel_deadline:
break
try:
new_token = next(stop_streamer)
except StopIteration:
generation_complete = True
break
except Empty:
if not thread.is_alive():
generation_complete = True
output = yield from self._drain_streamer_tail(
stop_streamer, output, active_stop_token_ids
)
break
if cancel_deadline is not None:
remaining = cancel_deadline - time.monotonic()
if remaining <= 0:
break
thread.join(timeout = remaining)
if thread.is_alive():
break
generation_complete = True
output = yield from self._drain_streamer_tail(
stop_streamer, output, active_stop_token_ids
)
break
continue
if new_token:
output, cleaned = self._append_stream_delta(
output, new_token, active_stop_token_ids
)
yield cleaned
finally:
if cancel_event is not None and not generation_complete:
cancel_event.set()
join_timeout = 10
if cancel_deadline is not None:
join_timeout = max(0, cancel_deadline - time.monotonic())
thread.join(timeout = join_timeout)
if thread.is_alive():
logger.warning(
"Vision generation thread did not exit after cancel/join timeout"
)
if "sequences" in gen_outputs:
self._record_generation_stats(
prompt_tokens = prompt_len,
completion_tokens = self._generated_token_count(
model, gen_outputs["sequences"], prompt_len
),
max_new_tokens = max_new_tokens,
ended_on_stop_token = stop_streamer.matched.is_set()
or self._ended_on_stop_token(gen_outputs["sequences"], active_stop_token_ids),
cancelled = not generation_complete
or (cancel_event is not None and cancel_event.is_set()),
timer = timer,
)
if err.get("msg"):
raise _GenerationThreadError(err["msg"])
except _GenerationThreadError:
raise
except Exception as e:
logger.error(f"Vision generation error: {e}")
raise
def generate_audio_input_response(
self,
messages,
system_prompt,
audio_array,
temperature,
top_p,
top_k,
min_p,
max_new_tokens,
repetition_penalty,
use_adapter: Optional[Union[bool, str]] = None,
cancel_event = None,
extra_audio_arrays: Optional[list] = None,
tools: Optional[list] = None,
speak: bool = False,
) -> Generator[str, None, None]:
"""Audio goes into the chat messages via processor.apply_chat_template (Gemma 3n pattern).
``tools`` (a list, empty for tool history alone) renders the catalogue and keeps
``messages`` as sent. ``speak`` answers in the model's own voice, as the MLX backend
does: the reply text, then one ``{"speech": <base64 wav>, "sample_rate"}`` item."""
import threading
import numpy as np
# Reset so a failed or uncountable run cannot surface stale stats.
self.last_generation_stats = None
if speak:
if extra_audio_arrays:
raise RuntimeError("A spoken reply answers one audio clip.")
yield from self._speak_audio_reply(
messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event
)
return
model_info = self.models[self.active_model_name]
model = model_info["model"]
processor = model_info.get("processor") or model_info.get("tokenizer")
raw_tokenizer = getattr(processor, "tokenizer", processor)
if not system_prompt:
system_prompt = "You are an assistant that transcribes speech accurately."
# Gemma 3n format: audio goes INTO apply_chat_template, one item per clip in order.
audio_messages = messages_with_attached_image(
alternating_turns(messages) if tools is None else messages,
system_prompt = system_prompt,
fallback_user_text = "Please transcribe this audio.",
structured_content = True,
image = 0,
audio = audio_array,
extra_audio = extra_audio_arrays or (),
)
# Direct processor render, so neutralize here with this processor's own profile (#7066).
from core.inference.chat_template_helpers import neutralize_for_render
audio_messages, tools, _ = neutralize_for_render(processor, audio_messages, tools)
inputs = processor.apply_chat_template(
audio_messages,
add_generation_prompt = True,
tokenize = True,
return_dict = True,
return_tensors = "pt",
truncation = False,
**({"tools": tools} if tools else {}),
).to(model.device)
try:
from queue import Empty
# As the text path: a rendered catalogue keeps native tool-call tokens for the healer.
streamer = IncrementalTextIteratorStreamer(
NativeToolTokenDecoder(raw_tokenizer) if tools else raw_tokenizer,
skip_prompt = True,
skip_special_tokens = not tools,
timeout = 0.2,
)
_audio_input_ids = inputs.get("input_ids") if hasattr(inputs, "get") else None
prompt_len = int(_audio_input_ids.shape[1]) if _audio_input_ids is not None else None
max_new_tokens = generation_budget_within_context(
model, prompt_len or 0, max_new_tokens
)
# Notebook uses do_sample=False (greedy) for ASR accuracy
generation_kwargs = dict(
**inputs,
streamer = streamer,
max_new_tokens = max_new_tokens,
use_cache = True,
do_sample = False,
)
timer = GenerationTimer()
generation_kwargs["logits_processor"] = with_prefill_boundary_processor(None, timer)
stopping_criteria = self._cancel_stopping_criteria(cancel_event)
if stopping_criteria is not None:
generation_kwargs["stopping_criteria"] = stopping_criteria
active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs)
err: dict[str, str] = {}
gen_outputs: dict = {}
def generate_fn():
with self._generation_lock, self._adapter_scope(use_adapter):
try:
# Started after the adapter swap so only model.generate() is timed.
timer.start()
# Only the returned sequences carry an exact generated-token count; see generate_stream.
sequences = _generate_supported(model, generation_kwargs)
if sequences is not None:
gen_outputs["sequences"] = sequences
except Exception as e:
err["msg"] = str(e)
logger.error(f"Audio input generation error in thread: {e}")
finally:
timer.finish()
try:
streamer.end()
except Exception:
pass
thread = threading.Thread(target = generate_fn)
thread.start()
output = ""
# The finally always sets cancel_event, so this flag tells a user stop from a normal end.
generation_complete = False
try:
while True:
if cancel_event is not None and cancel_event.is_set():
break
try:
new_token = next(streamer)
except StopIteration:
generation_complete = True
break
except Empty:
if not thread.is_alive():
generation_complete = True
break
continue
if new_token:
output += new_token
yield new_token
finally:
if cancel_event is not None:
cancel_event.set()
thread.join(timeout = 10)
if thread.is_alive():
logger.warning(
"Audio input generation thread did not exit after cancel/join timeout"
)
if "sequences" in gen_outputs:
self._record_generation_stats(
prompt_tokens = prompt_len,
completion_tokens = self._generated_token_count(
model, gen_outputs["sequences"], prompt_len
),
max_new_tokens = max_new_tokens,
ended_on_stop_token = self._ended_on_stop_token(
gen_outputs["sequences"], active_stop_token_ids
),
cancelled = not generation_complete,
timer = timer,
)
if err.get("msg"):
raise _GenerationThreadError(err["msg"])
except _GenerationThreadError:
raise
except Exception as e:
logger.error(f"Audio input generation error: {e}")
raise
def _speak_audio_reply(
self, messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event
):
import base64
system = "\n\n".join(p for p in (_SPEECH_REPLY_SYSTEM_PROMPT, system_prompt) if p)
conversation = messages_with_attached_image(
alternating_turns(messages),
system_prompt = system,
structured_content = True,
image = 0,
audio = audio_array,
)
result = self._generate_speech(
conversation, "<|mixed_start|>", max_new_tokens, use_adapter, cancel_event
)
if result is None:
return
text, wav, sample_rate = result
if text:
yield text
if wav:
yield {"speech": base64.b64encode(wav).decode("ascii"), "sample_rate": sample_rate}
def _speak_text(self, text, max_new_tokens, use_adapter, cancel_event, language, audio_options):
"""The loaded LFM3 model reads ``text`` aloud in its TTS mode: ``(wav_bytes, sample_rate)``."""
from core.inference.audio_errors import AudioGenerationCancelledError
conversation = [
{"role": "system", "content": _tts_system_prompt(language, audio_options)},
{"role": "user", "content": text},
]
result = self._generate_speech(
conversation, "<|audio_start|>", max_new_tokens, use_adapter, cancel_event
)
if result is None:
raise AudioGenerationCancelledError("Audio generation cancelled")
_, wav, sample_rate = result
if not wav:
raise RuntimeError("The model returned no speech.")
return wav, sample_rate
def _generate_speech(self, conversation, prefill, max_new_tokens, use_adapter, cancel_event):
"""Render ``conversation`` with the assistant turn opened by ``prefill`` and generate.
Returns ``(text, wav_bytes or None, sample_rate)``, or None once stopped."""
from core.inference.audio_codecs import _numpy_to_wav_bytes
from core.inference.chat_template_helpers import neutralize_for_render
model_info = self.models[self.active_model_name]
model = model_info["model"]
processor = model_info.get("processor") or model_info.get("tokenizer")
codec = _speech_codec(model, processor)
if codec is None:
raise RuntimeError("The loaded model cannot answer in speech.")
# Neutralize before the prefill, which is itself control markup (#7066).
conversation, _, _ = neutralize_for_render(processor, conversation, None)
conversation = [*conversation, {"role": "assistant", "content": prefill}]
inputs = processor.apply_chat_template(
conversation,
continue_final_message = True,
tokenize = True,
return_dict = True,
return_tensors = "pt",
truncation = False,
).to(model.device)
prompt_len = int(inputs["input_ids"].shape[1])
budget = generation_budget_within_context(model, prompt_len, max_new_tokens)
# Unset: the free window, capped as the MLX path caps it.
max_new_tokens = budget if max_new_tokens else min(budget, UNSET_GENERATION_BUDGET)
kwargs = dict(
**inputs,
max_new_tokens = max_new_tokens,
do_sample = False,
return_dict_in_generate = True,
)
if cancel_event is not None:
stopper = _speech_stop_streamer(cancel_event)
kwargs.update(streamer = stopper, audio_streamer = stopper)
timer = GenerationTimer()
with self._generation_lock, self._adapter_scope(use_adapter):
if cancel_event is not None and cancel_event.is_set():
return None
timer.start()
try:
out = model.generate(**kwargs)
except _SpeechStopped:
return None
finally:
timer.finish()
lengths = getattr(out, "audio_lengths", None)
frames = int(lengths[0]) if lengths is not None else 0
wav, sample_rate = None, int(getattr(codec.config, "sampling_rate", None) or 24000)
if frames > 0:
device = next(model.parameters()).device
if next(codec.parameters()).device != device:
codec.to(device = device)
with torch.inference_mode():
waveform = codec.decode(out.audio_codes[:1, :, :frames]).audio_values[0]
wav = _numpy_to_wav_bytes(waveform.float().cpu().numpy(), sample_rate)
if cancel_event is not None and cancel_event.is_set():
return None
raw_tokenizer = getattr(processor, "tokenizer", processor)
text = raw_tokenizer.decode(out.sequences[0, prompt_len:], skip_special_tokens = True)
self._record_generation_stats(
prompt_tokens = prompt_len,
completion_tokens = int(out.sequences.shape[1]) - prompt_len,
max_new_tokens = max_new_tokens,
timer = timer,
)
return text.strip(), wav, sample_rate
def generate_whisper_response(
self,
audio_array,
use_adapter: Optional[Union[bool, str]] = None,
cancel_event = None,
extra_audio_arrays: Optional[list] = None,
) -> Generator[str, None, None]:
"""Whisper ASR over an audio array; yields text from the pipeline built at model load."""
# No token counts here; clear so gen_done does not ship a previous turn's stats.
self.last_generation_stats = None
model_info = self.models[self.active_model_name]
whisper_pipe = model_info.get("whisper_pipeline")
if not whisper_pipe:
yield "Error: Whisper pipeline not initialized"
return
clips = [audio_array, *(extra_audio_arrays or [])]
try:
for index, clip in enumerate(clips):
if cancel_event is not None and cancel_event.is_set():
return
with self._generation_lock, self._adapter_scope(use_adapter):
result = whisper_pipe({"raw": clip, "sampling_rate": 16000})
text = result.get("text", "") if isinstance(result, dict) else str(result)
if text:
yield f"\n\n{text}" if index else text
except Exception as e:
logger.error(f"Whisper ASR error: {e}")
yield f"Error: {str(e)}"
def _is_gpt_oss_model(self, model_name: str = None) -> bool:
"""Whether the given (or active) model uses the gpt-oss harmony protocol."""
from utils.datasets import is_gpt_oss_model_name
return is_gpt_oss_model_name(model_name or self.active_model_name or "")
def _make_text_streamer(
self,
tokenizer,
*,
protocol_source = None,
reasoning_channel_markers = None,
reasoning_channel_markers_resolved: bool = False,
prompt: Optional[str] = None,
continued: bool = False,
skip_prompt: bool = True,
timeout: float = 0.2,
cancel_event = None,
use_harmony: bool = False,
preserve_tool_tokens: bool = False,
):
"""Create the streamer matching this model's native response protocol."""
if use_harmony:
try:
return HarmonyTextStreamer(
tokenizer,
skip_prompt = skip_prompt,
timeout = timeout,
)
except Exception as e:
logger.warning(f"HarmonyTextStreamer init failed, falling back: {e}")
return IncrementalTextIteratorStreamer(
tokenizer,
skip_prompt = skip_prompt,
skip_special_tokens = True,
timeout = timeout,
)
markers = (
reasoning_channel_markers
if reasoning_channel_markers_resolved
else reasoning_channel_markers
or detect_reasoning_channel_markers(protocol_source or tokenizer)
)
if markers is not None:
return ReasoningTextIteratorStreamer(
tokenizer,
markers = markers,
skip_prompt = skip_prompt,
timeout = timeout,
cancel_event = cancel_event,
in_reasoning = prompt_opens_reasoning_channel(prompt, markers, continued),
)
stream_tokenizer = NativeToolTokenDecoder(tokenizer) if preserve_tool_tokens else tokenizer
return IncrementalTextIteratorStreamer(
stream_tokenizer,
skip_prompt = skip_prompt,
skip_special_tokens = not preserve_tool_tokens,
timeout = timeout,
)
def _append_stream_delta(
self,
output: str,
new_token: str,
stop_token_ids = None,
):
"""Append a streamer delta and apply response-boundary cleanup."""
output += new_token
return output, self._clean_generated_text(output, stop_token_ids = stop_token_ids)
def _drain_streamer_tail(
self,
streamer,
output: str,
stop_token_ids = None,
):
"""Drain queued streamer text after the producer exits."""
while True:
try:
new_token = next(streamer)
except StopIteration:
return output
except Exception:
return output
if new_token:
output, cleaned = self._append_stream_delta(
output, new_token, stop_token_ids = stop_token_ids
)
yield cleaned
def _generate_diffusion_lm(
self,
model_info: dict,
prompt: Optional[str] = None,
messages: Optional[list] = None,
temperature: float = 0.0,
max_new_tokens: Optional[int] = 256,
cancel_event = None,
_adapter_state = None,
stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""A text-diffusion LM fills a whole block at once, so the answer arrives as one completion: there is
no left-to-right token stream to forward. Holds the generation lock until the sampler returns, and
the sampler polls ``cancel_event`` between denoising steps."""
# No token stats come back from the sampler; a previous request's must not be reported for this one.
self.last_generation_stats = None
from utils.models.diffusion_lm import require_diffusion_api
if messages:
# Same choke point as the left-to-right renderer: role / turn markup inside message text is
# neutralised before the sampler renders the chat template.
from core.inference.chat_template_helpers import neutralize_for_render
tokenizer = model_info["tokenizer"]
messages, _, _ = neutralize_for_render(
getattr(tokenizer, "tokenizer", tokenizer), messages, None
)
generate_text = getattr(require_diffusion_api("FastDiffusionModel"), "generate_text", None)
if generate_text is None:
raise RuntimeError(
"Unsloth: chatting with a text-diffusion model needs a newer Unsloth "
"(FastDiffusionModel.generate_text). Update with `unsloth studio update`."
)
try:
from unsloth.models.diffusion_samplers import (
DiffusionGenerationCancelled as cancelled_error,
)
except ImportError: # an older core: no cancellation error to catch
cancelled_error = ()
frame_kwargs = {}
sink = getattr(self, "diffusion_frame_sink", None)
if sink is not None and _accepts_on_frame(generate_text):
frame_kwargs["on_frame"] = _diffusion_frame_relay(
model_info["tokenizer"],
sink,
stop = stop,
eos_ids = _diffusion_eos_ids(
model_info["model"], model_info["tokenizer"], messages is not None
),
)
with self._generation_lock, self._adapter_scope(_adapter_state):
try:
text = generate_text(
model_info["model"],
model_info["tokenizer"],
prompt = prompt,
messages = messages,
max_new_tokens = int(max_new_tokens or 256),
temperature = float(temperature or 0.0),
stop_callback = (lambda: cancel_event is not None and cancel_event.is_set()),
**frame_kwargs,
)
except cancelled_error:
return
text = text or ""
for sequence in stop or ():
if sequence and sequence in text:
text = text[: text.index(sequence)]
if text and not (cancel_event is not None and cancel_event.is_set()):
yield text
def generate_stream(
self,
prompt: str,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
min_p: float = 0.0,
max_new_tokens: Optional[int] = 256,
repetition_penalty: float = 1.0,
cancel_event = None,
_adapter_state = None,
presence_penalty: float = 0.0,
reasoning_channel_markers = None,
reasoning_channel_markers_resolved: bool = False,
continued: bool = False,
preserve_tool_tokens: bool = False,
add_special_tokens: bool = True,
stop: Optional[list] = None,
_bos_probe = None,
) -> Generator[str, None, None]:
"""Rendered chat prompts pass add_special_tokens=False; raw prompts keep the tokenizer's BOS
default."""
if not self.active_model_name:
raise RuntimeError("No active model")
# Reset so a failed or uncountable run cannot surface stale stats.
self.last_generation_stats = None
model_info = self.models[self.active_model_name]
if model_info.get("diffusion_lm_profile"):
yield from self._generate_diffusion_lm(
model_info,
prompt = prompt,
temperature = temperature,
max_new_tokens = max_new_tokens,
cancel_event = cancel_event,
_adapter_state = _adapter_state,
stop = stop,
)
return
model = model_info["model"]
# For VLMs the stored tokenizer is the processor; unwrap so skip_prompt works.
tokenizer = model_info["tokenizer"]
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
try:
inputs = _tokenize_prompt(tokenizer, prompt, add_special_tokens, _bos_probe).to(
model.device
)
import threading
# skip_prompt swallows a template-prefilled <think>; re-emit it. gpt-oss emits its own tags.
think_prefix = (
""
if self._is_gpt_oss_model()
else detect_think_prefill(
prompt,
getattr(tokenizer, "all_special_tokens", None),
# Both streamers keep </think> through NativeToolTokenDecoder, so the opener must be re-emitted.
preserves_think_close = (
preserve_tool_tokens or reasoning_channel_markers is not None
)
and decoder_preserves_token(tokenizer, "</think>"),
)
)
streamer = self._make_text_streamer(
tokenizer,
protocol_source = model_info.get("tokenizer"),
reasoning_channel_markers = reasoning_channel_markers,
reasoning_channel_markers_resolved = reasoning_channel_markers_resolved,
prompt = prompt,
continued = continued,
skip_prompt = True,
timeout = 0.2,
cancel_event = cancel_event,
use_harmony = self._is_gpt_oss_model(),
preserve_tool_tokens = preserve_tool_tokens,
)
prompt_len = int(inputs["input_ids"].shape[1])
window = runtime_context_length(model)
if window and prompt_len > window:
if "truncates_past_window" not in model_info:
model_info["truncates_past_window"] = _truncates_past_window(model)
if model_info["truncates_past_window"]:
raise ContextBudgetExceeded(prompt_len, window)
max_new_tokens = generation_budget_within_context(model, prompt_len, max_new_tokens)
generation_kwargs = dict(
**inputs,
streamer = streamer,
max_new_tokens = max_new_tokens,
temperature = temperature,
top_p = top_p,
top_k = top_k,
min_p = min_p,
repetition_penalty = repetition_penalty,
do_sample = temperature > 0,
# Resolved once at load (chat_template-derived turn-end tokens).
eos_token_id = model_info.get("chat_turn_end_eos_ids") or tokenizer.eos_token_id,
pad_token_id = tokenizer.eos_token_id
if tokenizer.pad_token_id is None
else tokenizer.pad_token_id,
)
active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs)
# Presence penalty (GGUF parity); prompt_len excludes prompt tokens.
_pp = _make_presence_penalty_processor(presence_penalty, prompt_len)
timer = GenerationTimer()
generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)
stop_streamer = _StopSequenceStreamer(streamer, stop)
if stop_streamer.sequences:
generation_kwargs["streamer"] = stop_streamer
stopping_criteria = self._cancel_stopping_criteria(cancel_event, stop_streamer.matched)
if stopping_criteria is not None:
generation_kwargs["stopping_criteria"] = stopping_criteria
def generate_fn():
with self._generation_lock, self._adapter_scope(_adapter_state):
try:
# Started after the adapter swap so only model.generate() is timed.
timer.start()
# Only the returned sequences carry an exact token count.
with _decode_graph_scope():
sequences = _generate_supported(model, generation_kwargs)
if sequences is not None:
gen_outputs["sequences"] = sequences
except Exception as e:
err["msg"] = str(e)
stop_streamer.abort()
logger.error(f"Generation error: {e}")
finally:
timer.finish()
try:
stop_streamer.end()
except Exception:
pass
err: dict[str, str] = {}
gen_outputs: dict = {}
thread = threading.Thread(target = generate_fn)
thread.start()
output = think_prefix
# Emit prefilled <think> before the first token so the block renders during slow prompt prefill.
if think_prefix:
yield think_prefix
from queue import Empty
import time
generation_complete = False
cancel_deadline = None
try:
while True:
if cancel_event is not None and cancel_event.is_set():
if cancel_deadline is None:
cancel_deadline = time.monotonic() + 10
elif time.monotonic() >= cancel_deadline:
break
try:
new_token = next(stop_streamer)
except StopIteration:
generation_complete = True
break
except Empty:
if not thread.is_alive():
generation_complete = True
output = yield from self._drain_streamer_tail(
stop_streamer, output, active_stop_token_ids
)
break
if cancel_deadline is not None:
remaining = cancel_deadline - time.monotonic()
if remaining <= 0:
break
thread.join(timeout = remaining)
if thread.is_alive():
break
generation_complete = True
output = yield from self._drain_streamer_tail(
stop_streamer, output, active_stop_token_ids
)
break
continue
if new_token:
output, cleaned = self._append_stream_delta(
output, new_token, active_stop_token_ids
)
yield cleaned
finally:
# Set only on early exit: it is a shared mp.Event and would cancel the next serialized request.
if cancel_event is not None and not generation_complete:
cancel_event.set()
join_timeout = 10
if cancel_deadline is not None:
join_timeout = max(0, cancel_deadline - time.monotonic())
thread.join(timeout = join_timeout)
if thread.is_alive():
logger.warning("Generation thread did not exit after cancel/join timeout")
if "sequences" in gen_outputs:
self._record_generation_stats(
prompt_tokens = prompt_len,
completion_tokens = self._generated_token_count(
model, gen_outputs["sequences"], prompt_len
),
max_new_tokens = max_new_tokens,
ended_on_stop_token = stop_streamer.matched.is_set()
or self._ended_on_stop_token(gen_outputs["sequences"], active_stop_token_ids),
cancelled = not generation_complete
or (cancel_event is not None and cancel_event.is_set()),
timer = timer,
)
if err.get("msg"):
raise _GenerationThreadError(err["msg"])
except _GenerationThreadError:
raise
except Exception as e:
logger.error(f"Error during generation: {e}")
raise
def generate_audio_response(
self,
text: str,
temperature: float = 0.6,
top_p: float = 0.95,
top_k: int = 50,
min_p: float = 0.0,
max_new_tokens: int = 2048,
repetition_penalty: float = 1.0,
use_adapter: Optional[Union[bool, str]] = None,
cancel_event = None,
instructions: Optional[str] = None,
language: Optional[str] = None,
seed: Optional[int] = None,
audio_options: Optional[dict] = None,
) -> Tuple[bytes, int]:
"""Args unused: reserved for native audio models; codec TTS has no instructions, language or seed.
A chat model that speaks (LFM3) reads the text aloud in its TTS mode, in ``language``."""
del instructions, seed
if not self.active_model_name:
raise RuntimeError("No active model")
model_info = self.models[self.active_model_name]
audio_type = model_info.get("audio_type")
model = model_info["model"]
tokenizer = model_info.get("tokenizer")
if not audio_type:
raise RuntimeError(f"Model {self.active_model_name} is not an audio model")
if audio_type == "audio_vlm" and model_info.get("speaks"):
return self._speak_text(
text, max_new_tokens, use_adapter, cancel_event, language, audio_options
)
top_k = self._normalize_top_k(top_k)
# All four codecs concatenate prompts, not templates; this is their one choke point.
text = tts_prompt_with_voice(
neutralize_tts_prompt_text(text, audio_type), audio_type, audio_options
)
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
with self._generation_lock, self._adapter_scope(use_adapter):
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
stopping_criteria = self._cancel_stopping_criteria(cancel_event)
self.last_generation_stats = None
if audio_type == "snac":
result = self._generate_snac(
model,
tokenizer,
text,
temperature,
top_p,
max_new_tokens,
repetition_penalty,
stopping_criteria = stopping_criteria,
cancel_event = cancel_event,
)
elif audio_type == "csm":
processor = model_info.get("processor", tokenizer)
result = self._generate_csm(
model,
processor,
text,
max_new_tokens,
stopping_criteria = stopping_criteria,
cancel_event = cancel_event,
)
elif audio_type == "bicodec":
result = self._generate_bicodec(
model,
tokenizer,
text,
temperature,
top_k,
max_new_tokens,
stopping_criteria = stopping_criteria,
cancel_event = cancel_event,
)
elif audio_type == "dac":
result = self._generate_dac(
model,
tokenizer,
text,
temperature,
top_k,
top_p,
min_p,
max_new_tokens,
repetition_penalty,
stopping_criteria = stopping_criteria,
cancel_event = cancel_event,
)
else:
raise RuntimeError(f"Unknown audio_type: {audio_type}")
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
return result
def _generate_snac(
self,
model,
tokenizer,
text,
temperature,
top_p,
max_new_tokens,
repetition_penalty,
*,
stopping_criteria = None,
cancel_event = None,
):
"""Generate audio using SNAC codec (Orpheus)."""
device = model.device
start_token = torch.tensor([[128259]], device = device) # START_OF_HUMAN
end_tokens = torch.tensor([[128009, 128260]], device = device) # EOT, END_OF_HUMAN
text_ids = tokenizer(text, return_tensors = "pt").input_ids.to(device)
input_ids = torch.cat([start_token, text_ids, end_tokens], dim = 1)
attention_mask = torch.ones_like(input_ids)
generated = model.generate(
input_ids = input_ids,
attention_mask = attention_mask,
max_new_tokens = max_new_tokens,
do_sample = True,
temperature = temperature,
top_p = top_p,
repetition_penalty = repetition_penalty,
eos_token_id = 128258, # END_OF_SPEECH
use_cache = True,
**({"stopping_criteria": stopping_criteria} if stopping_criteria is not None else {}),
)
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
self._record_generation_stats(
prompt_tokens = input_ids.shape[1],
completion_tokens = self._generated_token_count(model, generated, input_ids.shape[1]),
max_new_tokens = max_new_tokens,
ended_on_stop_token = self._ended_on_stop_token(generated, 128258),
)
return self._audio_codec_manager.decode_snac(generated, str(device))
def _generate_csm(
self,
model,
processor,
text,
max_new_tokens,
*,
stopping_criteria = None,
cancel_event = None,
):
"""Generate audio using CSM (Sesame)."""
speaker_id = 0
inputs = processor(
f"[{speaker_id}]{text}", add_special_tokens = True, return_tensors = "pt"
).to(model.device)
audio_values = model.generate(
**inputs,
max_new_tokens = max_new_tokens,
output_audio = True,
**({"stopping_criteria": stopping_criteria} if stopping_criteria is not None else {}),
)
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
return self._audio_codec_manager.decode_csm(audio_values)
def _generate_bicodec(
self,
model,
tokenizer,
text,
temperature,
top_k,
max_new_tokens,
*,
stopping_criteria = None,
cancel_event = None,
):
"""Generate audio using BiCodec (Spark-TTS)."""
prompt = "<|task_tts|><|start_content|>" + text + "<|end_content|><|start_global_token|>"
inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
generated = model.generate(
**inputs,
max_new_tokens = max_new_tokens,
do_sample = True,
temperature = temperature,
top_k = top_k,
eos_token_id = tokenizer.eos_token_id,
pad_token_id = tokenizer.pad_token_id,
**({"stopping_criteria": stopping_criteria} if stopping_criteria is not None else {}),
)
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
prompt_len = inputs.input_ids.shape[1]
self._record_generation_stats(
prompt_tokens = prompt_len,
completion_tokens = self._generated_token_count(model, generated, prompt_len),
max_new_tokens = max_new_tokens,
ended_on_stop_token = self._ended_on_stop_token(generated, tokenizer.eos_token_id),
)
new_tokens = generated[:, prompt_len:]
decoded_text = tokenizer.batch_decode(new_tokens, skip_special_tokens = False)[0]
return self._audio_codec_manager.decode_bicodec(decoded_text, str(model.device))
def _generate_dac(
self,
model,
tokenizer,
text,
temperature,
top_k,
top_p,
min_p,
max_new_tokens,
repetition_penalty,
*,
stopping_criteria = None,
cancel_event = None,
):
"""Generate audio using DAC (OuteTTS). Follows Oute_TTS_(1B).ipynb exactly."""
# 64-token repetition window, same as the OuteTTS notebook.
self._patch_repetition_penalty_processor()
prompt = build_dac_tts_prompt(text)
with torch.inference_mode():
# Autocast device from the model, not the backend: a CPU-fallback DAC may sit on a GPU host.
device_type = (
model.device.type
if hasattr(model.device, "type")
else str(model.device).split(":", 1)[0]
)
# Clamp to autocast-supported backends (e.g. "meta" under offloaded loading raises).
if device_type not in ("cuda", "xpu", "mps", "cpu"):
device_type = "cpu"
# CPU/XPU autocast accept only bf16/fp16; skip autocast for fp32 models.
autocast_dtype_supported = model.dtype in (torch.bfloat16, torch.float16)
if device_type in ("cpu", "xpu") and not autocast_dtype_supported:
autocast_ctx = contextlib.nullcontext()
else:
autocast_ctx = torch.amp.autocast(device_type, dtype = model.dtype)
with autocast_ctx:
inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
generated = model.generate(
**inputs,
temperature = temperature,
top_k = top_k,
top_p = top_p,
min_p = min_p,
repetition_penalty = repetition_penalty,
max_new_tokens = max_new_tokens,
**(
{"stopping_criteria": stopping_criteria}
if stopping_criteria is not None
else {}
),
)
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
self._record_generation_stats(
prompt_tokens = inputs.input_ids.shape[1],
completion_tokens = self._generated_token_count(
model, generated, inputs.input_ids.shape[1]
),
max_new_tokens = max_new_tokens,
ended_on_stop_token = self._ended_on_stop_token(
generated, self._generation_stop_token_ids(model, {})
),
)
decoded_text = tokenizer.batch_decode(generated, skip_special_tokens = False)[0]
return self._audio_codec_manager.decode_dac(decoded_text, str(model.device))
_repetition_penalty_patched = False
@classmethod
def _patch_repetition_penalty_processor(cls):
"""Monkey-patch transformers' RepetitionPenaltyLogitsProcessor with the OuteTTS notebook's
64-token sliding-window variant. Applied once per process."""
if cls._repetition_penalty_patched:
return
cls._repetition_penalty_patched = True
from transformers import LogitsProcessor
import transformers.generation.utils as generation_utils
class RepetitionPenaltyLogitsProcessorPatch(LogitsProcessor):
def __init__(self, penalty: float):
self.penalty_last_n = 64
if not isinstance(penalty, float) or penalty <= 0:
raise ValueError(f"`penalty` has to be a positive float, but is {penalty}")
self.penalty = penalty
@torch.no_grad()
def __call__(
self, input_ids: torch.LongTensor, scores: torch.FloatTensor
) -> torch.FloatTensor:
if self.penalty_last_n == 0 or self.penalty == 1.0:
return scores
batch_size, seq_len = input_ids.shape
vocab_size = scores.shape[-1]
for b in range(batch_size):
start_index = max(0, seq_len - self.penalty_last_n)
window_indices = input_ids[b, start_index:]
if window_indices.numel() == 0:
continue
for token_id in set(window_indices.tolist()):
if token_id >= vocab_size:
continue
logit = scores[b, token_id]
scores[b, token_id] = (
logit * self.penalty if logit <= 0 else logit / self.penalty
)
return scores
generation_utils.RepetitionPenaltyLogitsProcessor = RepetitionPenaltyLogitsProcessorPatch
logger.info("Patched RepetitionPenaltyLogitsProcessor with 64-token window for OuteTTS")
def _apply_chat_template_for_generation(
self,
tokenizer,
messages: list,
*,
tools: Optional[list] = None,
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
continue_final_message: bool = False,
) -> str:
"""Peels kwargs the template does not understand; delegates to a dependency-light helper for
testing."""
from core.inference.chat_template_helpers import (
apply_chat_template_for_generation,
)
return apply_chat_template_for_generation(
tokenizer,
messages,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
continue_final_message = continue_final_message,
)
def format_chat_prompt(
self,
messages: list,
system_prompt: str = None,
continue_final_message: bool = False,
) -> str:
if not self.active_model_name or self.active_model_name not in self.models:
logger.error("No active model available")
return ""
if self.models[self.active_model_name].get("tokenizer") is None:
logger.error("Tokenizer not loaded for active model")
return ""
chat_template_info = self.models[self.active_model_name].get("chat_template_info", {})
tokenizer = self.models[self.active_model_name]["tokenizer"]
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
chat_messages = []
if system_prompt:
chat_messages.append({"role": "system", "content": system_prompt})
last_role = "system" if system_prompt else None
for msg in messages:
role = msg.get("role", "")
content = content_to_text(msg.get("content", ""))
reasoning_content = msg.get("reasoning_content")
has_reasoning_content = (
role == "assistant"
and isinstance(reasoning_content, str)
and bool(reasoning_content.strip())
)
if role in ["system", "user", "assistant"] and (
content.strip() or has_reasoning_content
):
if role == last_role:
logger.debug(f"Skipping consecutive {role} message to maintain alternation")
continue
if role == "user":
chat_messages.append(named_turn({"role": role, "content": content}, msg))
last_role = role
elif role == "assistant":
assistant_message = named_turn({"role": role, "content": content}, msg)
if has_reasoning_content:
assistant_message["reasoning_content"] = reasoning_content
chat_messages.append(assistant_message)
last_role = role
elif role == "system":
continue
# A continuation resumes that turn, so dropping it would restart the answer.
_continuing = continue_final_message and bool(
chat_messages and chat_messages[-1]["role"] == "assistant"
)
if not _continuing and chat_messages and chat_messages[-1]["role"] == "assistant":
logger.debug("Removing final assistant message to ensure proper alternation")
chat_messages.pop()
# Direct tokenizer render bypasses the choke point, so neutralize here too (#7066).
from core.inference.chat_template_helpers import (
markup_for_tokenizer,
render_prompt_with_boundary,
)
chat_messages = neutralize_control_markup_in_messages(
chat_messages, None, markup_for_tokenizer(tokenizer)
)
logger.info(f"Sending {len(chat_messages)} messages to tokenizer:")
for i, msg in enumerate(chat_messages):
logger.debug(f" {i}: {msg['role']} - {msg['content'][:50]}...")
try:
formatted_prompt = render_prompt_with_boundary(
tokenizer, chat_messages, continue_final_message = _continuing
)
logger.info(f"Successfully applied tokenizer's native chat template")
return formatted_prompt
except Exception as e:
error_msg = str(e).lower()
if "chat_template is not set" in error_msg or "no template argument" in error_msg:
logger.info(
f"Base model detected - no built-in chat template available, using fallback formatting"
)
else:
logger.warning(f"Failed to apply tokenizer chat template: {e}")
logger.debug(
f"""Failed with messages: {[f"{m['role']}: {m['content'][:30]}..." for m in chat_messages]}"""
)
# Manual formatters close the assistant turn, so render without the partial and append it.
partial = chat_messages[-1]["content"] if _continuing else None
manual_messages = chat_messages[:-1] if _continuing else chat_messages
if chat_template_info.get("has_template", False):
logger.info("Falling back to manual template formatting based on detected patterns")
template_type = chat_template_info.get("format_type", "generic")
manual_prompt = self._format_chat_manual(
manual_messages,
template_type,
chat_template_info.get("special_tokens", {}),
)
logger.debug(f"Manual template result: {manual_prompt[:200]}...")
else:
logger.info("Using generic chat formatting for base model")
manual_prompt = self._format_generic_template(manual_messages, {})
return f"{manual_prompt}{partial}" if partial else manual_prompt
def _format_chat_manual(self, messages: list, template_type: str, special_tokens: dict) -> str:
"""Manual chat-formatting fallback when the tokenizer template fails."""
if template_type == "llama3":
return self._format_llama3_template(messages, special_tokens)
elif template_type == "mistral":
return self._format_mistral_template(messages, special_tokens)
elif template_type == "chatml":
return self._format_chatml_template(messages, special_tokens)
elif template_type == "alpaca":
return self._format_alpaca_template(messages, special_tokens)
else:
return self._format_generic_template(messages, special_tokens)
def _format_llama3_template(self, messages: list, special_tokens: dict) -> str:
bos_token = special_tokens.get("bos_token", "<|begin_of_text|>")
formatted = bos_token
for msg in messages:
role = msg["role"]
content = content_to_text(msg["content"])
formatted += f"<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>"
formatted += "<|start_header_id|>assistant<|end_header_id|>\n\n"
return formatted
def _format_mistral_template(self, messages: list, special_tokens: dict) -> str:
bos_token = special_tokens.get("bos_token", "<s>")
formatted = bos_token
system_msg = None
conversation = []
for msg in messages:
if msg["role"] == "system":
system_msg = content_to_text(msg["content"])
else:
conversation.append(msg)
i = 0
while i < len(conversation):
if conversation[i]["role"] == "user":
user_content = content_to_text(conversation[i]["content"])
if system_msg and i == 0:
user_content = f"{system_msg}\n\n{user_content}"
formatted += f"[INST] {user_content} [/INST]"
if i + 1 < len(conversation) and conversation[i + 1]["role"] == "assistant":
formatted += f" {content_to_text(conversation[i + 1]['content'])}</s>"
i += 2
else:
formatted += " "
break
else:
i += 1
return formatted
def _format_chatml_template(self, messages: list, special_tokens: dict) -> str:
formatted = ""
for msg in messages:
role = msg["role"]
content = content_to_text(msg["content"])
formatted += f"<|im_start|>{role}\n{content}<|im_end|>\n"
formatted += "<|im_start|>assistant\n"
return formatted
def _format_alpaca_template(self, messages: list, special_tokens: dict) -> str:
formatted = ""
system_msg = None
for msg in messages:
content = content_to_text(msg["content"])
if msg["role"] == "system":
system_msg = content
elif msg["role"] == "user":
if system_msg:
formatted += f"### Instruction:\n{system_msg}\n\n### Input:\n{content}\n\n### Response:\n"
system_msg = None
else:
formatted += f"### Human:\n{content}\n\n### Assistant:\n"
elif msg["role"] == "assistant":
formatted += f"{content}\n\n"
return formatted
def _format_generic_template(self, messages: list, special_tokens: dict) -> str:
formatted = ""
for msg in messages:
role = msg["role"].title()
content = content_to_text(msg["content"])
formatted += f"{role}: {content}\n"
formatted += "Assistant: "
return formatted
def check_vision_model_compatibility(self) -> bool:
current_model = self.get_current_model()
if current_model and current_model in self.models:
return self.models[current_model].get("is_vision", False)
return False
def _reset_model_generation_state(self, model_name: str):
"""Reset generation state for a specific model to prevent contamination."""
if model_name not in self.models:
return
model = self.models[model_name].get("model")
if not model:
return
try:
if hasattr(model, "past_key_values"):
model.past_key_values = None
if hasattr(model, "generation_config"):
if hasattr(model.generation_config, "past_key_values"):
model.generation_config.past_key_values = None
logger.debug(f"Reset generation state for model: {model_name}")
except Exception as e:
logger.warning(f"Could not fully reset model state for {model_name}: {e}")
def reset_generation_state(self, caller_cancel_event = None):
"""caller_cancel_event is accepted for signature parity only; this never cancels a live
generation."""
try:
for model_name in self.models.keys():
self._reset_model_generation_state(model_name)
clear_gpu_cache()
logger.debug("Cleared GPU cache")
import gc
gc.collect()
logger.info("Performed comprehensive generation state reset")
except Exception as e:
logger.warning(f"Could not fully reset generation state: {e}")
def resize_image(
self,
img,
max_size: int = 800,
):
"""Resize image while maintaining aspect ratio if either dimension exceeds max_size"""
if img is None:
return None
if img.size[0] > max_size or img.size[1] > max_size:
from PIL import Image
ratio = min(max_size / img.size[0], max_size / img.size[1])
new_size = (int(img.size[0] * ratio), int(img.size[1] * ratio))
return img.resize(new_size, Image.Resampling.LANCZOS)
return img
def _generation_stop_token_ids(self, model, generation_kwargs: dict):
"""Return the stop-token ids active for a ``generate`` call."""
if "eos_token_id" in generation_kwargs:
return generation_kwargs.get("eos_token_id")
generation_config = getattr(model, "generation_config", None)
eos_token_id = getattr(generation_config, "eos_token_id", None)
if eos_token_id is not None:
return eos_token_id
config = getattr(model, "config", None)
return getattr(config, "eos_token_id", None)
def _generated_token_count(self, model, outputs, prompt_len) -> Optional[int]:
"""Returns None for unrecognised outputs, so the caller never reports a count it did not measure."""
sequences = getattr(outputs, "sequences", outputs)
shape = getattr(sequences, "shape", None)
if shape is None or len(shape) < 2:
return None
total = int(shape[-1])
if getattr(getattr(model, "config", None), "is_encoder_decoder", False):
return max(total - 1, 0)
if prompt_len is None or total < int(prompt_len):
return None
return total - int(prompt_len)
def _ended_on_stop_token(self, outputs, stop_token_ids) -> bool:
"""Whether the final token is a stop token rather than one cut off by the cap. A response
landing exactly on ``max_new_tokens`` ended naturally; only one that did NOT emit a stop
token ran out of budget."""
sequences = getattr(outputs, "sequences", outputs)
if isinstance(stop_token_ids, int):
stop_token_ids = (stop_token_ids,)
try:
last_id = int(sequences[0, -1])
return last_id in {int(token_id) for token_id in (stop_token_ids or ())}
except Exception:
return False
def _record_generation_stats(
self,
*,
prompt_tokens,
completion_tokens,
max_new_tokens,
ended_on_stop_token: bool = False,
cancelled: bool = False,
timer = None,
) -> None:
"""Cancelled runs never report finish_reason length: they stopped by request, not at the
token cap."""
if completion_tokens is None:
return
completion_tokens = int(completion_tokens)
prompt_tokens = int(prompt_tokens or 0)
stats = {
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
"truncated": (
not cancelled
and not ended_on_stop_token
and bool(max_new_tokens)
and completion_tokens >= int(max_new_tokens)
),
}
timings = build_generation_timings(
prompt_n = prompt_tokens,
predicted_n = completion_tokens,
prompt_ms = timer.prompt_ms if timer is not None else None,
predicted_ms = timer.predicted_ms if timer is not None else None,
)
if timings is not None:
stats["timings"] = timings
self.last_generation_stats = stats
def _cancel_stopping_criteria(self, *events):
"""Build a Transformers stopping criteria list for user cancellation."""
events = [ev for ev in events if ev is not None]
if not events:
return None
from transformers.generation.stopping_criteria import (
StoppingCriteria,
StoppingCriteriaList,
)
class _CancelCriteria(StoppingCriteria):
def __init__(self, ev):
self.ev = ev
def __call__(self, input_ids, scores, **kwargs):
return self.ev.is_set()
return StoppingCriteriaList([_CancelCriteria(ev) for ev in events])
def _clean_generated_text(
self,
text: str,
*,
stop_token_ids = None,
) -> str:
"""Strip leaked response-boundary tokens after streaming."""
if self._is_gpt_oss_model():
# Harmony vocabulary only: a literal <|im_end|> in an answer is content.
text = _HARMONY_PROTOCOL_TOKEN_RE.sub("", text)
return text.strip()
tokenizer = self.models.get(self.active_model_name, {}).get("tokenizer")
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
if tokenizer:
if stop_token_ids is None:
stop_token_ids = self.models.get(self.active_model_name, {}).get(
"chat_turn_end_eos_ids"
)
if isinstance(stop_token_ids, int):
stop_token_ids = (stop_token_ids,)
for token_id in stop_token_ids or ():
token = stop_token_text(tokenizer, token_id)
if isinstance(token, str) and token and text.endswith(token):
if closes_an_open_envelope(text, token):
# A native closer that is also the stop token still needs its opener present.
continue
text = text[: -len(token)]
elif (
isinstance(token, str)
and token
and text.endswith("</think>")
and text[: -len("</think>")].endswith(token)
):
text = text[: -len("</think>") - len(token)] + "</think>"
return text.strip()
def _load_chat_template_info(self, model_name: str):
if model_name not in self.models or not self.models[model_name].get("tokenizer"):
return
tokenizer = self.models[model_name]["tokenizer"]
chat_template_info = {
"has_template": False,
"template": None,
"format_type": "generic",
"special_tokens": {},
"template_name": None,
# Image turns render through the processor template, usually a different file (#10092).
"processor_template": None,
# Distinct from processor_template: a template-less processor still places the image.
"renders_image": False,
}
processor = self.models[model_name].get("processor")
# Same check as the image-ignoring fallback, since FastVisionModel may return a raw tokenizer.
try:
from transformers import ProcessorMixin
processes_images = processor is not None and (
isinstance(processor, ProcessorMixin) or hasattr(processor, "image_processor")
)
except Exception:
processes_images = processor is not None and hasattr(processor, "image_processor")
chat_template_info["renders_image"] = bool(processes_images)
# Not mlx-vlm's SINGLE_IMAGE_ONLY_MODELS: that list is its prompt builder's limit, while the
# transformers processors for llava_next and mllama take one image per marker.
chat_template_info["accepts_multiple_images"] = bool(processes_images) and bool(
_scaling_image_marker(tokenizer, processor, self.models[model_name].get("model"))
)
if processes_images:
processor_template = getattr(processor, "chat_template", None)
# Narrowing the named-template list form away disables the image-turn override.
if isinstance(processor_template, (str, dict, list, tuple)) and processor_template:
chat_template_info["processor_template"] = processor_template
try:
from utils.datasets import MODEL_TO_TEMPLATE_MAPPER
model_name_lower = model_name.lower()
if model_name_lower in MODEL_TO_TEMPLATE_MAPPER:
chat_template_info["template_name"] = MODEL_TO_TEMPLATE_MAPPER[model_name_lower]
logger.info(
f"Detected template '{chat_template_info['template_name']}' for {model_name} from mapper"
)
else:
# Partial match (for variants like model_name-bnb-4bit)
for key in MODEL_TO_TEMPLATE_MAPPER:
if key in model_name_lower or model_name_lower in key:
chat_template_info["template_name"] = MODEL_TO_TEMPLATE_MAPPER[key]
logger.info(
f"Detected template '{chat_template_info['template_name']}' for {model_name} (partial match)"
)
break
except Exception as e:
logger.warning(f"Could not detect template from mapper for {model_name}: {e}")
try:
if hasattr(tokenizer, "chat_template") and tokenizer.chat_template:
chat_template_info["has_template"] = True
chat_template_info["template"] = tokenizer.chat_template
template_str = tokenizer.chat_template.lower()
if "start_header_id" in template_str and "end_header_id" in template_str:
chat_template_info["format_type"] = "llama3"
elif "[inst]" in template_str and "[/inst]" in template_str:
chat_template_info["format_type"] = "mistral"
elif "<|im_start|>" in template_str and "<|im_end|>" in template_str:
chat_template_info["format_type"] = "chatml"
elif "### instruction:" in template_str or "### human:" in template_str:
chat_template_info["format_type"] = "alpaca"
else:
chat_template_info["format_type"] = "custom"
logger.info(
f"Loaded chat template for {model_name} (detected as {chat_template_info['format_type']} format)"
)
logger.debug(f"Template preview: {tokenizer.chat_template[:200]}...")
special_tokens = {}
if hasattr(tokenizer, "bos_token") and tokenizer.bos_token:
special_tokens["bos_token"] = tokenizer.bos_token
if hasattr(tokenizer, "eos_token") and tokenizer.eos_token:
special_tokens["eos_token"] = tokenizer.eos_token
if hasattr(tokenizer, "pad_token") and tokenizer.pad_token:
special_tokens["pad_token"] = tokenizer.pad_token
chat_template_info["special_tokens"] = special_tokens
else:
logger.info(f"No chat template found for {model_name}, will use generic formatting")
except Exception as e:
logger.error(f"Error loading chat template info for {model_name}: {e}")
self.models[model_name]["chat_template_info"] = chat_template_info
if chat_template_info["has_template"]:
logger.info(
f"Chat template loaded for {model_name}: {chat_template_info['format_type']} format"
)
else:
logger.info(f"No built-in chat template for {model_name}, will use generic formatting")
def get_current_model(self) -> Optional[str]:
"""Currently active model name."""
return self.active_model_name
def is_model_loading(self) -> bool:
return len(self.loading_models) > 0
def get_loading_model(self) -> Optional[str]:
return next(iter(self.loading_models)) if self.loading_models else None
def load_model_simple(
self,
model_path: str,
hf_token: Optional[str] = None,
max_seq_length: int = 2048,
load_in_4bit: bool = True,
) -> bool:
"""Simple model-loading wrapper for the chat interface: takes a string path and builds the
ModelConfig internally."""
try:
config = ModelConfig.from_ui_selection(
model_path,
lora_path = None, # No LoRA for chat
is_lora = False,
)
return self.load_model(
config = config,
max_seq_length = max_seq_length,
dtype = None,
load_in_4bit = load_in_4bit,
hf_token = hf_token,
)
except Exception as e:
logger.error(f"Error in load_model_simple: {e}")
return False
inference_backend = InferenceBackend()
def get_inference_backend() -> InferenceBackend:
return inference_backend