Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Turn-end stop tokens for chat models: some checkpoints set eos to a bare document terminator
while chat turns end with <|im_end|>, so generation runs on. Markers come from the chat_template,
not vocab membership, since base models may carry unused ChatML tokens. Dependency-light for unit
tests."""
from typing import Optional
# Canonical assistant-turn-end markers per chat family.
_CHAT_TURN_END_TOKENS = (
"<|im_end|>",
"<|eot_id|>",
"<|eom_id|>",
"<end_of_turn>",
"<turn|>",
"<|end|>",
"<|end_of_turn|>", # OpenChat, distinct from Gemma's
)
# harmony/gpt-oss uses <|end|> as a channel delimiter, not the turn end.
_HARMONY_MARKERS = ("<|channel|>", "<|constrain|>")
def _eos_id_set(eos_token_id) -> set:
if isinstance(eos_token_id, (list, tuple)):
return {int(t) for t in eos_token_id if t is not None}
if eos_token_id is not None:
return {int(eos_token_id)}
return set()
def _collect_template_text(chat_template) -> str:
"""Gathers every string leaf of chat_template, as multi-variant models store it as a dict or list."""
if isinstance(chat_template, str):
return chat_template
if isinstance(chat_template, dict):
values = chat_template.values()
elif isinstance(chat_template, (list, tuple)):
values = chat_template
else:
return ""
parts = [_collect_template_text(v) for v in values]
return "\n".join(p for p in parts if p)
def resolve_chat_turn_end_eos_ids_using(template_tokenizer, id_tokenizer) -> list:
"""Resolve ids on the original tokenizer; a mapped one can fold turn-end onto the doc-eos id."""
ids = _eos_id_set(getattr(id_tokenizer, "eos_token_id", None))
template = _collect_template_text(getattr(template_tokenizer, "chat_template", None))
if not template or any(h in template for h in _HARMONY_MARKERS):
return sorted(ids)
unk = getattr(id_tokenizer, "unk_token_id", None)
for marker in _CHAT_TURN_END_TOKENS:
if marker in template:
try:
tid = id_tokenizer.convert_tokens_to_ids(marker)
except Exception:
tid = None
if tid is not None and tid != unk and int(tid) >= 0:
ids.add(int(tid))
return sorted(ids)
def resolve_chat_turn_end_eos_ids(tokenizer) -> list:
"""tokenizer.eos plus turn-end markers the chat_template uses; resolve at load, harmony keeps eos."""
return resolve_chat_turn_end_eos_ids_using(tokenizer, tokenizer)
def chat_eos_repair(current_eos, turn_end_ids) -> Optional[list]:
"""Merged eos_token_id list, or None if ``current_eos`` already covers every
resolved turn-end id. Used to repair a model's generation_config at load so
every ``.generate()`` path (vision, tool loops) stops at the turn boundary."""
if not turn_end_ids:
return None
current_set = _eos_id_set(current_eos)
if set(turn_end_ids) <= current_set:
return None
return sorted(current_set | set(turn_end_ids))