Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

3.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Turn-end stop tokens for chat models: some checkpoints set eos to a bare document terminator
while chat turns end with <|im_end|>, so generation runs on. Markers come from the chat_template,
not vocab membership, since base models may carry unused ChatML tokens. Dependency-light for unit
tests."""

from typing import Optional

# Canonical assistant-turn-end markers per chat family.
_CHAT_TURN_END_TOKENS = (
    "<|im_end|>",
    "<|eot_id|>",
    "<|eom_id|>",
    "<end_of_turn>",
    "<turn|>",
    "<|end|>",
    "<|end_of_turn|>",  # OpenChat, distinct from Gemma's
)
# harmony/gpt-oss uses <|end|> as a channel delimiter, not the turn end.
_HARMONY_MARKERS = ("<|channel|>", "<|constrain|>")


def _eos_id_set(eos_token_id) -> set:
    if isinstance(eos_token_id, (list, tuple)):
        return {int(t) for t in eos_token_id if t is not None}
    if eos_token_id is not None:
        return {int(eos_token_id)}
    return set()


def _collect_template_text(chat_template) -> str:
    """Gathers every string leaf of chat_template, as multi-variant models store it as a dict or list."""
    if isinstance(chat_template, str):
        return chat_template
    if isinstance(chat_template, dict):
        values = chat_template.values()
    elif isinstance(chat_template, (list, tuple)):
        values = chat_template
    else:
        return ""
    parts = [_collect_template_text(v) for v in values]
    return "\n".join(p for p in parts if p)


def resolve_chat_turn_end_eos_ids_using(template_tokenizer, id_tokenizer) -> list:
    """Resolve ids on the original tokenizer; a mapped one can fold turn-end onto the doc-eos id."""
    ids = _eos_id_set(getattr(id_tokenizer, "eos_token_id", None))
    template = _collect_template_text(getattr(template_tokenizer, "chat_template", None))
    if not template or any(h in template for h in _HARMONY_MARKERS):
        return sorted(ids)
    unk = getattr(id_tokenizer, "unk_token_id", None)
    for marker in _CHAT_TURN_END_TOKENS:
        if marker in template:
            try:
                tid = id_tokenizer.convert_tokens_to_ids(marker)
            except Exception:
                tid = None
            if tid is not None and tid != unk and int(tid) >= 0:
                ids.add(int(tid))
    return sorted(ids)


def resolve_chat_turn_end_eos_ids(tokenizer) -> list:
    """tokenizer.eos plus turn-end markers the chat_template uses; resolve at load, harmony keeps eos."""
    return resolve_chat_turn_end_eos_ids_using(tokenizer, tokenizer)


def chat_eos_repair(current_eos, turn_end_ids) -> Optional[list]:
    """Merged eos_token_id list, or None if ``current_eos`` already covers every
    resolved turn-end id. Used to repair a model's generation_config at load so
    every ``.generate()`` path (vision, tool loops) stops at the turn boundary."""
    if not turn_end_ids:
        return None
    current_set = _eos_id_set(current_eos)
    if set(turn_end_ids) <= current_set:
        return None
    return sorted(current_set | set(turn_end_ids))