Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

8.2 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""dataset template fallback and consistency across splits (#11321)."""

from datasets import Dataset

from utils.datasets import apply_chat_template_to_dataset
from utils.datasets import chat_templates

OWN = "own-template"
OVERRIDE = "unsloth-override"

_WEATHER = {
    "type": "function",
    "function": {
        "name": "get_weather",
        "parameters": {"type": "object", "properties": {}},
    },
}


class _TemplatedTokenizer:
    eos_token = "</s>"

    def __init__(self):
        self.chat_template = OWN

    def apply_chat_template(self, conversation, **_kwargs):
        if any(turn["role"] == "broken" for turn in conversation):
            raise ValueError("Invalid content type")
        if self.chat_template == OVERRIDE and any(turn["role"] == "tool" for turn in conversation):
            raise ValueError("Conversation roles must alternate user/assistant/user/assistant/...")
        turns = "\n".join(f"{turn['role']}: {turn['content']}" for turn in conversation)
        return f"[{self.chat_template}] {turns}"


class _OwnToolTemplateTokenizer(_TemplatedTokenizer):
    def apply_chat_template(
        self,
        conversation,
        tools = None,
        **kwargs,
    ):
        rendered = super().apply_chat_template(conversation, **kwargs)
        if self.chat_template == OWN and tools:
            return f"[tools] {rendered}"
        return rendered


def _plain_convo(index):
    return [
        {"role": "user", "content": f"question {index}"},
        {"role": "assistant", "content": f"answer {index}"},
    ]


def _agentic_convo(index):
    return [
        {"role": "user", "content": f"question {index}"},
        {"role": "assistant", "content": "calling a tool"},
        {"role": "tool", "content": "tool result"},
        {"role": "assistant", "content": f"answer {index}"},
    ]


def _dataset_info(convo_factory, rows = 8):
    return {
        "dataset": Dataset.from_dict({"messages": [convo_factory(i) for i in range(rows)]}),
        "detected_format": "chatml_messages",
        "final_format": "chatml_messages",
        "chat_column": "messages",
        "is_standardized": True,
        "warnings": [],
    }


def _apply_override(tokenizer, _model_name):
    tokenizer.chat_template = OVERRIDE
    return tokenizer


def _format(dataset_info, tokenizer, monkeypatch):
    monkeypatch.setattr(chat_templates, "get_tokenizer_chat_template", _apply_override)
    return apply_chat_template_to_dataset(
        dataset_info,
        tokenizer,
        model_name = "unsloth/gemma-4-E4B-it",
        num_proc = 1,
    )


def test_agentic_rows_fall_back_to_the_checkpoints_own_template(monkeypatch):
    tokenizer = _TemplatedTokenizer()

    result = _format(_dataset_info(_agentic_convo), tokenizer, monkeypatch)

    assert result["success"] is True
    assert result["errors"] == []
    assert len(result["dataset"]) == 8
    assert all("tool: tool result" in text for text in result["dataset"]["text"])
    assert tokenizer.chat_template == OWN


def test_override_is_kept_when_it_renders_the_dataset(monkeypatch):
    tokenizer = _TemplatedTokenizer()

    result = _format(_dataset_info(_plain_convo), tokenizer, monkeypatch)

    assert result["success"] is True
    assert len(result["dataset"]) == 8
    assert tokenizer.chat_template == OVERRIDE


def test_own_template_replaces_an_override_that_silently_drops_tools(monkeypatch):
    tokenizer = _OwnToolTemplateTokenizer()
    dataset_info = _dataset_info(_plain_convo)
    dataset_info["dataset"] = dataset_info["dataset"].add_column(
        "tools", [[_WEATHER] for _ in range(8)]
    )

    result = _format(dataset_info, tokenizer, monkeypatch)

    assert result["success"] is True
    assert all(text.startswith("[tools]") for text in result["dataset"]["text"])
    assert tokenizer.chat_template == OWN


def test_sparse_tool_catalog_is_included_in_the_template_probe(monkeypatch):
    tokenizer = _OwnToolTemplateTokenizer()
    dataset_info = _dataset_info(_plain_convo, rows = 40)
    dataset_info["dataset"] = dataset_info["dataset"].add_column(
        "tools", [[_WEATHER] if index == 37 else [] for index in range(40)]
    )

    result = _format(dataset_info, tokenizer, monkeypatch)

    assert result["success"] is True
    assert result["dataset"][37]["text"].startswith("[tools]")
    assert tokenizer.chat_template == OWN


def test_override_is_kept_when_neither_template_renders(monkeypatch):
    tokenizer = _TemplatedTokenizer()
    broken = lambda index: [{"role": "broken", "content": f"row {index}"}]

    result = _format(_dataset_info(broken), tokenizer, monkeypatch)

    assert result["success"] is False
    assert "Chat template failed on all 8 rows" in result["errors"][0]
    assert tokenizer.chat_template == OVERRIDE


def test_only_the_rows_the_own_template_rejects_are_dropped(monkeypatch):
    tokenizer = _TemplatedTokenizer()
    mixed = lambda index: (
        [{"role": "broken", "content": f"row {index}"}]
        if index in (2, 5)
        else _agentic_convo(index)
    )

    result = _format(_dataset_info(mixed), tokenizer, monkeypatch)

    assert result["success"] is True
    assert len(result["dataset"]) == 6
    assert tokenizer.chat_template == OWN


def test_plain_rows_at_the_front_do_not_decide_it_for_the_agentic_rows_behind(monkeypatch):
    tokenizer = _TemplatedTokenizer()
    plain_then_agentic = lambda index: (_plain_convo(index) if index < 8 else _agentic_convo(index))

    result = _format(_dataset_info(plain_then_agentic, rows = 20), tokenizer, monkeypatch)

    assert result["success"] is True
    assert len(result["dataset"]) == 20
    assert tokenizer.chat_template == OWN


def test_a_small_dataset_probes_its_last_row(monkeypatch):
    tokenizer = _TemplatedTokenizer()
    agentic_tail = lambda index: (_agentic_convo(index) if index == 11 else _plain_convo(index))

    result = _format(_dataset_info(agentic_tail, rows = 12), tokenizer, monkeypatch)

    assert result["success"] is True
    assert len(result["dataset"]) == 12
    assert tokenizer.chat_template == OWN


def test_tool_rows_between_the_sampled_rows_are_still_dropped(monkeypatch):
    tokenizer = _TemplatedTokenizer()
    one_agentic = lambda index: (_agentic_convo(index) if index == 37 else _plain_convo(index))

    result = _format(_dataset_info(one_agentic, rows = 40), tokenizer, monkeypatch)

    assert result["success"] is True
    assert len(result["dataset"]) == 39
    assert tokenizer.chat_template == OVERRIDE
    assert "Dropped 1 of 40 rows" in result["dropped_rows_warning"]


def test_a_separate_eval_split_renders_with_the_template_training_chose(monkeypatch):
    tokenizer = _TemplatedTokenizer()

    train = _format(_dataset_info(_agentic_convo), tokenizer, monkeypatch)
    evaluation = _format(_dataset_info(_plain_convo), tokenizer, monkeypatch)

    assert all(text.startswith(f"[{OWN}]") for text in train["dataset"]["text"])
    assert all(text.startswith(f"[{OWN}]") for text in evaluation["dataset"]["text"])
    assert tokenizer.chat_template == OWN


def test_an_eval_split_does_not_move_the_template_training_chose(monkeypatch):
    tokenizer = _TemplatedTokenizer()

    train = _format(_dataset_info(_plain_convo), tokenizer, monkeypatch)
    evaluation = _format(_dataset_info(_agentic_convo), tokenizer, monkeypatch)

    assert all(text.startswith(f"[{OVERRIDE}]") for text in train["dataset"]["text"])
    assert evaluation["success"] is False
    assert "Chat template failed on all 8 rows" in evaluation["errors"][0]
    assert tokenizer.chat_template == OVERRIDE


def test_fallback_restores_the_template_on_a_processor_and_its_tokenizer():
    class _Processor(_TemplatedTokenizer):
        def __init__(self):
            super().__init__()
            self.tokenizer = _TemplatedTokenizer()

    processor = _Processor()
    processor.chat_template = OVERRIDE
    processor.tokenizer.chat_template = OVERRIDE
    dataset = Dataset.from_dict({"messages": [_agentic_convo(i) for i in range(4)]})

    note = chat_templates.keep_renderable_chat_template(processor, dataset, "messages", OWN)

    assert note
    assert processor.chat_template == OWN
    assert processor.tokenizer.chat_template == OWN