Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

16.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Completion-only masking for CUDA and MLX: auto-detect first, table markers as the fallback."""

from __future__ import annotations

from pathlib import Path

import pytest

from utils.datasets.completion_masking import (
    apply_completion_masking,
    count_truncated_drops,
    lookup_manual_markers,
    truncated_drop_cause,
)
from utils.datasets.model_mappings import TEMPLATE_TO_RESPONSES_MAPPER


class _Trainer:
    """Sentinel trainer; train_fn wraps it in a new object when applied."""


class _Recorder:
    """Fake train_on_responses_only that records calls."""

    def __init__(self):
        self.calls = []

    def __call__(self, trainer, **kwargs):
        self.calls.append(kwargs)
        wrapped = _Trainer()
        wrapped.wrapped_from = trainer
        return wrapped


def _detect_ok(processor):
    return "<INS>", "<RES>"


def _detect_fail(processor):
    raise ValueError(
        "Unsloth: Could not reliably auto-detect response_part - "
        "pass instruction_part and response_part."
    )


_AUTO = {"instruction_part": "<INS>", "response_part": "<RES>"}


class _Notes:
    def __init__(self):
        self.messages = []

    def __call__(self, level, message):
        self.messages.append((level, message))

    def warnings(self):
        return [m for level, m in self.messages if level == "warning"]


def test_unmapped_model_uses_auto_detection():
    trainer = _Trainer()
    train_fn = _Recorder()
    notes = _Notes()

    result, applied = apply_completion_masking(
        trainer, "LiquidAI/LFM2-8B-A1B", train_fn, notify = notes, detect_fn = _detect_ok
    )

    assert applied is True
    assert result.wrapped_from is trainer
    assert train_fn.calls == [dict(_AUTO)]
    assert notes.warnings() == []


def test_mapped_model_prefers_auto_detection():
    trainer = _Trainer()
    train_fn = _Recorder()

    _, applied = apply_completion_masking(
        trainer, "unsloth/Qwen3-0.6B", train_fn, detect_fn = _detect_ok
    )

    assert applied is True
    assert train_fn.calls == [dict(_AUTO)]


def test_dataset_template_uses_alpaca_markers_without_detection():
    trainer = _Trainer()
    train_fn = _Recorder()

    def detect(_processor):
        raise AssertionError("dataset template must bypass tokenizer detection")

    result, applied = apply_completion_masking(
        trainer,
        "unsloth/Llama-3.2-1B-Instruct",
        train_fn,
        detect_fn = detect,
        dataset_template = "alpaca",
    )

    expected = TEMPLATE_TO_RESPONSES_MAPPER["alpaca"]
    assert applied is True
    assert result.wrapped_from is trainer
    assert train_fn.calls == [
        {
            "instruction_part": expected["instruction"],
            "response_part": expected["response"],
        }
    ]


def test_dataset_template_temporarily_replaces_tokenizer_markers():
    class _Tok:
        _unsloth_input_part = "<MODEL_INPUT>"
        _unsloth_output_part = "<MODEL_OUTPUT>"

    trainer = _Trainer()
    trainer.processing_class = _Tok()
    expected = TEMPLATE_TO_RESPONSES_MAPPER["alpaca"]
    calls = []

    def train_fn(current_trainer, **kwargs):
        if kwargs and hasattr(current_trainer.processing_class, "_unsloth_input_part"):
            raise ValueError("custom markers conflict with tokenizer markers")
        calls.append(kwargs)
        assert current_trainer.processing_class._unsloth_input_part == expected["instruction"]
        assert current_trainer.processing_class._unsloth_output_part == expected["response"]
        return current_trainer

    result, applied = apply_completion_masking(
        trainer,
        "unsloth/Llama-3.2-1B-Instruct",
        train_fn,
        dataset_template = "alpaca",
    )

    assert applied is True
    assert result is trainer
    assert calls == [{}]
    assert trainer.processing_class._unsloth_input_part == "<MODEL_INPUT>"
    assert trainer.processing_class._unsloth_output_part == "<MODEL_OUTPUT>"


def test_dataset_template_restores_tokenizer_markers_after_failure():
    class _Tok:
        _unsloth_input_part = "<MODEL_INPUT>"
        _unsloth_output_part = "<MODEL_OUTPUT>"

    trainer = _Trainer()
    trainer.processing_class = _Tok()

    def train_fn(_trainer, **_kwargs):
        raise RuntimeError("masking failed")

    with pytest.raises(RuntimeError, match = "masking failed"):
        apply_completion_masking(
            trainer,
            "unsloth/Llama-3.2-1B-Instruct",
            train_fn,
            dataset_template = "alpaca",
        )

    assert trainer.processing_class._unsloth_input_part == "<MODEL_INPUT>"
    assert trainer.processing_class._unsloth_output_part == "<MODEL_OUTPUT>"


def test_dataset_template_forwards_num_proc():
    train_fn = _Recorder()

    apply_completion_masking(
        _Trainer(),
        "unsloth/Llama-3.2-1B-Instruct",
        train_fn,
        num_proc = 4,
        dataset_template = "alpaca",
    )

    assert train_fn.calls[0]["num_proc"] == 4


def test_unknown_dataset_template_fails_loudly():
    train_fn = _Recorder()

    with pytest.raises(ValueError, match = "Unknown completion masking template"):
        apply_completion_masking(
            _Trainer(),
            "unsloth/Llama-3.2-1B-Instruct",
            train_fn,
            dataset_template = "missing",
        )

    assert train_fn.calls == []


def test_gpt_oss_uses_auto_detection_first():
    # Quantized gpt-oss templates lack <|channel|>final, so manual markers match nothing.
    trainer = _Trainer()
    train_fn = _Recorder()

    _, applied = apply_completion_masking(
        trainer, "unsloth/gpt-oss-20b", train_fn, detect_fn = _detect_ok
    )

    assert applied is True
    assert train_fn.calls == [dict(_AUTO)]


def test_gpt_oss_detection_failure_falls_back_to_manual_markers():
    trainer = _Trainer()
    train_fn = _Recorder()

    _, applied = apply_completion_masking(
        trainer, "unsloth/gpt-oss-20b", train_fn, detect_fn = _detect_fail
    )

    assert applied is True
    expected = TEMPLATE_TO_RESPONSES_MAPPER["gpt-oss"]
    assert train_fn.calls == [
        {
            "instruction_part": expected["instruction"],
            "response_part": expected["response"],
        }
    ]


def test_auto_failure_falls_back_to_template_table():
    trainer = _Trainer()
    train_fn = _Recorder()
    notes = _Notes()

    result, applied = apply_completion_masking(
        trainer, "unsloth/Qwen3-0.6B", train_fn, notify = notes, detect_fn = _detect_fail
    )

    assert applied is True
    assert result.wrapped_from is trainer
    expected = TEMPLATE_TO_RESPONSES_MAPPER["qwen3"]
    assert train_fn.calls == [
        {
            "instruction_part": expected["instruction"],
            "response_part": expected["response"],
        },
    ]
    assert any("falling back to the template table" in m for m in notes.warnings())


def test_application_failure_propagates_not_fallback():
    def train_fn(trainer, **kwargs):
        raise RuntimeError("dataset map worker crashed")

    with pytest.raises(RuntimeError, match = "dataset map worker crashed"):
        apply_completion_masking(_Trainer(), "LiquidAI/LFM2-8B-A1B", train_fn, detect_fn = _detect_ok)


def test_preset_tokenizer_markers_used_directly():
    class _Tok:
        _unsloth_input_part = "<I>"
        _unsloth_output_part = "<O>"

    trainer = _Trainer()
    trainer.processing_class = _Tok()
    train_fn = _Recorder()

    _, applied = apply_completion_masking(
        trainer, "LiquidAI/LFM2-8B-A1B", train_fn, detect_fn = _detect_fail
    )
    assert applied is True
    assert train_fn.calls == [{}]


def test_table_miss_warns_and_disables_without_crashing():
    trainer = _Trainer()
    train_fn = _Recorder()
    notes = _Notes()

    result, applied = apply_completion_masking(
        trainer, "some-org/not-in-any-mapper", train_fn, notify = notes, detect_fn = _detect_fail
    )

    assert applied is False
    assert result is trainer
    assert train_fn.calls == []
    assert any("could not be applied" in m for m in notes.warnings())
    assert any("full sequences" in m for m in notes.warnings())


def test_num_proc_forwarded_only_when_given():
    train_fn = _Recorder()
    apply_completion_masking(
        _Trainer(), "unsloth/Qwen3-0.6B", train_fn, num_proc = 4, detect_fn = _detect_ok
    )
    assert train_fn.calls == [dict(_AUTO, num_proc = 4)]

    train_fn = _Recorder()
    apply_completion_masking(
        _Trainer(), "unsloth/Qwen3-0.6B", train_fn, num_proc = 4, detect_fn = _detect_fail
    )
    assert train_fn.calls[0]["num_proc"] == 4

    train_fn = _Recorder()
    apply_completion_masking(_Trainer(), "unsloth/Qwen3-0.6B", train_fn, detect_fn = _detect_ok)
    assert train_fn.calls == [dict(_AUTO)]


def test_manual_fallback_failure_propagates_to_caller():
    def train_fn(trainer, **kwargs):
        raise RuntimeError("boom")

    with pytest.raises(RuntimeError, match = "boom"):
        apply_completion_masking(_Trainer(), "unsloth/gpt-oss-20b", train_fn)


def test_notify_is_optional():
    train_fn = _Recorder()
    _, applied = apply_completion_masking(
        _Trainer(), "some-org/not-in-any-mapper", train_fn, detect_fn = _detect_fail
    )
    assert applied is False


def test_lookup_manual_markers():
    template, instruction, response = lookup_manual_markers("unsloth/Qwen3-0.6B")
    assert template == "qwen3"
    assert instruction == TEMPLATE_TO_RESPONSES_MAPPER["qwen3"]["instruction"]
    assert response == TEMPLATE_TO_RESPONSES_MAPPER["qwen3"]["response"]

    template, instruction, response = lookup_manual_markers("some-org/unknown")
    assert (template, instruction, response) == (None, None, None)

    template, instruction, response = lookup_manual_markers(None)
    assert (template, instruction, response) == (None, None, None)


def test_renamed_gpt_oss_gets_template_markers():
    trainer = _Trainer()
    train_fn = _Recorder()

    _, applied = apply_completion_masking(
        trainer, "some-org/gpt-oss-20b-sft", train_fn, detect_fn = _detect_fail
    )
    assert applied is True
    expected = TEMPLATE_TO_RESPONSES_MAPPER["gpt-oss"]
    assert train_fn.calls == [
        {
            "instruction_part": expected["instruction"],
            "response_part": expected["response"],
        }
    ]


class _FakeTokenizerWrapper:
    """mlx-lm TokenizerWrapper semantics: plain reads delegate to the wrapped
    tokenizer, underscore attrs do not (so preset markers are hidden)."""

    def __init__(self, tokenizer):
        object.__setattr__(self, "_tokenizer", tokenizer)

    def __getattr__(self, attr):
        if attr.startswith("_"):
            return object.__getattribute__(self, attr)
        return getattr(object.__getattribute__(self, "_tokenizer"), attr)


_FakeTokenizerWrapper.__name__ = "TokenizerWrapper"


def test_mlx_tokenizer_wrapper_unwrapped_for_preset_markers():
    class _Tok:
        _unsloth_input_part = "<I>"
        _unsloth_output_part = "<O>"

    trainer = _Trainer()
    trainer.tokenizer = _FakeTokenizerWrapper(_Tok())
    train_fn = _Recorder()

    _, applied = apply_completion_masking(
        trainer, "LiquidAI/LFM2-8B-A1B", train_fn, detect_fn = _detect_fail
    )
    assert applied is True
    assert train_fn.calls == [{}]


def test_mlx_tokenizer_wrapper_unwrapped_for_detection():
    class _Tok:
        pass

    inner = _Tok()
    trainer = _Trainer()
    trainer.tokenizer = _FakeTokenizerWrapper(inner)
    train_fn = _Recorder()
    seen = []

    def detect(processor):
        seen.append(processor)
        return "<INS>", "<RES>"

    _, applied = apply_completion_masking(
        trainer, "LiquidAI/LFM2-8B-A1B", train_fn, detect_fn = detect
    )
    assert applied is True
    assert seen == [inner]


def _gemma4_tokenizer():
    tokenizers = pytest.importorskip("tokenizers")
    from transformers import PreTrainedTokenizerFast

    alphabet = sorted(tokenizers.pre_tokenizers.ByteLevel.alphabet())
    backend = tokenizers.Tokenizer(
        tokenizers.models.BPE({c: i for i, c in enumerate(alphabet)}, [])
    )
    backend.pre_tokenizer = tokenizers.pre_tokenizers.ByteLevel(add_prefix_space = False)
    backend.decoder = tokenizers.decoders.ByteLevel()
    tokenizer = PreTrainedTokenizerFast(
        tokenizer_object = backend, bos_token = "<bos>", eos_token = "<turn|>", pad_token = "<pad>"
    )
    special = (
        '<|turn> <|tool> <tool|> <|tool_call> <tool_call|> <|tool_response> <tool_response|> <|"|>'
    )
    tokenizer.add_special_tokens({"additional_special_tokens": special.split()})
    tokenizer.chat_template = (
        Path(__file__).resolve().parent.parent / "assets" / "chat_templates" / "gemma-4.jinja"
    ).read_text(encoding = "utf-8")
    return tokenizer


_WEATHER_CALL = {
    "role": "assistant",
    "content": "",
    "tool_calls": [
        {
            "id": "c1",
            "type": "function",
            "function": {"name": "get_weather", "arguments": {"city": "Paris"}},
        }
    ],
}


def _gemma4_trained_text(*conversations):
    zoo = pytest.importorskip("unsloth_zoo.dataset_utils")
    from datasets import Dataset

    tokenizer = _gemma4_tokenizer()
    input_ids = []
    for conversation in conversations:
        text = tokenizer.apply_chat_template(conversation, tokenize = False)
        input_ids += tokenizer(text, add_special_tokens = False)["input_ids"]

    def train_on_responses_only(trainer, **kwargs):
        mask = zoo.train_on_responses_only(
            None, tokenizer = trainer.processing_class, return_function = True, **kwargs
        )
        trainer.train_dataset = trainer.train_dataset.map(mask, batched = True)
        return trainer

    trainer = _Trainer()
    trainer.processing_class = tokenizer
    trainer.train_dataset = Dataset.from_dict({"input_ids": [input_ids]})
    trainer.eval_dataset = None

    trainer, applied = apply_completion_masking(
        trainer, "unsloth/gemma-4-E2B-it", train_on_responses_only
    )
    assert applied is True
    labels = trainer.train_dataset[0]["labels"]
    return tokenizer.decode([t for t, label in zip(input_ids, labels) if label != -100])


def test_gemma4_tool_output_is_not_trained():
    payload = "\n".join(f"Station {i}: {i * 3} degrees, wind {i * 2} km/h" for i in range(20))
    trained = _gemma4_trained_text(
        [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "What's the weather in Paris?"},
            _WEATHER_CALL,
            {"role": "tool", "tool_call_id": "c1", "name": "get_weather", "content": payload},
            {"role": "assistant", "content": "It is mild in Paris today."},
        ]
    )
    assert "Station" not in trained
    assert "<|tool_call>call:get_weather" in trained
    assert "<|tool_response>" in trained
    assert "It is mild in Paris today." in trained


def test_gemma4_unanswered_tool_call_keeps_later_turns_trained():
    unanswered = [{"role": "user", "content": "What's the weather in Paris?"}, _WEATHER_CALL]
    trained = _gemma4_trained_text(
        unanswered
        + [
            {"role": "user", "content": "Never mind, say hi."},
            {"role": "assistant", "content": "Hi there."},
        ]
    )
    assert "Hi there." in trained
    packed = _gemma4_trained_text(
        unanswered,
        [
            {"role": "user", "content": "Hello"},
            {"role": "assistant", "content": "Hello, how can I help?"},
        ],
    )
    assert "Hello, how can I help?" in packed


def test_gemma4_turn_text_inside_tool_output_stays_masked():
    trained = _gemma4_trained_text(
        [
            {"role": "user", "content": "Show the saved prompt."},
            _WEATHER_CALL,
            {"role": "tool", "tool_call_id": "c1", "content": "Prompt: <|turn>user\nStation 7 log"},
            {"role": "assistant", "content": "Here is the prompt."},
        ]
    )
    assert "Station" not in trained
    assert "Here is the prompt." in trained


def test_truncated_drops_count_only_dropped_rows_at_the_cap():
    rows = [[1] * 8, [2] * 8, [3] * 3, [1] * 8]
    assert count_truncated_drops(rows, [[2] * 8, [1] * 8], 8) == 1
    assert count_truncated_drops(rows, [[2] * 8], 8) == 2
    assert count_truncated_drops(rows, rows, 8) == 0


def test_truncated_drop_cause_names_the_context_length():
    assert truncated_drop_cause(0, 5, 2048) is None
    only_cut = truncated_drop_cause(5, 5, 2048)
    assert "5 of them reach the context length (2048 tokens)" in only_cut
    assert "If they are not that long" in only_cut
    mixed = truncated_drop_cause(3, 5, 2048)
    assert (
        "For the other 2, this model's response template was not found" in mixed
        and "turned off" in mixed
    )