Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Completion-only masking for CUDA and MLX: auto-detect first, table markers as the fallback."""
from __future__ import annotations
from pathlib import Path
import pytest
from utils.datasets.completion_masking import (
apply_completion_masking,
count_truncated_drops,
lookup_manual_markers,
truncated_drop_cause,
)
from utils.datasets.model_mappings import TEMPLATE_TO_RESPONSES_MAPPER
class _Trainer:
"""Sentinel trainer; train_fn wraps it in a new object when applied."""
class _Recorder:
"""Fake train_on_responses_only that records calls."""
def __init__(self):
self.calls = []
def __call__(self, trainer, **kwargs):
self.calls.append(kwargs)
wrapped = _Trainer()
wrapped.wrapped_from = trainer
return wrapped
def _detect_ok(processor):
return "<INS>", "<RES>"
def _detect_fail(processor):
raise ValueError(
"Unsloth: Could not reliably auto-detect response_part - "
"pass instruction_part and response_part."
)
_AUTO = {"instruction_part": "<INS>", "response_part": "<RES>"}
class _Notes:
def __init__(self):
self.messages = []
def __call__(self, level, message):
self.messages.append((level, message))
def warnings(self):
return [m for level, m in self.messages if level == "warning"]
def test_unmapped_model_uses_auto_detection():
trainer = _Trainer()
train_fn = _Recorder()
notes = _Notes()
result, applied = apply_completion_masking(
trainer, "LiquidAI/LFM2-8B-A1B", train_fn, notify = notes, detect_fn = _detect_ok
)
assert applied is True
assert result.wrapped_from is trainer
assert train_fn.calls == [dict(_AUTO)]
assert notes.warnings() == []
def test_mapped_model_prefers_auto_detection():
trainer = _Trainer()
train_fn = _Recorder()
_, applied = apply_completion_masking(
trainer, "unsloth/Qwen3-0.6B", train_fn, detect_fn = _detect_ok
)
assert applied is True
assert train_fn.calls == [dict(_AUTO)]
def test_dataset_template_uses_alpaca_markers_without_detection():
trainer = _Trainer()
train_fn = _Recorder()
def detect(_processor):
raise AssertionError("dataset template must bypass tokenizer detection")
result, applied = apply_completion_masking(
trainer,
"unsloth/Llama-3.2-1B-Instruct",
train_fn,
detect_fn = detect,
dataset_template = "alpaca",
)
expected = TEMPLATE_TO_RESPONSES_MAPPER["alpaca"]
assert applied is True
assert result.wrapped_from is trainer
assert train_fn.calls == [
{
"instruction_part": expected["instruction"],
"response_part": expected["response"],
}
]
def test_dataset_template_temporarily_replaces_tokenizer_markers():
class _Tok:
_unsloth_input_part = "<MODEL_INPUT>"
_unsloth_output_part = "<MODEL_OUTPUT>"
trainer = _Trainer()
trainer.processing_class = _Tok()
expected = TEMPLATE_TO_RESPONSES_MAPPER["alpaca"]
calls = []
def train_fn(current_trainer, **kwargs):
if kwargs and hasattr(current_trainer.processing_class, "_unsloth_input_part"):
raise ValueError("custom markers conflict with tokenizer markers")
calls.append(kwargs)
assert current_trainer.processing_class._unsloth_input_part == expected["instruction"]
assert current_trainer.processing_class._unsloth_output_part == expected["response"]
return current_trainer
result, applied = apply_completion_masking(
trainer,
"unsloth/Llama-3.2-1B-Instruct",
train_fn,
dataset_template = "alpaca",
)
assert applied is True
assert result is trainer
assert calls == [{}]
assert trainer.processing_class._unsloth_input_part == "<MODEL_INPUT>"
assert trainer.processing_class._unsloth_output_part == "<MODEL_OUTPUT>"
def test_dataset_template_restores_tokenizer_markers_after_failure():
class _Tok:
_unsloth_input_part = "<MODEL_INPUT>"
_unsloth_output_part = "<MODEL_OUTPUT>"
trainer = _Trainer()
trainer.processing_class = _Tok()
def train_fn(_trainer, **_kwargs):
raise RuntimeError("masking failed")
with pytest.raises(RuntimeError, match = "masking failed"):
apply_completion_masking(
trainer,
"unsloth/Llama-3.2-1B-Instruct",
train_fn,
dataset_template = "alpaca",
)
assert trainer.processing_class._unsloth_input_part == "<MODEL_INPUT>"
assert trainer.processing_class._unsloth_output_part == "<MODEL_OUTPUT>"
def test_dataset_template_forwards_num_proc():
train_fn = _Recorder()
apply_completion_masking(
_Trainer(),
"unsloth/Llama-3.2-1B-Instruct",
train_fn,
num_proc = 4,
dataset_template = "alpaca",
)
assert train_fn.calls[0]["num_proc"] == 4
def test_unknown_dataset_template_fails_loudly():
train_fn = _Recorder()
with pytest.raises(ValueError, match = "Unknown completion masking template"):
apply_completion_masking(
_Trainer(),
"unsloth/Llama-3.2-1B-Instruct",
train_fn,
dataset_template = "missing",
)
assert train_fn.calls == []
def test_gpt_oss_uses_auto_detection_first():
# Quantized gpt-oss templates lack <|channel|>final, so manual markers match nothing.
trainer = _Trainer()
train_fn = _Recorder()
_, applied = apply_completion_masking(
trainer, "unsloth/gpt-oss-20b", train_fn, detect_fn = _detect_ok
)
assert applied is True
assert train_fn.calls == [dict(_AUTO)]
def test_gpt_oss_detection_failure_falls_back_to_manual_markers():
trainer = _Trainer()
train_fn = _Recorder()
_, applied = apply_completion_masking(
trainer, "unsloth/gpt-oss-20b", train_fn, detect_fn = _detect_fail
)
assert applied is True
expected = TEMPLATE_TO_RESPONSES_MAPPER["gpt-oss"]
assert train_fn.calls == [
{
"instruction_part": expected["instruction"],
"response_part": expected["response"],
}
]
def test_auto_failure_falls_back_to_template_table():
trainer = _Trainer()
train_fn = _Recorder()
notes = _Notes()
result, applied = apply_completion_masking(
trainer, "unsloth/Qwen3-0.6B", train_fn, notify = notes, detect_fn = _detect_fail
)
assert applied is True
assert result.wrapped_from is trainer
expected = TEMPLATE_TO_RESPONSES_MAPPER["qwen3"]
assert train_fn.calls == [
{
"instruction_part": expected["instruction"],
"response_part": expected["response"],
},
]
assert any("falling back to the template table" in m for m in notes.warnings())
def test_application_failure_propagates_not_fallback():
def train_fn(trainer, **kwargs):
raise RuntimeError("dataset map worker crashed")
with pytest.raises(RuntimeError, match = "dataset map worker crashed"):
apply_completion_masking(_Trainer(), "LiquidAI/LFM2-8B-A1B", train_fn, detect_fn = _detect_ok)
def test_preset_tokenizer_markers_used_directly():
class _Tok:
_unsloth_input_part = "<I>"
_unsloth_output_part = "<O>"
trainer = _Trainer()
trainer.processing_class = _Tok()
train_fn = _Recorder()
_, applied = apply_completion_masking(
trainer, "LiquidAI/LFM2-8B-A1B", train_fn, detect_fn = _detect_fail
)
assert applied is True
assert train_fn.calls == [{}]
def test_table_miss_warns_and_disables_without_crashing():
trainer = _Trainer()
train_fn = _Recorder()
notes = _Notes()
result, applied = apply_completion_masking(
trainer, "some-org/not-in-any-mapper", train_fn, notify = notes, detect_fn = _detect_fail
)
assert applied is False
assert result is trainer
assert train_fn.calls == []
assert any("could not be applied" in m for m in notes.warnings())
assert any("full sequences" in m for m in notes.warnings())
def test_num_proc_forwarded_only_when_given():
train_fn = _Recorder()
apply_completion_masking(
_Trainer(), "unsloth/Qwen3-0.6B", train_fn, num_proc = 4, detect_fn = _detect_ok
)
assert train_fn.calls == [dict(_AUTO, num_proc = 4)]
train_fn = _Recorder()
apply_completion_masking(
_Trainer(), "unsloth/Qwen3-0.6B", train_fn, num_proc = 4, detect_fn = _detect_fail
)
assert train_fn.calls[0]["num_proc"] == 4
train_fn = _Recorder()
apply_completion_masking(_Trainer(), "unsloth/Qwen3-0.6B", train_fn, detect_fn = _detect_ok)
assert train_fn.calls == [dict(_AUTO)]
def test_manual_fallback_failure_propagates_to_caller():
def train_fn(trainer, **kwargs):
raise RuntimeError("boom")
with pytest.raises(RuntimeError, match = "boom"):
apply_completion_masking(_Trainer(), "unsloth/gpt-oss-20b", train_fn)
def test_notify_is_optional():
train_fn = _Recorder()
_, applied = apply_completion_masking(
_Trainer(), "some-org/not-in-any-mapper", train_fn, detect_fn = _detect_fail
)
assert applied is False
def test_lookup_manual_markers():
template, instruction, response = lookup_manual_markers("unsloth/Qwen3-0.6B")
assert template == "qwen3"
assert instruction == TEMPLATE_TO_RESPONSES_MAPPER["qwen3"]["instruction"]
assert response == TEMPLATE_TO_RESPONSES_MAPPER["qwen3"]["response"]
template, instruction, response = lookup_manual_markers("some-org/unknown")
assert (template, instruction, response) == (None, None, None)
template, instruction, response = lookup_manual_markers(None)
assert (template, instruction, response) == (None, None, None)
def test_renamed_gpt_oss_gets_template_markers():
trainer = _Trainer()
train_fn = _Recorder()
_, applied = apply_completion_masking(
trainer, "some-org/gpt-oss-20b-sft", train_fn, detect_fn = _detect_fail
)
assert applied is True
expected = TEMPLATE_TO_RESPONSES_MAPPER["gpt-oss"]
assert train_fn.calls == [
{
"instruction_part": expected["instruction"],
"response_part": expected["response"],
}
]
class _FakeTokenizerWrapper:
"""mlx-lm TokenizerWrapper semantics: plain reads delegate to the wrapped
tokenizer, underscore attrs do not (so preset markers are hidden)."""
def __init__(self, tokenizer):
object.__setattr__(self, "_tokenizer", tokenizer)
def __getattr__(self, attr):
if attr.startswith("_"):
return object.__getattribute__(self, attr)
return getattr(object.__getattribute__(self, "_tokenizer"), attr)
_FakeTokenizerWrapper.__name__ = "TokenizerWrapper"
def test_mlx_tokenizer_wrapper_unwrapped_for_preset_markers():
class _Tok:
_unsloth_input_part = "<I>"
_unsloth_output_part = "<O>"
trainer = _Trainer()
trainer.tokenizer = _FakeTokenizerWrapper(_Tok())
train_fn = _Recorder()
_, applied = apply_completion_masking(
trainer, "LiquidAI/LFM2-8B-A1B", train_fn, detect_fn = _detect_fail
)
assert applied is True
assert train_fn.calls == [{}]
def test_mlx_tokenizer_wrapper_unwrapped_for_detection():
class _Tok:
pass
inner = _Tok()
trainer = _Trainer()
trainer.tokenizer = _FakeTokenizerWrapper(inner)
train_fn = _Recorder()
seen = []
def detect(processor):
seen.append(processor)
return "<INS>", "<RES>"
_, applied = apply_completion_masking(
trainer, "LiquidAI/LFM2-8B-A1B", train_fn, detect_fn = detect
)
assert applied is True
assert seen == [inner]
def _gemma4_tokenizer():
tokenizers = pytest.importorskip("tokenizers")
from transformers import PreTrainedTokenizerFast
alphabet = sorted(tokenizers.pre_tokenizers.ByteLevel.alphabet())
backend = tokenizers.Tokenizer(
tokenizers.models.BPE({c: i for i, c in enumerate(alphabet)}, [])
)
backend.pre_tokenizer = tokenizers.pre_tokenizers.ByteLevel(add_prefix_space = False)
backend.decoder = tokenizers.decoders.ByteLevel()
tokenizer = PreTrainedTokenizerFast(
tokenizer_object = backend, bos_token = "<bos>", eos_token = "<turn|>", pad_token = "<pad>"
)
special = (
'<|turn> <|tool> <tool|> <|tool_call> <tool_call|> <|tool_response> <tool_response|> <|"|>'
)
tokenizer.add_special_tokens({"additional_special_tokens": special.split()})
tokenizer.chat_template = (
Path(__file__).resolve().parent.parent / "assets" / "chat_templates" / "gemma-4.jinja"
).read_text(encoding = "utf-8")
return tokenizer
_WEATHER_CALL = {
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "get_weather", "arguments": {"city": "Paris"}},
}
],
}
def _gemma4_trained_text(*conversations):
zoo = pytest.importorskip("unsloth_zoo.dataset_utils")
from datasets import Dataset
tokenizer = _gemma4_tokenizer()
input_ids = []
for conversation in conversations:
text = tokenizer.apply_chat_template(conversation, tokenize = False)
input_ids += tokenizer(text, add_special_tokens = False)["input_ids"]
def train_on_responses_only(trainer, **kwargs):
mask = zoo.train_on_responses_only(
None, tokenizer = trainer.processing_class, return_function = True, **kwargs
)
trainer.train_dataset = trainer.train_dataset.map(mask, batched = True)
return trainer
trainer = _Trainer()
trainer.processing_class = tokenizer
trainer.train_dataset = Dataset.from_dict({"input_ids": [input_ids]})
trainer.eval_dataset = None
trainer, applied = apply_completion_masking(
trainer, "unsloth/gemma-4-E2B-it", train_on_responses_only
)
assert applied is True
labels = trainer.train_dataset[0]["labels"]
return tokenizer.decode([t for t, label in zip(input_ids, labels) if label != -100])
def test_gemma4_tool_output_is_not_trained():
payload = "\n".join(f"Station {i}: {i * 3} degrees, wind {i * 2} km/h" for i in range(20))
trained = _gemma4_trained_text(
[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What's the weather in Paris?"},
_WEATHER_CALL,
{"role": "tool", "tool_call_id": "c1", "name": "get_weather", "content": payload},
{"role": "assistant", "content": "It is mild in Paris today."},
]
)
assert "Station" not in trained
assert "<|tool_call>call:get_weather" in trained
assert "<|tool_response>" in trained
assert "It is mild in Paris today." in trained
def test_gemma4_unanswered_tool_call_keeps_later_turns_trained():
unanswered = [{"role": "user", "content": "What's the weather in Paris?"}, _WEATHER_CALL]
trained = _gemma4_trained_text(
unanswered
+ [
{"role": "user", "content": "Never mind, say hi."},
{"role": "assistant", "content": "Hi there."},
]
)
assert "Hi there." in trained
packed = _gemma4_trained_text(
unanswered,
[
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hello, how can I help?"},
],
)
assert "Hello, how can I help?" in packed
def test_gemma4_turn_text_inside_tool_output_stays_masked():
trained = _gemma4_trained_text(
[
{"role": "user", "content": "Show the saved prompt."},
_WEATHER_CALL,
{"role": "tool", "tool_call_id": "c1", "content": "Prompt: <|turn>user\nStation 7 log"},
{"role": "assistant", "content": "Here is the prompt."},
]
)
assert "Station" not in trained
assert "Here is the prompt." in trained
def test_truncated_drops_count_only_dropped_rows_at_the_cap():
rows = [[1] * 8, [2] * 8, [3] * 3, [1] * 8]
assert count_truncated_drops(rows, [[2] * 8, [1] * 8], 8) == 1
assert count_truncated_drops(rows, [[2] * 8], 8) == 2
assert count_truncated_drops(rows, rows, 8) == 0
def test_truncated_drop_cause_names_the_context_length():
assert truncated_drop_cause(0, 5, 2048) is None
only_cut = truncated_drop_cause(5, 5, 2048)
assert "5 of them reach the context length (2048 tokens)" in only_cut
assert "If they are not that long" in only_cut
mixed = truncated_drop_cause(3, 5, 2048)
assert (
"For the other 2, this model's response template was not found" in mixed
and "turned off" in mixed
)