Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from unsloth import FastLanguageModel
import unsloth.trainer as trainer_module
import unsloth.utils.packing as packing_module
from unsloth.utils import attention_dispatch as attention_dispatch_utils
from unsloth.utils.packing import (
configure_padding_free,
configure_sample_packing,
enable_padding_free_metadata,
enable_sample_packing,
mask_packed_boundary_labels,
mask_packed_sequence_boundaries,
patch_hybrid_linear_attention_varlen,
)
import copy
import inspect
import logging
from contextlib import ExitStack
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from real_accelerator import (
has_real_accelerator,
) # tests/_shared, on sys.path via tests/conftest.py
import torch
from datasets import Dataset, IterableDataset
from trl import SFTConfig, SFTTrainer
from trl.trainer.sft_trainer import DataCollatorForLanguageModeling
class _FakeConfig(SimpleNamespace):
# get_transformers_model_type() resolves through to_dict(), which SimpleNamespace lacks.
def to_dict(self):
return dict(self.__dict__)
def _build_packed_training_setup(tmp_path, device):
dtype = None
if device.type == "cuda":
if torch.cuda.is_bf16_supported():
dtype = torch.bfloat16
else:
dtype = torch.float16
elif device.type == "xpu":
dtype = torch.bfloat16
try:
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "hf-internal-testing/tiny-random-LlamaForCausalLM",
max_seq_length = 64,
load_in_4bit = False,
dtype = dtype,
)
except OSError as exc: # pragma: no cover - offline CI
pytest.skip(f"Requires access to tiny llama checkpoint: {exc}")
model.to(device)
dataset = Dataset.from_dict(
{
"text": [
"Hello world!",
"Short sample.",
"This is a slightly longer packed example to test batching.",
"Another response to include in the batch.",
]
}
)
training_args = SFTConfig(
per_device_train_batch_size = 1,
per_device_eval_batch_size = 1,
gradient_accumulation_steps = 1,
dataset_text_field = "text",
max_length = 64,
logging_steps = 1,
max_steps = 1,
fp16 = dtype == torch.float16,
bf16 = dtype == torch.bfloat16,
dataset_num_proc = 1,
output_dir = str(tmp_path),
packing = True,
)
trainer = SFTTrainer(
model = model,
processing_class = tokenizer,
train_dataset = dataset,
args = training_args,
)
enable_sample_packing(model, trainer)
dataloader = trainer.get_train_dataloader()
batch = next(iter(dataloader))
model_device = next(model.parameters()).device
for key, value in list(batch.items()):
if torch.is_tensor(value):
batch[key] = value.to(model_device)
from unsloth.models import llama as llama_mod
return model, batch, trainer, llama_mod
def _trim_batch_to_total_tokens(data, total_tokens):
def _trim_tensor(t: torch.Tensor):
if t.ndim >= 2 and t.size(1) > total_tokens:
return t[:, :total_tokens].contiguous()
return t
trimmed = {}
for key, value in data.items():
if torch.is_tensor(value):
trimmed[key] = _trim_tensor(value)
else:
trimmed[key] = value
return trimmed
def test_mask_packed_sequence_boundaries_marks_single_row():
shift_labels = torch.arange(6, dtype = torch.long).view(1, 6)
changed = mask_packed_sequence_boundaries(
shift_labels,
torch.tensor([2, 1, 3], dtype = torch.int32),
)
assert changed is True
flat = shift_labels.view(-1)
assert flat[1].item() == -100
assert flat[2].item() == -100
assert flat[5].item() == -100
assert flat[0].item() != -100
def test_mask_packed_sequence_boundaries_across_multiple_rows():
shift_labels = torch.arange(10, dtype = torch.long).view(2, 5)
lengths = torch.tensor([3, 2, 4, 1], dtype = torch.int32)
changed = mask_packed_sequence_boundaries(shift_labels, lengths)
assert changed is True
flat = shift_labels.view(-1)
for idx in (2, 4, 8, 9):
assert flat[idx].item() == -100
assert torch.any(flat != -100)
def test_enable_padding_free_metadata_does_not_mutate_examples():
collator = _PaddingFreeCollator()
trainer = SimpleNamespace(
data_collator = collator,
args = SimpleNamespace(remove_unused_columns = True),
)
enable_padding_free_metadata(_DummyModel(), trainer)
examples = [
{"input_ids": [1, 2, 3], "labels": [1, 2, 3]},
{"input_ids": [4, 5], "labels": [4, 5]},
]
before = copy.deepcopy(examples)
batch = trainer.data_collator.torch_call(examples)
assert examples == before, "collator wrapper mutated the caller's examples"
assert torch.equal(batch["packed_seq_lengths"], torch.tensor([3, 2], dtype = torch.int32))
explicit = [{"input_ids": [1, 2, 3], "seq_lengths": [2, 1]}]
explicit_before = copy.deepcopy(explicit)
batch = trainer.data_collator.torch_call(explicit)
assert explicit == explicit_before
assert torch.equal(batch["packed_seq_lengths"], torch.tensor([2, 1], dtype = torch.int32))
def test_enable_padding_free_metadata_still_hands_derived_lengths_to_the_collator():
collator = _PaddingFreeCollator()
trainer = SimpleNamespace(
data_collator = collator,
args = SimpleNamespace(remove_unused_columns = True),
)
enable_padding_free_metadata(_DummyModel(), trainer)
examples = [
{"input_ids": [1, 2, 3], "labels": [1, 2, 3]},
{"input_ids": [4, 5], "labels": [4, 5]},
]
before = copy.deepcopy(examples)
trainer.data_collator.torch_call(examples)
assert examples == before, "collator wrapper mutated the caller's examples"
assert [row["seq_lengths"] for row in collator.seen] == [[3], [2]]
assert [row["labels"] for row in collator.seen] == [[1, 2, 3], [4, 5]]
# seq_lengths=None counts as missing: TRL would sum(None).
nulled = [{"input_ids": [1, 2], "seq_lengths": None}]
nulled_before = copy.deepcopy(nulled)
trainer.data_collator.torch_call(nulled)
assert nulled == nulled_before
assert [row["seq_lengths"] for row in collator.seen] == [[2]]
explicit = [{"input_ids": [1, 2, 3], "seq_lengths": [2, 1]}]
trainer.data_collator.torch_call(explicit)
assert collator.seen is explicit
def test_configure_sample_packing():
config = SimpleNamespace()
configure_sample_packing(config)
assert config.packing is True
assert config.padding_free is True
assert config.remove_unused_columns is False
def test_configure_padding_free():
config = SimpleNamespace(remove_unused_columns = True)
configure_padding_free(config)
assert config.padding_free is True
assert config.remove_unused_columns is False
def _hybrid_config_model():
return SimpleNamespace(config = _FakeConfig(layer_types = ["linear_attention", "full_attention"]))
def _gemma3_model():
return SimpleNamespace(
config = _FakeConfig(
model_type = "gemma3", layer_types = ["sliding_attention", "full_attention"]
),
)
def _dense_qwen3_model():
return SimpleNamespace(
config = _FakeConfig(model_type = "qwen3", architectures = ["Qwen3ForCausalLM"])
)
class _FakeGatedDeltaNet(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv1d = torch.nn.Conv1d(4, 4, 3, groups = 4)
self.A_log = torch.nn.Parameter(torch.zeros(4))
def forward(self, hidden_states, **kwargs):
return self.chunk_gated_delta_rule(self.causal_conv1d_fn(hidden_states))
class _FakeHybridModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.config = SimpleNamespace()
self.linear_attn = _FakeGatedDeltaNet()
def test_is_hybrid_linear_attention_detects_and_excludes():
is_hybrid = trainer_module._is_hybrid_linear_attention_model
assert is_hybrid(_hybrid_config_model()) is True
assert is_hybrid(_FakeHybridModel()) is True
assert is_hybrid(_text_model()) is False
assert is_hybrid(_gemma3_model()) is False
assert is_hybrid(_dense_qwen3_model()) is False
assert is_hybrid(None) is False
def test_varlen_from_position_ids():
cu, seq_idx = packing_module._varlen_from_position_ids(torch.tensor([[0, 1, 0, 0, 1, 2]]))
assert cu.tolist() == [0, 2, 3, 6]
assert seq_idx.tolist() == [[0, 0, 1, 2, 2, 2]]
assert packing_module._varlen_from_position_ids(torch.tensor([[0, 1, 2, 3]])) is None
assert packing_module._varlen_from_position_ids(torch.tensor([[1, 2, 3]])) is None
assert packing_module._varlen_from_position_ids(torch.tensor([[0, 1], [0, 1]])) is None
assert packing_module._varlen_from_position_ids(None) is None
def test_get_packed_info_infers_lengths_from_reset_position_ids():
cpu = torch.device("cpu")
lengths, cu_seqlens, max_seqlen = packing_module.get_packed_info_from_kwargs(
{}, cpu, torch.tensor([[0, 1, 2, 0, 1]])
)
assert lengths.tolist() == [3, 2]
assert cu_seqlens.tolist() == [0, 3, 5]
assert max_seqlen == 3
# Opt-in only, and ordinary or batched positions are not a pack.
assert packing_module.get_packed_info_from_kwargs({}, cpu) is None
assert packing_module.get_packed_info_from_kwargs({}, cpu, torch.arange(5)[None]) is None
assert (
packing_module.get_packed_info_from_kwargs({}, cpu, torch.tensor([[0, 1], [0, 1]])) is None
)
explicit = torch.tensor([4, 1], dtype = torch.int32)
lengths, _, _ = packing_module.get_packed_info_from_kwargs(
{"packed_seq_lengths": explicit}, cpu, torch.tensor([[0, 1, 2, 0, 1]])
)
assert lengths.tolist() == [4, 1]
def test_seq_idx_from_cu_seqlens_handles_trailing_pad():
cu = torch.tensor([0, 2, 5], dtype = torch.int32)
boundaries, seq_idx = packing_module._seq_idx_from_cu_seqlens(cu, total = 8)
assert boundaries.tolist() == [0, 2, 5, 8]
assert seq_idx.tolist() == [[0, 0, 1, 1, 1, 2, 2, 2]]
boundaries2, _ = packing_module._seq_idx_from_cu_seqlens(cu, total = 5)
assert boundaries2.tolist() == [0, 2, 5]
assert (
packing_module._seq_idx_from_cu_seqlens(torch.tensor([1, 2], dtype = torch.int32), total = 2)
is None
)
assert packing_module._seq_idx_from_cu_seqlens(cu, total = 3) is None
def test_hybrid_varlen_metadata_prefers_packed_seq_lengths():
# A competing position_ids would segment [0, 3, 6]; packed_seq_lengths must win.
kwargs = {
"input_ids": torch.zeros(1, 6, dtype = torch.long),
"packed_seq_lengths": torch.tensor([2, 1, 3], dtype = torch.int32),
"position_ids": torch.tensor([[0, 1, 2, 0, 1, 2]]),
}
cu, seq_idx = packing_module._hybrid_varlen_metadata(kwargs)
assert cu.tolist() == [0, 2, 3, 6]
assert seq_idx.tolist() == [[0, 0, 1, 2, 2, 2]]
def test_hybrid_varlen_metadata_suppressed_when_cached():
base = {
"input_ids": torch.zeros(1, 6, dtype = torch.long),
"packed_seq_lengths": torch.tensor([2, 1, 3], dtype = torch.int32),
}
assert packing_module._hybrid_varlen_metadata({**base, "use_cache": True}) is None
assert packing_module._hybrid_varlen_metadata({**base, "past_key_values": object()}) is None
def test_hybrid_varlen_metadata_none_for_plain_batch():
kwargs = {
"input_ids": torch.zeros(1, 4, dtype = torch.long),
"position_ids": torch.tensor([[0, 1, 2, 3]]),
}
assert packing_module._hybrid_varlen_metadata(kwargs) is None
def _make_fake_kernels():
def causal_conv1d_fn(
x,
weight = None,
bias = None,
activation = None,
seq_idx = None,
):
causal_conv1d_fn.calls.append(seq_idx)
return x
causal_conv1d_fn.calls = []
def chunk_gated_delta_rule(
q,
k = None,
v = None,
cu_seqlens = None,
**kw,
):
chunk_gated_delta_rule.calls.append(cu_seqlens)
return q
chunk_gated_delta_rule.calls = []
return causal_conv1d_fn, chunk_gated_delta_rule
class _ShimGatedDeltaNet(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv1d = torch.nn.Conv1d(4, 4, 3, groups = 4)
self.causal_conv1d_fn, self.chunk_gated_delta_rule = _make_fake_kernels()
def forward(self, hidden_states, **kwargs):
return self.chunk_gated_delta_rule(self.causal_conv1d_fn(hidden_states))
class _ShimHybridModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.config = SimpleNamespace(layer_types = ["linear_attention", "full_attention"])
self.linear_attn = _ShimGatedDeltaNet()
def forward(
self,
input_ids = None,
position_ids = None,
packed_seq_lengths = None,
use_cache = None,
**kwargs,
):
return self.linear_attn(input_ids.float())
def test_patch_hybrid_varlen_flag_off(monkeypatch):
monkeypatch.delenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", raising = False)
model = _ShimHybridModel()
assert patch_hybrid_linear_attention_varlen(model) is False
assert not getattr(model, "_unsloth_varlen_forward_wrapped", False)
def test_patch_hybrid_varlen_active_and_idempotent(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
model = _ShimHybridModel()
conv_orig, scan_orig = (
model.linear_attn.causal_conv1d_fn,
model.linear_attn.chunk_gated_delta_rule,
)
assert patch_hybrid_linear_attention_varlen(model) is True
assert model._unsloth_varlen_forward_wrapped is True
assert model.linear_attn._unsloth_varlen_wrapped is True
assert patch_hybrid_linear_attention_varlen(model) is True
conv_orig.calls.clear()
scan_orig.calls.clear()
packing_module._HYBRID_WARNED.clear()
ids = torch.zeros(1, 6, dtype = torch.long)
model(
input_ids = ids,
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
assert conv_orig.calls[-1] is not None
assert scan_orig.calls[-1].tolist() == [0, 2, 3, 6]
assert not packing_module._HYBRID_WARNED
conv_orig.calls.clear()
scan_orig.calls.clear()
model(
input_ids = ids, packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32), use_cache = True
)
assert conv_orig.calls[-1] is None
assert scan_orig.calls[-1] is None
def test_patch_hybrid_varlen_survives_accelerate_unwrap(monkeypatch):
# TRL runs accelerate's extract_model_from_parallel(keep_fp32_wrapper = False) every step
# after a mixed-precision prepare_model; it rebinds the forward it finds with
# MethodType(forward, model). A plain-function wrapper then got the model positionally:
# "PeftModelForCausalLM.forward() got multiple values for argument 'input_ids'".
from types import MethodType
from accelerate.utils import extract_model_from_parallel
from accelerate.utils.operations import convert_outputs_to_fp32
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
model = _ShimHybridModel()
assert patch_hybrid_linear_attention_varlen(model) is True
assert list(inspect.signature(model.forward).parameters)[:3] == [
"input_ids",
"position_ids",
"packed_seq_lengths",
]
# Accelerator.prepare_model with native AMP (accelerate/accelerator.py).
model._original_forward = model.forward
if hasattr(model.forward, "__func__"):
model.forward = MethodType(convert_outputs_to_fp32(model.forward.__func__), model)
else:
model.forward = convert_outputs_to_fp32(model.forward)
model = extract_model_from_parallel(model, keep_fp32_wrapper = False)
out = model(input_ids = torch.zeros(1, 6, dtype = torch.long), use_cache = False)
assert out.shape == (1, 6)
def test_patch_hybrid_varlen_torch_fallback_fail_closed(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
model = _ShimHybridModel()
def torch_chunk_gated_delta_rule(
q,
cu_seqlens = None,
**kw,
):
return q
model.linear_attn.chunk_gated_delta_rule = torch_chunk_gated_delta_rule
assert patch_hybrid_linear_attention_varlen(model) is False
assert not getattr(model, "_unsloth_varlen_forward_wrapped", False)
def test_patch_hybrid_varlen_bad_signature_fail_closed(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
model = _ShimHybridModel()
def scan_no_cu(q, **kw):
return q
model.linear_attn.chunk_gated_delta_rule = scan_no_cu
assert patch_hybrid_linear_attention_varlen(model) is False
def _hybrid_model_with_gdn(gdn_forward):
class _GatedDeltaNet(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv1d = torch.nn.Conv1d(4, 4, 3, groups = 4)
self.causal_conv1d_fn, self.chunk_gated_delta_rule = _make_fake_kernels()
forward = gdn_forward
class _Model(torch.nn.Module):
def __init__(self):
super().__init__()
self.config = SimpleNamespace(layer_types = ["linear_attention", "full_attention"])
self.linear_attn = _GatedDeltaNet()
def forward(
self,
input_ids = None,
packed_seq_lengths = None,
use_cache = None,
**kwargs,
):
return self.linear_attn(input_ids.float())
return _Model()
def test_patch_hybrid_varlen_no_dispatch_aborts(monkeypatch):
# Dispatch is verified at runtime: both boundary kernels must be called.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
model = _hybrid_model_with_gdn(lambda self, hidden_states, **kw: hidden_states)
assert patch_hybrid_linear_attention_varlen(model) is True
with pytest.raises(RuntimeError, match = "were not invoked"):
model(
input_ids = torch.zeros(1, 6),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
def test_patch_hybrid_varlen_partial_dispatch_aborts(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
conv_only = _hybrid_model_with_gdn(
lambda self, hidden_states, **kw: self.causal_conv1d_fn(hidden_states)
)
assert patch_hybrid_linear_attention_varlen(conv_only) is True
with pytest.raises(RuntimeError, match = "were not invoked"):
conv_only(
input_ids = torch.zeros(1, 6),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
scan_only = _hybrid_model_with_gdn(
lambda self, hidden_states, **kw: self.chunk_gated_delta_rule(hidden_states)
)
assert patch_hybrid_linear_attention_varlen(scan_only) is True
with pytest.raises(RuntimeError, match = "were not invoked"):
scan_only(
input_ids = torch.zeros(1, 6),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
def _make_fake_mamba2_fused():
def mamba2_split_conv1d_scan_combined(
*args,
seq_idx = None,
**kwargs,
):
mamba2_split_conv1d_scan_combined.calls.append(seq_idx)
return args[0] if args else None
mamba2_split_conv1d_scan_combined.calls = []
return mamba2_split_conv1d_scan_combined
class _FakeNemotronHMamba2Mixer(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv1d = torch.nn.Conv1d(4, 4, 3, groups = 4)
self.A_log = torch.nn.Parameter(torch.zeros(4))
self.dt_bias = torch.nn.Parameter(torch.zeros(4))
self.mamba2_split_conv1d_scan_combined = _make_fake_mamba2_fused()
def forward(self, hidden_states, **kwargs):
fused_kwargs = dict(kwargs)
return self.mamba2_split_conv1d_scan_combined(hidden_states, **fused_kwargs)
class _FakeMamba2Model(torch.nn.Module):
def __init__(self):
super().__init__()
self.config = SimpleNamespace() # no markers -> module-level detection
self.mixer = _FakeNemotronHMamba2Mixer()
def forward(
self,
input_ids = None,
packed_seq_lengths = None,
use_cache = None,
**kwargs,
):
return self.mixer(input_ids.float(), **kwargs)
def test_is_hybrid_linear_attention_detects_mamba2_mixer():
is_hybrid = trainer_module._is_hybrid_linear_attention_model
assert is_hybrid(_FakeMamba2Model()) is True
def test_patch_mamba2_varlen_injects_seq_idx(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
model = _FakeMamba2Model()
fused_orig = model.mixer.mamba2_split_conv1d_scan_combined
assert patch_hybrid_linear_attention_varlen(model) is True
assert patch_hybrid_linear_attention_varlen(model) is True # idempotent
fused_orig.calls.clear()
packing_module._HYBRID_WARNED.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
seq_idx = fused_orig.calls[-1]
assert seq_idx is not None
assert seq_idx.tolist() == [[0, 0, 1, 2, 2, 2]]
assert not packing_module._HYBRID_WARNED
fused_orig.calls.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = True,
)
assert fused_orig.calls[-1] is None
def test_patch_mamba2_varlen_kwargs_only_stub_fail_closed(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
model = _FakeMamba2Model()
def stub(*args, **kwargs):
return None
model.mixer.mamba2_split_conv1d_scan_combined = stub
assert patch_hybrid_linear_attention_varlen(model) is False
def _fake_mamba2_model_with_modeling(name, **kernel_globals):
# The mixer's __init__ globals stand in for the modeling module the kernels load into.
import sys
import types
modeling = types.ModuleType(name)
modeling.__dict__.update(kernel_globals)
modeling._Base = _FakeNemotronHMamba2Mixer
exec(
"class NemotronHMamba2Mixer(_Base):\n"
" def __init__(self):\n"
" super().__init__()\n",
modeling.__dict__,
)
modeling.NemotronHMamba2Mixer.__module__ = name
sys.modules[name] = modeling
model = _FakeMamba2Model()
model.mixer = modeling.NemotronHMamba2Mixer()
return model, modeling
def test_patch_mamba2_varlen_rejects_unavailable_fast_path(monkeypatch):
# Kernels missing at mixer __init__: transformers routes to torch_forward, so decline up front.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
name = "fake_modeling_nemotron_h_nofast"
try:
model, modeling = _fake_mamba2_model_with_modeling(
name, is_fast_path_available = False, causal_conv1d_update = None
)
assert patch_hybrid_linear_attention_varlen(model) is False
assert modeling.is_fast_path_available is False
finally:
sys.modules.pop(name, None)
def test_patch_mamba2_varlen_syncs_stale_compiled_kernel_globals(monkeypatch):
# The compiled copy gets the real kernel globals, never a forced fast path.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
import types
def conv_update(*args, **kwargs):
return None
name = "fake_modeling_nemotron_h_fast"
compiled = types.ModuleType("unsloth_compiled_cache.fake_nemotron_h_sync")
compiled.is_fast_path_available = False
compiled.causal_conv1d_update = None
other = types.ModuleType("fake_nemotron_h_other")
other.is_fast_path_available = False
sys.modules[compiled.__name__] = compiled
sys.modules[other.__name__] = other
try:
model, modeling = _fake_mamba2_model_with_modeling(
name, is_fast_path_available = True, causal_conv1d_update = conv_update
)
assert patch_hybrid_linear_attention_varlen(model) is True
assert compiled.is_fast_path_available is True
assert compiled.causal_conv1d_update is conv_update
assert other.is_fast_path_available is False
finally:
for key in (name, compiled.__name__, other.__name__):
sys.modules.pop(key, None)
def test_patch_mamba2_varlen_rebinds_compiled_module_alias(monkeypatch):
# unsloth_compiled_cache holds its own fused-kernel alias; it must be rebound too.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
import types
fused = _make_fake_mamba2_fused()
compiled = types.ModuleType("unsloth_compiled_cache.NemotronHMamba2Mixer")
compiled.mamba_split_conv1d_scan_combined = fused
sys.modules[compiled.__name__] = compiled
class _CompiledNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def __init__(self):
super().__init__()
del self.mamba2_split_conv1d_scan_combined
self.mamba2_split_conv1d_scan_combined = fused
def forward(self, hidden_states, **fused_kwargs):
return compiled.mamba_split_conv1d_scan_combined(
hidden_states, seq_idx = None, **fused_kwargs
)
class _CompiledModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _CompiledNemotronHMamba2Mixer()
try:
model = _CompiledModel()
assert patch_hybrid_linear_attention_varlen(model) is True
fused.calls.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
assert fused.calls[-1] is not None
assert fused.calls[-1].tolist() == [[0, 0, 1, 2, 2, 2]]
assert compiled.mamba_split_conv1d_scan_combined is not fused
finally:
sys.modules.pop(compiled.__name__, None)
def _make_hub_mamba2_namespace(module_name):
# Functions exec'd into a registered module dict, so their LOAD_GLOBALs resolve through it.
import sys
import types
module = types.ModuleType(module_name)
sys.modules[module_name] = module
exec(
"""
def mamba_split_conv1d_scan_combined(*args, seq_idx=None, **kwargs):
mamba_split_conv1d_scan_combined.calls.append(seq_idx)
return args[0]
mamba_split_conv1d_scan_combined.calls = []
def cuda_kernels_forward(self, hidden_states, cache_params=None, attention_mask=None):
return mamba_split_conv1d_scan_combined(hidden_states, seq_idx=None)
""",
module.__dict__,
)
return module
def test_patch_mamba2_varlen_rewrites_cuda_kernels_forward_global(monkeypatch):
# transformers <= 5.15 cuda_kernels_forward LOAD_GLOBALs the kernel with seq_idx=None.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
import types
module = _make_hub_mamba2_namespace("transformers_modules.fake_nemotron_h")
ns = module.__dict__
class _HubNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def __init__(self):
super().__init__()
del self.mamba2_split_conv1d_scan_combined
self.cuda_kernels_forward = types.MethodType(ns["cuda_kernels_forward"], self)
def forward(self, hidden_states, **kwargs):
return self.cuda_kernels_forward(hidden_states)
_HubNemotronHMamba2Mixer.__module__ = module.__name__
class _HubModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _HubNemotronHMamba2Mixer()
try:
model = _HubModel()
assert patch_hybrid_linear_attention_varlen(model) is True
ns["mamba_split_conv1d_scan_combined"].calls.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
seq_idx = ns["mamba_split_conv1d_scan_combined"].calls[-1]
assert seq_idx is not None
assert seq_idx.tolist() == [[0, 0, 1, 2, 2, 2]]
finally:
sys.modules.pop(module.__name__, None)
def test_patch_mamba2_varlen_unreachable_kernel_namespace_aborts(monkeypatch):
# A kernel reachable only from an unregistered namespace must fail closed.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import types
ns: dict = {"__name__": "nowhere.fake_nemotron_h"}
exec(
"""
def mamba_split_conv1d_scan_combined(*args, seq_idx=None, **kwargs):
return args[0]
def cuda_kernels_forward(self, hidden_states, cache_params=None, attention_mask=None):
return mamba_split_conv1d_scan_combined(hidden_states, seq_idx=None)
""",
ns,
)
class _HiddenNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def __init__(self):
super().__init__()
self.cuda_kernels_forward = types.MethodType(ns["cuda_kernels_forward"], self)
def forward(self, hidden_states, **kwargs):
return self.cuda_kernels_forward(hidden_states)
class _HiddenModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _HiddenNemotronHMamba2Mixer()
model = _HiddenModel()
assert patch_hybrid_linear_attention_varlen(model) is True
with pytest.raises(RuntimeError, match = "were not invoked"):
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
def test_patch_mamba2_varlen_reaches_compiled_module_global(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
import types
compiled = types.ModuleType("unsloth_compiled_cache.NemotronHMamba2Mixer")
ns = compiled.__dict__
ns["__name__"] = compiled.__name__
exec(
"""
def mamba_split_conv1d_scan_combined(*args, seq_idx=None, **kwargs):
mamba_split_conv1d_scan_combined.calls.append(seq_idx)
return args[0]
mamba_split_conv1d_scan_combined.calls = []
def NemotronHMamba2Mixer_cuda_kernels_forward(self, hidden_states, cache_params=None, attention_mask=None):
return mamba_split_conv1d_scan_combined(hidden_states, seq_idx=None)
""",
ns,
)
sys.modules[compiled.__name__] = compiled
class _CompiledNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def __init__(self):
super().__init__()
del self.mamba2_split_conv1d_scan_combined
self.cuda_kernels_forward = types.MethodType(
ns["NemotronHMamba2Mixer_cuda_kernels_forward"], self
)
def forward(self, hidden_states, **kwargs):
return self.cuda_kernels_forward(hidden_states)
_CompiledNemotronHMamba2Mixer.__module__ = compiled.__name__
class _CompiledModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _CompiledNemotronHMamba2Mixer()
try:
model = _CompiledModel()
assert patch_hybrid_linear_attention_varlen(model) is True
ns["mamba_split_conv1d_scan_combined"].calls.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
seq_idx = ns["mamba_split_conv1d_scan_combined"].calls[-1]
assert seq_idx is not None
assert seq_idx.tolist() == [[0, 0, 1, 2, 2, 2]]
finally:
sys.modules.pop(compiled.__name__, None)
def test_patch_mamba2_varlen_overwrites_stale_compiled_import(monkeypatch):
# The compiled cache imported the kernel before mixer __init__ loaded the real one.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
import types
def stale(
*args,
seq_idx = None,
**kwargs,
):
stale.calls.append(seq_idx)
return args[0] if args else None
stale.calls = []
real = _make_fake_mamba2_fused()
compiled = types.ModuleType("unsloth_compiled_cache.NemotronHMamba2Mixer")
compiled.mamba_split_conv1d_scan_combined = stale
sys.modules[compiled.__name__] = compiled
class _StaleNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def __init__(self):
super().__init__()
self.mamba2_split_conv1d_scan_combined = real
def cuda_kernels_forward(
self,
hidden_states,
cache_params = None,
attention_mask = None,
):
return compiled.mamba_split_conv1d_scan_combined(hidden_states, seq_idx = None)
def forward(self, hidden_states, **kwargs):
return self.cuda_kernels_forward(hidden_states)
_StaleNemotronHMamba2Mixer.__module__ = compiled.__name__
class _StaleModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _StaleNemotronHMamba2Mixer()
try:
model = _StaleModel()
assert patch_hybrid_linear_attention_varlen(model) is True
real.calls.clear()
stale.calls.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
assert stale.calls == []
assert real.calls[-1] is not None
assert real.calls[-1].tolist() == [[0, 0, 1, 2, 2, 2]]
assert compiled.mamba_split_conv1d_scan_combined is not stale
finally:
sys.modules.pop(compiled.__name__, None)
def test_patch_mamba2_varlen_clears_padded_mask_for_fused_path(monkeypatch):
# The fused kernel needs an all-ones mask; packed batches carry pad zeros.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
def mamba_chunk_scan_combined(
*args,
seq_idx = None,
**kwargs,
):
mamba_chunk_scan_combined.calls.append(seq_idx)
return args[0] if args else None
mamba_chunk_scan_combined.calls = []
class _MaskedNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def cuda_kernels_forward(
self,
hidden_states,
cache_params = None,
attention_mask = None,
):
input_not_masked = attention_mask is None or bool(torch.all(attention_mask == 1))
if self.training and cache_params is None and input_not_masked:
return self.mamba2_split_conv1d_scan_combined(hidden_states, seq_idx = None)
return mamba_chunk_scan_combined(hidden_states, seq_idx = None)
def forward(
self,
hidden_states,
cache_params = None,
attention_mask = None,
**kwargs,
):
return self.cuda_kernels_forward(hidden_states, cache_params, attention_mask)
class _MaskedModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _MaskedNemotronHMamba2Mixer()
model = _MaskedModel()
fused_orig = model.mixer.mamba2_split_conv1d_scan_combined
model.train()
assert patch_hybrid_linear_attention_varlen(model) is True
fused_orig.calls.clear()
mamba_chunk_scan_combined.calls.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
attention_mask = torch.tensor([[1, 1, 1, 1, 1, 0]]),
use_cache = False,
)
assert mamba_chunk_scan_combined.calls == []
assert fused_orig.calls[-1] is not None
assert fused_orig.calls[-1].tolist() == [[0, 0, 1, 2, 2, 2]]
def test_patch_mamba2_varlen_clears_mask_on_compiled_mixer_forward(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
import types
compiled = types.ModuleType("unsloth_compiled_module_nemotron_h")
ns = compiled.__dict__
ns["__name__"] = compiled.__name__
ns["torch"] = torch
exec(
"""
def mamba_split_conv1d_scan_combined(*args, seq_idx=None, **kwargs):
mamba_split_conv1d_scan_combined.calls.append(seq_idx)
return args[0]
mamba_split_conv1d_scan_combined.calls = []
def mamba_chunk_scan_combined(*args, seq_idx=None, **kwargs):
mamba_chunk_scan_combined.calls.append(seq_idx)
return args[0]
mamba_chunk_scan_combined.calls = []
def cuda_kernels_forward(self, hidden_states, cache_params=None, attention_mask=None):
input_not_masked = attention_mask is None or bool(torch.all(attention_mask == 1))
if self.training and cache_params is None and input_not_masked:
return mamba_split_conv1d_scan_combined(hidden_states, seq_idx=None)
return mamba_chunk_scan_combined(hidden_states, seq_idx=None)
def NemotronHMamba2Mixer_forward(self, hidden_states, cache_params=None, attention_mask=None, **kwargs):
return self.cuda_kernels_forward(hidden_states, cache_params, attention_mask)
""",
ns,
)
sys.modules[compiled.__name__] = compiled
class _CompiledNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def __init__(self):
super().__init__()
del self.mamba2_split_conv1d_scan_combined
self.cuda_kernels_forward = types.MethodType(ns["cuda_kernels_forward"], self)
def forward(
self,
hidden_states,
cache_params = None,
attention_mask = None,
**kwargs,
):
return ns["NemotronHMamba2Mixer_forward"](
self,
hidden_states = hidden_states,
cache_params = cache_params,
attention_mask = attention_mask,
**kwargs,
)
_CompiledNemotronHMamba2Mixer.__module__ = compiled.__name__
class _EmptyCache:
def has_previous_state(self, layer_idx):
return False
class _CompiledModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _CompiledNemotronHMamba2Mixer()
def forward(
self,
input_ids = None,
packed_seq_lengths = None,
use_cache = None,
**kwargs,
):
# Nemotron-H hands mixers an empty Cache, which also blocks the fused kernel.
return self.mixer(
input_ids.float(),
cache_params = _EmptyCache(),
**kwargs,
)
try:
model = _CompiledModel()
model.train()
assert patch_hybrid_linear_attention_varlen(model) is True
ns["mamba_split_conv1d_scan_combined"].calls.clear()
ns["mamba_chunk_scan_combined"].calls.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
attention_mask = torch.tensor([[1, 1, 1, 1, 1, 0]]),
use_cache = False,
)
assert ns["mamba_chunk_scan_combined"].calls == []
seq_idx = ns["mamba_split_conv1d_scan_combined"].calls[-1]
assert seq_idx is not None
assert seq_idx.tolist() == [[0, 0, 1, 2, 2, 2]]
finally:
sys.modules.pop(compiled.__name__, None)
def test_patch_mamba2_varlen_wraps_shared_kernel_once_per_model(monkeypatch):
# Shared kernel name across layers: re-wrapping per mixer hit RecursionError.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
fused = _make_fake_mamba2_fused()
class _SharedNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def __init__(self):
super().__init__()
self.mamba2_split_conv1d_scan_combined = fused
class _ManyLayerModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _SharedNemotronHMamba2Mixer()
self.layers = torch.nn.ModuleList([_SharedNemotronHMamba2Mixer() for _ in range(24)])
def forward(
self,
input_ids = None,
packed_seq_lengths = None,
use_cache = None,
**kwargs,
):
hidden = self.mixer(input_ids.float(), **kwargs)
for layer in self.layers:
hidden = layer(hidden, **kwargs)
return hidden
model = _ManyLayerModel()
assert patch_hybrid_linear_attention_varlen(model) is True
installed = model.mixer.mamba2_split_conv1d_scan_combined
assert installed is not fused
assert installed.__wrapped__ is fused # exactly one wrapper deep
for layer in model.layers:
assert layer.mamba2_split_conv1d_scan_combined is installed
fused.calls.clear()
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
assert fused.calls[-1].tolist() == [[0, 0, 1, 2, 2, 2]]
def test_patch_mamba2_varlen_skips_stale_seq_idx_after_training(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
class _GeneratingModel(_FakeMamba2Model):
def generate(self, hidden_states):
return self.mixer(hidden_states)
model = _GeneratingModel()
fused_orig = model.mixer.mamba2_split_conv1d_scan_combined
assert patch_hybrid_linear_attention_varlen(model) is True
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
assert fused_orig.calls[-1].tolist() == [[0, 0, 1, 2, 2, 2]]
# A shorter prompt must not inherit the 6-token packed boundaries.
fused_orig.calls.clear()
model.generate(torch.zeros(1, 3, 4))
assert fused_orig.calls == [None]
def test_patch_mamba2_varlen_no_fused_dispatch_aborts(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
class _SilentNemotronHMamba2Mixer(_FakeNemotronHMamba2Mixer):
def forward(self, hidden_states, **kwargs):
return hidden_states
class _SilentModel(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = _SilentNemotronHMamba2Mixer()
model = _SilentModel()
assert patch_hybrid_linear_attention_varlen(model) is True
with pytest.raises(RuntimeError, match = "were not invoked"):
model(
input_ids = torch.zeros(1, 6),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
def test_varlen_from_position_ids_mrope_3d():
pos = torch.tensor([[0, 1, 0, 0, 1, 2]]).unsqueeze(0).expand(3, 1, 6).clone()
cu, seq_idx = packing_module._varlen_from_position_ids(pos)
assert cu.tolist() == [0, 2, 3, 6]
assert seq_idx.tolist() == [[0, 0, 1, 2, 2, 2]]
def test_hybrid_varlen_metadata_trailing_pad():
# Lengths sum to 6 but the input is 8 (pad_to_multiple_of).
kwargs = {
"input_ids": torch.zeros(1, 8, dtype = torch.long),
"packed_seq_lengths": torch.tensor([2, 1, 3], dtype = torch.int32),
}
cu, seq_idx = packing_module._hybrid_varlen_metadata(kwargs)
assert cu.tolist() == [0, 2, 3, 6, 8]
assert seq_idx.tolist() == [[0, 0, 1, 2, 2, 2, 3, 3]]
def _patch_fake_sft_trainer():
class FakeSFTTrainer:
def __init__(self, *args, **kwargs):
self.model = args[0] if len(args) >= 1 else kwargs["model"]
self.args = args[1] if len(args) >= 2 else kwargs["args"]
self.data_collator = args[2] if len(args) >= 3 else kwargs.get("data_collator")
trainer_module._patch_sft_trainer_auto_packing(SimpleNamespace(SFTTrainer = FakeSFTTrainer))
return FakeSFTTrainer
def _vlm_model():
return SimpleNamespace(
config = _FakeConfig(
architectures = ["Gemma4ForConditionalGeneration"],
model_type = "gemma4",
vision_config = SimpleNamespace(),
),
max_seq_length = 16,
)
def _text_model():
return SimpleNamespace(
config = _FakeConfig(
architectures = ["LlamaForCausalLM"],
model_type = "llama",
),
max_seq_length = 16,
)
class _CharacterTokenizer:
bos_token = None
eos_token = None
chat_template = None
def __call__(self, texts, **kwargs):
is_batched = isinstance(texts, list)
if not is_batched:
texts = [texts]
input_ids = [[ord(char) for char in text] for text in texts]
if kwargs.get("truncation") and kwargs.get("max_length") is not None:
input_ids = [ids[: kwargs["max_length"]] for ids in input_ids]
return {"input_ids": input_ids if is_batched else input_ids[0]}
def test_vlm_text_dataset_allows_explicit_packing():
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
trainer = fake_trainer(
model = _vlm_model(),
args = config,
processing_class = object(),
train_dataset = Dataset.from_dict({"text": ["text-only CPT sample"]}),
)
assert config.packing is True
assert config.padding_free is True
assert trainer.model._unsloth_allow_packed_overlength is True
def test_vlm_without_processing_class_still_disables_packing():
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
fake_trainer(
_vlm_model(),
config,
None,
Dataset.from_dict({"text": ["text-only sample"]}),
)
assert config.packing is False
assert config.padding_free is False
@pytest.mark.parametrize(
("model_type", "architecture"),
(
("t5", "T5ForConditionalGeneration"),
("bart", "BartForConditionalGeneration"),
("whisper", "WhisperForConditionalGeneration"),
),
)
def test_encoder_decoder_disables_packing(model_type, architecture):
# Encoder-decoder encoders attend across samples once padding-free drops the mask.
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
model = SimpleNamespace(
config = _FakeConfig(
model_type = model_type,
architectures = [architecture],
is_encoder_decoder = True,
),
max_seq_length = 16,
)
trainer = fake_trainer(model, config, None, Dataset.from_dict({"text": ["text-only sample"]}))
assert config.packing is False
assert config.padding_free is False
def test_decoder_only_conditional_generation_keeps_packing():
# CSM is decoder-only despite the ForConditionalGeneration name.
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
model = SimpleNamespace(
config = _FakeConfig(
model_type = "csm",
architectures = ["CsmForConditionalGeneration"],
is_encoder_decoder = False,
),
max_seq_length = 16,
)
trainer = fake_trainer(model, config, None, Dataset.from_dict({"text": ["text-only sample"]}))
assert config.packing is True
assert config.padding_free is True
assert trainer.model._unsloth_allow_packed_overlength is True
def _hybrid_trainer_model():
return SimpleNamespace(
config = _FakeConfig(
model_type = "qwen3_next",
architectures = ["Qwen3NextForCausalLM"],
layer_types = ["linear_attention", "full_attention"],
),
max_seq_length = 16,
)
def test_hybrid_varlen_active_enables_packing(monkeypatch):
monkeypatch.setattr(trainer_module, "_chunked_loss_bypasses_forward", lambda config: False)
monkeypatch.setattr(trainer_module, "patch_hybrid_linear_attention_varlen", lambda model: True)
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
fake_trainer(_hybrid_trainer_model(), config, None, Dataset.from_dict({"text": ["x"]}))
assert config.packing is True
assert config.padding_free is True
def test_hybrid_chunked_loss_stays_on_padded_path(monkeypatch):
# TRL's chunked-loss forward bypass skips the varlen shim.
monkeypatch.setattr(trainer_module, "_chunked_loss_bypasses_forward", lambda config: True)
monkeypatch.setattr(trainer_module, "patch_hybrid_linear_attention_varlen", lambda model: True)
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
fake_trainer(_hybrid_trainer_model(), config, None, Dataset.from_dict({"text": ["x"]}))
assert config.packing is False
assert config.padding_free is False
def test_string_hybrid_model_disables_packing(monkeypatch):
# A string model is built after init, so the shim cannot patch it.
monkeypatch.setattr(
trainer_module,
"_resolve_string_model_config",
lambda name, cfg: _FakeConfig(
model_type = "qwen3_next",
architectures = ["Qwen3NextForCausalLM"],
layer_types = ["linear_attention", "full_attention"],
),
)
monkeypatch.setattr(trainer_module, "patch_hybrid_linear_attention_varlen", lambda model: True)
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
fake_trainer("Qwen/Qwen3-Next-80B-A3B", config, None, Dataset.from_dict({"text": ["x"]}))
assert config.packing is False
assert config.padding_free is False
def test_vlm_vision_dataset_still_disables_packing():
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
fake_trainer(
_vlm_model(),
config,
None,
Dataset.from_dict({"images": [None], "text": ["multimodal sample"]}),
None,
object(),
)
assert config.packing is False
assert config.padding_free is False
@pytest.mark.parametrize(
"vision_column",
("pixel_values", "pixel_attention_mask", "image_grid_thw"),
)
def test_vlm_preprocessed_vision_dataset_disables_packing(vision_column):
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
fake_trainer(
model = _vlm_model(),
args = config,
processing_class = object(),
train_dataset = Dataset.from_dict({"input_ids": [[1]], vision_column: [None]}),
)
assert config.packing is False
assert config.padding_free is False
@pytest.mark.parametrize("dict_eval", (False, True))
def test_vlm_vision_eval_dataset_disables_packing(dict_eval):
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
eval_dataset = Dataset.from_dict({"input_ids": [[1]], "pixel_values": [None]})
if dict_eval:
eval_dataset = {"vision": eval_dataset}
fake_trainer(
model = _vlm_model(),
args = config,
processing_class = object(),
train_dataset = Dataset.from_dict({"text": ["text-only training sample"]}),
eval_dataset = eval_dataset,
)
assert config.packing is False
assert config.padding_free is False
def test_vlm_streaming_vision_dataset_without_metadata_disables_packing():
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
dataset = IterableDataset.from_generator(
lambda: iter([{"images": [None], "text": "multimodal sample"}])
)
assert dataset.column_names is None
fake_trainer(
model = _vlm_model(),
args = config,
processing_class = object(),
train_dataset = dataset,
)
assert config.packing is False
assert config.padding_free is False
assert next(iter(dataset))["text"] == "multimodal sample"
@pytest.mark.parametrize("data_collator", (None, object()))
def test_stateful_stream_is_not_consumed_during_detection(data_collator):
class StatefulDataset:
def __init__(self):
self.rows = iter([{"text": "first"}, {"text": "second"}])
def __iter__(self):
return (row for row in self.rows)
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
dataset = StatefulDataset()
fake_trainer(
model = _vlm_model(),
args = config,
processing_class = object(),
data_collator = data_collator,
train_dataset = dataset,
)
assert config.packing is False
assert config.padding_free is False
assert next(iter(dataset))["text"] == "first"
def test_text_model_stream_without_metadata_keeps_packing():
class StatefulDataset:
def __init__(self):
self.rows = iter([{"text": "first"}, {"text": "second"}])
def __iter__(self):
return (row for row in self.rows)
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
dataset = StatefulDataset()
trainer = fake_trainer(
model = _text_model(),
args = config,
processing_class = object(),
train_dataset = dataset,
)
assert config.packing is True
assert config.padding_free is True
assert trainer.model._unsloth_allow_packed_overlength is True
assert next(iter(dataset))["text"] == "first"
def _fake_sft_self():
"""Fail by name if SFTTrainer is not Unsloth's; padding _is_vlm would let tests pass against TRL."""
assert SFTTrainer.__name__ == "UnslothSFTTrainer", (
f"trl.SFTTrainer is {SFTTrainer.__name__!r}, so Unsloth's SFT patch did not "
"apply and these tests would be exercising TRL's _prepare_dataset instead of "
"the patched one. `import unsloth` reports the cause as a warning "
"('Could not build the patched trl.trainer.sft_trainer'), not an error."
)
return SimpleNamespace(model = None)
def test_bfd_packing_truncates_before_packing(monkeypatch):
args = SimpleNamespace(
dataset_num_proc = 1,
dataset_text_field = "text",
max_length = 4,
packing_strategy = "bfd",
)
trainer = _fake_sft_self()
dataset = Dataset.from_dict({"prompt": ["abc"], "completion": ["defghij"]})
prepare_globals = SFTTrainer._prepare_dataset.__globals__
def passthrough_pack_dataset(dataset, seq_length, strategy, map_kwargs):
return dataset
monkeypatch.setitem(prepare_globals, "pack_dataset", passthrough_pack_dataset)
packed = SFTTrainer._prepare_dataset(
trainer,
dataset,
_CharacterTokenizer(),
args,
True,
None,
"train",
)
assert len(packed["input_ids"][0]) == args.max_length
def test_wrapped_strategy_without_packing_still_truncates():
args = SimpleNamespace(
dataset_num_proc = 1,
dataset_text_field = "text",
max_length = 4,
packing_strategy = "wrapped",
)
trainer = _fake_sft_self()
dataset = Dataset.from_dict({"text": ["abcdefghi"]})
prepared = SFTTrainer._prepare_dataset(
trainer,
dataset,
_CharacterTokenizer(),
args,
False,
None,
"train",
)
assert len(prepared["input_ids"][0]) == args.max_length
@pytest.mark.parametrize("legacy_api", (False, True))
def test_wrapped_packing_preserves_overlength_tokens(monkeypatch, legacy_api):
args_kwargs = {
"dataset_num_proc": 1,
"dataset_text_field": "text",
"max_length": 4,
}
if not legacy_api:
args_kwargs["packing_strategy"] = "wrapped"
args = SimpleNamespace(**args_kwargs)
trainer = _fake_sft_self()
dataset = Dataset.from_dict({"text": ["abcdefghi"]})
prepare_globals = SFTTrainer._prepare_dataset.__globals__
pack_dataset = prepare_globals["pack_dataset"]
def legacy_pack_dataset(
dataset,
seq_length,
map_kwargs = None,
):
return pack_dataset(dataset, seq_length, "wrapped", map_kwargs)
if legacy_api:
monkeypatch.setitem(prepare_globals, "pack_dataset", legacy_pack_dataset)
packed = SFTTrainer._prepare_dataset(
trainer,
dataset,
_CharacterTokenizer(),
args,
True,
None,
"train",
)
packed_ids = packed["input_ids"]
assert sum(len(input_ids) for input_ids in packed_ids) == 9
assert all(len(input_ids) <= args.max_length for input_ids in packed_ids)
class _DummyChild(torch.nn.Module):
def __init__(self):
super().__init__()
self.max_seq_length = 8
class _DummyModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.max_seq_length = 16
self.child = _DummyChild()
self.config = SimpleNamespace(_attn_implementation = "sdpa")
self.generation_config = SimpleNamespace(attn_implementation = "sdpa")
def _build_trl_language_modeling_collator():
"""Filters kwargs against the installed collator signature, since TRL drops fields between releases."""
wanted = {
"pad_token_id": 0,
"completion_only_loss": False,
"return_tensors": "pt",
"padding_free": True,
"return_position_ids": False,
}
try:
accepted = set(inspect.signature(DataCollatorForLanguageModeling).parameters)
except (TypeError, ValueError):
accepted = {"pad_token_id"}
collator = DataCollatorForLanguageModeling(
**{key: value for key, value in wanted.items() if key in accepted}
)
if not hasattr(collator, "padding_free"):
collator.padding_free = True
if not hasattr(collator, "return_position_ids"):
collator.return_position_ids = False
return collator
class _DummyTrainer:
def __init__(self):
self.args = SimpleNamespace(remove_unused_columns = True)
self.data_collator = _build_trl_language_modeling_collator()
class _PaddingFreeCollator:
def __init__(self):
self.padding_free = True
self.return_position_ids = False
self.calls = 0
self.seen = None
def torch_call(self, examples):
self.calls += 1
self.seen = examples
return {
"input_ids": torch.tensor([[0]], dtype = torch.long),
"examples_seen": self.calls,
}
def test_enable_sample_packing():
model = _DummyModel()
trainer = _DummyTrainer()
enable_sample_packing(model, trainer)
assert getattr(model, "_unsloth_allow_packed_overlength") is True
assert getattr(model.child, "_unsloth_allow_packed_overlength") is True
collator = trainer.data_collator
assert collator.return_position_ids is True
assert getattr(collator, "_unsloth_packing_wrapped") is True
examples = [
{
"input_ids": [0, 1, 2],
"labels": [0, 1, 2],
"seq_lengths": [2, 1],
},
{
"input_ids": [3, 4, 5],
"labels": [3, 4, 5],
"seq_lengths": [3],
},
]
batch = collator.torch_call(examples)
assert "packed_seq_lengths" in batch
assert torch.equal(batch["packed_seq_lengths"], torch.tensor([2, 1, 3], dtype = torch.int32))
assert batch["input_ids"].shape == (1, 6)
expected_positions = torch.tensor([0, 1, 0, 0, 1, 2], dtype = torch.long)
assert torch.equal(batch["position_ids"].view(-1)[:6], expected_positions)
def test_enable_sample_packing_only_requires_torch_call():
"""Packing may require only torch_call, since TRL keeps changing its SFT collator fields."""
class _MinimalCollator:
def torch_call(self, examples):
return {"input_ids": torch.tensor([[0, 1, 2, 3, 4, 5]], dtype = torch.long)}
trainer = SimpleNamespace(
args = SimpleNamespace(remove_unused_columns = True),
data_collator = _MinimalCollator(),
)
enable_sample_packing(_DummyModel(), trainer)
collator = trainer.data_collator
assert getattr(collator, "_unsloth_packing_wrapped") is True
assert trainer.args.remove_unused_columns is False
batch = collator.torch_call(
[
{"input_ids": [0, 1, 2], "seq_lengths": [2, 1]},
{"input_ids": [3, 4, 5], "seq_lengths": [3]},
]
)
assert torch.equal(batch["packed_seq_lengths"], torch.tensor([2, 1, 3], dtype = torch.int32))
# has_real_accelerator: the body falls back to xpu, so this covers XPU too.
@pytest.mark.skipif(not has_real_accelerator(), reason = "builds a real 4bit model on an accelerator")
def test_enable_sample_packing_trl_collator(tmp_path):
if torch.cuda.is_available():
device = torch.device("cuda")
elif torch.xpu.is_available():
device = torch.device("xpu")
else:
device = torch.device("cpu")
model, _, trainer, _ = _build_packed_training_setup(tmp_path, device)
enable_sample_packing(model, trainer)
examples = [
{
"input_ids": [0, 1, 2],
"labels": [0, 1, 2],
"seq_lengths": [2, 1],
},
{
"input_ids": [3, 4, 5],
"labels": [3, 4, 5],
"seq_lengths": [3],
},
]
batch = trainer.data_collator.torch_call(examples)
assert batch["input_ids"].shape == (1, 6)
assert torch.equal(batch["packed_seq_lengths"], torch.tensor([2, 1, 3], dtype = torch.int32))
expected_positions = torch.tensor([0, 1, 0, 0, 1, 2], dtype = torch.long)
assert torch.equal(batch["position_ids"].view(-1)[:6], expected_positions)
if hasattr(trainer, "accelerator"):
trainer.accelerator.free_memory()
def test_enable_padding_free_metadata():
model = _DummyModel()
trainer = SimpleNamespace(
args = SimpleNamespace(remove_unused_columns = True),
data_collator = _PaddingFreeCollator(),
)
enable_padding_free_metadata(model, trainer)
assert getattr(model, "_unsloth_allow_packed_overlength") is True
assert getattr(model.child, "_unsloth_allow_packed_overlength") is True
collator = trainer.data_collator
assert collator.return_position_ids is True
assert getattr(collator, "_unsloth_padding_free_lengths_wrapped") is True
examples = [
{"input_ids": [0, 1, 2]},
{"input_ids": [3, 4]},
]
batch = collator.torch_call(examples)
assert torch.equal(batch["packed_seq_lengths"], torch.tensor([3, 2], dtype = torch.int32))
assert trainer.args.remove_unused_columns is False
# has_real_accelerator: the body falls back to xpu, so this covers XPU too.
@pytest.mark.skipif(not has_real_accelerator(), reason = "builds a real model on an accelerator")
def test_position_only_padding_free_keeps_samples_apart():
# TRL's padding-free DPO flattens chosen + rejected into one row with reset position_ids only.
device = torch.device("cuda" if torch.cuda.is_available() else "xpu")
try:
model, _ = FastLanguageModel.from_pretrained(
model_name = "hf-internal-testing/tiny-random-LlamaForCausalLM",
max_seq_length = 64,
load_in_4bit = False,
dtype = torch.float32,
)
except OSError as exc: # pragma: no cover - offline CI
pytest.skip(f"Requires access to tiny llama checkpoint: {exc}")
model.to(device)
model.train()
first, second = 13, 9
ids = torch.randint(5, 900, (1, first + second), generator = torch.Generator().manual_seed(0))
ids = ids.to(device)
arange = lambda n: torch.arange(n, device = device)[None]
def hidden(input_ids, position_ids):
with torch.no_grad():
out = model.model(input_ids = input_ids, position_ids = position_ids, use_cache = False)
return out.last_hidden_state.float()
with ExitStack() as stack:
stack.enter_context(patch.object(attention_dispatch_utils, "HAS_FLASH_ATTENTION", False))
stack.enter_context(patch.object(attention_dispatch_utils, "HAS_XFORMERS", False))
packed = hidden(ids, torch.cat([arange(first), arange(second)], dim = 1))[0, first:]
alone = hidden(ids[:, first:], arange(second))[0]
crossed = hidden(ids, arange(first + second))[0, first:]
leak = (packed - alone).abs().max().item()
control = (crossed - alone).abs().max().item()
assert control > 0
assert leak <= 1e-3 * control
# has_real_accelerator: the body falls back to xpu, so this covers XPU too.
@pytest.mark.skipif(not has_real_accelerator(), reason = "builds a real 4bit model on an accelerator")
def test_packing_sdpa(tmp_path, monkeypatch):
if torch.cuda.is_available():
device = torch.device("cuda")
elif torch.xpu.is_available():
device = torch.device("xpu")
else:
device = torch.device("cpu")
model, batch, trainer, llama_mod = _build_packed_training_setup(tmp_path, device)
assert "packed_seq_lengths" in batch
assert "attention_mask" not in batch
assert batch["packed_seq_lengths"].dtype == torch.int32
total_tokens = batch["input_ids"].size(-1)
assert int(batch["packed_seq_lengths"].sum().item()) == total_tokens
packed_tokens = int(batch["packed_seq_lengths"].sum().item())
assert "position_ids" in batch
flat_positions = batch["position_ids"].reshape(-1)[:packed_tokens]
expected_positions = torch.cat(
[torch.arange(length, dtype = torch.long) for length in batch["packed_seq_lengths"].tolist()]
)
assert torch.equal(flat_positions.cpu(), expected_positions)
inputs = _trim_batch_to_total_tokens(batch, packed_tokens)
seq_info = llama_mod.get_packed_info_from_kwargs(
{"packed_seq_lengths": batch["packed_seq_lengths"]},
inputs["input_ids"].device,
)
assert seq_info is not None
original_lengths = attention_dispatch_utils.packed_segment_lengths
segment_calls = []
captured_loss_labels = {}
def _capture_lengths(seq_info, total_tokens = None):
lengths = original_lengths(seq_info, total_tokens)
segment_calls.append(lengths)
return lengths
def _capture_loss(*, hidden_states, labels, **loss_kwargs):
captured_loss_labels["labels"] = labels.detach().to("cpu")
return torch.zeros((), device = hidden_states.device, dtype = hidden_states.dtype)
# Unmasked labels, so only the model's own boundary guard can mask the document starts.
inputs["labels"] = inputs["input_ids"].clone()
with ExitStack() as stack:
stack.enter_context(patch.object(attention_dispatch_utils, "HAS_FLASH_ATTENTION", False))
stack.enter_context(patch.object(attention_dispatch_utils, "HAS_XFORMERS", False))
stack.enter_context(
patch.object(
attention_dispatch_utils,
"packed_segment_lengths",
side_effect = _capture_lengths,
)
)
stack.enter_context(
patch.object(
llama_mod,
"unsloth_fused_ce_loss",
side_effect = _capture_loss,
)
)
# Packed training's loss path: the fused CE on raw labels (UNSLOTH_RETURN_LOGITS=1 blocks packing).
monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "0")
with torch.no_grad():
outputs = model(**inputs)
assert segment_calls, "SDPA packed path did not split the row into its segments"
packed = tuple(batch["packed_seq_lengths"].tolist())
assert all(lengths[: len(packed)] == packed for lengths in segment_calls)
assert outputs.loss is not None
assert "labels" in captured_loss_labels
flat_loss_labels = captured_loss_labels["labels"].reshape(-1)
# Raw labels (the kernel shifts): each later document's first token must not be a target.
starts = torch.cumsum(batch["packed_seq_lengths"].to(device = "cpu", dtype = torch.long), dim = 0)[
:-1
]
assert starts.numel() > 0
for idx in starts.tolist():
assert flat_loss_labels[idx].item() == -100
assert torch.any(flat_loss_labels != -100)
if hasattr(trainer, "accelerator"):
trainer.accelerator.free_memory()
# fmt: off
# Named to match the unsloth_zoo helper (sourced by name, "def sft_prepare_dataset" -> "def _prepare_dataset").
# Deliberately OMITS the "licensed under LGPLv3" header to emulate a newer Zoo whose header moved (dependency is only
# lower-bounded). Source only.
def sft_prepare_dataset(
self, dataset, processing_class, args, packing, formatting_func, dataset_text_field
):
do_truncation = True
max_seq_length = getattr(args, "max_length", 0)
if max_seq_length == 0: max_seq_length = getattr(args, "max_seq_length", 0)
used_column_names = ["text"]
map_kwargs = {}
dataset = processing_class(dataset, truncation = do_truncation,)
if do_truncation and max_seq_length > 0:
pass
if packing:
dataset = pack_dataset(
dataset.select_columns(used_column_names),
max_seq_length,
getattr(args, "packing_strategy", "bfd"),
map_kwargs,
)
return dataset
# fmt: on
def test_wrapped_packing_injection_is_drift_resistant(monkeypatch):
# Setup must install via the signature, not the Zoo license comment header.
import ast
import textwrap
import unsloth.models.rl_replacements as rlr
monkeypatch.setitem(rlr.RL_REPLACEMENTS, "sft_prepare_dataset", sft_prepare_dataset)
source = (
"def _prepare_dataset(self, dataset, processing_class, args, packing, "
"formatting_func, dataset_text_field):\n return dataset\n"
)
patched = rlr.sft_trainer_prepare_dataset("_prepare_dataset", source)
assert "_unsloth_wrapped_packing = packing" in patched
assert "import inspect as _inspect" in patched
assert patched.index("_unsloth_wrapped_packing = packing") < patched.index(
"truncation = do_truncation and not _unsloth_wrapped_packing"
)
# A padding-free None max_length would stop raw truncation.
assert 'max_seq_length = getattr(args, "max_length", 0) or 0' in patched
assert "if _unsloth_pack_has_strategy:" in patched
assert patched.count("_inspect.signature(pack_dataset)") == 1
ast.parse(textwrap.dedent(patched))
def test_require_replace_raises_on_missing_anchor():
from unsloth.models.rl_replacements import _require_replace
assert _require_replace("abc", "b", "B") == "aBc"
with pytest.raises(RuntimeError):
_require_replace("abc", "z", "Z", where = "unit test")
assert _require_replace("abc", "z", "Z", required = False, where = "optional") == "abc"
def test_require_replace_survives_a_trailing_comment_on_the_anchor():
"""A trailing comment on an anchored line must not stop _require_replace from matching the code."""
from unsloth.models.rl_replacements import _require_replace
anchor = "dataset = pack_dataset(\n a,\n b,\n)"
replacement = "dataset = pack_dataset(\n a,\n **kw,\n)"
commented = "x = 1\ndataset = pack_dataset( # noqa: F821 -- reached only past the probe\n a,\n b,\n)\ny = 2"
assert _require_replace(commented, anchor, replacement) == f"x = 1\n{replacement}\ny = 2"
inner = "dataset = pack_dataset(\n a, # the columns\n b,\n)"
assert _require_replace(inner, anchor, replacement) == replacement
with pytest.raises(RuntimeError):
_require_replace(
"dataset = pack_dataset(\n a,\n c,\n)",
anchor,
replacement,
where = "changed argument",
)
hashed = 'sep = "#"\n'
assert _require_replace(hashed + anchor, anchor, replacement) == hashed + replacement
def test_resolve_string_model_config_forwards_token(monkeypatch):
import transformers
captured = {}
class _FakeAutoConfig:
@staticmethod
def from_pretrained(name, **kwargs):
captured.update(kwargs)
return SimpleNamespace(is_encoder_decoder = False)
monkeypatch.setattr(transformers, "AutoConfig", _FakeAutoConfig)
config_arg = SimpleNamespace(
model_init_kwargs = {
"token": "hf_secret",
"trust_remote_code": True,
"cache_dir": "/tmp/cache",
"torch_dtype": "bfloat16", # not a config arg -> must NOT be forwarded
}
)
result = trainer_module._resolve_string_model_config("org/private-hybrid", config_arg)
assert result is not None
assert captured.get("token") == "hf_secret"
assert captured.get("trust_remote_code") is True
assert captured.get("cache_dir") == "/tmp/cache"
assert "torch_dtype" not in captured
def test_resolve_string_model_config_merges_top_level_trust_remote_code(monkeypatch):
import transformers
captured = {}
class _FakeAutoConfig:
@staticmethod
def from_pretrained(name, **kwargs):
captured.update(kwargs)
return SimpleNamespace(is_encoder_decoder = False)
monkeypatch.setattr(transformers, "AutoConfig", _FakeAutoConfig)
config_arg = SimpleNamespace(model_init_kwargs = {}, trust_remote_code = True)
trainer_module._resolve_string_model_config("org/remote-hybrid", config_arg)
assert captured.get("trust_remote_code") is True
# model_init_kwargs wins, mirroring TRL's setdefault.
captured.clear()
config_arg = SimpleNamespace(
model_init_kwargs = {"trust_remote_code": False}, trust_remote_code = True
)
trainer_module._resolve_string_model_config("org/remote-hybrid", config_arg)
assert captured.get("trust_remote_code") is False
def _warn_text_model():
return SimpleNamespace(
config = _FakeConfig(architectures = ["LlamaForCausalLM"], model_type = "llama"),
max_seq_length = 16,
)
def test_packing_skip_warning_is_accurate(monkeypatch, caplog):
monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(
packing = True,
padding_free = None,
remove_unused_columns = True,
max_seq_length = 4096,
max_length = 512,
)
with caplog.at_level(logging.WARNING, logger = "unsloth.trainer"):
fake_trainer(
model = _warn_text_model(),
args = config,
train_dataset = Dataset.from_dict({"text": ["sample"]}),
)
messages = [r.message for r in caplog.records if "packing=True ignored" in r.message]
assert len(messages) == 1
assert "UNSLOTH_RETURN_LOGITS" in messages[0]
assert "custom data collator" not in messages[0]
assert "4096" not in messages[0] and "512" not in messages[0]
# compute_metrics is one of several setters, so the message must not name it.
assert "compute_metrics" not in messages[0]
def test_packing_skip_warning_keeps_custom_collator_reason(monkeypatch, caplog):
monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True)
with caplog.at_level(logging.WARNING, logger = "unsloth.trainer"):
fake_trainer(
model = _warn_text_model(),
args = config,
data_collator = lambda features: features,
train_dataset = Dataset.from_dict({"text": ["sample"]}),
)
messages = [r.message for r in caplog.records if "packing=True ignored" in r.message]
assert len(messages) == 1
assert "custom data collator" in messages[0]
assert "UNSLOTH_RETURN_LOGITS" not in messages[0]
# Ring cannot keep packed rows apart, so it turns packing off. Ulysses keeps them, and 'auto' (the default
# since #16407) resolves to Ulysses unless the heads cannot be split over cp.
@pytest.mark.parametrize(
"backend, packs", [("ring", False), ("ulysses", True), ("auto", True), (None, True)]
)
def test_context_parallel_disables_eval_packing(monkeypatch, backend, packs):
monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)
fake_trainer = _patch_fake_sft_trainer()
config = SimpleNamespace(
packing = True,
eval_packing = True,
padding_free = None,
remove_unused_columns = True,
context_parallel_size = 2,
context_parallel_backend = backend,
)
fake_trainer(
model = _warn_text_model(),
args = config,
train_dataset = Dataset.from_dict({"text": ["sample"]}),
)
assert config.packing is packs and config.eval_packing is packs
# Fused-CE paths shift internally, so they mask boundaries on unshifted labels.
def test_mask_packed_boundary_labels_masks_next_document_first_token():
labels = torch.arange(6, dtype = torch.long).view(1, 6)
out = mask_packed_boundary_labels(labels, torch.tensor([2, 1, 3], dtype = torch.int32))
# Slot 0 is the out-of-range redirect: harmless, the shift discards labels[0].
assert out.reshape(-1).tolist() == [-100, 1, -100, -100, 4, 5]
assert labels.reshape(-1).tolist() == [0, 1, 2, 3, 4, 5]
assert out.shape == labels.shape
assert out.dtype == labels.dtype
def test_mask_packed_boundary_labels_matches_the_shifted_guard():
"""The two entry points must mask exactly the same CE targets."""
labels = torch.arange(100, 112, dtype = torch.long).view(1, 12)
lengths = torch.tensor([5, 4, 3], dtype = torch.int32)
shift_a = torch.empty_like(labels)
shift_a[..., :-1] = labels[..., 1:]
shift_a[..., -1] = -100
mask_packed_sequence_boundaries(shift_a, lengths)
masked = mask_packed_boundary_labels(labels, lengths)
shift_b = torch.empty_like(masked)
shift_b[..., :-1] = masked[..., 1:]
shift_b[..., -1] = -100
assert torch.equal(shift_a, shift_b)
def test_mask_packed_boundary_labels_is_idempotent_on_trl_masked_labels():
"""TRL already sets labels[position_ids == 0] = -100, so the guard is a no-op on it."""
lengths = torch.tensor([2, 1, 3], dtype = torch.int32)
labels = torch.arange(6, dtype = torch.long).view(1, 6)
position_ids = torch.tensor([[0, 1, 0, 0, 1, 2]], dtype = torch.long)
trl_labels = labels.clone()
trl_labels[position_ids == 0] = -100
once = mask_packed_boundary_labels(trl_labels, lengths)
twice = mask_packed_boundary_labels(once, lengths)
assert torch.equal(once, trl_labels)
assert torch.equal(twice, once)
def test_mask_packed_boundary_labels_is_a_noop_without_packing():
labels = torch.arange(6, dtype = torch.long).view(1, 6)
assert mask_packed_boundary_labels(labels, None) is labels
assert mask_packed_boundary_labels(labels, torch.tensor([], dtype = torch.int32)) is labels
assert mask_packed_boundary_labels(None, torch.tensor([2, 4])) is None
def test_mask_packed_boundary_labels_tolerates_pad_to_multiple_of():
labels = torch.tensor([[10, 11, 12, 13, -100, -100]], dtype = torch.long)
out = mask_packed_boundary_labels(labels, torch.tensor([2, 2], dtype = torch.int32))
assert out.reshape(-1).tolist() == [10, 11, -100, 13, -100, -100]
def test_mask_packed_boundary_labels_lengths_covering_whole_row():
labels = torch.arange(4, dtype = torch.long).view(1, 4)
out = mask_packed_boundary_labels(labels, [2, 2])
assert out.reshape(-1).tolist() == [-100, 1, -100, 3]
class _StubInner(torch.nn.Module):
def __init__(self, hidden):
super().__init__()
self.hidden = hidden
def forward(self, **kwargs):
from transformers.modeling_outputs import BaseModelOutputWithPast
return BaseModelOutputWithPast(
last_hidden_state = self.hidden,
past_key_values = None,
hidden_states = None,
attentions = None,
)
def _make_stub_causal_lm(
hidden_size = 8,
vocab = 16,
seq = 8,
):
hidden = torch.zeros(1, seq, hidden_size)
model = _StubInner(hidden)
lm_head = torch.nn.Linear(hidden_size, vocab, bias = False)
stub = SimpleNamespace(
model = model,
lm_head = lm_head,
# Mistral's training mask branch runs only without xformers, so set this for CI.
training = True,
config = SimpleNamespace(
output_attentions = False,
output_hidden_states = False,
use_return_dict = True,
model_type = "llama",
final_logit_softcapping = 0,
logit_scale = 0,
torch_dtype = torch.float32,
),
)
return stub
@pytest.mark.parametrize("module_name", ["llama", "mistral"])
def test_fused_ce_branch_masks_packed_boundaries(monkeypatch, module_name):
"""The fused-CE branch must hand boundary-masked labels to the kernel."""
import importlib
mod = importlib.import_module(f"unsloth.models.{module_name}")
seq = 8
stub = _make_stub_causal_lm(seq = seq)
seen = {}
def _fake_fused(**kwargs):
seen["labels"] = kwargs["labels"].clone()
return torch.zeros((), requires_grad = False)
monkeypatch.setattr(mod, "unsloth_fused_ce_loss", _fake_fused)
monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)
monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES", raising = False)
if module_name == "llama":
forward = mod.CausalLM_fast_forward(lambda *a, **k: None)
else:
forward = mod.MistralForCausalLM_fast_forward
labels = torch.arange(seq, dtype = torch.long).view(1, seq)
forward(
stub,
input_ids = torch.zeros(1, seq, dtype = torch.long),
labels = labels,
packed_seq_lengths = torch.tensor([3, 5], dtype = torch.int32),
)
got = seen["labels"].reshape(-1).tolist()
assert got == [-100, 1, 2, -100, 4, 5, 6, 7], got
assert labels.reshape(-1).tolist() == list(range(seq))
# wrappers match TRL >= 0.23.1 masking; the zoo count includes each real target once
class _UnmaskedPackingCollator:
"""padding-free collator with unmasked boundaries, like TRL < 0.23.1."""
def __init__(self):
self.padding_free = True
self.return_position_ids = False
def torch_call(self, examples):
ids = [i for ex in examples for i in ex["input_ids"]]
return {
"input_ids": torch.tensor([ids], dtype = torch.long),
"labels": torch.tensor([ids], dtype = torch.long),
}
def _zoo_num_items_in_batch(batch):
"""unsloth_zoo._unsloth_get_batch_samples' count: boundary slots zeroed, not subtracted."""
token_count = batch["labels"][..., 1:] != -100
lengths = batch.get("packed_seq_lengths")
if lengths is not None:
n_shift = token_count.shape[-1]
n_rows = token_count.numel() // n_shift
starts = torch.cumsum(lengths.long(), dim = 0)[:-1]
rows = torch.div(starts, n_shift + 1, rounding_mode = "floor")
cols = starts - rows * (n_shift + 1)
keep = (cols > 0) & (rows < n_rows)
token_count = token_count.reshape(n_rows, n_shift).clone()
token_count[rows[keep], cols[keep] - 1] = False
return int(token_count.sum())
@pytest.mark.parametrize("wrapper", [enable_sample_packing, enable_padding_free_metadata])
def test_collator_masks_boundary_targets_and_num_items_counts_them_once(wrapper):
model = SimpleNamespace(max_seq_length = 16, children = lambda: [])
trainer = SimpleNamespace(
args = SimpleNamespace(remove_unused_columns = True),
data_collator = _UnmaskedPackingCollator(),
)
wrapper(model, trainer)
examples = [
{"input_ids": [10, 11, 12], "seq_lengths": [2, 1]},
{"input_ids": [13, 14, 15], "seq_lengths": [3]},
]
batch = trainer.data_collator.torch_call(examples)
# boundary labels match TRL >= 0.23.1
assert batch["labels"].reshape(-1).tolist() == [-100, 11, -100, -100, 14, 15]
assert batch["input_ids"].reshape(-1).tolist() == [10, 11, 12, 13, 14, 15]
assert _zoo_num_items_in_batch(batch) == 3
assert examples[0] == {"input_ids": [10, 11, 12], "seq_lengths": [2, 1]}
@pytest.mark.parametrize("wrapper", [enable_sample_packing, enable_padding_free_metadata])
def test_collator_boundary_mask_is_a_no_op_on_trl_masked_labels(wrapper):
class _MaskedPackingCollator(_UnmaskedPackingCollator):
def torch_call(self, examples):
batch = super().torch_call(examples)
position_ids = torch.cat(
[torch.arange(n) for ex in examples for n in ex["seq_lengths"]]
).view(1, -1)
batch["labels"][position_ids == 0] = -100
return batch
model = SimpleNamespace(max_seq_length = 16, children = lambda: [])
trainer = SimpleNamespace(
args = SimpleNamespace(remove_unused_columns = True), data_collator = _MaskedPackingCollator()
)
wrapper(model, trainer)
batch = trainer.data_collator.torch_call(
[
{"input_ids": [10, 11, 12], "seq_lengths": [2, 1]},
{"input_ids": [13, 14, 15], "seq_lengths": [3]},
]
)
assert batch["labels"].reshape(-1).tolist() == [-100, 11, -100, -100, 14, 15]
assert _zoo_num_items_in_batch(batch) == 3
def test_collator_boundary_mask_leaves_unflattened_rows_alone():
class _RowCollator(_UnmaskedPackingCollator):
def torch_call(self, examples):
rows = [ex["input_ids"] for ex in examples]
return {"input_ids": torch.tensor(rows), "labels": torch.tensor(rows)}
model = SimpleNamespace(max_seq_length = 16, children = lambda: [])
trainer = SimpleNamespace(
args = SimpleNamespace(remove_unused_columns = True), data_collator = _RowCollator()
)
enable_sample_packing(model, trainer)
batch = trainer.data_collator.torch_call(
[{"input_ids": [10, 11, 12]}, {"input_ids": [13, 14, 15]}]
)
assert batch["labels"].tolist() == [[10, 11, 12], [13, 14, 15]]
def test_guard_is_idempotent_and_actually_masks():
lengths = torch.tensor([2, 1, 3], dtype = torch.int32)
labels = torch.arange(6, dtype = torch.long).view(1, 6)
once = mask_packed_boundary_labels(labels, lengths)
twice = mask_packed_boundary_labels(once, lengths)
assert torch.equal(twice, once)
# an identity helper would also pass the idempotence check
assert once.reshape(-1).tolist() == [-100, 1, -100, -100, 4, 5]
def test_call_as_packed_mamba2_prefill_propagates_inner_type_error():
calls = []
def cuda_kernels_forward(
hidden_states,
cache_params = None,
attention_mask = None,
):
calls.append((cache_params, attention_mask))
raise TypeError("'NoneType' object is not callable")
with pytest.raises(TypeError, match = "NoneType"):
packing_module._call_as_packed_mamba2_prefill(
cuda_kernels_forward, (torch.zeros(1), object(), torch.ones(1, 2)), {}
)
assert calls == [(None, None)]
class _FakeMamba1Mixer(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv1d = torch.nn.Conv1d(4, 4, 3, groups = 4)
self.A_log = torch.nn.Parameter(torch.zeros(4, 2))
self.dt_proj = torch.nn.Linear(2, 4)
class _FakeLfm2ShortConv(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv = torch.nn.Conv1d(4, 4, 3, groups = 4)
self.calls = []
def forward(
self,
hidden_states,
past_key_values = None,
attention_mask = None,
seq_idx = None,
):
self.calls.append(seq_idx)
return hidden_states
class _FakeFalconH1Mixer(_FakeNemotronHMamba2Mixer):
pass
class _FakeMiniMaxLightningAttention(torch.nn.Module):
pass
def _stateful_model(mixer):
model = _FakeMamba2Model()
model.mixer = mixer
return model
def test_stateful_mixer_kind_by_structure():
kind = packing_module._stateful_mixer_kind
assert kind(_FakeNemotronHMamba2Mixer()) == "ssd"
assert kind(_FakeFalconH1Mixer()) == "ssd" # name-independent
assert kind(_FakeMamba1Mixer()) == "unsupported"
assert kind(_FakeLfm2ShortConv()) == "short_conv"
assert kind(_FakeMiniMaxLightningAttention()) == "unsupported"
assert kind(torch.nn.Linear(2, 2)) is None
@pytest.mark.parametrize("mixer_cls", [_FakeFalconH1Mixer, _FakeMamba1Mixer, _FakeLfm2ShortConv])
def test_trainer_flags_every_stateful_mixer_as_hybrid(mixer_cls):
from unsloth.trainer import _is_hybrid_linear_attention_model
assert _is_hybrid_linear_attention_model(_stateful_model(mixer_cls())) is True
@pytest.mark.parametrize("mixer_cls", [_FakeMamba1Mixer, _FakeMiniMaxLightningAttention])
def test_patch_varlen_rejects_mixers_without_boundary_kernels(monkeypatch, mixer_cls):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
assert patch_hybrid_linear_attention_varlen(_stateful_model(mixer_cls())) is False
def test_patch_varlen_injects_seq_idx_into_short_conv(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
conv = _FakeLfm2ShortConv()
class _Lfm2Like(_FakeMamba2Model):
def __init__(self):
super().__init__()
self.mixer = conv
def forward(
self,
input_ids = None,
packed_seq_lengths = None,
use_cache = None,
**kwargs,
):
return self.mixer(input_ids.float()[..., None].expand(-1, -1, 4))
model = _Lfm2Like()
assert patch_hybrid_linear_attention_varlen(model) is True
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
assert conv.calls[-1].tolist() == [[0, 0, 1, 2, 2, 2]]
model(input_ids = torch.zeros(1, 4, dtype = torch.long), use_cache = True)
assert conv.calls[-1] is None
def _hub_wrapped(torch_function, implementation):
# Same closure shape as transformers' use_kernel_func_from_hub_with_fallback.
import functools
import inspect as _inspect
applicable_params = tuple(_inspect.signature(implementation).parameters)
@functools.wraps(torch_function)
def wrapped(*args, **kwargs):
kwargs = {k: v for k, v in kwargs.items() if k in applicable_params}
return implementation(*args, **kwargs)
return wrapped
def _kwargs_gdn_model(name, accelerated):
# transformers >= 5.16 shape: forward(**kwargs) splats into module-level hub kernels.
import sys
import types
calls = {"conv": [], "scan": []}
def conv_ref(
x,
weight,
bias = None,
activation = None,
**kwargs,
):
return x
def conv_fast(
x,
weight,
bias = None,
seq_idx = None,
activation = None,
):
calls["conv"].append(seq_idx)
return x
def scan_ref(q, **kwargs):
return q
def scan_fast(
q,
cu_seqlens = None,
**kwargs,
):
calls["scan"].append(cu_seqlens)
return q
modeling = types.ModuleType(name)
modeling.causal_conv1d_fn = _hub_wrapped(conv_ref, conv_fast if accelerated else conv_ref)
modeling.torch_chunk_gated_delta_rule = _hub_wrapped(
scan_ref, scan_fast if accelerated else scan_ref
)
exec(
"import torch\n"
"class FakeGatedDeltaNet(torch.nn.Module):\n"
" def __init__(self):\n"
" super().__init__()\n"
" self.conv1d = torch.nn.Conv1d(4, 4, 3, groups = 4)\n"
" def forward(self, hidden_states, **kwargs):\n"
" x = causal_conv1d_fn(hidden_states, self.conv1d.weight, **kwargs)\n"
" return torch_chunk_gated_delta_rule(x, cu_seqlens = kwargs.pop('cu_seq_lens_q', None), **kwargs)\n",
modeling.__dict__,
)
modeling.FakeGatedDeltaNet.__module__ = name
sys.modules[name] = modeling
model = _FakeMamba2Model()
model.mixer = modeling.FakeGatedDeltaNet()
return model, calls
def test_patch_varlen_kwargs_mixer_injects_boundaries(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
name = "fake_modeling_kwargs_gdn"
try:
model, calls = _kwargs_gdn_model(name, accelerated = True)
assert patch_hybrid_linear_attention_varlen(model) is True
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
assert calls["conv"][-1].tolist() == [[0, 0, 1, 2, 2, 2]]
assert calls["scan"][-1].tolist() == [0, 2, 3, 6]
finally:
sys.modules.pop(name, None)
def test_patch_varlen_kwargs_mixer_rejects_torch_fallback(monkeypatch):
# The hub wrapper would silently drop seq_idx / cu_seqlens on the reference kernels.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
name = "fake_modeling_kwargs_gdn_ref"
try:
model, _ = _kwargs_gdn_model(name, accelerated = False)
assert patch_hybrid_linear_attention_varlen(model) is False
finally:
sys.modules.pop(name, None)
def test_patch_varlen_kwargs_mixer_rejects_missing_recurrent_kernel(monkeypatch):
# A conv-only kernel table would let the first-forward handshake pass on the conv probe alone.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
name = "fake_modeling_kwargs_gdn_noscan"
try:
model, _ = _kwargs_gdn_model(name, accelerated = True)
del sys.modules[name].torch_chunk_gated_delta_rule
assert patch_hybrid_linear_attention_varlen(model) is False
finally:
sys.modules.pop(name, None)
def test_packed_mask_builder_gets_per_sequence_positions():
seen = []
def create_causal_mask(**kwargs):
seen.append(kwargs.get("position_ids"))
ns = {"create_causal_mask": create_causal_mask}
varlen = (torch.tensor([0, 2, 3, 6], dtype = torch.int32), torch.tensor([[0, 0, 1, 2, 2, 2]]))
packing_module._install_packed_mask_positions(ns, lambda: varlen)
ns["create_causal_mask"](inputs_embeds = None)
assert seen[-1].tolist() == [[0, 1, 0, 0, 1, 2]]
explicit = torch.arange(6)[None]
ns["create_causal_mask"](position_ids = explicit)
assert seen[-1] is explicit
def test_short_conv_only_model_packs_without_flag(monkeypatch):
monkeypatch.delenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", raising = False)
assert patch_hybrid_linear_attention_varlen(_stateful_model(_FakeLfm2ShortConv())) is True
assert patch_hybrid_linear_attention_varlen(_FakeMamba2Model()) is False
def test_short_conv_rejects_hub_conv_bound_to_torch_fallback(monkeypatch):
# transformers >= 5.16 LFM2 routes through the hub conv; its torch fallback drops seq_idx.
monkeypatch.delenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", raising = False)
import sys
import types
def conv_ref(
x,
weight,
bias = None,
activation = None,
**kwargs,
):
return x
name = "fake_modeling_lfm2_ref"
modeling = types.ModuleType(name)
modeling.causal_conv1d_fn = _hub_wrapped(conv_ref, conv_ref)
modeling._Base = _FakeLfm2ShortConv
exec("class Lfm2ShortConv(_Base):\n pass\n", modeling.__dict__)
modeling.Lfm2ShortConv.__module__ = name
sys.modules[name] = modeling
try:
assert (
patch_hybrid_linear_attention_varlen(_stateful_model(modeling.Lfm2ShortConv())) is False
)
finally:
sys.modules.pop(name, None)
def test_patch_mamba2_varlen_second_model_gets_its_own_boundaries(monkeypatch):
# The kernel is a module global, so its wrapper outlives the first model; a later patched model
# must get its own boundaries rather than the first model's context.
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
import sys
import types
name = "fake_modeling_nemotron_h_global"
modeling = types.ModuleType(name)
exec(
"def mamba_split_conv1d_scan_combined(*args, seq_idx = None, **kwargs):\n"
" mamba_split_conv1d_scan_combined.calls.append(seq_idx)\n"
" return args[0]\n"
"mamba_split_conv1d_scan_combined.calls = []\n",
modeling.__dict__,
)
modeling._Base = _FakeNemotronHMamba2Mixer
exec(
"class NemotronHMamba2Mixer(_Base):\n"
" def __init__(self):\n"
" super().__init__()\n"
" del self.mamba2_split_conv1d_scan_combined\n"
" def forward(self, hidden_states, **kwargs):\n"
" return mamba_split_conv1d_scan_combined(hidden_states, seq_idx = None)\n",
modeling.__dict__,
)
modeling.NemotronHMamba2Mixer.__module__ = name
sys.modules[name] = modeling
calls = modeling.mamba_split_conv1d_scan_combined.calls
def build():
model = _FakeMamba2Model()
model.mixer = modeling.NemotronHMamba2Mixer()
return model
try:
first, second = build(), build()
assert patch_hybrid_linear_attention_varlen(first) is True
first(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
assert patch_hybrid_linear_attention_varlen(second) is True
second(
input_ids = torch.zeros(1, 5, dtype = torch.long),
packed_seq_lengths = torch.tensor([4, 1], dtype = torch.int32),
use_cache = False,
)
assert calls[-1].tolist() == [[0, 0, 0, 0, 1]]
finally:
sys.modules.pop(name, None)
def test_generate_restores_packed_stash_for_pending_backward(monkeypatch):
monkeypatch.setenv("UNSLOTH_EXPERIMENTAL_HYBRID_PACKING", "1")
model = _FakeMamba2Model()
seen = []
model.generate = lambda *a, **k: seen.append(model.mixer._unsloth_varlen)
assert patch_hybrid_linear_attention_varlen(model) is True
model(
input_ids = torch.zeros(1, 6, dtype = torch.long),
packed_seq_lengths = torch.tensor([2, 1, 3], dtype = torch.int32),
use_cache = False,
)
stash = model.mixer._unsloth_varlen
model.generate()
assert seen == [None] # generate never sees training boundaries
assert model.mixer._unsloth_varlen is stash # recompute in backward still does
@pytest.mark.parametrize(
"model_type, stateful, extra",
[
("lfm2", True, {"num_hidden_layers": 2, "layer_types": ["conv", "full_attention"]}),
("mamba2", True, {}),
("falcon_h1", True, {}),
("llama", False, {}),
],
)
def test_string_model_config_detects_stateful_mixers(model_type, stateful, extra):
# A string model= reaches the detector as its config only.
from transformers import AutoConfig
from unsloth.trainer import _is_hybrid_linear_attention_model
# Keep the default layer count: shrinking it truncates hybrid layer schedules.
small = dict(
hidden_size = 64,
intermediate_size = 128,
num_attention_heads = 4,
num_key_value_heads = 2,
vocab_size = 128,
)
config = AutoConfig.for_model(model_type, **extra)
for key, value in small.items():
if hasattr(config, key):
setattr(config, key, value)
assert _is_hybrid_linear_attention_model(SimpleNamespace(config = config)) is stateful
_REMOTE_MODELING = """
import torch
from transformers import PretrainedConfig, PreTrainedModel
class RemoteSSMConfig(PretrainedConfig):
model_type = "remote_ssm_pr9812"
class RemoteMixer(torch.nn.Module):
def __init__(self):
super().__init__()
self.A_log = torch.nn.Parameter(torch.zeros(4))
self.dt_bias = torch.nn.Parameter(torch.zeros(4))
self.conv1d = torch.nn.Conv1d(4, 4, 3, groups = 4)
class RemoteSSMForCausalLM(PreTrainedModel):
config_class = RemoteSSMConfig
def __init__(self, config):
super().__init__(config)
self.mixer = RemoteMixer()
"""
@pytest.mark.parametrize("grant", [True, False])
def test_string_remote_code_model_forwards_trust_grant(tmp_path, grant):
# An auto_map-only checkpoint is built under the user's own trust_remote_code grant.
import json
from unsloth.trainer import _is_hybrid_linear_attention_model
(tmp_path / "modeling_remote_ssm.py").write_text(_REMOTE_MODELING, encoding = "utf-8")
auto_map = {
"AutoConfig": "modeling_remote_ssm.RemoteSSMConfig",
"AutoModelForCausalLM": "modeling_remote_ssm.RemoteSSMForCausalLM",
}
(tmp_path / "config.json").write_text(
json.dumps({"model_type": "remote_ssm_pr9812", "auto_map": auto_map}), encoding = "utf-8"
)
from transformers import AutoConfig
config = AutoConfig.from_pretrained(str(tmp_path), trust_remote_code = True)
target = SimpleNamespace(config = config, trust_remote_code = grant)
assert _is_hybrid_linear_attention_model(target) is grant