Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Pack Qwen3.5 / Qwen3-Next hybrids by default with document boundaries in the linear-attention kernels (#13383)
Daniel Hand5824d09e63e
diff --git a/tests/utils/test_hybrid_packing_default.py b/tests/utils/test_hybrid_packing_default.pynew file mode 100644index 00000000000..be369692319--- /dev/null+++ b/tests/utils/test_hybrid_packing_default.py@@ -0,0 +1,598 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Gated-delta hybrid packing on by default: the mode switch, the exactness probe and the+unsloth-zoo Triton conv acceptance, on small CPU stand-ins for the GatedDeltaNet mixer."""++import functools+import inspect+import logging+import sys+import types+from types import SimpleNamespace++import pytest+import torch++import unsloth.utils.packing as packing_module+from unsloth.utils.packing import patch_hybrid_linear_attention_varlen++ENV = "UNSLOTH_EXPERIMENTAL_HYBRID_PACKING"+++def _segments(T, seq_idx):+ if seq_idx is None:+ return [(0, T)]+ ids = seq_idx.reshape(-1)+ cuts = (torch.nonzero(ids[1:] != ids[:-1]).flatten() + 1).tolist()+ bounds = [0] + cuts + [T]+ return list(zip(bounds[:-1], bounds[1:]))+++def conv_exact(+ x,+ weight,+ bias = None,+ activation = None,+ seq_idx = None,+ **kwargs,+):+ import torch.nn.functional as F++ outs = []+ for s, e in _segments(x.shape[-1], seq_idx):+ seg = x[:, :, s:e]+ y = F.conv1d(+ seg, weight.unsqueeze(1), bias, padding = weight.shape[-1] - 1, groups = x.shape[1]+ )+ outs.append(y[:, :, : e - s])+ out = torch.cat(outs, dim = -1)+ return F.silu(out) if activation else out+++def conv_leaky(+ x,+ weight,+ bias = None,+ activation = None,+ seq_idx = None,+ **kwargs,+):+ # Names seq_idx (passes the static check) but ignores it.+ return conv_exact(x, weight, bias, activation, None)+++def scan_exact(+ x,+ cu_seqlens = None,+ **kwargs,+):+ # A decayed running sum that restarts at every cu_seqlens boundary.+ T = x.shape[1]+ starts = set(cu_seqlens.tolist()[:-1]) if cu_seqlens is not None else {0}+ out, state = [], torch.zeros_like(x[:, 0])+ for t in range(T):+ if t in starts:+ state = torch.zeros_like(state)+ state = 0.7 * state + x[:, t]+ out.append(state)+ return torch.stack(out, dim = 1)+++def scan_leaky(+ x,+ cu_seqlens = None,+ **kwargs,+):+ return scan_exact(x, None)+++class FakeGatedDeltaNet(torch.nn.Module):+ """transformers < 5.16 shape: instance kernels, the conv called with seq_idx=None."""++ def __init__(+ self,+ conv = conv_exact,+ scan = scan_exact,+ hidden = 8,+ width = 4,+ ):+ super().__init__()+ if hidden is not None:+ self.hidden_size = hidden+ self.conv1d = torch.nn.Conv1d(8, 8, width, groups = 8, bias = False)+ self.causal_conv1d_fn = conv+ self.chunk_gated_delta_rule = scan++ def forward(+ self,+ hidden_states,+ cache_params = None,+ attention_mask = None,+ ):+ x = hidden_states.transpose(1, 2)+ x = self.causal_conv1d_fn(+ x = x, weight = self.conv1d.weight.squeeze(1), bias = None, activation = "silu", seq_idx = None+ )+ return self.chunk_gated_delta_rule(x.transpose(1, 2), cu_seqlens = None)+++class FakeHybrid(torch.nn.Module):+ def __init__(self, mixer):+ super().__init__()+ self.config = SimpleNamespace(layer_types = ["linear_attention", "full_attention"])+ torch.manual_seed(0)+ self.embed = torch.nn.Embedding(32, 8)+ self.linear_attn = mixer++ def forward(+ self,+ input_ids = None,+ position_ids = None,+ packed_seq_lengths = None,+ use_cache = None,+ **kw,+ ):+ return self.linear_attn(self.embed(input_ids))++[email protected](autouse = True)+def _fresh_warnings(monkeypatch):+ packing_module._HYBRID_WARNED.clear()+ monkeypatch.setattr(packing_module, "_auto_hybrid_device_ok", lambda module: True)+ yield+ packing_module._HYBRID_WARNED.clear()+++def _untouched(model, conv, scan):+ mixer = model.linear_attn+ return (+ mixer.causal_conv1d_fn is conv+ and mixer.chunk_gated_delta_rule is scan+ and "forward" not in mixer.__dict__+ and not getattr(mixer, "_unsloth_varlen_wrapped", False)+ and not getattr(model, "_unsloth_varlen_forward_wrapped", False)+ )+++ "value,mode",+ [+ (None, "auto"),+ ("", "auto"),+ ("maybe", "auto"),+ ("1", "on"),+ ("TRUE", "on"),+ ("yes", "on"),+ ("on", "on"),+ ("0", "off"),+ ("false", "off"),+ ("No", "off"),+ (" off ", "off"),+ ],+)+def test_mode_parsing(monkeypatch, value, mode):+ if value is None:+ monkeypatch.delenv(ENV, raising = False)+ else:+ monkeypatch.setenv(ENV, value)+ assert packing_module._hybrid_packing_mode() == mode+ assert packing_module._hybrid_packing_enabled() is (mode == "on")+++def test_default_packs_exact_gated_delta(monkeypatch):+ monkeypatch.delenv(ENV, raising = False)+ model = FakeHybrid(FakeGatedDeltaNet())+ rng = torch.get_rng_state()+ assert patch_hybrid_linear_attention_varlen(model) is True+ assert torch.equal(rng, torch.get_rng_state()) # the probe uses its own generator+ assert not packing_module._HYBRID_WARNED+ ids = torch.randint(0, 32, (1, 9), generator = torch.Generator().manual_seed(1))+ lengths = [4, 1, 4]+ packed = model(+ input_ids = ids, packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32), use_cache = False+ )+ assert model._unsloth_varlen_handshake_done+ alone = torch.cat(+ [model(input_ids = ids[:, s : s + n]) for s, n in zip((0, 4, 5), lengths)], dim = 1+ )+ torch.testing.assert_close(packed, alone)+ plain = model(input_ids = ids, use_cache = True)+ assert not torch.allclose(plain, packed)++[email protected]("value", ["0", "false", "no", "off"])+def test_kill_switch_keeps_padded_path(monkeypatch, value):+ monkeypatch.setenv(ENV, value)+ model = FakeHybrid(FakeGatedDeltaNet())+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert _untouched(model, conv_exact, scan_exact)++[email protected]("mode", [None, "1"])[email protected]("conv,scan", [(conv_leaky, scan_exact), (conv_exact, scan_leaky)])+def test_probe_rejects_a_leaky_kernel_and_rolls_back(monkeypatch, caplog, mode, conv, scan):+ if mode is None:+ monkeypatch.delenv(ENV, raising = False)+ else:+ monkeypatch.setenv(ENV, mode)+ model = FakeHybrid(FakeGatedDeltaNet(conv = conv, scan = scan))+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert "exactness probe" in caplog.text and "differs" in caplog.text+ assert _untouched(model, conv, scan)+++def test_a_failed_probe_restores_the_modeling_namespace(monkeypatch):+ monkeypatch.delenv(ENV, raising = False)++ def create_causal_mask(*args, **kwargs):+ return None++ monkeypatch.setattr(+ sys.modules[__name__], "create_causal_mask", create_causal_mask, raising = False+ )+ model = FakeHybrid(FakeGatedDeltaNet(conv = conv_leaky))+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert sys.modules[__name__].create_causal_mask is create_causal_mask+++class DropoutGatedDeltaNet(FakeGatedDeltaNet):+ """LoRA-style input dropout: in train mode the packed and per-document passes would differ."""++ def __init__(self, **kwargs):+ super().__init__(**kwargs)+ self.dropout = torch.nn.Dropout(0.5)++ def forward(+ self,+ hidden_states,+ cache_params = None,+ attention_mask = None,+ ):+ return super().forward(self.dropout(hidden_states), cache_params, attention_mask)+++def test_probe_runs_in_eval_mode_and_keeps_the_rng(monkeypatch):+ monkeypatch.delenv(ENV, raising = False)+ mixer = DropoutGatedDeltaNet()+ model = FakeHybrid(mixer)+ model.train()+ rng = torch.get_rng_state()+ assert patch_hybrid_linear_attention_varlen(model) is True+ assert torch.equal(rng, torch.get_rng_state())+ assert mixer.training and mixer.dropout.training+++def test_probe_failure_to_run_is_a_rejection(monkeypatch, caplog):+ monkeypatch.delenv(ENV, raising = False)++ def conv_raises(+ x,+ weight,+ bias = None,+ activation = None,+ seq_idx = None,+ **kwargs,+ ):+ raise RuntimeError("kernel exploded")++ model = FakeHybrid(FakeGatedDeltaNet(conv = conv_raises))+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert "kernel exploded" in caplog.text+ assert _untouched(model, conv_raises, scan_exact)+++def test_probe_requires_dispatch(monkeypatch, caplog):+ monkeypatch.delenv(ENV, raising = False)++ class BypassGatedDeltaNet(FakeGatedDeltaNet):+ def forward(+ self,+ hidden_states,+ cache_params = None,+ attention_mask = None,+ ):+ return conv_exact(+ hidden_states.transpose(1, 2), self.conv1d.weight.squeeze(1)+ ).transpose(1, 2)++ mixer = BypassGatedDeltaNet()+ model = FakeHybrid(mixer)+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert "not dispatched" in caplog.text+++def test_default_needs_a_probeable_mixer(monkeypatch, caplog):+ monkeypatch.delenv(ENV, raising = False)+ model = FakeHybrid(FakeGatedDeltaNet(hidden = None))+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert "probe" in caplog.text+ assert _untouched(model, conv_exact, scan_exact)+ monkeypatch.setenv(ENV, "1")+ assert patch_hybrid_linear_attention_varlen(FakeHybrid(FakeGatedDeltaNet(hidden = None))) is True+++def test_default_rejects_torch_fallback(monkeypatch):+ monkeypatch.delenv(ENV, raising = False)++ def torch_chunk_gated_delta_rule(+ x,+ cu_seqlens = None,+ **kwargs,+ ):+ return scan_exact(x, cu_seqlens)++ model = FakeHybrid(FakeGatedDeltaNet(scan = torch_chunk_gated_delta_rule))+ assert patch_hybrid_linear_attention_varlen(model) is False+++# transformers >= 5.16: forward(**kwargs) splats into module-level hub kernels.+++def _hub_wrapped(torch_function, implementation):+ applicable_params = tuple(inspect.signature(implementation).parameters)++ @functools.wraps(torch_function)+ def wrapped(*args, **kwargs):+ if implementation is None: # transformers references torch_function here too (export path)+ return torch_function(*args, **kwargs)+ kwargs = {k: v for k, v in kwargs.items() if k in applicable_params}+ return implementation(*args, **kwargs)++ return wrapped+++def _zoo_style_dispatch(torch_function, name_seq_idx = True):+ # unsloth-zoo gdn_causal_conv1d._make_hub_dispatch: marked, own signature naming seq_idx.+ if name_seq_idx:++ @functools.wraps(torch_function)+ def causal_conv1d_fn(+ hidden_states,+ weight,+ bias = None,+ activation = None,+ seq_idx = None,+ **kwargs,+ ):+ return conv_exact(hidden_states, weight, bias, activation, seq_idx)+ else:++ @functools.wraps(torch_function)+ def causal_conv1d_fn(+ hidden_states,+ weight,+ bias = None,+ activation = None,+ **kwargs,+ ):+ return conv_exact(hidden_states, weight, bias, activation)++ causal_conv1d_fn._unsloth_triton_causal_conv1d = True+ causal_conv1d_fn.__signature__ = inspect.signature(causal_conv1d_fn, follow_wrapped = False)+ return causal_conv1d_fn+++def _kwargs_model(name, conv_impl):+ def conv_ref(+ hidden_states,+ weight,+ bias = None,+ activation = None,+ **kwargs,+ ):+ return conv_exact(hidden_states, weight, bias, activation)++ def scan_ref(x, **kwargs):+ return scan_exact(x)++ def scan_fast(+ x,+ cu_seqlens = None,+ **kwargs,+ ):+ return scan_exact(x, cu_seqlens)++ modeling = types.ModuleType(name)+ impl = conv_impl(conv_ref)+ modeling.causal_conv1d_fn = _hub_wrapped(impl, impl)+ modeling.torch_chunk_gated_delta_rule = _hub_wrapped(scan_ref, scan_fast)+ exec(+ "import torch\n"+ "class FakeKwargsGatedDeltaNet(torch.nn.Module):\n"+ " def __init__(self):\n"+ " super().__init__()\n"+ " self.hidden_size = 8\n"+ " self.conv1d = torch.nn.Conv1d(8, 8, 4, groups = 8, bias = False)\n"+ " def forward(self, hidden_states, cache_params = None, attention_mask = None, **kwargs):\n"+ " x = causal_conv1d_fn(hidden_states.transpose(1, 2), self.conv1d.weight.squeeze(1), None,\n"+ " activation = 'silu', **kwargs).transpose(1, 2)\n"+ " return torch_chunk_gated_delta_rule(x, cu_seqlens = kwargs.pop('cu_seq_lens_q', None), **kwargs)\n",+ modeling.__dict__,+ )+ modeling.FakeKwargsGatedDeltaNet.__module__ = name+ sys.modules[name] = modeling+ return FakeHybrid(modeling.FakeKwargsGatedDeltaNet())++[email protected]("mode", [None, "1"])+def test_zoo_triton_conv_is_accepted(monkeypatch, mode):+ if mode is None:+ monkeypatch.delenv(ENV, raising = False)+ else:+ monkeypatch.setenv(ENV, mode)+ name = "fake_modeling_zoo_conv_gdn"+ try:+ model = _kwargs_model(name, _zoo_style_dispatch)+ assert patch_hybrid_linear_attention_varlen(model) is True+ ids = torch.randint(0, 32, (1, 9), generator = torch.Generator().manual_seed(2))+ packed = model(+ input_ids = ids,+ packed_seq_lengths = torch.tensor([4, 1, 4], dtype = torch.int32),+ use_cache = False,+ )+ alone = torch.cat(+ [model(input_ids = ids[:, s:e]) for s, e in ((0, 4), (4, 5), (5, 9))], dim = 1+ )+ torch.testing.assert_close(packed, alone)+ assert model.linear_attn._unsloth_varlen_kwargs_hit+ finally:+ sys.modules.pop(name, None)+++def test_zoo_triton_conv_without_seq_idx_is_rejected(monkeypatch, caplog):+ # An older unsloth-zoo dispatch (marked, no seq_idx) would be filtered by the hub wrapper.+ monkeypatch.delenv(ENV, raising = False)+ name = "fake_modeling_zoo_conv_old"+ try:+ model = _kwargs_model(name, functools.partial(_zoo_style_dispatch, name_seq_idx = False))+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert "causal_conv1d_fn" in caplog.text+ finally:+ sys.modules.pop(name, None)+++def test_unmarked_torch_conv_is_rejected(monkeypatch, caplog):+ monkeypatch.delenv(ENV, raising = False)+ name = "fake_modeling_torch_conv"+ try:+ model = _kwargs_model(name, lambda fn: fn)+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert "has no accelerated kernel" in caplog.text+ finally:+ sys.modules.pop(name, None)+++def test_real_zoo_dispatch_is_recognised():+ gcc = pytest.importorskip("unsloth_zoo.temporary_patches.gdn_causal_conv1d")+ if "seq_idx" not in inspect.signature(gcc._legacy_causal_conv1d_fn).parameters:+ pytest.skip("unsloth-zoo without the seq_idx Triton conv")++ def causal_conv1d_fn(+ hidden_states,+ weight,+ bias = None,+ activation = None,+ **kwargs,+ ):+ return hidden_states++ assert packing_module._is_unsloth_seq_idx_conv(gcc._make_hub_dispatch(causal_conv1d_fn))+ assert packing_module._is_unsloth_seq_idx_conv(gcc._legacy_causal_conv1d_fn)+ assert not packing_module._is_unsloth_seq_idx_conv(causal_conv1d_fn)+++def test_default_on_is_cuda_only(monkeypatch, caplog):+ monkeypatch.delenv(ENV, raising = False)+ monkeypatch.undo() # the real device check+ monkeypatch.delenv(ENV, raising = False)+ mixer = FakeGatedDeltaNet()+ assert not packing_module._auto_hybrid_device_ok(mixer) # CPU+ monkeypatch.setattr(torch.version, "hip", "6.4")+ assert not packing_module._auto_hybrid_device_ok(mixer)+ monkeypatch.setattr(torch.version, "hip", None)+ model = FakeHybrid(mixer)+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert "only on CUDA" in caplog.text+ assert _untouched(model, conv_exact, scan_exact)+ monkeypatch.setenv(ENV, "1")+ assert patch_hybrid_linear_attention_varlen(FakeHybrid(FakeGatedDeltaNet())) is True+++def test_default_on_is_cuda_only_for_kwargs_mixers(monkeypatch, caplog):+ # transformers 5.16+ kwargs-mode Qwen mixers leave gated_delta_modules; the default must still cover them.+ monkeypatch.undo()+ monkeypatch.delenv(ENV, raising = False)+ name = "fake_modeling_kwargs_cpu_gdn"+ try:+ model = _kwargs_model(name, _zoo_style_dispatch)+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):+ assert patch_hybrid_linear_attention_varlen(model) is False+ assert "only on CUDA" in caplog.text+ monkeypatch.setenv(ENV, "1")+ assert (+ patch_hybrid_linear_attention_varlen(_kwargs_model(name, _zoo_style_dispatch)) is True+ )+ finally:+ sys.modules.pop(name, None)+++def test_engagement_counters(monkeypatch):+ monkeypatch.delenv(ENV, raising = False)+ stats = dict(packing_module.HYBRID_PACKING_STATS)+ assert patch_hybrid_linear_attention_varlen(FakeHybrid(FakeGatedDeltaNet())) is True+ assert (+ patch_hybrid_linear_attention_varlen(FakeHybrid(FakeGatedDeltaNet(conv = conv_leaky)))+ is False+ )+ now = packing_module.HYBRID_PACKING_STATS+ assert now["probe_runs"] == stats["probe_runs"] + 2+ assert now["probe_passed"] == stats["probe_passed"] + 1+ assert now["probe_failed"] == stats["probe_failed"] + 1+ assert now["patched"] == stats["patched"] + 1+++def test_hub_kernel_behind_compile_disable_wrapper(monkeypatch):+ # The gate must look through torch_compiler_disable_unless_decode to the hub closure.+ monkeypatch.delenv(ENV, raising = False)+ name = "fake_unsloth_compiled_module_gdn"++ def disable_unless_decode(func):+ @functools.wraps(func)+ def forward(*args, **kwargs):+ return func(*args, **kwargs)++ forward._unsloth_undisabled = func+ return forward++ try:+ model = _kwargs_model(name, _zoo_style_dispatch)+ modeling = sys.modules[name]+ modeling.torch_chunk_gated_delta_rule = disable_unless_decode(+ modeling.torch_chunk_gated_delta_rule+ )+ assert "implementation" in packing_module._hub_closure(+ modeling.torch_chunk_gated_delta_rule+ )+ assert patch_hybrid_linear_attention_varlen(model) is True+ finally:+ sys.modules.pop(name, None)+++def test_boundaries_follow_the_call_onto_its_device():+ varlen = (+ torch.tensor([0, 4, 9], dtype = torch.int32),+ torch.tensor([[0] * 4 + [1] * 5], dtype = torch.int32),+ )+ x = torch.empty(1, 8, 9, device = "meta")+ cu, seq_idx = packing_module._varlen_on(varlen, (), {"x": x})+ assert cu.device.type == "meta" and seq_idx.device.type == "meta"+ assert packing_module._varlen_on(varlen, (torch.zeros(1, 8, 9),), {}) is varlen+++def test_a_stale_stash_never_reaches_a_call_of_another_length(monkeypatch):+ monkeypatch.delenv(ENV, raising = False)+ model = FakeHybrid(FakeGatedDeltaNet())+ assert patch_hybrid_linear_attention_varlen(model) is True+ ids = torch.randint(0, 32, (1, 9), generator = torch.Generator().manual_seed(2))+ model(+ input_ids = ids,+ packed_seq_lengths = torch.tensor([4, 1, 4], dtype = torch.int32),+ use_cache = False,+ )+ mixer = model.linear_attn+ stale = mixer._unsloth_varlen+ hidden = model.embed(ids[:, :5])+ mixer._unsloth_varlen = None+ plain = mixer(hidden)+ mixer._unsloth_varlen = stale+ mixer._unsloth_varlen_conv_hit = mixer._unsloth_varlen_scan_hit = False+ torch.testing.assert_close(mixer(hidden), plain)+ assert not mixer._unsloth_varlen_conv_hit and not mixer._unsloth_varlen_scan_hitdiff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex a8f19947c0e..dee005012c5 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -1559,8 +1559,7 @@ def _patch_sft_trainer_auto_packing(trl_module):else model)is_hybrid = _is_hybrid_linear_attention_model(hybrid_target)- # Hybrid models corrupt packed batches unless the gated-delta / Mamba2 conv and scan reset at- # sequence boundaries, so enable the experimental varlen shim (flag plus kernels) or keep them blocked.+ # Hybrid models pack only when the varlen shim's kernel checks and exactness probe pass.if (is_hybridand not isinstance(model, str)diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex e505f9afe45..b91ae1bfbe4 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -321,9 +321,8 @@ def enable_padding_free_metadata(model, trainer):collator._unsloth_padding_free_lengths_wrapped = True-# experimental hybrid packing: Qwen3.5 / Qwen3-Next gated-delta and Nemotron-H Mamba2.-# recurrent and conv state leak unless seq_idx / cu_seqlens reach accelerated kernels.-# env-gated; pure-torch fallbacks fail closed to prevent cross-sequence state leaks.+# Hybrid linear-attention packing leaks conv/recurrent state across documents unless seq_idx / cu_seqlens+# reach the accelerated kernels, so it fails closed on torch fallbacks. =0 disables, =1 opts in all mixers._MAMBA2_FUSED_NAMES = ("mamba2_split_conv1d_scan_combined","mamba_split_conv1d_scan_combined",@@ -333,18 +332,36 @@ _HYBRID_LOGGER = logging.getLogger("unsloth.hybrid_packing")_HYBRID_WARNED: set = set()+def _hybrid_packing_mode() -> str:+ """ "on" / "off" / "auto"; read at call time so setting the flag after import still works."""+ value = os.environ.get(_HYBRID_PACKING_ENV_VAR, "").strip().lower()+ if value in {"1", "true", "yes", "on"}:+ return "on"+ if value in {"0", "false", "no", "off"}:+ return "off"+ return "auto"++def _hybrid_packing_enabled() -> bool:- # Read at call time so setting the flag after `import unsloth` still takes effect.- return os.environ.get(_HYBRID_PACKING_ENV_VAR, "0").strip().lower() in {- "1",- "true",- "yes",- "on",- }+ return _hybrid_packing_mode() == "on"+++# Auto-packed mixers (CUDA only). MoE router aux loss is computed over the whole packed row, not per document.+_HYBRID_AUTO_KINDS = ("gated_delta",)++HYBRID_PACKING_STATS = {+ "probe_runs": 0,+ "probe_passed": 0,+ "probe_failed": 0,+ "patched": 0,+ "rejected": 0,+ "last_probe_rel_err": None,+}def _hybrid_reject(reason: str) -> bool:# One deduped diagnostic explaining why hybrid packing stayed on the padded path.+ HYBRID_PACKING_STATS["rejected"] += 1if reason not in _HYBRID_WARNED:_HYBRID_WARNED.add(reason)_HYBRID_LOGGER.warning(@@ -711,22 +728,31 @@ def _hybrid_varlen_dispatched(module) -> bool:)+def _varlen_input(args, kwargs):+ for name in ("x", "hidden_states", "zxbcdt", "query", "q"):+ candidate = kwargs.get(name)+ if isinstance(candidate, torch.Tensor):+ return candidate+ for arg in args:+ if isinstance(arg, torch.Tensor):+ return arg+ return None+++def _varlen_on(varlen, args, kwargs):+ """Move varlen to this call's input device (multi-GPU splits run mixers away from input ids)."""+ tensor = _varlen_input(args, kwargs)+ if tensor is None or varlen[1].device == tensor.device:+ return varlen+ return tuple(t.to(tensor.device, non_blocking = True) for t in varlen)++def _varlen_seq_idx_applies(seq_idx, args, kwargs) -> bool:# generate() bypasses model.forward, so a stale training seq_idx must not reach a prompt.if seq_idx is None:return Falsetotal = seq_idx.shape[-1]- tensor = None- for name in ("x", "hidden_states", "zxbcdt"):- candidate = kwargs.get(name)- if isinstance(candidate, torch.Tensor):- tensor = candidate- break- if tensor is None:- for arg in args:- if isinstance(arg, torch.Tensor):- tensor = arg- break+ tensor = _varlen_input(args, kwargs)if tensor is None or tensor.dim() < 2 or seq_idx.shape[0] != tensor.shape[0]:return False# causal_conv1d_fn takes (batch, dim, seqlen), the others (batch, seqlen, ...).@@ -826,11 +852,22 @@ _KWARGS_TRACE: list = [None]def _hub_closure(fn) -> dict:- fn = getattr(fn, "_unsloth_varlen_probe_of", fn)- try:- return inspect.getclosurevars(fn).nonlocals- except (TypeError, ValueError):- return {}+ """Kernel-hub wrapper closure nonlocals, unwrapping Unsloth's probes and pass-through wrappers; {} if none."""+ for _ in range(8):+ if fn is None:+ break+ try:+ nonlocals = inspect.getclosurevars(fn).nonlocals+ except (TypeError, ValueError):+ nonlocals = {}+ if "implementation" in nonlocals:+ return nonlocals+ fn = (+ getattr(fn, "_unsloth_varlen_probe_of", None)+ or getattr(fn, "_unsloth_undisabled", None)+ or getattr(fn, "__wrapped__", None)+ )+ return {}def _is_kwargs_mixer(module) -> bool:@@ -846,6 +883,20 @@ def _is_kwargs_mixer(module) -> bool:return "implementation" in _hub_closure(globs.get("causal_conv1d_fn"))+# unsloth-zoo's Triton GDN conv looks like the hub "torch function" but honours seq_idx on all paths.+_ZOO_TRITON_CONV_MARK = "_unsloth_triton_causal_conv1d"+++def _is_unsloth_seq_idx_conv(fn) -> bool:+ try:+ fn = inspect.unwrap(fn, stop = lambda f: getattr(f, _ZOO_TRITON_CONV_MARK, False))+ return bool(getattr(fn, _ZOO_TRITON_CONV_MARK, False)) and (+ "seq_idx" in inspect.signature(fn).parameters+ )+ except (TypeError, ValueError):+ return False++def _kwargs_kernels_available(modules, recurrent_ids = ()) -> Optional[str]:for module in modules:globs = getattr(inspect.getmodule(type(module)), "__dict__", {})@@ -858,7 +909,10 @@ def _kwargs_kernels_available(modules, recurrent_ids = ()) -> Optional[str]:closure = _hub_closure(fn)# The hub wrapper drops kwargs its bound implementation does not name, silently.impl = closure.get("implementation")- if impl is None or impl is closure.get("torch_function"):+ if impl is None or (+ impl is closure.get("torch_function")+ and not (name == "causal_conv1d_fn" and _is_unsloth_seq_idx_conv(impl))+ ):return f"{name} has no accelerated kernel"if _KWARGS_KERNELS[name] not in closure.get("applicable_params", ()):return f"{name} kernel does not accept {_KWARGS_KERNELS[name]}"@@ -897,6 +951,7 @@ def _wrap_kwargs_mixer_forward(module) -> None:varlen = getattr(module, "_unsloth_varlen", None)if varlen is None or not _varlen_seq_idx_applies(varlen[1], args, kwargs):return forward_orig(*args, **kwargs)+ varlen = _varlen_on(varlen, args, kwargs)if kwargs.get("seq_idx") is None:kwargs["seq_idx"] = varlen[1]if kwargs.get("cu_seq_lens_q") is None:@@ -1083,15 +1138,133 @@ def _hybrid_varlen_metadata(kwargs):return _varlen_from_position_ids(kwargs.get("position_ids"))+# A 1-token document, one shorter than the conv width, and one crossing fla's 64-token chunk boundary.+_HYBRID_PROBE_LENGTHS = (5, 1, 70, 7)+_HYBRID_PROBE_RTOL = {torch.float32: 1e-4, torch.bfloat16: 2e-2, torch.float16: 1e-2}+_HYBRID_SNAPSHOT_ATTRS = (+ "forward",+ "causal_conv1d_fn",+ "chunk_gated_delta_rule",+ "_unsloth_varlen",+ "_unsloth_varlen_wrapped",+ "_unsloth_varlen_orig_scan",+ "_unsloth_varlen_orig_conv",+ "_unsloth_varlen_kwargs_mode",+ "_unsloth_varlen_conv_hit",+ "_unsloth_varlen_scan_hit",+ "_unsloth_varlen_kwargs_hit",+)+_MISSING = object()+++def _snapshot_mixers(modules):+ saved = []+ for module in modules:+ owners = [module] + [getattr(module, c, None) for c in _QKV_CONVS]+ for owner in owners:+ if isinstance(owner, torch.nn.Module):+ state = {k: owner.__dict__.get(k, _MISSING) for k in _HYBRID_SNAPSHOT_ATTRS}+ saved.append((owner, state))+ return saved+++def _restore_mixers(saved) -> None:+ for owner, state in saved:+ for key, value in state.items():+ if value is _MISSING:+ owner.__dict__.pop(key, None)+ else:+ owner.__dict__[key] = value+++def _auto_hybrid_device_ok(module) -> bool:+ weight = next(module.parameters(), None)+ return torch.version.hip is None and weight is not None and weight.device.type == "cuda"+++def _probe_target(modules):+ for module in modules:+ if _stateful_mixer_kind(module) != "gated_delta":+ continue+ hidden = getattr(module, "hidden_size", None)+ weight = getattr(getattr(module, "conv1d", None), "weight", None)+ if isinstance(hidden, int) and hidden > 0 and isinstance(weight, torch.Tensor):+ if weight.device.type != "meta" and weight.dtype in _HYBRID_PROBE_RTOL:+ return module+ return None+++def _hybrid_exactness_probe(module) -> Optional[str]:+ """Packed vs per-document mixer output: None if exact, else the reason. Eval mode, forked RNG."""+ weight = module.conv1d.weight+ dtype, device = weight.dtype, weight.device+ lengths = torch.tensor(_HYBRID_PROBE_LENGTHS, dtype = torch.int32)+ cu_seqlens = torch.cat(+ [torch.zeros(1, dtype = torch.int32), lengths.cumsum(0, dtype = torch.int32)]+ )+ total = int(cu_seqlens[-1])+ seq_idx = torch.repeat_interleave(+ torch.arange(len(_HYBRID_PROBE_LENGTHS), dtype = torch.int32), lengths+ )+ generator = torch.Generator().manual_seed(0)+ hidden = torch.randn(1, total, module.hidden_size, generator = generator)+ hidden = hidden.to(device = device, dtype = dtype)+ varlen = (cu_seqlens.to(device), seq_idx[None].to(device))++ def run(x):+ out = module(x)+ return out[0] if isinstance(out, (tuple, list)) else out++ saved = _snapshot_mixers([module])+ modes = [(m, m.training) for m in module.modules()]+ HYBRID_PACKING_STATS["probe_runs"] += 1+ rng_devices = [device] if device.type == "cuda" else []+ try:+ module.eval()+ with torch.no_grad(), torch.random.fork_rng(devices = rng_devices):+ for flag in (+ "_unsloth_varlen_conv_hit",+ "_unsloth_varlen_scan_hit",+ "_unsloth_varlen_kwargs_hit",+ ):+ module.__dict__[flag] = False+ module.__dict__["_unsloth_varlen"] = varlen+ packed = run(hidden)+ dispatched = _hybrid_varlen_dispatched(module)+ module.__dict__["_unsloth_varlen"] = None+ starts = cu_seqlens.tolist()+ alone = torch.cat([run(hidden[:, s:e]) for s, e in zip(starts[:-1], starts[1:])], dim = 1)+ except Exception as exc:+ return f"exactness probe failed to run ({type(exc).__name__}: {str(exc)[:200]})"+ finally:+ _restore_mixers(saved)+ for m, training in modes:+ m.training = training+ if not dispatched:+ return "exactness probe: boundary kernels were not dispatched"+ if not isinstance(packed, torch.Tensor) or packed.shape != alone.shape:+ return "exactness probe: unexpected mixer output"+ packed, alone = packed.float(), alone.float()+ if not bool(torch.isfinite(packed).all()) or not bool(torch.isfinite(alone).all()):+ return "exactness probe: non-finite mixer output"+ scale = alone.abs().max().clamp_min(1e-6)+ err = ((packed - alone).abs().max() / scale).item()+ HYBRID_PACKING_STATS["last_probe_rel_err"] = err+ if err > _HYBRID_PROBE_RTOL[dtype]:+ return f"exactness probe: packed output differs from per-document output (rel {err:.2e})"+ return None++def patch_hybrid_linear_attention_varlen(model) -> bool:- """Feed seq_idx / cu_seqlens to hybrid mixers so packing resets state at sequence boundaries.- Gated by UNSLOTH_EXPERIMENTAL_HYBRID_PACKING (short-conv-only models pack without it), fail-closed,- idempotent. True when active."""+ """Feed seq_idx / cu_seqlens to hybrid mixers; True when packing is active. Fail-closed, idempotent."""mixers = _iter_stateful_mixers(model)# Short-conv-only models (LFM2) reset at seq_idx on both kernel and torch paths, so they pack by default.short_conv_only = bool(mixers) and all(kind == "short_conv" for _, kind in mixers)- if not (_hybrid_packing_enabled() or short_conv_only):- return False+ mode = _hybrid_packing_mode()+ auto_eligible = bool(mixers) and all(kind in _HYBRID_AUTO_KINDS for _, kind in mixers)+ if not short_conv_only:+ if mode == "off" or (mode == "auto" and not auto_eligible):+ return Falseunsupported = sorted({type(m).__name__ for m, kind in mixers if kind == "unsupported"})if unsupported:return _hybrid_reject(f"no varlen kernel path for {unsupported}")@@ -1117,6 +1290,16 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:return Truehub_short_convs = [m for m in short_conv_modules if _uses_hub_conv(m)]+ probe_target = _probe_target(gated_delta_modules + kwargs_modules)+ if mode == "auto" and not short_conv_only:+ if probe_target is None:+ return _hybrid_reject("no gated-delta mixer the exactness probe can run")+ # Every stateful mixer, kwargs-mode ones included (transformers 5.16+ Qwen mixers move there).+ if not all(_auto_hybrid_device_ok(m) for m in hybrid_modules):+ return _hybrid_reject(+ f"default-on only on CUDA (set {_HYBRID_PACKING_ENV_VAR}=1 to opt in on this device)"+ )+ rollback = _snapshot_mixers(hybrid_modules)if kwargs_modules or hub_short_convs:reason = _kwargs_kernels_available(kwargs_modules + hub_short_convs, recurrent_ids = {id(m) for m in kwargs_modules}@@ -1154,10 +1337,10 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:**kwargs,):varlen = getattr(_module, "_unsloth_varlen", None)- if varlen is not None:+ if varlen is not None and _varlen_seq_idx_applies(varlen[1], args, kwargs):_module._unsloth_varlen_conv_hit = True # runtime dispatch handshakeif kwargs.get("seq_idx") is None:- kwargs["seq_idx"] = varlen[1]+ kwargs["seq_idx"] = _varlen_on(varlen, args, kwargs)[1]return _orig(*args, **kwargs)@wraps(scan_orig)@@ -1168,10 +1351,10 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:**kwargs,):varlen = getattr(_module, "_unsloth_varlen", None)- if varlen is not None:+ if varlen is not None and _varlen_seq_idx_applies(varlen[1], args, kwargs):_module._unsloth_varlen_scan_hit = Trueif kwargs.get("cu_seqlens") is None:- kwargs["cu_seqlens"] = varlen[0]+ kwargs["cu_seqlens"] = _varlen_on(varlen, args, kwargs)[0]return _orig(*args, **kwargs)if conv_orig is not None:@@ -1180,9 +1363,16 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:module._unsloth_varlen = Nonemodule._unsloth_varlen_wrapped = True- for ns in _iter_mamba2_install_namespaces(hybrid_modules):+ mask_namespaces = list(_iter_mamba2_install_namespaces(hybrid_modules))+ kwargs_namespaces = list(_iter_mamba2_install_namespaces(kwargs_modules + hub_short_convs))+ namespace_rollback = {}+ for ns in mask_namespaces + kwargs_namespaces:+ if isinstance(ns, dict) and id(ns) not in namespace_rollback:+ names = tuple(_MASK_BUILDERS) + tuple(_KWARGS_KERNELS)+ namespace_rollback[id(ns)] = (ns, {name: ns[name] for name in names if name in ns})+ for ns in mask_namespaces:_install_packed_mask_positions(ns, lambda: _PACKED[0])- for ns in _iter_mamba2_install_namespaces(kwargs_modules + hub_short_convs):+ for ns in kwargs_namespaces:_install_kwargs_probes(ns)for module in kwargs_modules:if not getattr(module, "_unsloth_varlen_wrapped", False):@@ -1195,6 +1385,16 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:module._unsloth_varlen = Nonemodule._unsloth_varlen_wrapped = True+ if probe_target is not None:+ reason = _hybrid_exactness_probe(probe_target)+ if reason is not None:+ HYBRID_PACKING_STATS["probe_failed"] += 1+ _restore_mixers(rollback)+ for ns, before in namespace_rollback.values():+ ns.update(before)+ return _hybrid_reject(reason)+ HYBRID_PACKING_STATS["probe_passed"] += 1+wrapped_fused: dict[int, Any] = {}def _ensure_mamba2_fused_wrapped(fn):@@ -1291,9 +1491,9 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:m._unsloth_varlen = None_hybrid_reject("varlen kernels not dispatched (dispatch changed?)")raise RuntimeError(- "Unsloth: experimental hybrid packing cannot continue because the "+ "Unsloth: hybrid packing cannot continue because the ""varlen boundary kernels were not invoked for "- f"{sorted(set(missing))}. Unset UNSLOTH_EXPERIMENTAL_HYBRID_PACKING "+ f"{sorted(set(missing))}. Set UNSLOTH_EXPERIMENTAL_HYBRID_PACKING=0 ""to train these models on the padded path.\n"+ _mamba2_handshake_debug([m for m in hybrid_modules if _stateful_mixer_kind(m) == "ssd"]@@ -1306,6 +1506,7 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:model._unsloth_varlen_forward_wrapped = True_wrap_generate_clears_varlen(model, hybrid_modules)+ HYBRID_PACKING_STATS["patched"] += 1return True