Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pack Qwen3.5 / Qwen3-Next hybrids by default with document boundaries in the linear-attention kernels (#13383)

Daniel Hand5824d09e63e
diff --git a/tests/utils/test_hybrid_packing_default.py b/tests/utils/test_hybrid_packing_default.py
new file mode 100644
index 00000000000..be369692319
--- /dev/null
+++ b/tests/utils/test_hybrid_packing_default.py
@@ -0,0 +1,598 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Gated-delta hybrid packing on by default: the mode switch, the exactness probe and the
+unsloth-zoo Triton conv acceptance, on small CPU stand-ins for the GatedDeltaNet mixer."""
+
+import functools
+import inspect
+import logging
+import sys
+import types
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+import unsloth.utils.packing as packing_module
+from unsloth.utils.packing import patch_hybrid_linear_attention_varlen
+
+ENV = "UNSLOTH_EXPERIMENTAL_HYBRID_PACKING"
+
+
+def _segments(T, seq_idx):
+ if seq_idx is None:
+ return [(0, T)]
+ ids = seq_idx.reshape(-1)
+ cuts = (torch.nonzero(ids[1:] != ids[:-1]).flatten() + 1).tolist()
+ bounds = [0] + cuts + [T]
+ return list(zip(bounds[:-1], bounds[1:]))
+
+
+def conv_exact(
+ x,
+ weight,
+ bias = None,
+ activation = None,
+ seq_idx = None,
+ **kwargs,
+):
+ import torch.nn.functional as F
+
+ outs = []
+ for s, e in _segments(x.shape[-1], seq_idx):
+ seg = x[:, :, s:e]
+ y = F.conv1d(
+ seg, weight.unsqueeze(1), bias, padding = weight.shape[-1] - 1, groups = x.shape[1]
+ )
+ outs.append(y[:, :, : e - s])
+ out = torch.cat(outs, dim = -1)
+ return F.silu(out) if activation else out
+
+
+def conv_leaky(
+ x,
+ weight,
+ bias = None,
+ activation = None,
+ seq_idx = None,
+ **kwargs,
+):
+ # Names seq_idx (passes the static check) but ignores it.
+ return conv_exact(x, weight, bias, activation, None)
+
+
+def scan_exact(
+ x,
+ cu_seqlens = None,
+ **kwargs,
+):
+ # A decayed running sum that restarts at every cu_seqlens boundary.
+ T = x.shape[1]
+ starts = set(cu_seqlens.tolist()[:-1]) if cu_seqlens is not None else {0}
+ out, state = [], torch.zeros_like(x[:, 0])
+ for t in range(T):
+ if t in starts:
+ state = torch.zeros_like(state)
+ state = 0.7 * state + x[:, t]
+ out.append(state)
+ return torch.stack(out, dim = 1)
+
+
+def scan_leaky(
+ x,
+ cu_seqlens = None,
+ **kwargs,
+):
+ return scan_exact(x, None)
+
+
+class FakeGatedDeltaNet(torch.nn.Module):
+ """transformers < 5.16 shape: instance kernels, the conv called with seq_idx=None."""
+
+ def __init__(
+ self,
+ conv = conv_exact,
+ scan = scan_exact,
+ hidden = 8,
+ width = 4,
+ ):
+ super().__init__()
+ if hidden is not None:
+ self.hidden_size = hidden
+ self.conv1d = torch.nn.Conv1d(8, 8, width, groups = 8, bias = False)
+ self.causal_conv1d_fn = conv
+ self.chunk_gated_delta_rule = scan
+
+ def forward(
+ self,
+ hidden_states,
+ cache_params = None,
+ attention_mask = None,
+ ):
+ x = hidden_states.transpose(1, 2)
+ x = self.causal_conv1d_fn(
+ x = x, weight = self.conv1d.weight.squeeze(1), bias = None, activation = "silu", seq_idx = None
+ )
+ return self.chunk_gated_delta_rule(x.transpose(1, 2), cu_seqlens = None)
+
+
+class FakeHybrid(torch.nn.Module):
+ def __init__(self, mixer):
+ super().__init__()
+ self.config = SimpleNamespace(layer_types = ["linear_attention", "full_attention"])
+ torch.manual_seed(0)
+ self.embed = torch.nn.Embedding(32, 8)
+ self.linear_attn = mixer
+
+ def forward(
+ self,
+ input_ids = None,
+ position_ids = None,
+ packed_seq_lengths = None,
+ use_cache = None,
+ **kw,
+ ):
+ return self.linear_attn(self.embed(input_ids))
+
+
[email protected](autouse = True)
+def _fresh_warnings(monkeypatch):
+ packing_module._HYBRID_WARNED.clear()
+ monkeypatch.setattr(packing_module, "_auto_hybrid_device_ok", lambda module: True)
+ yield
+ packing_module._HYBRID_WARNED.clear()
+
+
+def _untouched(model, conv, scan):
+ mixer = model.linear_attn
+ return (
+ mixer.causal_conv1d_fn is conv
+ and mixer.chunk_gated_delta_rule is scan
+ and "forward" not in mixer.__dict__
+ and not getattr(mixer, "_unsloth_varlen_wrapped", False)
+ and not getattr(model, "_unsloth_varlen_forward_wrapped", False)
+ )
+
+
+ "value,mode",
+ [
+ (None, "auto"),
+ ("", "auto"),
+ ("maybe", "auto"),
+ ("1", "on"),
+ ("TRUE", "on"),
+ ("yes", "on"),
+ ("on", "on"),
+ ("0", "off"),
+ ("false", "off"),
+ ("No", "off"),
+ (" off ", "off"),
+ ],
+)
+def test_mode_parsing(monkeypatch, value, mode):
+ if value is None:
+ monkeypatch.delenv(ENV, raising = False)
+ else:
+ monkeypatch.setenv(ENV, value)
+ assert packing_module._hybrid_packing_mode() == mode
+ assert packing_module._hybrid_packing_enabled() is (mode == "on")
+
+
+def test_default_packs_exact_gated_delta(monkeypatch):
+ monkeypatch.delenv(ENV, raising = False)
+ model = FakeHybrid(FakeGatedDeltaNet())
+ rng = torch.get_rng_state()
+ assert patch_hybrid_linear_attention_varlen(model) is True
+ assert torch.equal(rng, torch.get_rng_state()) # the probe uses its own generator
+ assert not packing_module._HYBRID_WARNED
+ ids = torch.randint(0, 32, (1, 9), generator = torch.Generator().manual_seed(1))
+ lengths = [4, 1, 4]
+ packed = model(
+ input_ids = ids, packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32), use_cache = False
+ )
+ assert model._unsloth_varlen_handshake_done
+ alone = torch.cat(
+ [model(input_ids = ids[:, s : s + n]) for s, n in zip((0, 4, 5), lengths)], dim = 1
+ )
+ torch.testing.assert_close(packed, alone)
+ plain = model(input_ids = ids, use_cache = True)
+ assert not torch.allclose(plain, packed)
+
+
[email protected]("value", ["0", "false", "no", "off"])
+def test_kill_switch_keeps_padded_path(monkeypatch, value):
+ monkeypatch.setenv(ENV, value)
+ model = FakeHybrid(FakeGatedDeltaNet())
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert _untouched(model, conv_exact, scan_exact)
+
+
[email protected]("mode", [None, "1"])
[email protected]("conv,scan", [(conv_leaky, scan_exact), (conv_exact, scan_leaky)])
+def test_probe_rejects_a_leaky_kernel_and_rolls_back(monkeypatch, caplog, mode, conv, scan):
+ if mode is None:
+ monkeypatch.delenv(ENV, raising = False)
+ else:
+ monkeypatch.setenv(ENV, mode)
+ model = FakeHybrid(FakeGatedDeltaNet(conv = conv, scan = scan))
+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert "exactness probe" in caplog.text and "differs" in caplog.text
+ assert _untouched(model, conv, scan)
+
+
+def test_a_failed_probe_restores_the_modeling_namespace(monkeypatch):
+ monkeypatch.delenv(ENV, raising = False)
+
+ def create_causal_mask(*args, **kwargs):
+ return None
+
+ monkeypatch.setattr(
+ sys.modules[__name__], "create_causal_mask", create_causal_mask, raising = False
+ )
+ model = FakeHybrid(FakeGatedDeltaNet(conv = conv_leaky))
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert sys.modules[__name__].create_causal_mask is create_causal_mask
+
+
+class DropoutGatedDeltaNet(FakeGatedDeltaNet):
+ """LoRA-style input dropout: in train mode the packed and per-document passes would differ."""
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.dropout = torch.nn.Dropout(0.5)
+
+ def forward(
+ self,
+ hidden_states,
+ cache_params = None,
+ attention_mask = None,
+ ):
+ return super().forward(self.dropout(hidden_states), cache_params, attention_mask)
+
+
+def test_probe_runs_in_eval_mode_and_keeps_the_rng(monkeypatch):
+ monkeypatch.delenv(ENV, raising = False)
+ mixer = DropoutGatedDeltaNet()
+ model = FakeHybrid(mixer)
+ model.train()
+ rng = torch.get_rng_state()
+ assert patch_hybrid_linear_attention_varlen(model) is True
+ assert torch.equal(rng, torch.get_rng_state())
+ assert mixer.training and mixer.dropout.training
+
+
+def test_probe_failure_to_run_is_a_rejection(monkeypatch, caplog):
+ monkeypatch.delenv(ENV, raising = False)
+
+ def conv_raises(
+ x,
+ weight,
+ bias = None,
+ activation = None,
+ seq_idx = None,
+ **kwargs,
+ ):
+ raise RuntimeError("kernel exploded")
+
+ model = FakeHybrid(FakeGatedDeltaNet(conv = conv_raises))
+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert "kernel exploded" in caplog.text
+ assert _untouched(model, conv_raises, scan_exact)
+
+
+def test_probe_requires_dispatch(monkeypatch, caplog):
+ monkeypatch.delenv(ENV, raising = False)
+
+ class BypassGatedDeltaNet(FakeGatedDeltaNet):
+ def forward(
+ self,
+ hidden_states,
+ cache_params = None,
+ attention_mask = None,
+ ):
+ return conv_exact(
+ hidden_states.transpose(1, 2), self.conv1d.weight.squeeze(1)
+ ).transpose(1, 2)
+
+ mixer = BypassGatedDeltaNet()
+ model = FakeHybrid(mixer)
+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert "not dispatched" in caplog.text
+
+
+def test_default_needs_a_probeable_mixer(monkeypatch, caplog):
+ monkeypatch.delenv(ENV, raising = False)
+ model = FakeHybrid(FakeGatedDeltaNet(hidden = None))
+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert "probe" in caplog.text
+ assert _untouched(model, conv_exact, scan_exact)
+ monkeypatch.setenv(ENV, "1")
+ assert patch_hybrid_linear_attention_varlen(FakeHybrid(FakeGatedDeltaNet(hidden = None))) is True
+
+
+def test_default_rejects_torch_fallback(monkeypatch):
+ monkeypatch.delenv(ENV, raising = False)
+
+ def torch_chunk_gated_delta_rule(
+ x,
+ cu_seqlens = None,
+ **kwargs,
+ ):
+ return scan_exact(x, cu_seqlens)
+
+ model = FakeHybrid(FakeGatedDeltaNet(scan = torch_chunk_gated_delta_rule))
+ assert patch_hybrid_linear_attention_varlen(model) is False
+
+
+# transformers >= 5.16: forward(**kwargs) splats into module-level hub kernels.
+
+
+def _hub_wrapped(torch_function, implementation):
+ applicable_params = tuple(inspect.signature(implementation).parameters)
+
+ @functools.wraps(torch_function)
+ def wrapped(*args, **kwargs):
+ if implementation is None: # transformers references torch_function here too (export path)
+ return torch_function(*args, **kwargs)
+ kwargs = {k: v for k, v in kwargs.items() if k in applicable_params}
+ return implementation(*args, **kwargs)
+
+ return wrapped
+
+
+def _zoo_style_dispatch(torch_function, name_seq_idx = True):
+ # unsloth-zoo gdn_causal_conv1d._make_hub_dispatch: marked, own signature naming seq_idx.
+ if name_seq_idx:
+
+ @functools.wraps(torch_function)
+ def causal_conv1d_fn(
+ hidden_states,
+ weight,
+ bias = None,
+ activation = None,
+ seq_idx = None,
+ **kwargs,
+ ):
+ return conv_exact(hidden_states, weight, bias, activation, seq_idx)
+ else:
+
+ @functools.wraps(torch_function)
+ def causal_conv1d_fn(
+ hidden_states,
+ weight,
+ bias = None,
+ activation = None,
+ **kwargs,
+ ):
+ return conv_exact(hidden_states, weight, bias, activation)
+
+ causal_conv1d_fn._unsloth_triton_causal_conv1d = True
+ causal_conv1d_fn.__signature__ = inspect.signature(causal_conv1d_fn, follow_wrapped = False)
+ return causal_conv1d_fn
+
+
+def _kwargs_model(name, conv_impl):
+ def conv_ref(
+ hidden_states,
+ weight,
+ bias = None,
+ activation = None,
+ **kwargs,
+ ):
+ return conv_exact(hidden_states, weight, bias, activation)
+
+ def scan_ref(x, **kwargs):
+ return scan_exact(x)
+
+ def scan_fast(
+ x,
+ cu_seqlens = None,
+ **kwargs,
+ ):
+ return scan_exact(x, cu_seqlens)
+
+ modeling = types.ModuleType(name)
+ impl = conv_impl(conv_ref)
+ modeling.causal_conv1d_fn = _hub_wrapped(impl, impl)
+ modeling.torch_chunk_gated_delta_rule = _hub_wrapped(scan_ref, scan_fast)
+ exec(
+ "import torch\n"
+ "class FakeKwargsGatedDeltaNet(torch.nn.Module):\n"
+ " def __init__(self):\n"
+ " super().__init__()\n"
+ " self.hidden_size = 8\n"
+ " self.conv1d = torch.nn.Conv1d(8, 8, 4, groups = 8, bias = False)\n"
+ " def forward(self, hidden_states, cache_params = None, attention_mask = None, **kwargs):\n"
+ " x = causal_conv1d_fn(hidden_states.transpose(1, 2), self.conv1d.weight.squeeze(1), None,\n"
+ " activation = 'silu', **kwargs).transpose(1, 2)\n"
+ " return torch_chunk_gated_delta_rule(x, cu_seqlens = kwargs.pop('cu_seq_lens_q', None), **kwargs)\n",
+ modeling.__dict__,
+ )
+ modeling.FakeKwargsGatedDeltaNet.__module__ = name
+ sys.modules[name] = modeling
+ return FakeHybrid(modeling.FakeKwargsGatedDeltaNet())
+
+
[email protected]("mode", [None, "1"])
+def test_zoo_triton_conv_is_accepted(monkeypatch, mode):
+ if mode is None:
+ monkeypatch.delenv(ENV, raising = False)
+ else:
+ monkeypatch.setenv(ENV, mode)
+ name = "fake_modeling_zoo_conv_gdn"
+ try:
+ model = _kwargs_model(name, _zoo_style_dispatch)
+ assert patch_hybrid_linear_attention_varlen(model) is True
+ ids = torch.randint(0, 32, (1, 9), generator = torch.Generator().manual_seed(2))
+ packed = model(
+ input_ids = ids,
+ packed_seq_lengths = torch.tensor([4, 1, 4], dtype = torch.int32),
+ use_cache = False,
+ )
+ alone = torch.cat(
+ [model(input_ids = ids[:, s:e]) for s, e in ((0, 4), (4, 5), (5, 9))], dim = 1
+ )
+ torch.testing.assert_close(packed, alone)
+ assert model.linear_attn._unsloth_varlen_kwargs_hit
+ finally:
+ sys.modules.pop(name, None)
+
+
+def test_zoo_triton_conv_without_seq_idx_is_rejected(monkeypatch, caplog):
+ # An older unsloth-zoo dispatch (marked, no seq_idx) would be filtered by the hub wrapper.
+ monkeypatch.delenv(ENV, raising = False)
+ name = "fake_modeling_zoo_conv_old"
+ try:
+ model = _kwargs_model(name, functools.partial(_zoo_style_dispatch, name_seq_idx = False))
+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert "causal_conv1d_fn" in caplog.text
+ finally:
+ sys.modules.pop(name, None)
+
+
+def test_unmarked_torch_conv_is_rejected(monkeypatch, caplog):
+ monkeypatch.delenv(ENV, raising = False)
+ name = "fake_modeling_torch_conv"
+ try:
+ model = _kwargs_model(name, lambda fn: fn)
+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert "has no accelerated kernel" in caplog.text
+ finally:
+ sys.modules.pop(name, None)
+
+
+def test_real_zoo_dispatch_is_recognised():
+ gcc = pytest.importorskip("unsloth_zoo.temporary_patches.gdn_causal_conv1d")
+ if "seq_idx" not in inspect.signature(gcc._legacy_causal_conv1d_fn).parameters:
+ pytest.skip("unsloth-zoo without the seq_idx Triton conv")
+
+ def causal_conv1d_fn(
+ hidden_states,
+ weight,
+ bias = None,
+ activation = None,
+ **kwargs,
+ ):
+ return hidden_states
+
+ assert packing_module._is_unsloth_seq_idx_conv(gcc._make_hub_dispatch(causal_conv1d_fn))
+ assert packing_module._is_unsloth_seq_idx_conv(gcc._legacy_causal_conv1d_fn)
+ assert not packing_module._is_unsloth_seq_idx_conv(causal_conv1d_fn)
+
+
+def test_default_on_is_cuda_only(monkeypatch, caplog):
+ monkeypatch.delenv(ENV, raising = False)
+ monkeypatch.undo() # the real device check
+ monkeypatch.delenv(ENV, raising = False)
+ mixer = FakeGatedDeltaNet()
+ assert not packing_module._auto_hybrid_device_ok(mixer) # CPU
+ monkeypatch.setattr(torch.version, "hip", "6.4")
+ assert not packing_module._auto_hybrid_device_ok(mixer)
+ monkeypatch.setattr(torch.version, "hip", None)
+ model = FakeHybrid(mixer)
+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert "only on CUDA" in caplog.text
+ assert _untouched(model, conv_exact, scan_exact)
+ monkeypatch.setenv(ENV, "1")
+ assert patch_hybrid_linear_attention_varlen(FakeHybrid(FakeGatedDeltaNet())) is True
+
+
+def test_default_on_is_cuda_only_for_kwargs_mixers(monkeypatch, caplog):
+ # transformers 5.16+ kwargs-mode Qwen mixers leave gated_delta_modules; the default must still cover them.
+ monkeypatch.undo()
+ monkeypatch.delenv(ENV, raising = False)
+ name = "fake_modeling_kwargs_cpu_gdn"
+ try:
+ model = _kwargs_model(name, _zoo_style_dispatch)
+ with caplog.at_level(logging.WARNING, logger = "unsloth.hybrid_packing"):
+ assert patch_hybrid_linear_attention_varlen(model) is False
+ assert "only on CUDA" in caplog.text
+ monkeypatch.setenv(ENV, "1")
+ assert (
+ patch_hybrid_linear_attention_varlen(_kwargs_model(name, _zoo_style_dispatch)) is True
+ )
+ finally:
+ sys.modules.pop(name, None)
+
+
+def test_engagement_counters(monkeypatch):
+ monkeypatch.delenv(ENV, raising = False)
+ stats = dict(packing_module.HYBRID_PACKING_STATS)
+ assert patch_hybrid_linear_attention_varlen(FakeHybrid(FakeGatedDeltaNet())) is True
+ assert (
+ patch_hybrid_linear_attention_varlen(FakeHybrid(FakeGatedDeltaNet(conv = conv_leaky)))
+ is False
+ )
+ now = packing_module.HYBRID_PACKING_STATS
+ assert now["probe_runs"] == stats["probe_runs"] + 2
+ assert now["probe_passed"] == stats["probe_passed"] + 1
+ assert now["probe_failed"] == stats["probe_failed"] + 1
+ assert now["patched"] == stats["patched"] + 1
+
+
+def test_hub_kernel_behind_compile_disable_wrapper(monkeypatch):
+ # The gate must look through torch_compiler_disable_unless_decode to the hub closure.
+ monkeypatch.delenv(ENV, raising = False)
+ name = "fake_unsloth_compiled_module_gdn"
+
+ def disable_unless_decode(func):
+ @functools.wraps(func)
+ def forward(*args, **kwargs):
+ return func(*args, **kwargs)
+
+ forward._unsloth_undisabled = func
+ return forward
+
+ try:
+ model = _kwargs_model(name, _zoo_style_dispatch)
+ modeling = sys.modules[name]
+ modeling.torch_chunk_gated_delta_rule = disable_unless_decode(
+ modeling.torch_chunk_gated_delta_rule
+ )
+ assert "implementation" in packing_module._hub_closure(
+ modeling.torch_chunk_gated_delta_rule
+ )
+ assert patch_hybrid_linear_attention_varlen(model) is True
+ finally:
+ sys.modules.pop(name, None)
+
+
+def test_boundaries_follow_the_call_onto_its_device():
+ varlen = (
+ torch.tensor([0, 4, 9], dtype = torch.int32),
+ torch.tensor([[0] * 4 + [1] * 5], dtype = torch.int32),
+ )
+ x = torch.empty(1, 8, 9, device = "meta")
+ cu, seq_idx = packing_module._varlen_on(varlen, (), {"x": x})
+ assert cu.device.type == "meta" and seq_idx.device.type == "meta"
+ assert packing_module._varlen_on(varlen, (torch.zeros(1, 8, 9),), {}) is varlen
+
+
+def test_a_stale_stash_never_reaches_a_call_of_another_length(monkeypatch):
+ monkeypatch.delenv(ENV, raising = False)
+ model = FakeHybrid(FakeGatedDeltaNet())
+ assert patch_hybrid_linear_attention_varlen(model) is True
+ ids = torch.randint(0, 32, (1, 9), generator = torch.Generator().manual_seed(2))
+ model(
+ input_ids = ids,
+ packed_seq_lengths = torch.tensor([4, 1, 4], dtype = torch.int32),
+ use_cache = False,
+ )
+ mixer = model.linear_attn
+ stale = mixer._unsloth_varlen
+ hidden = model.embed(ids[:, :5])
+ mixer._unsloth_varlen = None
+ plain = mixer(hidden)
+ mixer._unsloth_varlen = stale
+ mixer._unsloth_varlen_conv_hit = mixer._unsloth_varlen_scan_hit = False
+ torch.testing.assert_close(mixer(hidden), plain)
+ assert not mixer._unsloth_varlen_conv_hit and not mixer._unsloth_varlen_scan_hit
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index a8f19947c0e..dee005012c5 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -1559,8 +1559,7 @@ def _patch_sft_trainer_auto_packing(trl_module):
else model
)
is_hybrid = _is_hybrid_linear_attention_model(hybrid_target)
- # Hybrid models corrupt packed batches unless the gated-delta / Mamba2 conv and scan reset at
- # sequence boundaries, so enable the experimental varlen shim (flag plus kernels) or keep them blocked.
+ # Hybrid models pack only when the varlen shim's kernel checks and exactness probe pass.
if (
is_hybrid
and not isinstance(model, str)
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index e505f9afe45..b91ae1bfbe4 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -321,9 +321,8 @@ def enable_padding_free_metadata(model, trainer):
collator._unsloth_padding_free_lengths_wrapped = True
-# experimental hybrid packing: Qwen3.5 / Qwen3-Next gated-delta and Nemotron-H Mamba2.
-# recurrent and conv state leak unless seq_idx / cu_seqlens reach accelerated kernels.
-# env-gated; pure-torch fallbacks fail closed to prevent cross-sequence state leaks.
+# Hybrid linear-attention packing leaks conv/recurrent state across documents unless seq_idx / cu_seqlens
+# reach the accelerated kernels, so it fails closed on torch fallbacks. =0 disables, =1 opts in all mixers.
_MAMBA2_FUSED_NAMES = (
"mamba2_split_conv1d_scan_combined",
"mamba_split_conv1d_scan_combined",
@@ -333,18 +332,36 @@ _HYBRID_LOGGER = logging.getLogger("unsloth.hybrid_packing")
_HYBRID_WARNED: set = set()
+def _hybrid_packing_mode() -> str:
+ """ "on" / "off" / "auto"; read at call time so setting the flag after import still works."""
+ value = os.environ.get(_HYBRID_PACKING_ENV_VAR, "").strip().lower()
+ if value in {"1", "true", "yes", "on"}:
+ return "on"
+ if value in {"0", "false", "no", "off"}:
+ return "off"
+ return "auto"
+
+
def _hybrid_packing_enabled() -> bool:
- # Read at call time so setting the flag after `import unsloth` still takes effect.
- return os.environ.get(_HYBRID_PACKING_ENV_VAR, "0").strip().lower() in {
- "1",
- "true",
- "yes",
- "on",
- }
+ return _hybrid_packing_mode() == "on"
+
+
+# Auto-packed mixers (CUDA only). MoE router aux loss is computed over the whole packed row, not per document.
+_HYBRID_AUTO_KINDS = ("gated_delta",)
+
+HYBRID_PACKING_STATS = {
+ "probe_runs": 0,
+ "probe_passed": 0,
+ "probe_failed": 0,
+ "patched": 0,
+ "rejected": 0,
+ "last_probe_rel_err": None,
+}
def _hybrid_reject(reason: str) -> bool:
# One deduped diagnostic explaining why hybrid packing stayed on the padded path.
+ HYBRID_PACKING_STATS["rejected"] += 1
if reason not in _HYBRID_WARNED:
_HYBRID_WARNED.add(reason)
_HYBRID_LOGGER.warning(
@@ -711,22 +728,31 @@ def _hybrid_varlen_dispatched(module) -> bool:
)
+def _varlen_input(args, kwargs):
+ for name in ("x", "hidden_states", "zxbcdt", "query", "q"):
+ candidate = kwargs.get(name)
+ if isinstance(candidate, torch.Tensor):
+ return candidate
+ for arg in args:
+ if isinstance(arg, torch.Tensor):
+ return arg
+ return None
+
+
+def _varlen_on(varlen, args, kwargs):
+ """Move varlen to this call's input device (multi-GPU splits run mixers away from input ids)."""
+ tensor = _varlen_input(args, kwargs)
+ if tensor is None or varlen[1].device == tensor.device:
+ return varlen
+ return tuple(t.to(tensor.device, non_blocking = True) for t in varlen)
+
+
def _varlen_seq_idx_applies(seq_idx, args, kwargs) -> bool:
# generate() bypasses model.forward, so a stale training seq_idx must not reach a prompt.
if seq_idx is None:
return False
total = seq_idx.shape[-1]
- tensor = None
- for name in ("x", "hidden_states", "zxbcdt"):
- candidate = kwargs.get(name)
- if isinstance(candidate, torch.Tensor):
- tensor = candidate
- break
- if tensor is None:
- for arg in args:
- if isinstance(arg, torch.Tensor):
- tensor = arg
- break
+ tensor = _varlen_input(args, kwargs)
if tensor is None or tensor.dim() < 2 or seq_idx.shape[0] != tensor.shape[0]:
return False
# causal_conv1d_fn takes (batch, dim, seqlen), the others (batch, seqlen, ...).
@@ -826,11 +852,22 @@ _KWARGS_TRACE: list = [None]
def _hub_closure(fn) -> dict:
- fn = getattr(fn, "_unsloth_varlen_probe_of", fn)
- try:
- return inspect.getclosurevars(fn).nonlocals
- except (TypeError, ValueError):
- return {}
+ """Kernel-hub wrapper closure nonlocals, unwrapping Unsloth's probes and pass-through wrappers; {} if none."""
+ for _ in range(8):
+ if fn is None:
+ break
+ try:
+ nonlocals = inspect.getclosurevars(fn).nonlocals
+ except (TypeError, ValueError):
+ nonlocals = {}
+ if "implementation" in nonlocals:
+ return nonlocals
+ fn = (
+ getattr(fn, "_unsloth_varlen_probe_of", None)
+ or getattr(fn, "_unsloth_undisabled", None)
+ or getattr(fn, "__wrapped__", None)
+ )
+ return {}
def _is_kwargs_mixer(module) -> bool:
@@ -846,6 +883,20 @@ def _is_kwargs_mixer(module) -> bool:
return "implementation" in _hub_closure(globs.get("causal_conv1d_fn"))
+# unsloth-zoo's Triton GDN conv looks like the hub "torch function" but honours seq_idx on all paths.
+_ZOO_TRITON_CONV_MARK = "_unsloth_triton_causal_conv1d"
+
+
+def _is_unsloth_seq_idx_conv(fn) -> bool:
+ try:
+ fn = inspect.unwrap(fn, stop = lambda f: getattr(f, _ZOO_TRITON_CONV_MARK, False))
+ return bool(getattr(fn, _ZOO_TRITON_CONV_MARK, False)) and (
+ "seq_idx" in inspect.signature(fn).parameters
+ )
+ except (TypeError, ValueError):
+ return False
+
+
def _kwargs_kernels_available(modules, recurrent_ids = ()) -> Optional[str]:
for module in modules:
globs = getattr(inspect.getmodule(type(module)), "__dict__", {})
@@ -858,7 +909,10 @@ def _kwargs_kernels_available(modules, recurrent_ids = ()) -> Optional[str]:
closure = _hub_closure(fn)
# The hub wrapper drops kwargs its bound implementation does not name, silently.
impl = closure.get("implementation")
- if impl is None or impl is closure.get("torch_function"):
+ if impl is None or (
+ impl is closure.get("torch_function")
+ and not (name == "causal_conv1d_fn" and _is_unsloth_seq_idx_conv(impl))
+ ):
return f"{name} has no accelerated kernel"
if _KWARGS_KERNELS[name] not in closure.get("applicable_params", ()):
return f"{name} kernel does not accept {_KWARGS_KERNELS[name]}"
@@ -897,6 +951,7 @@ def _wrap_kwargs_mixer_forward(module) -> None:
varlen = getattr(module, "_unsloth_varlen", None)
if varlen is None or not _varlen_seq_idx_applies(varlen[1], args, kwargs):
return forward_orig(*args, **kwargs)
+ varlen = _varlen_on(varlen, args, kwargs)
if kwargs.get("seq_idx") is None:
kwargs["seq_idx"] = varlen[1]
if kwargs.get("cu_seq_lens_q") is None:
@@ -1083,15 +1138,133 @@ def _hybrid_varlen_metadata(kwargs):
return _varlen_from_position_ids(kwargs.get("position_ids"))
+# A 1-token document, one shorter than the conv width, and one crossing fla's 64-token chunk boundary.
+_HYBRID_PROBE_LENGTHS = (5, 1, 70, 7)
+_HYBRID_PROBE_RTOL = {torch.float32: 1e-4, torch.bfloat16: 2e-2, torch.float16: 1e-2}
+_HYBRID_SNAPSHOT_ATTRS = (
+ "forward",
+ "causal_conv1d_fn",
+ "chunk_gated_delta_rule",
+ "_unsloth_varlen",
+ "_unsloth_varlen_wrapped",
+ "_unsloth_varlen_orig_scan",
+ "_unsloth_varlen_orig_conv",
+ "_unsloth_varlen_kwargs_mode",
+ "_unsloth_varlen_conv_hit",
+ "_unsloth_varlen_scan_hit",
+ "_unsloth_varlen_kwargs_hit",
+)
+_MISSING = object()
+
+
+def _snapshot_mixers(modules):
+ saved = []
+ for module in modules:
+ owners = [module] + [getattr(module, c, None) for c in _QKV_CONVS]
+ for owner in owners:
+ if isinstance(owner, torch.nn.Module):
+ state = {k: owner.__dict__.get(k, _MISSING) for k in _HYBRID_SNAPSHOT_ATTRS}
+ saved.append((owner, state))
+ return saved
+
+
+def _restore_mixers(saved) -> None:
+ for owner, state in saved:
+ for key, value in state.items():
+ if value is _MISSING:
+ owner.__dict__.pop(key, None)
+ else:
+ owner.__dict__[key] = value
+
+
+def _auto_hybrid_device_ok(module) -> bool:
+ weight = next(module.parameters(), None)
+ return torch.version.hip is None and weight is not None and weight.device.type == "cuda"
+
+
+def _probe_target(modules):
+ for module in modules:
+ if _stateful_mixer_kind(module) != "gated_delta":
+ continue
+ hidden = getattr(module, "hidden_size", None)
+ weight = getattr(getattr(module, "conv1d", None), "weight", None)
+ if isinstance(hidden, int) and hidden > 0 and isinstance(weight, torch.Tensor):
+ if weight.device.type != "meta" and weight.dtype in _HYBRID_PROBE_RTOL:
+ return module
+ return None
+
+
+def _hybrid_exactness_probe(module) -> Optional[str]:
+ """Packed vs per-document mixer output: None if exact, else the reason. Eval mode, forked RNG."""
+ weight = module.conv1d.weight
+ dtype, device = weight.dtype, weight.device
+ lengths = torch.tensor(_HYBRID_PROBE_LENGTHS, dtype = torch.int32)
+ cu_seqlens = torch.cat(
+ [torch.zeros(1, dtype = torch.int32), lengths.cumsum(0, dtype = torch.int32)]
+ )
+ total = int(cu_seqlens[-1])
+ seq_idx = torch.repeat_interleave(
+ torch.arange(len(_HYBRID_PROBE_LENGTHS), dtype = torch.int32), lengths
+ )
+ generator = torch.Generator().manual_seed(0)
+ hidden = torch.randn(1, total, module.hidden_size, generator = generator)
+ hidden = hidden.to(device = device, dtype = dtype)
+ varlen = (cu_seqlens.to(device), seq_idx[None].to(device))
+
+ def run(x):
+ out = module(x)
+ return out[0] if isinstance(out, (tuple, list)) else out
+
+ saved = _snapshot_mixers([module])
+ modes = [(m, m.training) for m in module.modules()]
+ HYBRID_PACKING_STATS["probe_runs"] += 1
+ rng_devices = [device] if device.type == "cuda" else []
+ try:
+ module.eval()
+ with torch.no_grad(), torch.random.fork_rng(devices = rng_devices):
+ for flag in (
+ "_unsloth_varlen_conv_hit",
+ "_unsloth_varlen_scan_hit",
+ "_unsloth_varlen_kwargs_hit",
+ ):
+ module.__dict__[flag] = False
+ module.__dict__["_unsloth_varlen"] = varlen
+ packed = run(hidden)
+ dispatched = _hybrid_varlen_dispatched(module)
+ module.__dict__["_unsloth_varlen"] = None
+ starts = cu_seqlens.tolist()
+ alone = torch.cat([run(hidden[:, s:e]) for s, e in zip(starts[:-1], starts[1:])], dim = 1)
+ except Exception as exc:
+ return f"exactness probe failed to run ({type(exc).__name__}: {str(exc)[:200]})"
+ finally:
+ _restore_mixers(saved)
+ for m, training in modes:
+ m.training = training
+ if not dispatched:
+ return "exactness probe: boundary kernels were not dispatched"
+ if not isinstance(packed, torch.Tensor) or packed.shape != alone.shape:
+ return "exactness probe: unexpected mixer output"
+ packed, alone = packed.float(), alone.float()
+ if not bool(torch.isfinite(packed).all()) or not bool(torch.isfinite(alone).all()):
+ return "exactness probe: non-finite mixer output"
+ scale = alone.abs().max().clamp_min(1e-6)
+ err = ((packed - alone).abs().max() / scale).item()
+ HYBRID_PACKING_STATS["last_probe_rel_err"] = err
+ if err > _HYBRID_PROBE_RTOL[dtype]:
+ return f"exactness probe: packed output differs from per-document output (rel {err:.2e})"
+ return None
+
+
def patch_hybrid_linear_attention_varlen(model) -> bool:
- """Feed seq_idx / cu_seqlens to hybrid mixers so packing resets state at sequence boundaries.
- Gated by UNSLOTH_EXPERIMENTAL_HYBRID_PACKING (short-conv-only models pack without it), fail-closed,
- idempotent. True when active."""
+ """Feed seq_idx / cu_seqlens to hybrid mixers; True when packing is active. Fail-closed, idempotent."""
mixers = _iter_stateful_mixers(model)
# Short-conv-only models (LFM2) reset at seq_idx on both kernel and torch paths, so they pack by default.
short_conv_only = bool(mixers) and all(kind == "short_conv" for _, kind in mixers)
- if not (_hybrid_packing_enabled() or short_conv_only):
- return False
+ mode = _hybrid_packing_mode()
+ auto_eligible = bool(mixers) and all(kind in _HYBRID_AUTO_KINDS for _, kind in mixers)
+ if not short_conv_only:
+ if mode == "off" or (mode == "auto" and not auto_eligible):
+ return False
unsupported = sorted({type(m).__name__ for m, kind in mixers if kind == "unsupported"})
if unsupported:
return _hybrid_reject(f"no varlen kernel path for {unsupported}")
@@ -1117,6 +1290,16 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
return True
hub_short_convs = [m for m in short_conv_modules if _uses_hub_conv(m)]
+ probe_target = _probe_target(gated_delta_modules + kwargs_modules)
+ if mode == "auto" and not short_conv_only:
+ if probe_target is None:
+ return _hybrid_reject("no gated-delta mixer the exactness probe can run")
+ # Every stateful mixer, kwargs-mode ones included (transformers 5.16+ Qwen mixers move there).
+ if not all(_auto_hybrid_device_ok(m) for m in hybrid_modules):
+ return _hybrid_reject(
+ f"default-on only on CUDA (set {_HYBRID_PACKING_ENV_VAR}=1 to opt in on this device)"
+ )
+ rollback = _snapshot_mixers(hybrid_modules)
if kwargs_modules or hub_short_convs:
reason = _kwargs_kernels_available(
kwargs_modules + hub_short_convs, recurrent_ids = {id(m) for m in kwargs_modules}
@@ -1154,10 +1337,10 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
**kwargs,
):
varlen = getattr(_module, "_unsloth_varlen", None)
- if varlen is not None:
+ if varlen is not None and _varlen_seq_idx_applies(varlen[1], args, kwargs):
_module._unsloth_varlen_conv_hit = True # runtime dispatch handshake
if kwargs.get("seq_idx") is None:
- kwargs["seq_idx"] = varlen[1]
+ kwargs["seq_idx"] = _varlen_on(varlen, args, kwargs)[1]
return _orig(*args, **kwargs)
@wraps(scan_orig)
@@ -1168,10 +1351,10 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
**kwargs,
):
varlen = getattr(_module, "_unsloth_varlen", None)
- if varlen is not None:
+ if varlen is not None and _varlen_seq_idx_applies(varlen[1], args, kwargs):
_module._unsloth_varlen_scan_hit = True
if kwargs.get("cu_seqlens") is None:
- kwargs["cu_seqlens"] = varlen[0]
+ kwargs["cu_seqlens"] = _varlen_on(varlen, args, kwargs)[0]
return _orig(*args, **kwargs)
if conv_orig is not None:
@@ -1180,9 +1363,16 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
module._unsloth_varlen = None
module._unsloth_varlen_wrapped = True
- for ns in _iter_mamba2_install_namespaces(hybrid_modules):
+ mask_namespaces = list(_iter_mamba2_install_namespaces(hybrid_modules))
+ kwargs_namespaces = list(_iter_mamba2_install_namespaces(kwargs_modules + hub_short_convs))
+ namespace_rollback = {}
+ for ns in mask_namespaces + kwargs_namespaces:
+ if isinstance(ns, dict) and id(ns) not in namespace_rollback:
+ names = tuple(_MASK_BUILDERS) + tuple(_KWARGS_KERNELS)
+ namespace_rollback[id(ns)] = (ns, {name: ns[name] for name in names if name in ns})
+ for ns in mask_namespaces:
_install_packed_mask_positions(ns, lambda: _PACKED[0])
- for ns in _iter_mamba2_install_namespaces(kwargs_modules + hub_short_convs):
+ for ns in kwargs_namespaces:
_install_kwargs_probes(ns)
for module in kwargs_modules:
if not getattr(module, "_unsloth_varlen_wrapped", False):
@@ -1195,6 +1385,16 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
module._unsloth_varlen = None
module._unsloth_varlen_wrapped = True
+ if probe_target is not None:
+ reason = _hybrid_exactness_probe(probe_target)
+ if reason is not None:
+ HYBRID_PACKING_STATS["probe_failed"] += 1
+ _restore_mixers(rollback)
+ for ns, before in namespace_rollback.values():
+ ns.update(before)
+ return _hybrid_reject(reason)
+ HYBRID_PACKING_STATS["probe_passed"] += 1
+
wrapped_fused: dict[int, Any] = {}
def _ensure_mamba2_fused_wrapped(fn):
@@ -1291,9 +1491,9 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
m._unsloth_varlen = None
_hybrid_reject("varlen kernels not dispatched (dispatch changed?)")
raise RuntimeError(
- "Unsloth: experimental hybrid packing cannot continue because the "
+ "Unsloth: hybrid packing cannot continue because the "
"varlen boundary kernels were not invoked for "
- f"{sorted(set(missing))}. Unset UNSLOTH_EXPERIMENTAL_HYBRID_PACKING "
+ f"{sorted(set(missing))}. Set UNSLOTH_EXPERIMENTAL_HYBRID_PACKING=0 "
"to train these models on the padded path.\n"
+ _mamba2_handshake_debug(
[m for m in hybrid_modules if _stateful_mixer_kind(m) == "ssd"]
@@ -1306,6 +1506,7 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
model._unsloth_varlen_forward_wrapped = True
_wrap_generate_clears_varlen(model, hybrid_modules)
+ HYBRID_PACKING_STATS["patched"] += 1
return True