Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Hybrid packing + block compile: fix the guard crash, the skipped varlen, and the cross-document leak (#13956)

Etherla33463fb165b
diff --git a/tests/utils/test_hybrid_packing_default.py b/tests/utils/test_hybrid_packing_default.py
index be369692319..f6a17f51ddc 100644
--- a/tests/utils/test_hybrid_packing_default.py
+++ b/tests/utils/test_hybrid_packing_default.py
@@ -377,7 +377,27 @@ def _zoo_style_dispatch(torch_function, name_seq_idx = True):
return causal_conv1d_fn
+def scan_fast(
+ x,
+ cu_seqlens = None,
+ **kwargs,
+):
+ return scan_exact(x, cu_seqlens)
+
+
def _kwargs_model(name, conv_impl):
+ modeling = types.ModuleType(name)
+ _install_kwargs_mixer(modeling.__dict__, conv_impl)
+ modeling.FakeKwargsGatedDeltaNet.__module__ = name
+ sys.modules[name] = modeling
+ return FakeHybrid(modeling.FakeKwargsGatedDeltaNet())
+
+
+def _install_kwargs_mixer(
+ namespace,
+ conv_impl,
+ scan_impl = scan_fast,
+):
def conv_ref(
hidden_states,
weight,
@@ -390,17 +410,9 @@ def _kwargs_model(name, conv_impl):
def scan_ref(x, **kwargs):
return scan_exact(x)
- def scan_fast(
- x,
- cu_seqlens = None,
- **kwargs,
- ):
- return scan_exact(x, cu_seqlens)
-
- modeling = types.ModuleType(name)
impl = conv_impl(conv_ref)
- modeling.causal_conv1d_fn = _hub_wrapped(impl, impl)
- modeling.torch_chunk_gated_delta_rule = _hub_wrapped(scan_ref, scan_fast)
+ namespace["causal_conv1d_fn"] = _hub_wrapped(impl, impl)
+ namespace["torch_chunk_gated_delta_rule"] = _hub_wrapped(scan_ref, scan_impl)
exec(
"import torch\n"
"class FakeKwargsGatedDeltaNet(torch.nn.Module):\n"
@@ -412,11 +424,8 @@ def _kwargs_model(name, conv_impl):
" x = causal_conv1d_fn(hidden_states.transpose(1, 2), self.conv1d.weight.squeeze(1), None,\n"
" activation = 'silu', **kwargs).transpose(1, 2)\n"
" return torch_chunk_gated_delta_rule(x, cu_seqlens = kwargs.pop('cu_seq_lens_q', None), **kwargs)\n",
- modeling.__dict__,
+ namespace,
)
- modeling.FakeKwargsGatedDeltaNet.__module__ = name
- sys.modules[name] = modeling
- return FakeHybrid(modeling.FakeKwargsGatedDeltaNet())
@pytest.mark.parametrize("mode", [None, "1"])
@@ -596,3 +605,129 @@ def test_a_stale_stash_never_reaches_a_call_of_another_length(monkeypatch):
mixer._unsloth_varlen_conv_hit = mixer._unsloth_varlen_scan_hit = False
torch.testing.assert_close(mixer(hidden), plain)
assert not mixer._unsloth_varlen_conv_hit and not mixer._unsloth_varlen_scan_hit
+
+
+def test_boundary_check_holds_after_a_dynamic_recompile():
+ # Traced, `total in (dim, dim)` came out False once the conv layout recompiled with dynamic shapes.
+ seen = []
+
+ def check(seq_idx, x):
+ seen.append(packing_module._varlen_seq_idx_applies(seq_idx, (x,), {}))
+ return x + 1
+
+ seq_idx = torch.zeros(1, 300, dtype = torch.int32)
+ torch._dynamo.reset()
+ try:
+ compiled = torch.compile(check, backend = "eager")
+ compiled(seq_idx, torch.zeros(1, 300, 8)) # (batch, seqlen, ...)
+ compiled(seq_idx, torch.zeros(1, 8, 300)) # causal_conv1d_fn: (batch, dim, seqlen)
+ finally:
+ torch._dynamo.reset()
+ assert seen == [True, True]
+
+
[email protected]("folds", [True, False])
+def test_a_packed_mixer_runs_outside_a_compiled_block(monkeypatch, folds):
+ # Traced, transformers' hub-kernel wrappers run their torch reference where Dynamo folds
+ # torch.compiler.is_exporting() to True (torch 2.10 / 2.11), and it ignores seq_idx / cu_seqlens.
+ # Where it does not, the packed mixer stays in the graph.
+ monkeypatch.setattr(packing_module, "_FOLDS_IS_EXPORTING", [folds])
+ traced = []
+
+ class Mixer(torch.nn.Module):
+ def forward(self, hidden_states, **kwargs):
+ traced.append(torch.compiler.is_compiling())
+ return hidden_states * 2
+
+ mixer = Mixer()
+ packing_module._wrap_kwargs_mixer_forward(mixer)
+ mixer._unsloth_varlen = (
+ torch.tensor([0, 4, 9], dtype = torch.int32),
+ torch.tensor([[0] * 4 + [1] * 5], dtype = torch.int32),
+ )
+ hidden = torch.randn(1, 9, 8)
+ torch._dynamo.reset()
+ try:
+ block = torch.compile(lambda h: mixer(h), backend = "eager")
+ block(hidden)
+ mixer._unsloth_varlen = None
+ block(hidden)
+ finally:
+ torch._dynamo.reset()
+ assert traced == ([False, True] if folds else [True, True]) # only the packed call leaves it
+
+
+def test_is_exporting_fold_probe_matches_this_torch():
+ from packaging.version import Version
+ folds = packing_module._dynamo_folds_is_exporting()
+ assert folds == (Version(torch.__version__.split("+")[0]).release[:2] in ((2, 10), (2, 11)))
+
+
+def test_probes_and_block_adapters_match_the_namespace_dynamo_reads(monkeypatch):
+ # Unsloth's compiled modules run `from <modeling> import __name__`, so Dynamo resolves their globals
+ # through the modeling module; a kernel bound differently there failed the block's closure guard.
+ from unsloth.models import _block_compile as block
+
+ monkeypatch.delenv(ENV, raising = False)
+ names = (
+ "fake_alias.fla.ops.gated_delta_rule.chunk",
+ "fake_modeling_kwargs_alias_gdn",
+ "fake_unsloth_compiled_module_alias_gdn",
+ )
+ fla, modeling, compiled = (types.ModuleType(n) for n in names)
+
+ def chunk_gated_delta_rule(
+ x,
+ cu_seqlens = None,
+ **kwargs,
+ ):
+ return scan_exact(x, cu_seqlens)
+
+ fla.chunk_gated_delta_rule = chunk_gated_delta_rule
+ compiled.__name__ = modeling.__name__
+ for module in (modeling, compiled):
+ _install_kwargs_mixer(vars(module), _zoo_style_dispatch, scan_impl = chunk_gated_delta_rule)
+ for n, module in zip(names, (fla, modeling, compiled)):
+ monkeypatch.setitem(sys.modules, n, module)
+ model = FakeHybrid(compiled.FakeKwargsGatedDeltaNet())
+ assert patch_hybrid_linear_attention_varlen(model) is True
+ # Route only these modules: routing a real fla import would outlive this test.
+ monkeypatch.setattr(block, "sys", SimpleNamespace(modules = {n: sys.modules[n] for n in names}))
+ block._route_fla_kernel()
+ block._route_causal_conv(model.linear_attn)
+ for kernel in ("causal_conv1d_fn", "torch_chunk_gated_delta_rule"):
+ probe = vars(compiled)[kernel]
+ assert vars(modeling)[kernel] is probe, kernel
+ assert probe._unsloth_varlen_probe_of is not None, kernel
+ assert getattr(probe.__kwdefaults__["_fn"], "_unsloth_block_adapter", False), kernel
+
+
+def test_a_failed_probe_restores_the_mirrored_alias_namespace(monkeypatch):
+ # A stale compiled module of another model family is walked too (its name says unsloth_compiled), and its
+ # probes are mirrored into the modeling module it imports. A failed exactness probe rolls back both, or
+ # the compiled global and sys.modules[__name__] disagree and the block's closure guard fails again.
+ monkeypatch.delenv(ENV, raising = False)
+ names = (
+ "fake_modeling_rollback_gdn",
+ "fake_other_family_modeling",
+ "fake_unsloth_compiled_module_stale",
+ )
+ modeling, other, stale = (types.ModuleType(n) for n in names)
+ for module in (modeling, other, stale):
+ _install_kwargs_mixer(vars(module), _zoo_style_dispatch)
+ stale.__name__ = other.__name__
+ modeling.FakeKwargsGatedDeltaNet.__module__ = modeling.__name__
+ for n, module in zip(names, (modeling, other, stale)):
+ monkeypatch.setitem(sys.modules, n, module)
+ before = {
+ n: {k: vars(m)[k] for k in packing_module._KWARGS_KERNELS if k in vars(m)}
+ for n, m in zip(names, (modeling, other, stale))
+ }
+ monkeypatch.setattr(packing_module, "_hybrid_exactness_probe", lambda target: "forced failure")
+ assert (
+ patch_hybrid_linear_attention_varlen(FakeHybrid(modeling.FakeKwargsGatedDeltaNet()))
+ is False
+ )
+ for n, m in zip(names, (modeling, other, stale)):
+ for k, fn in before[n].items():
+ assert vars(m)[k] is fn, (n, k)
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index 297317d9d26..00e4678dafd 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -206,10 +206,11 @@ def _route_fla_kernel():
if not isinstance(space, dict) or space is adapter.__dict__:
continue
for key, value in list(space.items()):
- if (
- value is caller
- and isinstance(key, str)
- and key.endswith("chunk_gated_delta_rule")
+ if value is not caller or not isinstance(key, str):
+ continue
+ # A packing varlen probe keeps the kernel as its `_fn` keyword default; route under it.
+ if key.endswith("chunk_gated_delta_rule") or (
+ key == "_fn" and str(space.get("_name", "")).endswith("chunk_gated_delta_rule")
):
space[key] = adapter
routed += 1
@@ -268,12 +269,38 @@ def _namespaces(fn, depth = 3):
return out
+def _alias_namespace(space):
+ """The namespace Dynamo resolves `space`'s globals through: an inlined function's globals are
+ read as sys.modules[__name__], and Unsloth's compiled modules import transformers' __name__."""
+ module = (
+ sys.modules.get(space.get("__name__")) if isinstance(space.get("__name__"), str) else None
+ )
+ alias = getattr(module, "__dict__", None)
+ return alias if isinstance(alias, dict) and alias is not space else None
+
+
def _route_causal_conv(gdn):
from ._gdn_op import _causal_depthwise_conv
for space in _namespaces(type(gdn).forward):
fn = space.get("causal_conv1d_fn")
- if callable(fn) and not getattr(fn, "_unsloth_block_adapter", False):
+ if not callable(fn) or getattr(fn, "_unsloth_block_adapter", False):
+ continue
+ probe_defaults = (
+ getattr(fn, "__kwdefaults__", None)
+ if getattr(fn, "_unsloth_varlen_probe_of", None) is not None
+ else None
+ )
+ if probe_defaults is not None:
+ # Packing's varlen probe stays outermost so its boundary handshake still sees the call.
+ inner = probe_defaults.get("_fn")
+ if callable(inner) and not getattr(inner, "_unsloth_block_adapter", False):
+ probe_defaults["_fn"] = _causal_conv1d_adapter(inner, _causal_depthwise_conv)
+ else:
space["causal_conv1d_fn"] = _causal_conv1d_adapter(fn, _causal_depthwise_conv)
+ alias = _alias_namespace(space)
+ if alias is not None and callable(alias.get("causal_conv1d_fn")):
+ # Dynamo guards the alias binding: it must be the object the forward actually calls.
+ alias["causal_conv1d_fn"] = space["causal_conv1d_fn"]
def _route_sm_count(modules = None):
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index 202c67c346d..5ab59a8ad90 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -767,8 +767,9 @@ def _varlen_seq_idx_applies(seq_idx, args, kwargs) -> bool:
tensor = _varlen_input(args, kwargs)
if tensor is None or tensor.dim() < 2 or seq_idx.shape[0] != tensor.shape[0]:
return False
- # causal_conv1d_fn takes (batch, dim, seqlen), the others (batch, seqlen, ...).
- return total in (tensor.shape[1], tensor.shape[-1])
+ # causal_conv1d_fn takes (batch, dim, seqlen), the others (batch, seqlen, ...). Compared with ==:
+ # after a dynamic-shape recompile Dynamo evaluated `total in (...)` False for a matching row.
+ return total == tensor.shape[1] or total == tensor.shape[-1]
def _wrap_mamba2_seq_idx_call(orig, *, hit_attr = "_unsloth_varlen_fused_hit"):
@@ -931,6 +932,13 @@ def _kwargs_kernels_available(modules, recurrent_ids = ()) -> Optional[str]:
return None
+def _kwargs_alias_namespace(namespace):
+ """The namespace Dynamo resolves `namespace`'s globals through (sys.modules[__name__]), when it differs."""
+ name = namespace.get("__name__") if isinstance(namespace, dict) else None
+ alias = getattr(sys.modules.get(name), "__dict__", None) if isinstance(name, str) else None
+ return alias if isinstance(alias, dict) and alias is not namespace else None
+
+
def _install_kwargs_probes(namespace) -> None:
if not isinstance(namespace, dict):
return
@@ -953,6 +961,33 @@ def _install_kwargs_probes(namespace) -> None:
probe._unsloth_varlen_probe_of = fn
namespace[name] = probe
+ # Dynamo reads an inlined function's globals through sys.modules[__name__]; Unsloth's compiled
+ # modules import transformers' __name__, so both namespaces must hold the same probe or the
+ # block compile guards one object and traces the other.
+ alias = _kwargs_alias_namespace(namespace)
+ if alias is not None and callable(alias.get(name)):
+ alias[name] = probe
+
+
+_FOLDS_IS_EXPORTING = []
+
+
+def _dynamo_folds_is_exporting() -> bool:
+ # torch 2.10 / 2.11 Dynamo constant-folds torch.compiler.is_exporting() to True inside any
+ # compiled region; 2.12+ reads the real flag. Probed once, eagerly, outside any trace.
+ if not _FOLDS_IS_EXPORTING:
+
+ def probe(x):
+ return x + 1 if torch.compiler.is_exporting() else x
+
+ try:
+ folds = bool(
+ torch.compile(probe, backend = "eager", fullgraph = True)(torch.zeros(1)).item()
+ )
+ except Exception:
+ folds = True # unknown: keep the packed mixer eager, which is always correct
+ _FOLDS_IS_EXPORTING.append(folds)
+ return _FOLDS_IS_EXPORTING[0]
def _wrap_kwargs_mixer_forward(module) -> None:
@@ -963,6 +998,9 @@ def _wrap_kwargs_mixer_forward(module) -> None:
varlen = getattr(module, "_unsloth_varlen", None)
if varlen is None or not _varlen_seq_idx_applies(varlen[1], args, kwargs):
return forward_orig(*args, **kwargs)
+ return _packed_forward(varlen, args, kwargs)
+
+ def _packed_forward(varlen, args, kwargs):
varlen = _varlen_on(varlen, args, kwargs)
if kwargs.get("seq_idx") is None:
kwargs["seq_idx"] = varlen[1]
@@ -978,6 +1016,13 @@ def _wrap_kwargs_mixer_forward(module) -> None:
module._unsloth_varlen_kwargs_hit = True
return out
+ if _dynamo_folds_is_exporting():
+ # Eager even inside a compiled block: Dynamo folds torch.compiler.is_exporting() to True, so
+ # the hub-kernel wrappers trace their torch reference, which drops seq_idx / cu_seqlens and
+ # runs the packed documents into each other (Qwen3.5-0.8B per-document cosine 0.71). Newer
+ # torch traces the real kernels, and the eager mixer costs 6-20% per step there.
+ _packed_forward = torch.compiler.disable(_packed_forward)
+
module.forward = forward
module._unsloth_varlen_kwargs_mode = True
@@ -1442,9 +1487,14 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
kwargs_namespaces = list(_iter_mamba2_install_namespaces(kwargs_modules + hub_short_convs))
namespace_rollback = {}
for ns in mask_namespaces + kwargs_namespaces:
- if isinstance(ns, dict) and id(ns) not in namespace_rollback:
- names = tuple(_MASK_BUILDERS) + tuple(_KWARGS_KERNELS)
- namespace_rollback[id(ns)] = (ns, {name: ns[name] for name in names if name in ns})
+ # The alias namespace too: _install_kwargs_probes mirrors into it, and a failed probe must undo that.
+ for space in (ns, _kwargs_alias_namespace(ns)):
+ if isinstance(space, dict) and id(space) not in namespace_rollback:
+ names = tuple(_MASK_BUILDERS) + tuple(_KWARGS_KERNELS)
+ namespace_rollback[id(space)] = (
+ space,
+ {name: space[name] for name in names if name in space},
+ )
for ns in mask_namespaces:
_install_packed_mask_positions(ns, lambda: _PACKED[0])
for ns in kwargs_namespaces: