Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Hybrid packing + block compile: fix the guard crash, the skipped varlen, and the cross-document leak (#13956)
Etherla33463fb165b
diff --git a/tests/utils/test_hybrid_packing_default.py b/tests/utils/test_hybrid_packing_default.pyindex be369692319..f6a17f51ddc 100644--- a/tests/utils/test_hybrid_packing_default.py+++ b/tests/utils/test_hybrid_packing_default.py@@ -377,7 +377,27 @@ def _zoo_style_dispatch(torch_function, name_seq_idx = True):return causal_conv1d_fn+def scan_fast(+ x,+ cu_seqlens = None,+ **kwargs,+):+ return scan_exact(x, cu_seqlens)++def _kwargs_model(name, conv_impl):+ modeling = types.ModuleType(name)+ _install_kwargs_mixer(modeling.__dict__, conv_impl)+ modeling.FakeKwargsGatedDeltaNet.__module__ = name+ sys.modules[name] = modeling+ return FakeHybrid(modeling.FakeKwargsGatedDeltaNet())+++def _install_kwargs_mixer(+ namespace,+ conv_impl,+ scan_impl = scan_fast,+):def conv_ref(hidden_states,weight,@@ -390,17 +410,9 @@ def _kwargs_model(name, conv_impl):def scan_ref(x, **kwargs):return scan_exact(x)- def scan_fast(- x,- cu_seqlens = None,- **kwargs,- ):- return scan_exact(x, cu_seqlens)-- modeling = types.ModuleType(name)impl = conv_impl(conv_ref)- modeling.causal_conv1d_fn = _hub_wrapped(impl, impl)- modeling.torch_chunk_gated_delta_rule = _hub_wrapped(scan_ref, scan_fast)+ namespace["causal_conv1d_fn"] = _hub_wrapped(impl, impl)+ namespace["torch_chunk_gated_delta_rule"] = _hub_wrapped(scan_ref, scan_impl)exec("import torch\n""class FakeKwargsGatedDeltaNet(torch.nn.Module):\n"@@ -412,11 +424,8 @@ def _kwargs_model(name, conv_impl):" x = causal_conv1d_fn(hidden_states.transpose(1, 2), self.conv1d.weight.squeeze(1), None,\n"" activation = 'silu', **kwargs).transpose(1, 2)\n"" return torch_chunk_gated_delta_rule(x, cu_seqlens = kwargs.pop('cu_seq_lens_q', None), **kwargs)\n",- modeling.__dict__,+ namespace,)- modeling.FakeKwargsGatedDeltaNet.__module__ = name- sys.modules[name] = modeling- return FakeHybrid(modeling.FakeKwargsGatedDeltaNet())@pytest.mark.parametrize("mode", [None, "1"])@@ -596,3 +605,129 @@ def test_a_stale_stash_never_reaches_a_call_of_another_length(monkeypatch):mixer._unsloth_varlen_conv_hit = mixer._unsloth_varlen_scan_hit = Falsetorch.testing.assert_close(mixer(hidden), plain)assert not mixer._unsloth_varlen_conv_hit and not mixer._unsloth_varlen_scan_hit+++def test_boundary_check_holds_after_a_dynamic_recompile():+ # Traced, `total in (dim, dim)` came out False once the conv layout recompiled with dynamic shapes.+ seen = []++ def check(seq_idx, x):+ seen.append(packing_module._varlen_seq_idx_applies(seq_idx, (x,), {}))+ return x + 1++ seq_idx = torch.zeros(1, 300, dtype = torch.int32)+ torch._dynamo.reset()+ try:+ compiled = torch.compile(check, backend = "eager")+ compiled(seq_idx, torch.zeros(1, 300, 8)) # (batch, seqlen, ...)+ compiled(seq_idx, torch.zeros(1, 8, 300)) # causal_conv1d_fn: (batch, dim, seqlen)+ finally:+ torch._dynamo.reset()+ assert seen == [True, True]++[email protected]("folds", [True, False])+def test_a_packed_mixer_runs_outside_a_compiled_block(monkeypatch, folds):+ # Traced, transformers' hub-kernel wrappers run their torch reference where Dynamo folds+ # torch.compiler.is_exporting() to True (torch 2.10 / 2.11), and it ignores seq_idx / cu_seqlens.+ # Where it does not, the packed mixer stays in the graph.+ monkeypatch.setattr(packing_module, "_FOLDS_IS_EXPORTING", [folds])+ traced = []++ class Mixer(torch.nn.Module):+ def forward(self, hidden_states, **kwargs):+ traced.append(torch.compiler.is_compiling())+ return hidden_states * 2++ mixer = Mixer()+ packing_module._wrap_kwargs_mixer_forward(mixer)+ mixer._unsloth_varlen = (+ torch.tensor([0, 4, 9], dtype = torch.int32),+ torch.tensor([[0] * 4 + [1] * 5], dtype = torch.int32),+ )+ hidden = torch.randn(1, 9, 8)+ torch._dynamo.reset()+ try:+ block = torch.compile(lambda h: mixer(h), backend = "eager")+ block(hidden)+ mixer._unsloth_varlen = None+ block(hidden)+ finally:+ torch._dynamo.reset()+ assert traced == ([False, True] if folds else [True, True]) # only the packed call leaves it+++def test_is_exporting_fold_probe_matches_this_torch():+ from packaging.version import Version+ folds = packing_module._dynamo_folds_is_exporting()+ assert folds == (Version(torch.__version__.split("+")[0]).release[:2] in ((2, 10), (2, 11)))+++def test_probes_and_block_adapters_match_the_namespace_dynamo_reads(monkeypatch):+ # Unsloth's compiled modules run `from <modeling> import __name__`, so Dynamo resolves their globals+ # through the modeling module; a kernel bound differently there failed the block's closure guard.+ from unsloth.models import _block_compile as block++ monkeypatch.delenv(ENV, raising = False)+ names = (+ "fake_alias.fla.ops.gated_delta_rule.chunk",+ "fake_modeling_kwargs_alias_gdn",+ "fake_unsloth_compiled_module_alias_gdn",+ )+ fla, modeling, compiled = (types.ModuleType(n) for n in names)++ def chunk_gated_delta_rule(+ x,+ cu_seqlens = None,+ **kwargs,+ ):+ return scan_exact(x, cu_seqlens)++ fla.chunk_gated_delta_rule = chunk_gated_delta_rule+ compiled.__name__ = modeling.__name__+ for module in (modeling, compiled):+ _install_kwargs_mixer(vars(module), _zoo_style_dispatch, scan_impl = chunk_gated_delta_rule)+ for n, module in zip(names, (fla, modeling, compiled)):+ monkeypatch.setitem(sys.modules, n, module)+ model = FakeHybrid(compiled.FakeKwargsGatedDeltaNet())+ assert patch_hybrid_linear_attention_varlen(model) is True+ # Route only these modules: routing a real fla import would outlive this test.+ monkeypatch.setattr(block, "sys", SimpleNamespace(modules = {n: sys.modules[n] for n in names}))+ block._route_fla_kernel()+ block._route_causal_conv(model.linear_attn)+ for kernel in ("causal_conv1d_fn", "torch_chunk_gated_delta_rule"):+ probe = vars(compiled)[kernel]+ assert vars(modeling)[kernel] is probe, kernel+ assert probe._unsloth_varlen_probe_of is not None, kernel+ assert getattr(probe.__kwdefaults__["_fn"], "_unsloth_block_adapter", False), kernel+++def test_a_failed_probe_restores_the_mirrored_alias_namespace(monkeypatch):+ # A stale compiled module of another model family is walked too (its name says unsloth_compiled), and its+ # probes are mirrored into the modeling module it imports. A failed exactness probe rolls back both, or+ # the compiled global and sys.modules[__name__] disagree and the block's closure guard fails again.+ monkeypatch.delenv(ENV, raising = False)+ names = (+ "fake_modeling_rollback_gdn",+ "fake_other_family_modeling",+ "fake_unsloth_compiled_module_stale",+ )+ modeling, other, stale = (types.ModuleType(n) for n in names)+ for module in (modeling, other, stale):+ _install_kwargs_mixer(vars(module), _zoo_style_dispatch)+ stale.__name__ = other.__name__+ modeling.FakeKwargsGatedDeltaNet.__module__ = modeling.__name__+ for n, module in zip(names, (modeling, other, stale)):+ monkeypatch.setitem(sys.modules, n, module)+ before = {+ n: {k: vars(m)[k] for k in packing_module._KWARGS_KERNELS if k in vars(m)}+ for n, m in zip(names, (modeling, other, stale))+ }+ monkeypatch.setattr(packing_module, "_hybrid_exactness_probe", lambda target: "forced failure")+ assert (+ patch_hybrid_linear_attention_varlen(FakeHybrid(modeling.FakeKwargsGatedDeltaNet()))+ is False+ )+ for n, m in zip(names, (modeling, other, stale)):+ for k, fn in before[n].items():+ assert vars(m)[k] is fn, (n, k)diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 297317d9d26..00e4678dafd 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -206,10 +206,11 @@ def _route_fla_kernel():if not isinstance(space, dict) or space is adapter.__dict__:continuefor key, value in list(space.items()):- if (- value is caller- and isinstance(key, str)- and key.endswith("chunk_gated_delta_rule")+ if value is not caller or not isinstance(key, str):+ continue+ # A packing varlen probe keeps the kernel as its `_fn` keyword default; route under it.+ if key.endswith("chunk_gated_delta_rule") or (+ key == "_fn" and str(space.get("_name", "")).endswith("chunk_gated_delta_rule")):space[key] = adapterrouted += 1@@ -268,12 +269,38 @@ def _namespaces(fn, depth = 3):return out+def _alias_namespace(space):+ """The namespace Dynamo resolves `space`'s globals through: an inlined function's globals are+ read as sys.modules[__name__], and Unsloth's compiled modules import transformers' __name__."""+ module = (+ sys.modules.get(space.get("__name__")) if isinstance(space.get("__name__"), str) else None+ )+ alias = getattr(module, "__dict__", None)+ return alias if isinstance(alias, dict) and alias is not space else None++def _route_causal_conv(gdn):from ._gdn_op import _causal_depthwise_convfor space in _namespaces(type(gdn).forward):fn = space.get("causal_conv1d_fn")- if callable(fn) and not getattr(fn, "_unsloth_block_adapter", False):+ if not callable(fn) or getattr(fn, "_unsloth_block_adapter", False):+ continue+ probe_defaults = (+ getattr(fn, "__kwdefaults__", None)+ if getattr(fn, "_unsloth_varlen_probe_of", None) is not None+ else None+ )+ if probe_defaults is not None:+ # Packing's varlen probe stays outermost so its boundary handshake still sees the call.+ inner = probe_defaults.get("_fn")+ if callable(inner) and not getattr(inner, "_unsloth_block_adapter", False):+ probe_defaults["_fn"] = _causal_conv1d_adapter(inner, _causal_depthwise_conv)+ else:space["causal_conv1d_fn"] = _causal_conv1d_adapter(fn, _causal_depthwise_conv)+ alias = _alias_namespace(space)+ if alias is not None and callable(alias.get("causal_conv1d_fn")):+ # Dynamo guards the alias binding: it must be the object the forward actually calls.+ alias["causal_conv1d_fn"] = space["causal_conv1d_fn"]def _route_sm_count(modules = None):diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex 202c67c346d..5ab59a8ad90 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -767,8 +767,9 @@ def _varlen_seq_idx_applies(seq_idx, args, kwargs) -> bool:tensor = _varlen_input(args, kwargs)if tensor is None or tensor.dim() < 2 or seq_idx.shape[0] != tensor.shape[0]:return False- # causal_conv1d_fn takes (batch, dim, seqlen), the others (batch, seqlen, ...).- return total in (tensor.shape[1], tensor.shape[-1])+ # causal_conv1d_fn takes (batch, dim, seqlen), the others (batch, seqlen, ...). Compared with ==:+ # after a dynamic-shape recompile Dynamo evaluated `total in (...)` False for a matching row.+ return total == tensor.shape[1] or total == tensor.shape[-1]def _wrap_mamba2_seq_idx_call(orig, *, hit_attr = "_unsloth_varlen_fused_hit"):@@ -931,6 +932,13 @@ def _kwargs_kernels_available(modules, recurrent_ids = ()) -> Optional[str]:return None+def _kwargs_alias_namespace(namespace):+ """The namespace Dynamo resolves `namespace`'s globals through (sys.modules[__name__]), when it differs."""+ name = namespace.get("__name__") if isinstance(namespace, dict) else None+ alias = getattr(sys.modules.get(name), "__dict__", None) if isinstance(name, str) else None+ return alias if isinstance(alias, dict) and alias is not namespace else None++def _install_kwargs_probes(namespace) -> None:if not isinstance(namespace, dict):return@@ -953,6 +961,33 @@ def _install_kwargs_probes(namespace) -> None:probe._unsloth_varlen_probe_of = fnnamespace[name] = probe+ # Dynamo reads an inlined function's globals through sys.modules[__name__]; Unsloth's compiled+ # modules import transformers' __name__, so both namespaces must hold the same probe or the+ # block compile guards one object and traces the other.+ alias = _kwargs_alias_namespace(namespace)+ if alias is not None and callable(alias.get(name)):+ alias[name] = probe+++_FOLDS_IS_EXPORTING = []+++def _dynamo_folds_is_exporting() -> bool:+ # torch 2.10 / 2.11 Dynamo constant-folds torch.compiler.is_exporting() to True inside any+ # compiled region; 2.12+ reads the real flag. Probed once, eagerly, outside any trace.+ if not _FOLDS_IS_EXPORTING:++ def probe(x):+ return x + 1 if torch.compiler.is_exporting() else x++ try:+ folds = bool(+ torch.compile(probe, backend = "eager", fullgraph = True)(torch.zeros(1)).item()+ )+ except Exception:+ folds = True # unknown: keep the packed mixer eager, which is always correct+ _FOLDS_IS_EXPORTING.append(folds)+ return _FOLDS_IS_EXPORTING[0]def _wrap_kwargs_mixer_forward(module) -> None:@@ -963,6 +998,9 @@ def _wrap_kwargs_mixer_forward(module) -> None:varlen = getattr(module, "_unsloth_varlen", None)if varlen is None or not _varlen_seq_idx_applies(varlen[1], args, kwargs):return forward_orig(*args, **kwargs)+ return _packed_forward(varlen, args, kwargs)++ def _packed_forward(varlen, args, kwargs):varlen = _varlen_on(varlen, args, kwargs)if kwargs.get("seq_idx") is None:kwargs["seq_idx"] = varlen[1]@@ -978,6 +1016,13 @@ def _wrap_kwargs_mixer_forward(module) -> None:module._unsloth_varlen_kwargs_hit = Truereturn out+ if _dynamo_folds_is_exporting():+ # Eager even inside a compiled block: Dynamo folds torch.compiler.is_exporting() to True, so+ # the hub-kernel wrappers trace their torch reference, which drops seq_idx / cu_seqlens and+ # runs the packed documents into each other (Qwen3.5-0.8B per-document cosine 0.71). Newer+ # torch traces the real kernels, and the eager mixer costs 6-20% per step there.+ _packed_forward = torch.compiler.disable(_packed_forward)+module.forward = forwardmodule._unsloth_varlen_kwargs_mode = True@@ -1442,9 +1487,14 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:kwargs_namespaces = list(_iter_mamba2_install_namespaces(kwargs_modules + hub_short_convs))namespace_rollback = {}for ns in mask_namespaces + kwargs_namespaces:- if isinstance(ns, dict) and id(ns) not in namespace_rollback:- names = tuple(_MASK_BUILDERS) + tuple(_KWARGS_KERNELS)- namespace_rollback[id(ns)] = (ns, {name: ns[name] for name in names if name in ns})+ # The alias namespace too: _install_kwargs_probes mirrors into it, and a failed probe must undo that.+ for space in (ns, _kwargs_alias_namespace(ns)):+ if isinstance(space, dict) and id(space) not in namespace_rollback:+ names = tuple(_MASK_BUILDERS) + tuple(_KWARGS_KERNELS)+ namespace_rollback[id(space)] = (+ space,+ {name: space[name] for name in names if name in space},+ )for ns in mask_namespaces:_install_packed_mask_positions(ns, lambda: _PACKED[0])for ns in kwargs_namespaces: