Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallelism: drop the SDPA wrappers each ring step leaves in torch's registry (#14193)
Daniel Han7ca1b3d80dbb
diff --git a/tests/test_context_parallel_wrapper_registry.py b/tests/test_context_parallel_wrapper_registry.pynew file mode 100644index 00000000000..c0989dd0017--- /dev/null+++ b/tests/test_context_parallel_wrapper_registry.py@@ -0,0 +1,80 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++import pytest+import torch++import unsloth # noqa: F401+from unsloth import context_parallel as cp+++def _torch_cp_registry():+ try:+ from torch.distributed.tensor.experimental._context_parallel import _attention+ except ImportError:+ from torch.distributed.tensor.experimental import _attention+ return getattr(_attention, "_replaced_functions", None)+++def gloo_cp_manager(monkeypatch, tmp_path):+ import torch.distributed as dist++ if _torch_cp_registry() is None or not cp._torch_supports_context_parallel():+ pytest.skip("this torch has no CP wrapper registry")+ if dist.is_initialized():+ pytest.skip("a process group is already initialized")+ dist.init_process_group(+ "gloo", init_method = f"file://{tmp_path / 'rendezvous'}", rank = 0, world_size = 1+ )+ monkeypatch.setattr(cp, "DEVICE_TYPE_TORCH", "cpu")+ try:+ yield cp.ContextParallelManager(1)+ finally:+ dist.destroy_process_group()+++def _cp_step(manager):+ with manager.apply({"input_ids": torch.ones(1, 8, dtype = torch.long)}):+ pass+++def test_ring_steps_do_not_grow_the_cp_wrapper_registry(gloo_cp_manager):+ import torch.nn.functional as F++ registry = _torch_cp_registry()+ original = F.scaled_dot_product_attention+ before = dict(registry)+ seen = []+ with gloo_cp_manager.apply({"input_ids": torch.ones(1, 8, dtype = torch.long)}):+ seen.append(F.scaled_dot_product_attention in registry) # the real torch wrapper engaged+ for _ in range(50):+ _cp_step(gloo_cp_manager)+ assert seen == [True]+ assert registry == before+ assert F.scaled_dot_product_attention is original+++def test_cp_wrappers_created_outside_unsloth_are_kept(gloo_cp_manager):+ import torch.nn.functional as F++ registry = _torch_cp_registry()+ original = F.scaled_dot_product_attention+ before = set(registry)+ foreign = lambda *a, **k: None+ registry[foreign] = ("scaled_dot_product_attention", original)+ try:+ buffer = torch.ones(1, 8)+ with cp.context_parallel(gloo_cp_manager.mesh, buffers = [buffer], buffer_seq_dims = [1]):+ outer = F.scaled_dot_product_attention+ assert outer in registry+ _cp_step(gloo_cp_manager) # nested inside a user's own CP context+ assert F.scaled_dot_product_attention is outer and outer in registry+ with gloo_cp_manager.apply({"input_ids": torch.ones(1, 8, dtype = torch.long)}):+ _cp_step(gloo_cp_manager) # re-entrant Unsloth step+ assert F.scaled_dot_product_attention is outer and outer in registry+ assert foreign in registry and outer in registry+ finally:+ for wrapper in set(registry) - before:+ registry.pop(wrapper, None)+ F.scaled_dot_product_attention = originaldiff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 7e7ddd36aa2..9b34ebbd90b 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -65,6 +65,28 @@ def _ring_sdpa_backends():return backends+def _cp_replaced_functions():+ # torch's CP registry of SDPA wrappers; restore never drops entries, so each step adds one.+ # torch >= 2.10 keeps it in _context_parallel._attention, 2.7 - 2.9 in experimental._attention+ # (a stub without it on 2.10+).+ modules = []+ try:+ from torch.distributed.tensor.experimental._context_parallel import _attention as new_layout+ modules.append(new_layout)+ except ImportError:+ pass+ try:+ from torch.distributed.tensor.experimental import _attention as old_layout+ modules.append(old_layout)+ except ImportError:+ pass+ for module in modules:+ registry = getattr(module, "_replaced_functions", None)+ if isinstance(registry, dict):+ return registry+ return None++def context_parallel_options(config) -> tuple:"""(size, backend, rotate_method) from a training config, validated before any dataset work."""size = getattr(config, "context_parallel_size", 1)@@ -411,6 +433,8 @@ class ContextParallelManager:dispatcher = getattr(DTensor, "_op_dispatcher", None)handlers = getattr(dispatcher, "_custom_op_handlers", None)ring_backends = _ring_sdpa_backends()+ registry = _cp_replaced_functions()+ wrappers = set(registry) if registry is not None else Noneraised = Falsetry:with (@@ -438,6 +462,11 @@ class ContextParallelManager:if handlers is not None:dispatcher._custom_op_handlers = handlers_unregister_cp_sharding_rules()+ if wrappers is not None:+ # Drop only the wrappers this step's context added, never one still installed.+ for wrapper in [w for w in registry if w not in wrappers]:+ if wrapper is not F.scaled_dot_product_attention:+ registry.pop(wrapper, None)def _refuse_iterable_datasets(*datasets) -> None: