Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallelism: drop the SDPA wrappers each ring step leaves in torch's registry (#14193)

Daniel Han7ca1b3d80dbb
diff --git a/tests/test_context_parallel_wrapper_registry.py b/tests/test_context_parallel_wrapper_registry.py
new file mode 100644
index 00000000000..c0989dd0017
--- /dev/null
+++ b/tests/test_context_parallel_wrapper_registry.py
@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+
+
+def _torch_cp_registry():
+ try:
+ from torch.distributed.tensor.experimental._context_parallel import _attention
+ except ImportError:
+ from torch.distributed.tensor.experimental import _attention
+ return getattr(_attention, "_replaced_functions", None)
+
+
+def gloo_cp_manager(monkeypatch, tmp_path):
+ import torch.distributed as dist
+
+ if _torch_cp_registry() is None or not cp._torch_supports_context_parallel():
+ pytest.skip("this torch has no CP wrapper registry")
+ if dist.is_initialized():
+ pytest.skip("a process group is already initialized")
+ dist.init_process_group(
+ "gloo", init_method = f"file://{tmp_path / 'rendezvous'}", rank = 0, world_size = 1
+ )
+ monkeypatch.setattr(cp, "DEVICE_TYPE_TORCH", "cpu")
+ try:
+ yield cp.ContextParallelManager(1)
+ finally:
+ dist.destroy_process_group()
+
+
+def _cp_step(manager):
+ with manager.apply({"input_ids": torch.ones(1, 8, dtype = torch.long)}):
+ pass
+
+
+def test_ring_steps_do_not_grow_the_cp_wrapper_registry(gloo_cp_manager):
+ import torch.nn.functional as F
+
+ registry = _torch_cp_registry()
+ original = F.scaled_dot_product_attention
+ before = dict(registry)
+ seen = []
+ with gloo_cp_manager.apply({"input_ids": torch.ones(1, 8, dtype = torch.long)}):
+ seen.append(F.scaled_dot_product_attention in registry) # the real torch wrapper engaged
+ for _ in range(50):
+ _cp_step(gloo_cp_manager)
+ assert seen == [True]
+ assert registry == before
+ assert F.scaled_dot_product_attention is original
+
+
+def test_cp_wrappers_created_outside_unsloth_are_kept(gloo_cp_manager):
+ import torch.nn.functional as F
+
+ registry = _torch_cp_registry()
+ original = F.scaled_dot_product_attention
+ before = set(registry)
+ foreign = lambda *a, **k: None
+ registry[foreign] = ("scaled_dot_product_attention", original)
+ try:
+ buffer = torch.ones(1, 8)
+ with cp.context_parallel(gloo_cp_manager.mesh, buffers = [buffer], buffer_seq_dims = [1]):
+ outer = F.scaled_dot_product_attention
+ assert outer in registry
+ _cp_step(gloo_cp_manager) # nested inside a user's own CP context
+ assert F.scaled_dot_product_attention is outer and outer in registry
+ with gloo_cp_manager.apply({"input_ids": torch.ones(1, 8, dtype = torch.long)}):
+ _cp_step(gloo_cp_manager) # re-entrant Unsloth step
+ assert F.scaled_dot_product_attention is outer and outer in registry
+ assert foreign in registry and outer in registry
+ finally:
+ for wrapper in set(registry) - before:
+ registry.pop(wrapper, None)
+ F.scaled_dot_product_attention = original
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 7e7ddd36aa2..9b34ebbd90b 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -65,6 +65,28 @@ def _ring_sdpa_backends():
return backends
+def _cp_replaced_functions():
+ # torch's CP registry of SDPA wrappers; restore never drops entries, so each step adds one.
+ # torch >= 2.10 keeps it in _context_parallel._attention, 2.7 - 2.9 in experimental._attention
+ # (a stub without it on 2.10+).
+ modules = []
+ try:
+ from torch.distributed.tensor.experimental._context_parallel import _attention as new_layout
+ modules.append(new_layout)
+ except ImportError:
+ pass
+ try:
+ from torch.distributed.tensor.experimental import _attention as old_layout
+ modules.append(old_layout)
+ except ImportError:
+ pass
+ for module in modules:
+ registry = getattr(module, "_replaced_functions", None)
+ if isinstance(registry, dict):
+ return registry
+ return None
+
+
def context_parallel_options(config) -> tuple:
"""(size, backend, rotate_method) from a training config, validated before any dataset work."""
size = getattr(config, "context_parallel_size", 1)
@@ -411,6 +433,8 @@ class ContextParallelManager:
dispatcher = getattr(DTensor, "_op_dispatcher", None)
handlers = getattr(dispatcher, "_custom_op_handlers", None)
ring_backends = _ring_sdpa_backends()
+ registry = _cp_replaced_functions()
+ wrappers = set(registry) if registry is not None else None
raised = False
try:
with (
@@ -438,6 +462,11 @@ class ContextParallelManager:
if handlers is not None:
dispatcher._custom_op_handlers = handlers
_unregister_cp_sharding_rules()
+ if wrappers is not None:
+ # Drop only the wrappers this step's context added, never one still installed.
+ for wrapper in [w for w in registry if w not in wrappers]:
+ if wrapper is not F.scaled_dot_product_attention:
+ registry.pop(wrapper, None)
def _refuse_iterable_datasets(*datasets) -> None: