Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Ulysses: dynamic and longrope RoPE see the whole sequence's positions (#17051)

Shubhankar Tripathyccbd70b8bc88
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
index fbe1c20cb41..d41ea23d5bc 100644
--- a/tests/test_context_parallel.py
+++ b/tests/test_context_parallel.py
@@ -1633,6 +1633,56 @@ def test_a_forwarded_cp_size_of_one_installs_no_wrappers(monkeypatch, forwarded)
assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)
[email protected]("rope", ["dynamic", "longrope"])
+def test_ulysses_seq_len_rope_uses_the_whole_sequence(monkeypatch, rope):
+ # dynamic / longrope choose frequencies from max(position_ids) + 1: rank 0 of 2 sees positions
+ # 0..15 (within the original 16), the whole sequence 0..31 (beyond it).
+ import copy
+ import types
+ from transformers import LlamaConfig, LlamaForCausalLM, Phi3Config, Phi3ForCausalLM
+
+ small = dict(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 4,
+ vocab_size = 64,
+ )
+ if rope == "dynamic":
+ config = LlamaConfig(
+ max_position_embeddings = 16,
+ rope_scaling = {"rope_type": "dynamic", "factor": 2.0},
+ **small,
+ )
+ model = LlamaForCausalLM(config)
+ else:
+ factors = [1.0 + i for i in range(4)]
+ config = Phi3Config(
+ max_position_embeddings = 64,
+ original_max_position_embeddings = 16,
+ rope_scaling = {"type": "longrope", "short_factor": [1.0] * 4, "long_factor": factors},
+ pad_token_id = 0,
+ **small,
+ )
+ model = Phi3ForCausalLM(config)
+ rotary = model.model.rotary_emb
+ # rope_scaling: the form transformers 4.57 reads and 5.x converts, so neither corner falls back to default.
+ assert rotary.rope_type == rope
+ x = torch.ones(1, 32, 8)
+ full = torch.arange(32)[None]
+ # A copy: dynamic NTK caches the frequencies it last computed.
+ expected = [t[:, :16] for t in copy.deepcopy(rotary)(x, full)]
+ manager = _manager(size = 2)
+ manager.backend = "ulysses"
+ manager.attach_attention_hooks(model)
+ manager.mesh = types.SimpleNamespace(get_local_rank = lambda: 0)
+ manager.full_position_ids = full
+ monkeypatch.setattr(cp, "_ACTIVE_MANAGER", manager)
+ for got, want in zip(rotary(x[:, :16], full[:, :16]), expected):
+ torch.testing.assert_close(got, want)
+
+
@pytest.mark.parametrize(
"model_type, blocker",
[
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 7e2c292084f..03c985af769 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -313,6 +313,42 @@ def _unregister_cp_sharding_rules() -> None:
_ACTIVE_MANAGER: Optional["ContextParallelManager"] = None
+def _seq_len_rope(module) -> bool:
+ kinds = getattr(module, "rope_type", None)
+ kinds = kinds.values() if isinstance(kinds, dict) else (kinds,)
+ return any(kind in ("dynamic", "longrope") for kind in kinds)
+
+
+def _rope_over_full_positions(module) -> None:
+ """dynamic / longrope pick their frequencies from max(position_ids) + 1 (transformers'
+ dynamic_rope_update): under Ulysses each rank would see only its shard and rotate with its own
+ frequencies. Append the sequence's largest position so the frequencies match, then drop it: the
+ tables stay this rank's size (slicing full-sequence tables would keep them alive for backward)."""
+ forward = module.forward
+
+ @functools.wraps(forward)
+ def full_positions_forward(x, position_ids, *args, **kwargs):
+ manager = get_cp_manager()
+ full = getattr(manager, "full_position_ids", None)
+ local = position_ids.shape[-1]
+ if (
+ not isinstance(full, torch.Tensor)
+ or full.ndim != 2
+ or full.shape != (position_ids.shape[0], local * manager.size)
+ ):
+ return forward(x, position_ids, *args, **kwargs)
+ start = manager.mesh.get_local_rank() * local
+ full = full.to(position_ids.device)
+ ids = torch.cat(
+ [full[:, start : start + local], full.max().expand(full.shape[0], 1)], dim = -1
+ )
+ cos, sin = forward(x, ids, *args, **kwargs)
+ return cos[:, :local], sin[:, :local]
+
+ module.forward = full_positions_forward
+ module._unsloth_cp_full_rope = True
+
+
# Before the imports below: on the CPU route they bind this from the partially initialised module.
def get_cp_manager() -> Optional["ContextParallelManager"]:
return _ACTIVE_MANAGER
@@ -673,6 +709,9 @@ class ContextParallelManager:
wrap_ulysses_mixer_forward(module, get_cp_manager)
# Fast forwards exchange inside run_attention; other HF layers run their attention after the exchange.
generic = not _uses_unsloth_attention(model)
+ for module in model.modules() if generic else ():
+ if _seq_len_rope(module) and not getattr(module, "_unsloth_cp_full_rope", False):
+ _rope_over_full_positions(module)
for name, module in model.named_modules():
if (
name.endswith("self_attn")
@@ -874,6 +913,7 @@ class ContextParallelManager:
def _ulysses_shard(self, inputs: dict) -> None:
# Contiguous shards; packed_seq_lengths stays global.
rank = self.mesh.get_local_rank()
+ self.full_position_ids = inputs.get("position_ids") # for _rope_over_full_positions
self._note_caller_padding(inputs.get("attention_mask"))
real = self._real_tokens(inputs.get("input_ids"))
if real is not None:
@@ -905,6 +945,7 @@ class ContextParallelManager:
finally:
_ACTIVE_MANAGER = previous
self.packed_seq_lengths = None
+ self.full_position_ids = None
@contextlib.contextmanager
def apply(self, inputs: dict) -> Iterator[None]: