Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Ulysses: dynamic and longrope RoPE see the whole sequence's positions (#17051)
Shubhankar Tripathyccbd70b8bc88
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.pyindex fbe1c20cb41..d41ea23d5bc 100644--- a/tests/test_context_parallel.py+++ b/tests/test_context_parallel.py@@ -1633,6 +1633,56 @@ def test_a_forwarded_cp_size_of_one_installs_no_wrappers(monkeypatch, forwarded)assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)[email protected]("rope", ["dynamic", "longrope"])+def test_ulysses_seq_len_rope_uses_the_whole_sequence(monkeypatch, rope):+ # dynamic / longrope choose frequencies from max(position_ids) + 1: rank 0 of 2 sees positions+ # 0..15 (within the original 16), the whole sequence 0..31 (beyond it).+ import copy+ import types+ from transformers import LlamaConfig, LlamaForCausalLM, Phi3Config, Phi3ForCausalLM++ small = dict(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ num_key_value_heads = 4,+ vocab_size = 64,+ )+ if rope == "dynamic":+ config = LlamaConfig(+ max_position_embeddings = 16,+ rope_scaling = {"rope_type": "dynamic", "factor": 2.0},+ **small,+ )+ model = LlamaForCausalLM(config)+ else:+ factors = [1.0 + i for i in range(4)]+ config = Phi3Config(+ max_position_embeddings = 64,+ original_max_position_embeddings = 16,+ rope_scaling = {"type": "longrope", "short_factor": [1.0] * 4, "long_factor": factors},+ pad_token_id = 0,+ **small,+ )+ model = Phi3ForCausalLM(config)+ rotary = model.model.rotary_emb+ # rope_scaling: the form transformers 4.57 reads and 5.x converts, so neither corner falls back to default.+ assert rotary.rope_type == rope+ x = torch.ones(1, 32, 8)+ full = torch.arange(32)[None]+ # A copy: dynamic NTK caches the frequencies it last computed.+ expected = [t[:, :16] for t in copy.deepcopy(rotary)(x, full)]+ manager = _manager(size = 2)+ manager.backend = "ulysses"+ manager.attach_attention_hooks(model)+ manager.mesh = types.SimpleNamespace(get_local_rank = lambda: 0)+ manager.full_position_ids = full+ monkeypatch.setattr(cp, "_ACTIVE_MANAGER", manager)+ for got, want in zip(rotary(x[:, :16], full[:, :16]), expected):+ torch.testing.assert_close(got, want)++@pytest.mark.parametrize("model_type, blocker",[diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 7e2c292084f..03c985af769 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -313,6 +313,42 @@ def _unregister_cp_sharding_rules() -> None:_ACTIVE_MANAGER: Optional["ContextParallelManager"] = None+def _seq_len_rope(module) -> bool:+ kinds = getattr(module, "rope_type", None)+ kinds = kinds.values() if isinstance(kinds, dict) else (kinds,)+ return any(kind in ("dynamic", "longrope") for kind in kinds)+++def _rope_over_full_positions(module) -> None:+ """dynamic / longrope pick their frequencies from max(position_ids) + 1 (transformers'+ dynamic_rope_update): under Ulysses each rank would see only its shard and rotate with its own+ frequencies. Append the sequence's largest position so the frequencies match, then drop it: the+ tables stay this rank's size (slicing full-sequence tables would keep them alive for backward)."""+ forward = module.forward++ @functools.wraps(forward)+ def full_positions_forward(x, position_ids, *args, **kwargs):+ manager = get_cp_manager()+ full = getattr(manager, "full_position_ids", None)+ local = position_ids.shape[-1]+ if (+ not isinstance(full, torch.Tensor)+ or full.ndim != 2+ or full.shape != (position_ids.shape[0], local * manager.size)+ ):+ return forward(x, position_ids, *args, **kwargs)+ start = manager.mesh.get_local_rank() * local+ full = full.to(position_ids.device)+ ids = torch.cat(+ [full[:, start : start + local], full.max().expand(full.shape[0], 1)], dim = -1+ )+ cos, sin = forward(x, ids, *args, **kwargs)+ return cos[:, :local], sin[:, :local]++ module.forward = full_positions_forward+ module._unsloth_cp_full_rope = True++# Before the imports below: on the CPU route they bind this from the partially initialised module.def get_cp_manager() -> Optional["ContextParallelManager"]:return _ACTIVE_MANAGER@@ -673,6 +709,9 @@ class ContextParallelManager:wrap_ulysses_mixer_forward(module, get_cp_manager)# Fast forwards exchange inside run_attention; other HF layers run their attention after the exchange.generic = not _uses_unsloth_attention(model)+ for module in model.modules() if generic else ():+ if _seq_len_rope(module) and not getattr(module, "_unsloth_cp_full_rope", False):+ _rope_over_full_positions(module)for name, module in model.named_modules():if (name.endswith("self_attn")@@ -874,6 +913,7 @@ class ContextParallelManager:def _ulysses_shard(self, inputs: dict) -> None:# Contiguous shards; packed_seq_lengths stays global.rank = self.mesh.get_local_rank()+ self.full_position_ids = inputs.get("position_ids") # for _rope_over_full_positionsself._note_caller_padding(inputs.get("attention_mask"))real = self._real_tokens(inputs.get("input_ids"))if real is not None:@@ -905,6 +945,7 @@ class ContextParallelManager:finally:_ACTIVE_MANAGER = previousself.packed_seq_lengths = None+ self.full_position_ids = None@contextlib.contextmanagerdef apply(self, inputs: dict) -> Iterator[None]: