Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
"""Context parallelism plumbing that runs without a process group (the ring itself needs >= 2 GPUs)."""
import contextlib
import pytest
import torch
import unsloth # noqa: F401
from unsloth import context_parallel as cp
from unsloth.utils import attention_dispatch as ad
def _manager(size = 2):
manager = object.__new__(cp.ContextParallelManager)
manager.size = size
return manager
def _fake_context_parallel(sharded):
@contextlib.contextmanager
def fake(mesh, buffers, buffer_seq_dims, no_restore_buffers):
sharded.extend(buffers)
yield
return fake
def test_none_position_ids_and_shift_labels_are_rebuilt():
inputs = {
"input_ids": torch.ones(1, 4, dtype = torch.long),
"labels": torch.tensor([[1, 2, 3, 4]]),
"position_ids": None,
"shift_labels": None,
}
_manager()._prepare_inputs(inputs)
assert inputs["position_ids"].tolist() == [[0, 1, 2, 3]]
assert inputs["shift_labels"].tolist() == [[2, 3, 4, -100]]
def test_labels_shift_before_sharding_and_pad_to_load_balancer_chunks():
manager = _manager(size = 2)
labels = torch.tensor([[-100, 5, 6, 7, 8, 9]])
inputs = {"input_ids": torch.tensor([[1, 5, 6, 7, 8, 9]]), "labels": labels.clone()}
manager._prepare_inputs(inputs)
assert inputs["shift_labels"].tolist() == [[5, 6, 7, 8, 9, -100, -100, -100]]
assert inputs["labels"].tolist() == [[-100, 5, 6, 7, 8, 9, -100, -100]]
assert inputs["position_ids"].tolist() == [[0, 1, 2, 3, 4, 5, 6, 7]]
assert inputs["input_ids"].shape == (1, 8)
@pytest.mark.parametrize("mask", [[[0, 1, 1, 1]], [[1, 0, 1, 1]], [[[1, 1, 1, 1]] * 4]])
def test_left_padded_or_holed_masks_are_refused(mask):
inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long), "attention_mask": torch.tensor(mask)}
with pytest.raises(ValueError, match = "right-padded"):
_manager()._prepare_inputs(inputs)
right = {
"input_ids": torch.ones(1, 4, dtype = torch.long),
"attention_mask": torch.tensor([[1, 1, 0, 0]]),
}
_manager()._prepare_inputs(right)
def test_inputs_embeds_batches_are_refused():
with pytest.raises(ValueError, match = "input_ids"):
_manager()._prepare_inputs(
{"inputs_embeds": torch.zeros(1, 4, 8), "labels": torch.ones(1, 4)}
)
def test_active_manager_resets_when_the_step_raises(monkeypatch):
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
manager = _manager()
manager.mesh = None
inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long)}
with pytest.raises(RuntimeError, match = "boom"):
with manager.apply(inputs):
assert cp.get_cp_manager() is manager
raise RuntimeError("boom")
assert cp.get_cp_manager() is None
def test_attention_hook_only_drops_the_mask_inside_a_cp_step(monkeypatch):
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
mask = torch.ones(1, 4)
_, kwargs = cp._self_attn_pre_forward_hook(None, (), {"attention_mask": mask})
assert kwargs["attention_mask"] is mask
manager = _manager()
manager.mesh = None
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
_, kwargs = cp._self_attn_pre_forward_hook(None, (), {"attention_mask": mask})
assert kwargs["attention_mask"] is None
def test_backend_is_sdpa_under_cp_and_packing_is_refused(monkeypatch):
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
manager = _manager()
manager.mesh = None
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
assert ad.select_attention_backend(False) == ad.SDPA
with pytest.raises(ValueError, match = "packing"):
ad.select_attention_backend(True)
def test_sdpa_is_looked_up_per_call(monkeypatch):
# A name bound at import would keep attending to the local shard only.
calls = []
monkeypatch.setattr(
torch.nn.functional, "scaled_dot_product_attention", lambda *a, **k: calls.append(1)
)
ad.scaled_dot_product_attention(None, None, None)
assert calls == [1]
def test_training_step_divides_the_pre_shard_token_count_by_cp_size(monkeypatch):
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
seen = []
class Trainer:
def __init__(self):
pass
def training_step(
self,
model,
inputs,
num_items_in_batch = None,
):
seen.append(("train", num_items_in_batch))
assert inputs["use_cache"] is False
def prediction_step(self, model, inputs, prediction_loss_only):
return (None, None, None)
def train(self):
pass
def evaluate(self):
pass
def evaluation_loop(self, *a, **k):
pass
import trl
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
cp.patch_sft_trainer(install_cp_methods = True)
trainer = Trainer()
manager = _manager(size = 4)
manager.mesh = None
trainer._context_parallel_manager = manager
batch = lambda: {"input_ids": torch.ones(1, 8, dtype = torch.long)}
trainer.training_step(None, batch(), torch.tensor(12.0))
trainer.training_step(None, batch(), num_items_in_batch = torch.tensor(12.0))
trainer.training_step(None, batch(), torch.tensor(12))
assert [(k, float(n)) for k, n in seen] == [("train", 3.0)] * 3
assert seen[-1][1].dtype == torch.int64
def test_training_step_installs_the_idle_expert_hook_once(monkeypatch):
# Under CP every-expert MoE blocks get the optimizer pre-hook that drops globally idle experts'
# zero gradients; through accelerate's wrapper, once, and not for a model without such blocks.
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
class Experts(torch.nn.Module):
def __init__(self):
super().__init__()
self.experts = torch.nn.ModuleList(torch.nn.Linear(2, 2) for _ in range(3))
def forward(self, x):
return x
Experts.forward._unsloth_remote_moe_shim = True
class Wrapped: # accelerate's AcceleratedOptimizer steps the optimizer it wraps
def __init__(self, optimizer):
self.optimizer = optimizer
class Trainer:
def __init__(self):
pass
def training_step(
self,
model,
inputs,
num_items_in_batch = None,
):
pass
def prediction_step(self, model, inputs, prediction_loss_only):
return (None, None, None)
def train(self):
pass
def evaluate(self):
pass
def evaluation_loop(self, *a, **k):
pass
import trl
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
cp.patch_sft_trainer(install_cp_methods = True)
trainer = Trainer()
trainer.model = torch.nn.Sequential(Experts())
inner = torch.optim.AdamW(trainer.model.parameters())
trainer.optimizer = Wrapped(inner)
manager = _manager(size = 2)
manager.mesh = None
trainer._context_parallel_manager = manager
for _ in range(2):
trainer.training_step(None, {"input_ids": torch.ones(1, 4, dtype = torch.long)}, 4)
assert len(inner._optimizer_step_pre_hooks) == 1
plain = torch.optim.AdamW(torch.nn.Linear(2, 2).parameters())
trainer.model, trainer.optimizer = torch.nn.Linear(2, 2), plain
trainer.training_step(None, {"input_ids": torch.ones(1, 4, dtype = torch.long)}, 4)
assert not plain._optimizer_step_pre_hooks
def test_shift_labels_is_not_an_eval_label_name():
# A named shift_labels made find_labels treat it as a required eval label.
from transformers.utils import find_labels
from unsloth.models.llama import CausalLM_fast_forward, LlamaModel_fast_forward_inference
class LM(torch.nn.Module):
forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference)
assert find_labels(LM) == ["labels"]
@pytest.mark.parametrize("opt_out", [False, True])
def test_gqa_stays_native_under_cp_unless_opted_out(monkeypatch, opt_out):
if opt_out:
monkeypatch.setenv("UNSLOTH_CP_GQA", "0")
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
monkeypatch.setattr(ad, "SDPA_HAS_GQA", True)
seen = []
monkeypatch.setattr(
ad,
"scaled_dot_product_attention",
lambda Q, K, V, **k: (seen.append((K.shape[1], k.get("enable_gqa"))), Q)[1],
)
config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 2, n_groups = 2)
context = ad.AttentionContext(
bsz = 1,
q_len = 4,
kv_seq_len = 4,
n_heads = 4,
head_dim = 8,
requires_grad = True,
seq_info = None,
attention_mask = None,
causal_mask = None,
)
Q, K = torch.zeros(1, 4, 4, 8), torch.zeros(1, 2, 4, 8)
manager = _manager()
manager.mesh = None
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
ad.run_attention(config = config, context = context, Q = Q, K = K, V = K)
assert seen == ([(4, None)] if opt_out else [(2, True)])
def test_every_rank_builds_the_same_global_mesh(monkeypatch):
built = []
class FakeMesh:
def __init__(
self,
device_type,
mesh,
mesh_dim_names = None,
):
built.append((mesh.tolist(), mesh_dim_names))
def __getitem__(self, name):
return ("submesh", name)
monkeypatch.setattr(cp, "DeviceMesh", FakeMesh)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: 4)
for rank in range(4):
monkeypatch.setattr(cp.dist, "get_rank", lambda rank = rank: rank)
manager = cp.ContextParallelManager(2)
assert manager.mesh == ("submesh", "cp")
assert built == [([[0, 1], [2, 3]], ("dp_replicate", "cp"))] * 4
def _patched_trainer(
monkeypatch,
install_cp_methods = True,
on_init = None,
**init_attrs,
):
import trl
class Trainer:
def __init__(self, *a, **k):
for key, value in init_attrs.items():
setattr(self, key, value)
if on_init is not None:
on_init(self)
def training_step(
self,
model,
inputs,
num_items_in_batch = None,
):
pass
def prediction_step(
self,
model,
inputs,
prediction_loss_only,
ignore_keys = None,
):
return (torch.tensor(1.0), None, None)
def _prepare_inputs(self, inputs):
return inputs
def compute_loss_context_manager(self):
return contextlib.nullcontext()
def compute_loss(
self,
model,
inputs,
num_items_in_batch = None,
):
# Like TRL's SFTTrainer: a per-token loss of 1 over this shard's targets, divided by the
# num_items_in_batch argument (an inputs["num_items_in_batch"] key is not read).
assert inputs["use_cache"] is False
targets = inputs["shift_labels"].ne(-100).sum()
if num_items_in_batch is None:
num_items_in_batch = targets.clamp_min(1)
return targets.float() / num_items_in_batch
def train(self, *a, **k):
return "trained"
def evaluate(self, *a, **k):
return "evaluated"
def predict(self, *a, **k):
return "predicted"
def evaluation_loop(self, *a, **k):
return self.gather_function(torch.arange(1.0, 3.0))
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
cp.patch_sft_trainer(install_cp_methods = install_cp_methods)
return Trainer
@pytest.mark.parametrize("world_size", [2, 4])
def test_eval_loss_is_the_token_mean_even_with_an_empty_shard(monkeypatch, world_size):
import types
@contextlib.contextmanager
def first_half(mesh, buffers, buffer_seq_dims, no_restore_buffers):
for buffer in buffers:
buffer.data = buffer[:, : buffer.shape[1] // 2].clone()
yield
monkeypatch.setattr(cp, "context_parallel", first_half)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: world_size)
Trainer = _patched_trainer(monkeypatch)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
manager = _manager(size = 2)
manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")
trainer._context_parallel_manager = manager
counts = []
def all_reduce(tensor, group = None):
assert group == "cp"
counts.append(group)
tensor.add_(2 / 2)
monkeypatch.setattr(cp.dist, "all_reduce", all_reduce)
labels = torch.tensor([[-100] * 7 + [1, 1]])
batch = lambda: {"input_ids": torch.ones(1, 9, dtype = torch.long), "labels": labels.clone()}
loss, logits, labels_out = trainer.prediction_step(None, batch(), True)
assert counts == ["cp"] and loss.item() == pytest.approx(1.0)
assert logits is None and labels_out is None
with pytest.raises(NotImplementedError, match = "loss-only"):
trainer.prediction_step(None, batch(), False)
def _model_normalised_loss(
self,
model,
inputs,
num_items_in_batch = None,
):
# TRL < 1 hands inputs to the model: sum / inputs["num_items_in_batch"] when the key is there, else
# transformers' token mean over this shard, 0 / 0 on a shard without targets.
targets = inputs["shift_labels"].ne(-100).sum().float()
count = inputs.get("num_items_in_batch")
return targets / (targets if count is None else count)
@pytest.mark.parametrize("compute_loss", ["trl", "model"])
def test_eval_loss_is_not_multiplied_by_the_cp_size(monkeypatch, compute_loss):
import types
@contextlib.contextmanager
def first_half(mesh, buffers, buffer_seq_dims, no_restore_buffers):
for buffer in buffers:
buffer.data = buffer[:, : buffer.shape[1] // 2].clone()
yield
monkeypatch.setattr(cp, "context_parallel", first_half)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
Trainer = _patched_trainer(monkeypatch)
if compute_loss == "model":
monkeypatch.setattr(Trainer, "compute_loss", _model_normalised_loss)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
manager = _manager(size = 2)
manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")
trainer._context_parallel_manager = manager
# Two ranks holding 2 of the 4 targets each, with the same per-token loss: the sum is 1, not 2.
monkeypatch.setattr(cp.dist, "all_reduce", lambda tensor, group = None: tensor.mul_(2))
labels = torch.tensor([[-100, 1, 1, -100, -100, -100, -100, 1, 1]]) # padded to 12, halves of 6
batch = {"input_ids": torch.ones(1, 9, dtype = torch.long), "labels": labels}
loss, _, _ = trainer.prediction_step(None, batch, True)
assert loss.item() == pytest.approx(1.0)
def test_eval_loss_of_a_shard_without_targets_is_zero_even_when_its_mean_is_nan(monkeypatch):
import types
@contextlib.contextmanager
def first_half(mesh, buffers, buffer_seq_dims, no_restore_buffers):
for buffer in buffers:
buffer.data = buffer[:, : buffer.shape[1] // 2].clone()
yield
monkeypatch.setattr(cp, "context_parallel", first_half)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
Trainer = _patched_trainer(monkeypatch)
monkeypatch.setattr(Trainer, "compute_loss", _model_normalised_loss)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
manager = _manager(size = 2)
manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")
trainer._context_parallel_manager = manager
# This rank holds the prompt (no targets), the other rank's share is the whole mean, 1.
monkeypatch.setattr(cp.dist, "all_reduce", lambda tensor, group = None: tensor.add_(1.0))
labels = torch.tensor([[-100] * 7 + [1, 1]])
batch = {"input_ids": torch.ones(1, 9, dtype = torch.long), "labels": labels}
loss, _, _ = trainer.prediction_step(None, batch, True)
assert loss.item() == pytest.approx(1.0)
@pytest.mark.parametrize(
"attrs",
[
{"label_smoothing_factor": 0.1},
{"compute_loss_func": lambda *a, **k: 0},
{"loss_type": "chunked_nll"},
],
)
def test_label_dropping_loss_paths_are_refused(monkeypatch, attrs):
import types
monkeypatch.setattr(cp.dist, "is_available", lambda: True)
monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
for key in ("label_smoothing_factor", "loss_type"):
if key in attrs:
setattr(args, key, attrs[key])
init = {"args": args}
if "compute_loss_func" in attrs:
init["compute_loss_func"] = attrs["compute_loss_func"]
Trainer = _patched_trainer(monkeypatch, **init)
with pytest.raises(NotImplementedError, match = "loss_type = 'nll'"):
Trainer()
def test_unaveraged_token_counts_are_refused(monkeypatch):
import types
monkeypatch.setattr(cp.dist, "is_available", lambda: True)
monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
args = types.SimpleNamespace(
context_parallel_size = 2,
label_smoothing_factor = 0.0,
average_tokens_across_devices = False,
)
Trainer = _patched_trainer(monkeypatch, args = args)
with pytest.raises(NotImplementedError, match = "average_tokens_across_devices"):
Trainer()
def test_old_accelerate_is_refused(monkeypatch):
import types, accelerate
monkeypatch.setattr(cp.dist, "is_available", lambda: True)
monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
monkeypatch.setattr(accelerate, "__version__", "1.9.0")
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
Trainer = _patched_trainer(monkeypatch, args = args)
with pytest.raises(NotImplementedError, match = "accelerate >= 1.10.0"):
Trainer()
def test_fp32_qkv_is_downcast_under_cp(monkeypatch):
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
seen = []
monkeypatch.setattr(
ad, "scaled_dot_product_attention", lambda Q, K, V, **k: (seen.append(Q.dtype), Q)[1]
)
config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 4, n_groups = 1)
context = ad.AttentionContext(
bsz = 1,
q_len = 4,
kv_seq_len = 4,
n_heads = 4,
head_dim = 8,
requires_grad = True,
seq_info = None,
attention_mask = None,
causal_mask = None,
)
Q = torch.zeros(1, 4, 4, 8, dtype = torch.float32)
ad.run_attention(config = config, context = context, Q = Q, K = Q, V = Q)
manager = _manager()
manager.mesh = None
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
out = ad.run_attention(config = config, context = context, Q = Q, K = Q, V = Q)
assert seen[0] == torch.float32 and seen[1] in (torch.bfloat16, torch.float16)
assert out.dtype == torch.float32
@pytest.mark.parametrize("distributed_type, fsdp_version", [("DEEPSPEED", None), ("FSDP", 1)])
def test_deepspeed_and_fsdp1_are_refused(monkeypatch, distributed_type, fsdp_version):
import types
monkeypatch.setattr(cp.dist, "is_available", lambda: True)
monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
monkeypatch.setattr(cp, "_supports_context_parallel", lambda model, backend = "ring": True)
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
plugin = None if fsdp_version is None else types.SimpleNamespace(fsdp_version = fsdp_version)
accelerator = types.SimpleNamespace(
distributed_type = types.SimpleNamespace(name = distributed_type),
state = types.SimpleNamespace(fsdp_plugin = plugin),
)
Trainer = _patched_trainer(monkeypatch, args = args, accelerator = accelerator, model = None)
with pytest.raises(NotImplementedError, match = "DDP and FSDP2 only"):
Trainer()
def _fsdp2_accelerator(parallelism_config = None):
import types
return types.SimpleNamespace(
distributed_type = types.SimpleNamespace(name = "FSDP"),
state = types.SimpleNamespace(
device_mesh = None,
fsdp_plugin = types.SimpleNamespace(fsdp_version = 2),
parallelism_config = parallelism_config,
),
)
def test_fsdp2_shards_over_dp_shard_cp_and_a_later_trainer_restores_the_config(monkeypatch):
import types
monkeypatch.setattr(cp, "_INSTALLED_MESH", [])
monkeypatch.setattr(cp, "_INSTALLED_PARALLELISM_CONFIG", [])
user_pc = types.SimpleNamespace(dp_replicate_size = 1, tp_size = 1, sp_size = 1, cp_size = 1)
accelerator = _fsdp2_accelerator(user_pc)
Trainer, _, built = _gated_trainer(monkeypatch, accelerator = accelerator)
trainer = Trainer()
assert built == [(2, "ulysses", "allgather", "fsdp2")] # the default backend: auto
# Installed only while a CP entry point (train / evaluate / predict / save_model) runs.
assert accelerator.state.parallelism_config is user_pc
assert accelerator.state.device_mesh is None
with cp._cp_state_scope(trainer):
pc = accelerator.state.parallelism_config
# What accelerate's fsdp2_prepare_model indexes the mesh with; cp stays Unsloth's own.
assert pc.fsdp_dim_names == ["dp_shard_cp"]
assert (pc.dp_shard_size, pc.cp_size, pc.non_data_parallel_size) == (4, 1, 1)
assert accelerator.state.device_mesh is not None
later = _patched_trainer(
monkeypatch,
args = types.SimpleNamespace(context_parallel_size = 1),
accelerator = accelerator,
)
later()
assert accelerator.state.parallelism_config is user_pc
assert accelerator.state.device_mesh is None
assert accelerator.state.parallelism_config is user_pc
assert accelerator.state.device_mesh is None
@pytest.mark.parametrize("dim", ["dp_replicate", "tp", "sp"])
def test_fsdp2_refuses_other_parallel_dims(monkeypatch, dim):
import types
sizes = {"dp_replicate_size": 1, "tp_size": 1, "sp_size": 1, f"{dim}_size": 2}
accelerator = _fsdp2_accelerator(types.SimpleNamespace(cp_size = 1, **sizes))
Trainer, _, built = _gated_trainer(monkeypatch, accelerator = accelerator)
with pytest.raises(NotImplementedError, match = "dp_shard, cp"):
Trainer()
assert built == []
def _cp_env(monkeypatch):
monkeypatch.setattr(cp.dist, "is_available", lambda: True)
monkeypatch.setattr(cp.dist, "is_initialized", lambda: True)
monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
monkeypatch.setattr(cp, "_supports_context_parallel", lambda model, backend = "ring": True)
@pytest.mark.parametrize("case", ["iterable", "dispatch", "eval_dict"])
def test_dispatched_or_iterable_loaders_are_refused(monkeypatch, case):
import types
_cp_env(monkeypatch)
class Stream(torch.utils.data.IterableDataset):
def __iter__(self):
return iter(())
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
accelerator = types.SimpleNamespace(
distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),
dispatch_batches = case == "dispatch",
)
Trainer = _patched_trainer(
monkeypatch,
args = args,
accelerator = accelerator,
model = None,
train_dataset = Stream() if case == "iterable" else [1],
eval_dataset = {"a": [1], "b": Stream()} if case == "eval_dict" else None,
)
with pytest.raises(NotImplementedError, match = "iterable datasets|dispatch_batches"):
Trainer()
def test_iterable_eval_dataset_passed_to_evaluate_is_refused(monkeypatch):
class Stream(torch.utils.data.IterableDataset):
def __iter__(self):
return iter(())
Trainer = _patched_trainer(monkeypatch)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = _manager()
with pytest.raises(NotImplementedError, match = "iterable datasets"):
trainer.evaluate(Stream())
with pytest.raises(NotImplementedError, match = "iterable datasets"):
trainer.evaluate(eval_dataset = {"a": Stream()})
trainer._context_parallel_manager = None
assert trainer.evaluate(Stream()) == "evaluated"
def test_iterable_predict_dataset_is_refused(monkeypatch):
class Stream(torch.utils.data.IterableDataset):
def __iter__(self):
return iter(())
Trainer = _patched_trainer(monkeypatch)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = _manager()
with pytest.raises(NotImplementedError, match = "iterable datasets"):
trainer.predict(Stream())
trainer._context_parallel_manager = None
assert trainer.predict(Stream()) == "predicted"
def test_sliding_window_models_are_refused(monkeypatch):
import types
_cp_env(monkeypatch)
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
accelerator = types.SimpleNamespace(distributed_type = types.SimpleNamespace(name = "MULTI_GPU"))
model = types.SimpleNamespace(config = types.SimpleNamespace(sliding_window = 4096))
Trainer = _patched_trainer(monkeypatch, args = args, accelerator = accelerator, model = model)
with pytest.raises(NotImplementedError, match = "sliding window"):
Trainer()
def test_a_later_non_cp_trainer_drops_the_installed_mesh(monkeypatch):
import types
mesh = object()
monkeypatch.setattr(cp, "_INSTALLED_MESH", [mesh])
state = types.SimpleNamespace(device_mesh = mesh)
args = types.SimpleNamespace(context_parallel_size = 1)
Trainer = _patched_trainer(
monkeypatch, args = args, accelerator = types.SimpleNamespace(state = state)
)
Trainer()
assert state.device_mesh is None
other = object()
state.device_mesh = other
Trainer()
assert state.device_mesh is other
def test_a_later_non_cp_trainer_restores_the_mesh_cp_replaced(monkeypatch):
import types
# An accepted FSDP2 ParallelismConfig came with its own mesh; CP swapped both.
prior, cp_mesh = object(), object()
monkeypatch.setattr(cp, "_INSTALLED_MESH", [cp_mesh, prior])
state = types.SimpleNamespace(device_mesh = cp_mesh)
args = types.SimpleNamespace(context_parallel_size = 1)
Trainer = _patched_trainer(
monkeypatch, args = args, accelerator = types.SimpleNamespace(state = state)
)
Trainer()
assert state.device_mesh is prior
def test_every_attention_layer_must_be_the_llama_forward():
from unsloth.models.llama import LlamaAttention_fast_forward
class Llama(torch.nn.Module):
forward = LlamaAttention_fast_forward
class Other(torch.nn.Module):
def forward(self, x):
return x
def model(*attns):
root = torch.nn.Module()
root.layers = torch.nn.ModuleList()
for attn in attns:
layer = torch.nn.Module()
layer.self_attn = attn()
root.layers.append(layer)
return root
assert cp._supports_context_parallel(model(Llama, Llama))
assert not cp._supports_context_parallel(model(Llama, Other))
assert not cp._supports_context_parallel(model())
@pytest.mark.parametrize("user_value", [None, True])
def test_find_unused_parameters_is_off_under_cp_unless_the_user_set_it(monkeypatch, user_value):
import types
_cp_env(monkeypatch)
monkeypatch.setattr(
cp,
"ContextParallelManager",
lambda size, *rest, **kwargs: types.SimpleNamespace(
size = size,
backend = "ring",
device_mesh = object(),
attach_attention_hooks = lambda model: None,
),
)
args = types.SimpleNamespace(
context_parallel_size = 2,
label_smoothing_factor = 0.0,
ddp_find_unused_parameters = user_value,
)
handler = types.SimpleNamespace(
find_unused_parameters = True if user_value is None else user_value
)
accelerator = types.SimpleNamespace(
distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),
state = types.SimpleNamespace(device_mesh = None),
ddp_handler = handler,
)
Trainer = _patched_trainer(
monkeypatch, args = args, accelerator = accelerator, model = None, train_dataset = [1]
)
Trainer()
expected = False if user_value is None else True
# 5.x reads the handler built at init; 4.x rebuilds it from the argument at train time.
assert handler.find_unused_parameters is expected
assert args.ddp_find_unused_parameters is expected
def test_active_manager_is_visible_from_the_autograd_device_thread(monkeypatch):
import threading
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
manager = _manager()
manager.mesh = None
seen = []
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
thread = threading.Thread(target = lambda: seen.append(cp.get_cp_manager()))
thread.start()
thread.join()
assert seen == [manager]
assert cp.get_cp_manager() is None
def test_transformers_cp_size_is_refused(monkeypatch):
import types
args = types.SimpleNamespace(
context_parallel_size = 1, parallelism_config = types.SimpleNamespace(cp_size = 2)
)
Trainer = _patched_trainer(monkeypatch, args = args)
with pytest.raises(NotImplementedError, match = "context_parallel_size"):
Trainer()
@pytest.mark.parametrize("cp_size", [1, 2])
def test_accelerate_config_cp_size_is_refused(monkeypatch, cp_size):
# `accelerate launch` with parallelism_config_cp_size: the Accelerator builds its
# ParallelismConfig from the environment and TrainingArguments.parallelism_config stays None.
import types
accelerator = types.SimpleNamespace(
parallelism_config = types.SimpleNamespace(cp_size = cp_size),
state = types.SimpleNamespace(device_mesh = None),
)
Trainer = _patched_trainer(
monkeypatch,
args = types.SimpleNamespace(context_parallel_size = 1, parallelism_config = None),
accelerator = accelerator,
)
if cp_size == 1:
Trainer()
return
with pytest.raises(NotImplementedError, match = "accelerate config"):
Trainer()
def test_cp_state_is_reinstalled_at_train_after_another_trainer(monkeypatch):
import types
mesh = object()
state = types.SimpleNamespace(device_mesh = None)
Trainer = _patched_trainer(monkeypatch, args = types.SimpleNamespace(context_parallel_size = 1))
trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer.accelerator = types.SimpleNamespace(state = state)
trainer.args = types.SimpleNamespace(ddp_find_unused_parameters = None)
trainer._context_parallel_manager = types.SimpleNamespace(
size = 2, backend = "ring", device_mesh = mesh
)
# A later non-CP trainer cleared the shared state; train() must put the CP mesh back while it runs.
seen = []
release = cp._release_accelerator_state
monkeypatch.setattr(
cp, "_release_accelerator_state", lambda s: (seen.append(s.device_mesh), release(s))
)
assert trainer.train() == "trained"
assert seen == [mesh]
assert state.device_mesh is None
assert trainer.args.ddp_find_unused_parameters is False
def test_attention_hooks_are_not_stacked_by_a_second_manager():
model = torch.nn.Module()
model.self_attn = torch.nn.Linear(2, 2)
for _ in range(2):
_manager().attach_attention_hooks(model)
assert len(model.self_attn._forward_pre_hooks) == 1
def test_decoder_layers_see_no_mask_inside_a_cp_step_only():
class Layer(torch.nn.Module):
def __init__(self, linear = False):
super().__init__()
self.self_attn = torch.nn.Identity()
if linear:
self.linear_attn = torch.nn.Identity()
self.seen = []
def forward(
self,
hidden_states,
attention_mask = None,
):
self.seen.append(attention_mask)
return hidden_states
model = torch.nn.Module()
model.layers = torch.nn.ModuleList([Layer(), Layer(linear = True)])
for _ in range(2):
_manager().attach_attention_hooks(model)
assert len(model.layers[0]._forward_pre_hooks) == 1
assert (
len(model.layers[1]._forward_pre_hooks) == 0
) # a linear attention scan reads the padding mask
x, mask = torch.ones(1, 4, 2), torch.ones(1, 1, 4, 4, dtype = torch.bool)
def run(masks):
for layer in model.layers:
layer.seen.clear()
for m in masks:
layer(x, attention_mask = m)
return [layer.seen for layer in model.layers]
# No CP manager: the exact same object passes through.
plain = run([mask, None])
assert plain[0][0] is mask and plain[0][1] is None and plain[1][0] is mask
with cp._manager_scope(_manager()):
inside = run([mask, None])
assert inside[0] == [None, None]
assert inside[1][0] is mask
assert run([mask])[0][0] is mask
def test_sdpa_is_restored_when_the_cp_step_raises(monkeypatch):
import torch.nn.functional as F
original = F.scaled_dot_product_attention
@contextlib.contextmanager
def leaky(mesh, buffers, buffer_seq_dims, no_restore_buffers):
F.scaled_dot_product_attention = lambda *a, **k: None # torch < 2.13
yield
F.scaled_dot_product_attention = original
monkeypatch.setattr(cp, "context_parallel", leaky)
manager = _manager()
manager.mesh = None
with pytest.raises(RuntimeError):
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
raise RuntimeError("OOM")
assert F.scaled_dot_product_attention is original
@pytest.mark.parametrize("remainder", [0, 3])
def test_eval_gather_keeps_one_loss_per_cp_group(monkeypatch, remainder):
import types
per_rank = [torch.tensor([10.0 * r + 1, 10.0 * r + 2]) for r in (0, 0, 1, 1, 2, 2)]
accelerator = types.SimpleNamespace(
gather = lambda t: torch.cat(per_rank),
gather_for_metrics = lambda t, *a, **k: t,
gradient_state = types.SimpleNamespace(end_of_dataloader = True, remainder = remainder),
)
Trainer = _patched_trainer(monkeypatch)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer.accelerator = accelerator
trainer._context_parallel_manager = _manager(size = 2)
out = trainer.evaluation_loop()
expected = [1.0, 2.0, 11.0, 12.0, 21.0, 22.0]
assert out.tolist() == (expected[:remainder] if remainder else expected)
def test_replaced_iterable_train_dataset_is_refused_at_train(monkeypatch):
class Stream(torch.utils.data.IterableDataset):
def __iter__(self):
return iter(())
Trainer = _patched_trainer(monkeypatch)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = _manager()
trainer.train_dataset = Stream()
with pytest.raises(NotImplementedError, match = "iterable datasets"):
trainer.train()
def test_null_inputs_embeds_placeholder_is_allowed():
inputs = {"input_ids": torch.ones(1, 4, dtype = torch.long), "inputs_embeds": None}
_manager()._prepare_inputs(inputs)
assert inputs["position_ids"].shape == (1, 4)
def test_replaced_iterable_eval_dataset_is_refused_at_evaluate(monkeypatch):
class Stream(torch.utils.data.IterableDataset):
def __iter__(self):
return iter(())
Trainer = _patched_trainer(monkeypatch)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
trainer._context_parallel_manager = _manager()
trainer.eval_dataset = Stream()
with pytest.raises(NotImplementedError, match = "iterable datasets"):
trainer.evaluate()
trainer.eval_dataset = {"a": [1], "b": Stream()}
with pytest.raises(NotImplementedError, match = "iterable datasets"):
trainer.evaluate("b")
assert trainer.evaluate("a") == "evaluated"
def test_model_init_is_refused(monkeypatch):
import types
_cp_env(monkeypatch)
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
Trainer = _patched_trainer(monkeypatch, args = args, model_init = lambda: None)
with pytest.raises(NotImplementedError, match = "model_init"):
Trainer()
def test_cp_dtensor_dispatcher_is_torn_down_when_the_step_raises(monkeypatch):
dispatcher = cp.DTensor._op_dispatcher
original = dispatcher._custom_op_handlers
unregistered = []
monkeypatch.setattr(cp, "_unregister_cp_sharding_rules", lambda: unregistered.append(1))
@contextlib.contextmanager
def leaky(mesh, buffers, buffer_seq_dims, no_restore_buffers):
dispatcher._custom_op_handlers = {**original, "cp": None} # torch < 2.13: no finally
yield
dispatcher._custom_op_handlers = original
monkeypatch.setattr(cp, "context_parallel", leaky)
manager = _manager()
manager.mesh = None
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
pass
assert dispatcher._custom_op_handlers is original and unregistered == []
with pytest.raises(RuntimeError):
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
raise RuntimeError("OOM")
assert dispatcher._custom_op_handlers is original and unregistered == [1]
def _gated_trainer(
monkeypatch,
model = None,
accelerator = None,
install_cp_methods = True,
on_init = None,
**args_overrides,
):
"""A trainer that clears every CP gate, with the manager stubbed (no process group)."""
import types
_cp_env(monkeypatch)
built = []
def manager(
size,
backend = "ring",
rotate_method = "allgather",
fsdp2 = False,
):
built.append((size, backend, rotate_method) + (("fsdp2",) if fsdp2 else ()))
return types.SimpleNamespace(
size = size,
backend = backend,
fsdp2 = fsdp2,
device_mesh = {"dp_shard_cp": types.SimpleNamespace(size = lambda: 4)},
attach_attention_hooks = lambda model: None,
)
monkeypatch.setattr(cp, "ContextParallelManager", manager)
args = types.SimpleNamespace(context_parallel_size = 2, label_smoothing_factor = 0.0)
for key, value in args_overrides.items():
setattr(args, key, value)
if accelerator is None:
accelerator = types.SimpleNamespace(
distributed_type = types.SimpleNamespace(name = "MULTI_GPU"),
state = types.SimpleNamespace(device_mesh = None),
)
init = {"args": args, "accelerator": accelerator, "model": model, "train_dataset": [1]}
return _patched_trainer(monkeypatch, install_cp_methods, on_init, **init), args, built
_CP_METHODS = (
"train",
"training_step",
"prediction_step",
"evaluate",
"predict",
"evaluation_loop",
)
def test_cp_methods_are_bound_only_on_the_trainer_that_enables_cp(monkeypatch):
import inspect
# With CP off class wrappers slowed torch 2.10 LoRA steps by ~35%, so only __init__ is patched on the
# class and the CP wrappers are bound per trainer that enables CP.
import types
Trainer = _patched_trainer(monkeypatch, install_cp_methods = False)
assert hasattr(Trainer.__init__, "__wrapped__")
assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)
trainer = Trainer(args = types.SimpleNamespace(context_parallel_size = 1))
assert trainer._context_parallel_manager is None
assert trainer.train() == "trained"
assert not any(name in vars(trainer) for name in _CP_METHODS)
Trainer, _, built = _gated_trainer(monkeypatch, install_cp_methods = False)
plain = {name: Trainer.__dict__[name] for name in _CP_METHODS}
cp_trainer = Trainer()
assert built == [(2, "ulysses", "allgather")] # the default backend: auto
for name in _CP_METHODS:
assert Trainer.__dict__[name] is plain[name]
# Bound per instance (train is wrapped again by the flash attention scope).
assert inspect.unwrap(vars(cp_trainer)[name]) is plain[name]
@pytest.mark.parametrize(
"overrides, error",
[
({"context_parallel_size": True}, ValueError),
({"context_parallel_size": 0}, ValueError),
({"context_parallel_size": -2}, ValueError),
({"context_parallel_size": 2.0}, ValueError),
({"context_parallel_size": "2"}, ValueError),
({"context_parallel_backend": "tree"}, ValueError),
({"context_parallel_rotate_method": "ring"}, ValueError),
],
)
def test_bad_cp_options_fail_before_the_trainer_builds(monkeypatch, overrides, error):
import types, trl
seen = []
class Trainer:
def __init__(self, *a, **k):
seen.append(1)
training_step = prediction_step = train = evaluate = evaluation_loop = None
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
cp.patch_sft_trainer()
args = types.SimpleNamespace(**{"context_parallel_size": 2, **overrides})
with pytest.raises(error):
Trainer(None, args)
with pytest.raises(error):
Trainer(None, args = args)
assert seen == []
def test_cp_option_defaults_and_size_one():
import types
assert cp.context_parallel_options(None) == (1, "auto", "allgather")
config = types.SimpleNamespace(context_parallel_size = 1, context_parallel_backend = "ulysses")
assert cp.context_parallel_options(config) == (1, "ulysses", "allgather")
config = types.SimpleNamespace(
context_parallel_size = 4, context_parallel_rotate_method = "alltoall"
)
assert cp.context_parallel_options(config) == (4, "auto", "alltoall")
def test_manager_gets_the_backend_and_rotate_method(monkeypatch):
Trainer, _, built = _gated_trainer(
monkeypatch, context_parallel_backend = "ring", context_parallel_rotate_method = "alltoall"
)
Trainer()
assert built == [(2, "ring", "alltoall")]
@pytest.mark.parametrize("version", ["2.6.0", "2.6.0+cu124", "2.5.1"])
def test_torch_older_than_2_7_is_refused(monkeypatch, version):
Trainer, _, _ = _gated_trainer(monkeypatch)
monkeypatch.setattr(cp.torch, "__version__", version)
with pytest.raises(RuntimeError, match = "PyTorch >= 2.7"):
Trainer()
def test_torch_2_7_dev_builds_pass_the_gate(monkeypatch):
monkeypatch.setattr(cp.torch, "__version__", "2.7.0a0+git1234")
assert cp._torch_supports_context_parallel()
@pytest.mark.parametrize(
"config, reason",
[
({"layer_types": ["full_attention", "sliding_attention"]}, "sliding_attention"),
({"layer_types": ["chunked_attention", "full_attention"]}, "chunked_attention"),
({"attn_logit_softcapping": 50.0}, "softcapping"),
({"attention_chunk_size": 8192}, "chunked attention"),
({"use_sliding_window": True}, "sliding window"),
({"text_config": {"layer_types": ["sliding_attention"]}}, "sliding_attention"),
],
)
def test_windowed_or_softcapped_attention_is_refused_for_ring(monkeypatch, config, reason):
import types
if "text_config" in config:
config = {"text_config": types.SimpleNamespace(**config["text_config"])}
model = types.SimpleNamespace(config = types.SimpleNamespace(**config))
Trainer, _, _ = _gated_trainer(monkeypatch, model = model)
with pytest.raises(NotImplementedError, match = reason):
Trainer()
def test_full_attention_layer_types_pass(monkeypatch):
import types
config = types.SimpleNamespace(
layer_types = ["full_attention"] * 2, sliding_window = None, use_sliding_window = False
)
Trainer, _, built = _gated_trainer(monkeypatch, model = types.SimpleNamespace(config = config))
Trainer()
assert built
def test_legacy_prediction_loop_is_refused(monkeypatch):
Trainer, _, built = _gated_trainer(monkeypatch, use_legacy_prediction_loop = True)
with pytest.raises(NotImplementedError, match = "use_legacy_prediction_loop"):
Trainer()
assert built == []
def test_model_without_loss_kwargs_is_refused(monkeypatch):
Trainer, _, _ = _gated_trainer(monkeypatch)
monkeypatch.setattr(Trainer, "model_accepts_loss_kwargs", False, raising = False)
with pytest.raises(NotImplementedError, match = "loss kwargs"):
Trainer()
def test_flash_attention_impl_is_scoped_to_train_without_packing(monkeypatch):
from unsloth.utils import packing
scoped = []
monkeypatch.setattr(
packing, "_scope_flash_attention_impl", lambda model, trainer: scoped.append(trainer)
)
Trainer, _, _ = _gated_trainer(monkeypatch)
trainer = Trainer()
assert scoped == [trainer]
def test_training_step_refuses_a_missing_token_count_with_labels(monkeypatch):
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
Trainer = _patched_trainer(monkeypatch)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
manager = _manager()
manager.mesh = None
trainer._context_parallel_manager = manager
batch = {"input_ids": torch.ones(1, 4, dtype = torch.long), "labels": torch.ones(1, 4)}
with pytest.raises(RuntimeError, match = "num_items_in_batch"):
trainer.training_step(None, dict(batch))
with pytest.raises(RuntimeError, match = "num_items_in_batch"):
trainer.training_step(None, dict(batch), None)
shifted = {"input_ids": torch.ones(1, 4, dtype = torch.long), "shift_labels": torch.ones(1, 4)}
with pytest.raises(RuntimeError, match = "num_items_in_batch"):
trainer.training_step(None, shifted)
assert cp.get_cp_manager() is None
trainer.training_step(None, {"input_ids": torch.ones(1, 4, dtype = torch.long)})
trainer.training_step(None, dict(batch), torch.tensor(4.0))
@pytest.mark.parametrize("raises", [False, True])
def test_rotate_method_is_set_for_the_step_and_restored(monkeypatch, raises):
if cp._cp_options is None:
pytest.skip("this torch has no ring rotate option")
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
manager = _manager()
manager.mesh = None
manager.rotate_method = "alltoall"
before = cp._cp_options.rotate_method
seen = []
with contextlib.suppress(RuntimeError):
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
seen.append(cp._cp_options.rotate_method.name)
if raises:
raise RuntimeError("boom")
assert seen == ["ALL_TO_ALL"]
assert cp._cp_options.rotate_method == before
@pytest.mark.parametrize("raises", [False, True])
def test_ring_pins_sdpa_to_kernels_the_dispatcher_rings(monkeypatch, raises):
if cp.sdpa_kernel is None:
pytest.skip("no torch.nn.attention.sdpa_kernel")
monkeypatch.setattr(cp, "context_parallel", _fake_context_parallel([]))
manager = _manager()
manager.mesh = None
cuda = torch.backends.cuda
before = (cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled(), cuda.math_sdp_enabled())
seen = []
with contextlib.suppress(RuntimeError):
with manager.apply({"input_ids": torch.ones(1, 4, dtype = torch.long)}):
seen.append(
(
cuda.flash_sdp_enabled(),
cuda.mem_efficient_sdp_enabled(),
cuda.math_sdp_enabled(),
)
)
if raises:
raise RuntimeError("boom")
assert seen == [(True, True, False)]
assert (
cuda.flash_sdp_enabled(),
cuda.mem_efficient_sdp_enabled(),
cuda.math_sdp_enabled(),
) == before
def test_cudnn_is_pinned_only_where_the_ring_handles_it():
backends = cp._ring_sdpa_backends()
if backends is None:
pytest.skip("no SDPBackend")
from torch.nn.attention import SDPBackend
assert backends[:2] == [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]
from packaging.version import Version
if Version(Version(torch.__version__).base_version) >= Version("2.13"):
assert SDPBackend.CUDNN_ATTENTION in backends
def test_unsloth_training_arguments_keep_the_cp_options():
from unsloth.trainer import UnslothTrainingArguments
args = UnslothTrainingArguments(
output_dir = "unused",
context_parallel_size = 2,
context_parallel_rotate_method = "alltoall",
report_to = "none",
)
assert cp.context_parallel_options(args) == (2, "auto", "alltoall")
def test_cli_forwards_the_cp_options():
import importlib.util, pathlib, types
path = pathlib.Path(__file__).resolve().parents[1] / "unsloth-cli.py"
spec = importlib.util.spec_from_file_location("unsloth_cli_for_test", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
args = types.SimpleNamespace(
per_device_train_batch_size = 1,
gradient_accumulation_steps = 1,
warmup_steps = 0,
max_steps = 1,
learning_rate = 1e-4,
logging_steps = 1,
optim = "adamw_torch",
weight_decay = 0.0,
lr_scheduler_type = "linear",
seed = 0,
output_dir = "unused",
report_to = "none",
max_seq_length = 64,
packing = False,
per_device_eval_batch_size = 4,
context_parallel_size = 2,
context_parallel_backend = "ring",
context_parallel_rotate_method = "alltoall",
)
config = module._build_sft_config(lambda **kwargs: kwargs, args, False, True)
assert config["context_parallel_size"] == 2
assert config["context_parallel_backend"] == "ring"
assert config["context_parallel_rotate_method"] == "alltoall"
assert config["average_tokens_across_devices"] is True
def test_generated_sft_config_carries_the_cp_options(tmp_path):
import dataclasses
import trl.trainer.sft_config as config_module
names = {f.name for f in dataclasses.fields(config_module.SFTConfig)}
if "context_parallel_size" not in names:
pytest.skip("SFTConfig is not the Unsloth-generated class in this process")
assert {"context_parallel_backend", "context_parallel_rotate_method"} <= names
config = config_module.SFTConfig(
output_dir = str(tmp_path),
context_parallel_size = 2,
context_parallel_rotate_method = "alltoall",
report_to = "none",
)
assert cp.context_parallel_options(config) == (2, "auto", "alltoall")
default = config_module.SFTConfig(output_dir = str(tmp_path), report_to = "none")
assert cp.context_parallel_options(default) == (1, "auto", "allgather")
class _FakeCPMesh:
def __init__(self, rank):
self.rank = rank
def get_local_rank(self):
return self.rank
def get_group(self):
return ("group", self.rank)
def _ulysses_manager(rank, size = 2):
manager = _manager(size)
manager.backend = "ulysses"
manager.mesh = _FakeCPMesh(rank)
return manager
def test_ulysses_shards_contiguously_after_global_shift():
labels = torch.tensor([[-100, 5, 6, 7, 8, 9, 10]])
shards = []
for rank in range(2):
inputs = {"input_ids": torch.arange(1, 8)[None], "labels": labels.clone()}
manager = _ulysses_manager(rank)
with manager.apply(inputs):
assert cp.get_cp_manager() is manager
shards.append(inputs)
assert cp.get_cp_manager() is None
assert [s["input_ids"].tolist() for s in shards] == [[[1, 2, 3, 4]], [[5, 6, 7, 0]]]
assert [s["shift_labels"].tolist() for s in shards] == [[[5, 6, 7, 8]], [[9, 10, -100, -100]]]
assert [s["position_ids"].tolist() for s in shards] == [[[0, 1, 2, 3]], [[4, 5, 6, 7]]]
def test_ulysses_packed_batch_keeps_global_lengths_and_masks_document_starts():
lengths = torch.tensor([3, 4], dtype = torch.int32)
labels = torch.tensor([[10, 11, 12, 20, 21, 22, 23]])
shards = []
for rank in range(2):
inputs = {
"input_ids": torch.arange(7)[None],
"labels": labels.clone(),
"packed_seq_lengths": lengths,
}
with _ulysses_manager(rank).apply(inputs):
shards.append(inputs)
assert all(s["packed_seq_lengths"] is lengths for s in shards)
assert torch.cat([s["position_ids"] for s in shards], 1).tolist() == [[0, 1, 2, 0, 1, 2, 3, 4]]
shift = torch.cat([s["shift_labels"] for s in shards], 1).tolist()
assert shift == [[11, 12, -100, 21, 22, 23, -100, -100]]
def test_ulysses_refuses_packed_lengths_longer_than_the_row():
inputs = {
"input_ids": torch.ones(1, 4, dtype = torch.long),
"position_ids": torch.arange(4)[None],
"packed_seq_lengths": torch.tensor([3, 10**9]),
}
with pytest.raises(ValueError, match = "packed row has 4 tokens"):
_ulysses_manager(0)._prepare_inputs(inputs)
def test_ring_refuses_packed_batches():
inputs = {
"input_ids": torch.ones(1, 4, dtype = torch.long),
"packed_seq_lengths": torch.tensor([4]),
}
manager = _manager()
manager.backend = "ring"
with pytest.raises(ValueError, match = "ring context parallelism does not support packing"):
manager._prepare_inputs(inputs)
def test_ulysses_keeps_varlen_and_routes_attention_through_the_exchange(monkeypatch):
calls = []
def fake_ulysses(run_attention_fn, config, context, Q, K, V, group):
calls.append((run_attention_fn, group))
return "exchanged"
monkeypatch.setattr(ad, "ulysses_attention", fake_ulysses)
manager = _ulysses_manager(1)
monkeypatch.setattr(cp, "_ACTIVE_MANAGER", manager)
if ad.HAS_FLASH_ATTENTION:
assert ad.select_attention_backend(True) == ad.FLASH_VARLEN
else:
assert ad.select_attention_backend(True) != ad.FLASH_VARLEN
assert ad.run_attention(config = None, context = None, Q = 1, K = 2, V = 3) == "exchanged"
assert calls == [(ad._run_attention, ("group", 1))]
manager.backend = "ring"
with pytest.raises(ValueError, match = "context_parallel_backend = 'ulysses'"):
ad.select_attention_backend(True)
def test_inherited_trainer_methods_are_resolved_per_call(monkeypatch):
import trl
class Base:
def train(self):
return "base"
def training_step(self, *a, **k):
pass
evaluate = evaluation_loop = prediction_step = training_step
class Trainer(Base):
def __init__(self):
pass
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
cp.patch_sft_trainer(install_cp_methods = True)
monkeypatch.setattr(Base, "train", lambda self: "patched later")
assert Trainer().train() == "patched later"
def test_ulysses_head_counts_are_checked_at_init(monkeypatch):
import types
config = types.SimpleNamespace(num_attention_heads = 3, num_key_value_heads = 3)
model = types.SimpleNamespace(config = config)
Trainer, _, built = _gated_trainer(monkeypatch, model = model, context_parallel_backend = "ulysses")
with pytest.raises(ValueError, match = "divisible by context_parallel_size"):
Trainer()
assert built == []
config.num_attention_heads = config.num_key_value_heads = 4
Trainer()
assert built == [(2, "ulysses", "allgather")]
@pytest.mark.parametrize("backend", ["ring", "ulysses"])
def test_only_ring_needs_sm80(monkeypatch, backend):
import types
config = types.SimpleNamespace(num_attention_heads = 4, num_key_value_heads = 4)
Trainer, _, built = _gated_trainer(
monkeypatch, model = types.SimpleNamespace(config = config), context_parallel_backend = backend
)
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (7, 5))
if backend == "ring":
with pytest.raises(NotImplementedError, match = "compute capability"):
Trainer()
else:
Trainer()
assert built == [(2, "ulysses", "allgather")]
@pytest.mark.parametrize("fsdp2", [False, True])
def test_total_train_batch_size_counts_cp_groups_not_ranks(monkeypatch, fsdp2):
import types
import transformers
import trl
# The real transformers formula: DDP leaves parallelism_config unset, FSDP2 installs cp_size = 1.
class Trainer(transformers.Trainer):
def __init__(self, *a, **k):
pass
monkeypatch.setattr(trl, "SFTTrainer", Trainer)
cp.patch_sft_trainer(install_cp_methods = True)
trainer = cp._bind_cp_methods(object.__new__(Trainer))
pc = types.SimpleNamespace(cp_size = 1, sp_size = 1, tp_size = 1) if fsdp2 else None
trainer.accelerator = types.SimpleNamespace(parallelism_config = pc)
trainer.model = types.SimpleNamespace()
trainer.is_deepspeed_enabled = False
trainer._train_batch_size = 2
args = types.SimpleNamespace(world_size = 8, gradient_accumulation_steps = 3)
trainer._context_parallel_manager = None
assert trainer.get_total_train_batch_size(args) == 2 * 3 * 8
# D = 2 data parallel groups of N = 4 CP peers that each consume the same batch.
trainer._context_parallel_manager = _manager(size = 4)
assert trainer.get_total_train_batch_size(args) == 2 * 3 * 2
def test_instance_wrappers_made_in_init_capture_the_cp_methods(monkeypatch):
# Auto packing (self.train) and the token budget (self.training_step) wrap the class methods on
# the instance inside __init__; with lazy installation they must already see the CP wrappers.
import functools
captured = {}
def on_init(self):
for name in ("train", "training_step"):
method = getattr(self, name)
captured[name] = method
setattr(self, name, functools.partial(method))
import types
for forwarded in (False, True):
captured.clear()
Trainer, args, built = _gated_trainer(
monkeypatch, install_cp_methods = False, on_init = on_init
)
plain = {name: Trainer.__dict__[name] for name in ("train", "training_step")}
if forwarded:
# SFTTrainer(model, args = base_args, context_parallel_size = 2): moved into args in __init__
Trainer(
None, args = types.SimpleNamespace(context_parallel_size = 1), context_parallel_size = 2
)
else:
Trainer(
None, args = args
) # SFTTrainer(model, args = SFTConfig(context_parallel_size = 2))
assert built == [(2, "ulysses", "allgather")] # the default backend: auto
for name, method in captured.items():
assert method.__func__ is not plain[name] and method.__func__.__wrapped__ is plain[name]
# The class keeps its own methods.
assert {name: Trainer.__dict__[name] for name in plain} == plain
@pytest.mark.parametrize("unhashable", [False, True])
def test_a_trainer_built_before_cp_still_restores_the_accelerator_state(monkeypatch, unhashable):
# A non-CP trainer whose __init__ wrapped self.train (auto packing) before any CP trainer existed
# must still drop the mesh a later CP trainer installs, as patched_train does.
import functools
import types
plain_args = [types.SimpleNamespace(context_parallel_size = 1)]
def on_init(self):
if plain_args: # the first trainer has CP off
self.args = plain_args.pop()
self.train = functools.partial(type(self).train, self)
if unhashable:
# A subclass defining __eq__ without __hash__ is unhashable.
on_init_plain = on_init
def on_init(self):
type(self).__eq__ = lambda a, b: a is b
type(self).__hash__ = None
on_init_plain(self)
Trainer, args, built = _gated_trainer(monkeypatch, install_cp_methods = False, on_init = on_init)
plain_train = Trainer.__dict__["train"]
early = Trainer(None, args = types.SimpleNamespace(context_parallel_size = 1))
assert early._context_parallel_manager is None
Trainer(None, args = args) # enables CP: binds the wrappers on that trainer only
assert built == [(2, "ulysses", "allgather")] # the default backend: auto
# The CP trainer put the shared state back at the end of its __init__ ...
assert early.accelerator.state.device_mesh is None
# ... and the earlier trainer runs the class's own train.
assert Trainer.__dict__["train"] is plain_train and early.train.func is plain_train
assert early.train() == "trained"
@pytest.mark.parametrize("forwarded", [1, None])
def test_a_forwarded_cp_size_of_one_installs_no_wrappers(monkeypatch, forwarded):
# SFTTrainer(..., args = config_with_cp_2, context_parallel_size = 1): the keyword wins in
# _backwards_compatible_trainer, so CP is off and the class must stay unwrapped.
import types
def on_init(self):
self.args = types.SimpleNamespace(context_parallel_size = 1)
Trainer, args, built = _gated_trainer(monkeypatch, install_cp_methods = False, on_init = on_init)
Trainer(None, args = args, context_parallel_size = forwarded)
assert built == []
assert not any(hasattr(Trainer.__dict__[name], "__wrapped__") for name in _CP_METHODS)
@pytest.mark.parametrize("rope", ["dynamic", "longrope"])
def test_ulysses_seq_len_rope_uses_the_whole_sequence(monkeypatch, rope):
# dynamic / longrope choose frequencies from max(position_ids) + 1: rank 0 of 2 sees positions
# 0..15 (within the original 16), the whole sequence 0..31 (beyond it).
import copy
import types
from transformers import LlamaConfig, LlamaForCausalLM, Phi3Config, Phi3ForCausalLM
small = dict(
hidden_size = 32,
intermediate_size = 64,
num_hidden_layers = 1,
num_attention_heads = 4,
num_key_value_heads = 4,
vocab_size = 64,
)
if rope == "dynamic":
config = LlamaConfig(
max_position_embeddings = 16,
rope_scaling = {"rope_type": "dynamic", "factor": 2.0},
**small,
)
model = LlamaForCausalLM(config)
else:
factors = [1.0 + i for i in range(4)]
config = Phi3Config(
max_position_embeddings = 64,
original_max_position_embeddings = 16,
rope_scaling = {"type": "longrope", "short_factor": [1.0] * 4, "long_factor": factors},
pad_token_id = 0,
**small,
)
model = Phi3ForCausalLM(config)
rotary = model.model.rotary_emb
# rope_scaling: the form transformers 4.57 reads and 5.x converts, so neither corner falls back to default.
assert rotary.rope_type == rope
x = torch.ones(1, 32, 8)
full = torch.arange(32)[None]
# A copy: dynamic NTK caches the frequencies it last computed.
expected = [t[:, :16] for t in copy.deepcopy(rotary)(x, full)]
manager = _manager(size = 2)
manager.backend = "ulysses"
manager.attach_attention_hooks(model)
manager.mesh = types.SimpleNamespace(get_local_rank = lambda: 0)
manager.full_position_ids = full
monkeypatch.setattr(cp, "_ACTIVE_MANAGER", manager)
for got, want in zip(rotary(x[:, :16], full[:, :16]), expected):
torch.testing.assert_close(got, want)
@pytest.mark.parametrize(
"model_type, blocker",
[
("kimi_linear", "KimiDeltaAttention"),
("kimi_k3", "KimiDeltaAttention"),
("minimax", "MiniMaxLightningAttention"),
("deepseek_v32", "DeepseekV32Attention"),
("llama", None),
("minimax_m2", None),
],
)
def test_ulysses_packing_blocker_classifies_model_id_strings(model_type, blocker):
# A model ID string has no modules yet: its config's model type must give the instance outcome,
# so SFT also turns eval_packing off for hybrids whose Ulysses mixers refuse packed batches.
assert cp.ulysses_packing_blocker("some/model-id", model_types = (model_type,)) == blocker