Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallel: allow text-only Qwen3.5 checkpoints that carry a vision_config (#16072)
Daniel Hanc41ed6ad9c14
diff --git a/tests/test_ulysses_hybrid.py b/tests/test_ulysses_hybrid.pyindex b2f730acb4f..fe794b46b54 100644--- a/tests/test_ulysses_hybrid.py+++ b/tests/test_ulysses_hybrid.py@@ -56,6 +56,22 @@ def _config(kind, **overrides):return transformers.Qwen3_5TextConfig(**common), transformers.Qwen3_5ForCausalLM+def _vlm(**overrides):+ # Shipped Qwen3.5 checkpoints carry a vision_config even for text-only training.+ torch.manual_seed(0)+ text_config, _ = _config("qwen3_5", **overrides)+ config = transformers.Qwen3_5Config(+ text_config = text_config.to_dict(),+ vision_config = dict(+ depth = 1, hidden_size = 16, intermediate_size = 32, num_heads = 2, out_hidden_size = 32+ ),+ tie_word_embeddings = False,+ )+ model = transformers.Qwen3_5ForConditionalGeneration(config).train()+ model.config._attn_implementation = "sdpa"+ return model++def _model(kind, **overrides):torch.manual_seed(0)config, cls = _config(kind, **overrides)@@ -104,14 +120,27 @@ def _loss(logits, shift_labels, count):def _parity_worker(rank, world, kind, control):ad.HAS_FLASH_ATTENTION = Falsead.HAS_XFORMERS = False- model = _model(kind)+ model = _vlm() if kind == "qwen3_5_vlm" else _model(kind)+ if kind == "qwen3_5_vlm":+ assert model.config.vision_config is not None+ cp._check_cp_model(model, world, "ulysses")_force_torch_kernels(model)+ calls = []+ exchange = uh.ulysses_gated_delta_forward++ def counted(*args, **kwargs):+ calls.append(1)+ return exchange(*args, **kwargs)++ uh.ulysses_gated_delta_forward = countedids = torch.randint(0, 64, (2, 13), generator = torch.Generator().manual_seed(1))shift = F.pad(ids, (0, 1), value = -100)[:, 1:]count = shift.ne(-100).sum()- _loss(model(input_ids = ids, use_cache = False).logits, shift, count).backward()- reference = {n: p.grad.clone() for n, p in model.named_parameters()}+ reference_loss = _loss(model(input_ids = ids, use_cache = False).logits, shift, count)+ reference_loss.backward()+ reference = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}model.zero_grad(set_to_none = True)+ assert not callsmanager = _manager(world, rank)manager.attach_attention_hooks(model)@@ -125,10 +154,23 @@ def _parity_worker(rank, world, kind, control):out = model(input_ids = inputs["input_ids"], position_ids = inputs["position_ids"], use_cache = False)- _loss(out.logits, inputs["shift_labels"], count).backward()+ loss = _loss(out.logits, inputs["shift_labels"], count)+ loss.backward()+ if not control:+ config_kind = "qwen3_5" if kind == "qwen3_5_vlm" else kind+ assert len(calls) == sum(+ t == "linear_attention" for t in _config(config_kind)[0].layer_types+ )+ total = loss.detach().clone()+ dist.all_reduce(total)+ torch.testing.assert_close(total, reference_loss.detach(), rtol = 1e-5, atol = 1e-6)for name, param in model.named_parameters():grad = torch.zeros_like(param) if param.grad is None else param.graddist.all_reduce(grad)+ if name not in reference:+ # The vision tower gets no gradient from text-only batches, on either side.+ assert not grad.any(), name+ continuetorch.testing.assert_close(grad, reference[name], rtol = 1e-4, atol = 1e-6, msg = name)@@ -160,6 +202,109 @@ def test_ulysses_hybrid_gradients_match_one_rank(kind):_spawn(_parity_worker, 2, kind, False)+ not hasattr(transformers, "Qwen3_5Config"), reason = "transformers without Qwen3.5"+)+def test_text_only_qwen3_5_checkpoint_with_a_vision_config_matches_one_rank():+ _spawn(_parity_worker, 2, "qwen3_5_vlm", False)+++ not hasattr(transformers, "Qwen3_5Config"), reason = "transformers without Qwen3.5"+)+def test_qwen3_5_checkpoint_with_a_vision_config_passes_the_gate():+ model = _vlm()+ assert cp._hybrid_config_blocker(model) is None+ assert cp._supports_context_parallel(model, "ulysses")+ cp._check_cp_model(model, 2, "ulysses")+++def _ddp_text_only_worker(rank, world, freeze_vision):+ from torch.nn.parallel import DistributedDataParallel++ ad.HAS_FLASH_ATTENTION = False+ ad.HAS_XFORMERS = False+ model = _vlm()+ _force_torch_kernels(model)+ if freeze_vision:+ model.model.visual.requires_grad_(False)+ manager = _manager(world, rank)+ manager.mesh = types.SimpleNamespace(+ get_group = lambda: dist.group.WORLD, get_local_rank = lambda: rank+ )+ manager.attach_attention_hooks(model)+ ddp = DistributedDataParallel(model, find_unused_parameters = False)+ for step in range(2):+ ids = torch.randint(0, 64, (2, 13), generator = torch.Generator().manual_seed(step))+ shift = F.pad(ids, (0, 1), value = -100)[:, 1:]+ inputs = {"input_ids": ids, "shift_labels": shift}+ manager._prepare_inputs(inputs)+ with manager.apply(inputs):+ out = ddp(+ input_ids = inputs["input_ids"], position_ids = inputs["position_ids"], use_cache = False+ )+ _loss(out.logits, inputs["shift_labels"], shift.ne(-100).sum()).backward()+++ not hasattr(transformers, "Qwen3_5Config"), reason = "transformers without Qwen3.5"+)+def test_ddp_text_only_fails_at_step_two_with_a_trainable_vision_tower():+ # The unused vision tower means DDP's reduction never finishes.+ with pytest.raises(Exception, match = "finished reduction in the prior iteration"):+ _spawn(_ddp_text_only_worker, 2, False)+ _spawn(_ddp_text_only_worker, 2, True)+++def _cp_trainer(+ model,+ distributed_type,+ num_processes = 2,+ fsdp_version = None,+):+ plugin = None if fsdp_version is None else types.SimpleNamespace(fsdp_version = fsdp_version)+ accelerator = types.SimpleNamespace(+ distributed_type = types.SimpleNamespace(name = distributed_type),+ num_processes = num_processes,+ state = types.SimpleNamespace(fsdp_plugin = plugin, parallelism_config = None),+ dispatch_batches = None,+ )+ return types.SimpleNamespace(accelerator = accelerator, model = model)+++ not hasattr(transformers, "Qwen3_5Config"), reason = "transformers without Qwen3.5"+)+def test_ddp_refuses_a_trainable_vision_tower_but_not_fsdp2_frozen_or_one_process():+ pytest.importorskip("accelerate", minversion = "1.10.0")+ model = _vlm()+ trainer = _cp_trainer(model, "MULTI_CPU")+ with pytest.raises(NotImplementedError, match = "finetune_vision_layers = False"):+ cp._check_cp_trainer(trainer, 2, "ulysses", [model], allow_fsdp2 = True)+ # FSDP2 reduces unused parameters as zeros; no DDP reducer runs without several processes.+ fsdp2 = _cp_trainer(model, "FSDP", fsdp_version = 2)+ assert cp._check_cp_trainer(fsdp2, 2, "ulysses", [model], allow_fsdp2 = True)+ for single in (_cp_trainer(model, "NO"), _cp_trainer(model, "MULTI_CPU", num_processes = 1)):+ cp._check_cp_trainer(single, 2, "ulysses", [model], allow_fsdp2 = True)+ model.model.visual.requires_grad_(False)+ cp._check_cp_trainer(trainer, 2, "ulysses", [model], allow_fsdp2 = True)+++ "key", ["pixel_values", "pixel_values_videos", "image_grid_thw", "video_grid_thw"]+)+def test_batches_with_image_or_video_inputs_are_refused(key):+ inputs = {"input_ids": torch.ones(1, 6, dtype = torch.long), key: torch.ones(1, 3)}+ with pytest.raises(ValueError, match = key):+ _manager(2, 0)._prepare_inputs(inputs)+ with pytest.raises(ValueError, match = "image / video"):+ with _manager(2, 0).apply(dict(inputs)):+ pass+ # A collator's None placeholder is a text-only batch.+ inputs[key] = None+ _manager(2, 0)._prepare_inputs(inputs)++def test_shard_local_gated_delta_is_wrong():# Negative control: the stock forward on a sequence shard restarts the scan at the shard seam.with pytest.raises(Exception, match = "AssertionError"):@@ -236,7 +381,7 @@ def test_ulysses_accepts_right_padding_with_mask_derived_positions():_manager(2, 0)._prepare_inputs(inputs)-def test_hybrid_refuses_multimodal_and_router_aux_loss():+def test_hybrid_refuses_router_aux_loss_but_not_a_vision_config():model = _model(KINDS[0])assert cp._supports_context_parallel(model, "ulysses")model.config.output_router_logits = True@@ -244,8 +389,8 @@ def test_hybrid_refuses_multimodal_and_router_aux_loss():assert not cp._supports_context_parallel(model, "ulysses")model.config.output_router_logits = Falsemodel.config.vision_config = {"model_type": "qwen3_5_vision"}- assert cp._hybrid_config_blocker(model) == "multimodal models"- assert not cp._supports_context_parallel(model, "ulysses")+ assert cp._hybrid_config_blocker(model) is None+ assert cp._supports_context_parallel(model, "ulysses")model.config.vision_config = None# transformers 4.x Qwen3-Next: per-expert modules, each run only when routed on that rank.model.mlp_holder = torch.nn.Module()diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 88dd53c6a20..dc4f8226615 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -168,6 +168,22 @@ _ACTIVE_MANAGER: Optional["ContextParallelManager"] = None# shift_labels is built before sharding so each shard's last token keeps its next-token target._BUFFER_NAMES = ("input_ids", "attention_mask", "labels", "position_ids", "shift_labels")_PAD_VALUES = {"labels": -100, "shift_labels": -100, "attention_mask": 0, "input_ids": 0}+# Only the text buffers above are sharded; image / video / audio features would stay whole on every rank.+_MULTIMODAL_INPUTS = (+ "pixel_values",+ "pixel_values_videos",+ "image_grid_thw",+ "video_grid_thw",+ "input_features",+)+++def _multimodal_inputs(inputs: dict) -> list:+ return sorted(+ name+ for name, value in inputs.items()+ if value is not None and (name in _MULTIMODAL_INPUTS or name.startswith("pixel_values"))+ )_INSTALLED_MESH = []@@ -194,9 +210,7 @@ def _positions_restart(position_ids, attention_mask) -> bool:def _hybrid_config_blocker(model) -> Optional[str]:config = getattr(model, "config", None)- if getattr(config, "vision_config", None) is not None:- # Only text buffers are sharded; image / video features would stay whole on every rank.- return "multimodal models"+ # vision_config is allowed (Qwen3.5 ships one); image / video batches are refused in _prepare_inputs.text_config = getattr(config, "text_config", None) or configif getattr(text_config, "output_router_logits", False):# The load balancing loss is a product of per-sequence means, not a sum over shards.@@ -307,6 +321,13 @@ class ContextParallelManager:raise ValueError("Unsloth: context parallelism needs input_ids batches (not inputs_embeds).")+ multimodal = _multimodal_inputs(inputs)+ if multimodal:+ raise ValueError(+ "Unsloth: context parallelism shards text tokens only; batches with image / video "+ f"inputs ({', '.join(multimodal)}) are not supported. Train on text-only data, or "+ "use context_parallel_size = 1."+ )bsz, seq_len = input_ids.shapepacked_seq_lengths = inputs.get("packed_seq_lengths")ulysses = self.backend == "ulysses"@@ -729,6 +750,33 @@ def _check_cp_model()+# Encoders and projectors a text-only batch never runs; context parallelism only takes text batches.+_MEDIA_TOWER_NAMES = frozenset(+ (+ "visual",+ "vision_tower",+ "vision_model",+ "vision_encoder",+ "multi_modal_projector",+ "mm_projector",+ "embed_vision",+ "audio_tower",+ "audio_model",+ "embed_audio",+ )+)+++def _trainable_media_tower_parameters(model) -> list:+ if not isinstance(model, torch.nn.Module):+ return []+ return [+ name+ for name, param in model.named_parameters()+ if param.requires_grad and not _MEDIA_TOWER_NAMES.isdisjoint(name.split("."))+ ]++def _check_cp_trainer(self,size: int,@@ -760,6 +808,20 @@ def _check_cp_trainer()if fsdp2:_check_fsdp2_parallelism_config(accelerator)+ elif (+ distributed_type.startswith("MULTI_")+ and (getattr(accelerator, "num_processes", 2) or 2) > 1+ ):+ # Unused trainable towers break DDP (find_unused_parameters = False for reentrant checkpointing).+ for model in models:+ unused = _trainable_media_tower_parameters(model)+ if unused:+ raise NotImplementedError(+ "Unsloth: context parallelism trains on text-only batches, so the trainable "+ f"vision / audio tower ({len(unused)} parameters, e.g. {unused[0]}) never gets a "+ "gradient and DDP fails at the second step. Freeze it (for LoRA, "+ "finetune_vision_layers = False) or train with FSDP2."+ )if getattr(accelerator, "dispatch_batches", None):raise NotImplementedError("Unsloth: context parallelism does not support dispatch_batches.")_refuse_iterable_datasets(