Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

4.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team.
"""One gloo rank for test_plain_trainer_ddp_gc.py. No unsloth import at module level: a spawned rank must set up the GPU-free harness first."""

from __future__ import annotations

import os
import runpy
import tempfile


def _report(queue, item):
    """Deliver the result before teardown. put() hands it to a feeder thread that needs the GIL, and a
    teardown blocking in C while holding it left both ranks' results unsent after they had trained."""
    queue.put(item)
    queue.close()
    queue.join_thread()


def run_rank(rank, port, cases, queue):
    os.environ.update(
        MASTER_ADDR = "127.0.0.1",
        MASTER_PORT = str(port),
        RANK = str(rank),
        LOCAL_RANK = str(rank),
        WORLD_SIZE = "2",
        LOCAL_WORLD_SIZE = "2",
        ACCELERATE_USE_CPU = "true",
    )
    runpy.run_path(
        os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "conftest.py")
    )
    import torch

    torch.set_num_threads(1)
    import unsloth  # noqa: F401
    import peft
    import transformers
    from unsloth.models._utils import patch_gradient_accumulation_fix

    # What FastLanguageModel / FastModel.from_pretrained installs on transformers.Trainer.
    patch_gradient_accumulation_fix(transformers.Trainer)

    out = {}
    try:
        for name, trainer_name, user_value in cases:
            torch.manual_seed(0)
            config = transformers.LlamaConfig(
                vocab_size = 64,
                hidden_size = 16,
                intermediate_size = 32,
                num_hidden_layers = 2,
                num_attention_heads = 2,
                num_key_value_heads = 1,
            )
            model = transformers.LlamaForCausalLM(config)
            # Reentrant checkpointing armed on the model (Unsloth's get_peft_model does this), not in the args.
            model.gradient_checkpointing_enable(
                gradient_checkpointing_kwargs = {"use_reentrant": True}
            )
            model.enable_input_require_grads()
            model = peft.get_peft_model(
                model,
                peft.LoraConfig(
                    r = 4, lora_alpha = 8, target_modules = ["q_proj", "v_proj", "gate_proj"]
                ),
            )
            g = torch.Generator().manual_seed(7)
            rows = []
            for _ in range(8):
                ids = torch.randint(1, 64, (12,), generator = g)
                rows.append(
                    {"input_ids": ids, "attention_mask": torch.ones_like(ids), "labels": ids}
                )
            extra = {} if user_value is None else {"ddp_find_unused_parameters": user_value}
            args_cls = (
                transformers.Seq2SeqTrainingArguments
                if trainer_name == "Seq2SeqTrainer"
                else transformers.TrainingArguments
            )
            args = args_cls(
                output_dir = tempfile.mkdtemp(prefix = "ddp_gc_"),
                per_device_train_batch_size = 2,
                max_steps = 2,
                learning_rate = 1e-2,
                use_cpu = True,
                report_to = "none",
                save_strategy = "no",
                logging_steps = 1,
                dataloader_num_workers = 0,
                **extra,
            )
            trainer = getattr(transformers, trainer_name)(
                model = model, args = args, train_dataset = rows
            )
            error = None
            try:
                trainer.train()
            except RuntimeError as e:
                error = str(e).splitlines()[0] or type(e).__name__
            ddp = trainer.model_wrapped
            out[name] = dict(
                error = error,
                find_unused = getattr(ddp, "find_unused_parameters", None),
                is_ddp = isinstance(ddp, torch.nn.parallel.DistributedDataParallel),
                args_value = args.ddp_find_unused_parameters,
                losses = [h["loss"] for h in trainer.state.log_history if "loss" in h],
            )
            if error is not None:
                # A rank that raised leaves the group unusable for the next case.
                break
        _report(queue, (rank, out))
    except BaseException:
        import traceback
        _report(queue, (rank, {"error": traceback.format_exc()}))
        raise
    if not any(case.get("error") for case in out.values()):
        import torch.distributed as dist
        if dist.is_initialized():
            dist.destroy_process_group()