Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

3.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team.
"""A plain transformers Trainer (and Seq2SeqTrainer) on a LoRA model with reentrant gradient
checkpointing must train under DDP: Unsloth picks find_unused_parameters = False as it does for its
TRL trainers, where transformers' True fails with "marked as ready twice". An explicit value is kept."""

from __future__ import annotations

import os
import queue as queue_module
import sys

import pytest

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))  # spawned ranks inherit sys.path
import _plain_trainer_ddp_gc_worker as worker  # noqa: E402

pytest.importorskip("peft")
pytest.importorskip("accelerate")
pytestmark = pytest.mark.skipif(sys.platform == "win32", reason = "two gloo ranks over TCP localhost")


def _failed(results):
    """A rank that raised (in backward, in the save) can leave its peer blocked in a collective:
    once one has, wait only briefly for the other."""
    return any(
        out.get("error") or any(isinstance(c, dict) and c.get("error") for c in out.values())
        for out in results.values()
    )


def _free_port():
    from tests._rendezvous_port import free_port
    return free_port()


def _run(cases):
    import torch.multiprocessing as mp

    ctx = mp.get_context("spawn")
    queue = ctx.Queue()
    port = _free_port()
    procs = [ctx.Process(target = worker.run_rank, args = (r, port, cases, queue)) for r in range(2)]
    for p in procs:
        p.start()
    results = {}
    try:
        while len(results) < len(procs):
            try:
                # Below the 330 s pytest timeout, so a hang fails here with what did arrive.
                rank, out = queue.get(timeout = 60 if _failed(results) else 240)
            except queue_module.Empty:
                break
            results[rank] = out
    finally:
        for p in procs:
            p.join(5 if len(results) < len(procs) else 30)
            if p.is_alive():
                p.kill()
    assert results, "no rank reported"
    for out in results.values():
        assert "error" not in out, out.get("error")
    return results


def test_plain_trainers_train_under_ddp_with_reentrant_checkpointing():
    results = _run([("trainer", "Trainer", None), ("seq2seq", "Seq2SeqTrainer", None)])
    for rank in results:
        for name in ("trainer", "seq2seq"):
            case = results[rank].get(name)
            assert case is not None, (rank, name, results[rank])
            assert case["error"] is None, (rank, name, case["error"])
            assert case["is_ddp"] and case["find_unused"] is False
            assert case["args_value"] is False
            assert len(case["losses"]) == 2
    assert sorted(results) == [0, 1]
    # DDP averages the gradients: both ranks log the same losses.
    assert results[0]["trainer"]["losses"] == results[1]["trainer"]["losses"]


def test_explicit_user_value_is_kept():
    results = _run([("user_true", "Trainer", True)])
    for rank in results:
        case = results[rank]["user_true"]
        assert case["args_value"] is True
        assert case["find_unused"] is True
        # The user's choice is honoured, so the reentrant checkpointing failure is theirs to see.
        assert case["error"] is not None and "ready" in case["error"], case["error"]