Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MFU logging under fullgraph, NCCL logs under distributed debug, token-budget metrics across ranks (#14207)

Daniel Han9f78f5e5803f
diff --git a/tests/test_debug_distributed_env.py b/tests/test_debug_distributed_env.py
index a96b97b9c27..13bfb04307b 100644
--- a/tests/test_debug_distributed_env.py
+++ b/tests/test_debug_distributed_env.py
@@ -83,3 +83,69 @@ def test_existing_process_group_leaves_the_environment_untouched(monkeypatch, ca
assert _module().apply_debug_distributed_env(env) == []
assert env == {"UNSLOTH_DEBUG_DISTRIBUTED": "1"}
assert "already exists" in capsys.readouterr().err
+
+
[email protected]("user_value", [None, "TRACE"])
+def test_nccl_debug_survives_zoo_logging_switches(user_value):
+ # unsloth_zoo sets NCCL_DEBUG=WARN on import under UNSLOTH_ENABLE_LOGGING / UNSLOTH_COMPILE_DEBUG.
+ mod = _module()
+ env = {"UNSLOTH_DEBUG_DISTRIBUTED": "1", "UNSLOTH_ENABLE_LOGGING": "1"}
+ if user_value is not None:
+ env["NCCL_DEBUG"] = user_value
+ mod.apply_debug_distributed_env(env)
+ restore = mod.keep_debug_nccl_env(env)
+ env["NCCL_DEBUG"] = "WARN"
+ restore()
+ assert env["NCCL_DEBUG"] == (user_value or "INFO")
+
+
+def test_keep_nccl_debug_is_a_no_op_when_off():
+ mod = _module()
+ env = {"UNSLOTH_ENABLE_LOGGING": "1"}
+ restore = mod.keep_debug_nccl_env(env)
+ env["NCCL_DEBUG"] = "WARN"
+ restore()
+ assert env["NCCL_DEBUG"] == "WARN"
+
+
+def test_gpu_init_restores_nccl_debug_around_zoo_import():
+ source = (_UNSLOTH / "_gpu_init.py").read_text(encoding = "utf-8")
+ zoo = source.index("\n import unsloth_zoo\n")
+ keep = source.rfind("keep_debug_nccl_env()", 0, zoo)
+ restore = source.find("_restore_nccl_debug()", zoo)
+ assert keep != -1 and restore != -1
+ assert source.count("\n import unsloth_zoo\n") == 1
+ assert restore < source.index("except PackageNotFoundError", zoo)
+
+
[email protected]("switch", ["UNSLOTH_ENABLE_LOGGING", "UNSLOTH_COMPILE_DEBUG"])
+def test_real_zoo_import_keeps_nccl_info(switch):
+ import os
+ import subprocess
+ import sys
+
+ if importlib.util.find_spec("unsloth_zoo") is None:
+ pytest.skip("unsloth_zoo not installed")
+ code = (
+ "import importlib.util, os, sys\n"
+ "spec = importlib.util.spec_from_file_location('dd', sys.argv[1])\n"
+ "m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n"
+ "m.apply_debug_distributed_env()\n"
+ "restore = m.keep_debug_nccl_env()\n"
+ "try:\n"
+ " import unsloth_zoo\n"
+ "except BaseException:\n"
+ " pass\n" # CPU hosts raise after the env block ran; only NCCL_DEBUG matters here.
+ "restore()\n"
+ "print('NCCL_DEBUG=' + os.environ.get('NCCL_DEBUG', ''))\n"
+ )
+ env = {k: v for k, v in os.environ.items() if k != "NCCL_DEBUG"}
+ env.update({"UNSLOTH_DEBUG_DISTRIBUTED": "1", switch: "1"})
+ out = subprocess.run(
+ [sys.executable, "-c", code, str(_UNSLOTH / "distributed_debug.py")],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ ).stdout
+ assert "NCCL_DEBUG=INFO" in out.splitlines()
diff --git a/tests/test_mfu_logging.py b/tests/test_mfu_logging.py
index 6a16a0f2928..331840de963 100644
--- a/tests/test_mfu_logging.py
+++ b/tests/test_mfu_logging.py
@@ -99,7 +99,7 @@ def test_sequence_lengths_exclude_padding_and_split_packing():
assert mfu._sequence_lengths({"attention_mask": mask}, ()).tolist() == [3, 2]
packed = torch.tensor([[0, 1, 2, 0, 1, 0, 1, 2, 3]])
lengths = mfu._sequence_lengths({"input_ids": packed, "position_ids": packed}, ())
- assert lengths.tolist() == [3, 2, 4]
+ assert [n for n in lengths.tolist() if n] == [3, 2, 4]
ids = torch.zeros(2, 5, dtype = torch.long)
assert mfu._sequence_lengths({"input_ids": ids}, ()).tolist() == [5, 5]
@@ -571,3 +571,51 @@ def test_diffusion_and_data_parallel_report_nothing():
callback = mfu.UnslothMFUCallback(peak_tflops = 1)
callback.on_train_begin(SimpleNamespace(n_gpu = 1), _State(), None, model = model)
assert not callback._enabled and callback._handle is None
+
+
+def test_position_id_packing_counts_under_fullgraph_compile():
+ # Position-id-only padding free batches (TRL padding_free) must not add a data dependent
+ # branch to a model compiled with fullgraph = True.
+ from types import SimpleNamespace
+
+ class Tiny(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(4, 4, bias = False)
+ self.config = SimpleNamespace(num_hidden_layers = 1, num_attention_heads = 1, hidden_size = 4)
+
+ def forward(
+ self,
+ input_ids,
+ position_ids = None,
+ ):
+ return self.proj(torch.nn.functional.one_hot(input_ids, 4).float())
+
+ torch._dynamo.reset()
+ model = Tiny().train()
+ callback = mfu.UnslothMFUCallback()
+ callback.on_train_begin(SimpleNamespace(n_gpu = 1), None, None, model = model)
+ try:
+ compiled = torch.compile(model, backend = "eager", fullgraph = True)
+ ids = torch.tensor([[0, 1, 2, 1, 2, 3, 0]])
+ positions = torch.tensor([[0, 1, 2, 0, 1, 0, 1]])
+ compiled(input_ids = ids, position_ids = positions).sum().backward()
+ assert int(callback._tokens) == 7
+ assert int(callback._pairs) == 6 + 3 + 3
+ finally:
+ callback.on_train_end(None, None, None)
+ torch._dynamo.reset()
+ # Leading tokens before the first restart are skipped; no restart counts the whole row.
+ lead = mfu._sequence_lengths({"position_ids": torch.tensor([[3, 4, 0, 1]])}, ())
+ assert [n for n in lead.tolist() if n] == [2]
+ none = mfu._sequence_lengths({"position_ids": torch.tensor([[1, 2, 3]])}, ())
+ assert [n for n in none.tolist() if n] == [3]
+
+
+def test_h200_nvl_uses_its_own_dense_peak(monkeypatch):
+ # NVIDIA H200 datasheet: NVL 1,671 TFLOPS BF16 with sparsity -> 835.5 dense; SXM 1,979 -> 989.
+ monkeypatch.delenv("UNSLOTH_PEAK_TFLOPS", raising = False)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ for name, peak in (("NVIDIA H200 NVL", 835.5), ("NVIDIA H200", 989.0)):
+ monkeypatch.setattr(torch.cuda, "get_device_name", lambda device = None, n = name: n)
+ assert mfu.get_peak_tflops(dtype = torch.bfloat16) == peak, name
diff --git a/tests/test_token_budget_metrics_distributed.py b/tests/test_token_budget_metrics_distributed.py
new file mode 100644
index 00000000000..352c53d450d
--- /dev/null
+++ b/tests/test_token_budget_metrics_distributed.py
@@ -0,0 +1,224 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_TOKEN_BUDGET under several processes: TRL gathers each forward's metric sums across ranks,
+so split-chunk entries must be folded with token counts summed over ranks to match an unsplit step.
+Two CPU ranks over gloo."""
+
+from __future__ import annotations
+
+import contextlib
+import importlib.util
+import multiprocessing
+import os
+import sys
+import tempfile
+from collections import defaultdict
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+dist = pytest.importorskip("torch.distributed")
+
+if not dist.is_available() or not dist.is_gloo_available() or sys.platform != "linux":
+ pytest.skip("needs torch.distributed with gloo on Linux (fork)", allow_module_level = True)
+
+_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "utils" / "token_budget.py"
+_spec = importlib.util.spec_from_file_location("_unsloth_token_budget_dist", _PATH)
+tb = importlib.util.module_from_spec(_spec)
+sys.modules[_spec.name] = tb
+_spec.loader.exec_module(tb)
+
+_BUDGET = 36
+_WORLD = 2
+
+
+class _Accelerator:
+ distributed_type = "MULTI_GPU"
+ num_processes = _WORLD
+ device = torch.device("cpu")
+
+ @contextlib.contextmanager
+ def no_sync(self, model):
+ yield
+
+ def gather_sum(self, value):
+ # accelerator.gather_for_metrics(x).sum() for a scalar.
+ value = torch.as_tensor(value, dtype = torch.float64).clone()
+ dist.all_reduce(value, op = dist.ReduceOp.SUM)
+ return value
+
+
+class _TRL115Trainer:
+ """TRL 1.15 SFTTrainer.compute_loss (sft_trainer.py:1638-1642): accuracy and entropy are global
+ sums over valid labels divided by the global valid-label count."""
+
+ num_virtual_tokens = 0
+
+ def __init__(self):
+ self._metrics = {"train": defaultdict(list), "eval": defaultdict(list)}
+ self.accelerator = _Accelerator()
+ self.args = type("Args", (), {"gradient_accumulation_steps": 1})()
+ self.model_accepts_loss_kwargs = True
+ self.label_smoother = None
+
+ def compute_loss(self, inputs):
+ ids = inputs["input_ids"]
+ correct, entropy = (ids % 2 == 0).double(), ids.double()
+ label_mask = inputs["labels"][..., 1:] != -100
+ num_valid = self.accelerator.gather_sum(label_mask.sum())
+ entropy_sum = self.accelerator.gather_sum((entropy[..., 1:] * label_mask).sum())
+ num_correct = self.accelerator.gather_sum((correct[..., 1:] * label_mask).sum())
+ self._metrics["train"]["entropy"].append(float(entropy_sum / num_valid))
+ self._metrics["train"]["mean_token_accuracy"].append(float(num_correct / num_valid))
+
+ def training_step(
+ self,
+ model,
+ inputs,
+ num_items_in_batch = None,
+ ):
+ self.compute_loss(inputs)
+ return torch.tensor(1.0)
+
+
+class _TRL022Trainer(_TRL115Trainer):
+ """TRL 0.22.2 (sft_trainer.py:1056-1061, 1099-1105): entropy is the mean of per-rank attention
+ ratios (not refoldable), accuracy is global correct over global shifted labels."""
+
+ def compute_loss(self, inputs):
+ ids = inputs["input_ids"]
+ correct, entropy = (ids % 2 == 0).double(), ids.double()
+ attention_mask = inputs["attention_mask"].double()
+ local_entropy = (entropy * attention_mask).sum() / attention_mask.sum()
+ entropy = self.accelerator.gather_sum(local_entropy) / _WORLD
+ self._metrics["train"]["entropy"].append(float(entropy))
+ shift_labels = inputs["labels"][..., 1:]
+ valid = shift_labels != -100
+ correct_tokens = self.accelerator.gather_sum((correct[..., 1:].bool() & valid).sum())
+ total_tokens = self.accelerator.gather_sum(valid.sum())
+ self._metrics["train"]["mean_token_accuracy"].append(float(correct_tokens / total_tokens))
+
+
+_TRAINERS = {"trl115": _TRL115Trainer, "trl022": _TRL022Trainer}
+
+
+def _rank_batch(rank):
+ # Unequal row lengths per rank, so chunks differ in token count within and across ranks.
+ lengths = [[2, 3, 3, 12], [9, 2, 12, 5]][rank]
+ width = 12
+ gen = torch.Generator().manual_seed(rank)
+ input_ids = torch.randint(1, 50, (len(lengths), width), generator = gen)
+ attention_mask = torch.zeros(len(lengths), width, dtype = torch.long)
+ for row, n in enumerate(lengths):
+ attention_mask[row, :n] = 1
+ labels = input_ids.masked_fill(attention_mask == 0, -100)
+ labels[:, 0] = -100
+ return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}
+
+
+def _worker(
+ rank,
+ init_file,
+ trainer_name,
+ queue,
+ shift_labels_on_rank = None,
+):
+ try:
+ dist.init_process_group(
+ "gloo", init_method = f"file://{init_file}", rank = rank, world_size = _WORLD
+ )
+ trainer_cls = _TRAINERS[trainer_name]
+ batch = _rank_batch(rank)
+ if rank == shift_labels_on_rank:
+ # A second label source makes only this rank decline to fold.
+ batch["shift_labels"] = batch["labels"][..., 1:].contiguous()
+
+ os.environ.pop("UNSLOTH_TOKEN_BALANCE", None)
+ os.environ.pop("UNSLOTH_TOKEN_BUDGET", None)
+ reference = trainer_cls()
+ tb.install_token_budget(reference)
+ reference.training_step(None, dict(batch), 10)
+
+ os.environ["UNSLOTH_TOKEN_BUDGET"] = str(_BUDGET)
+ trainer = trainer_cls()
+ tb.install_token_budget(trainer)
+ trainer.training_step(None, dict(batch), 10)
+ n_chunks = len(tb.split_batch_by_token_budget(dict(batch), _BUDGET))
+ queue.put(
+ (
+ rank,
+ {
+ "reference": {k: list(v) for k, v in reference._metrics["train"].items()},
+ "split": {k: list(v) for k, v in trainer._metrics["train"].items()},
+ "splits": trainer._unsloth_token_budget_splits,
+ "n_chunks": n_chunks,
+ },
+ )
+ )
+ dist.destroy_process_group()
+ except BaseException as error:
+ queue.put((rank, repr(error)))
+ raise
+
+
+def _run(trainer_name, shift_labels_on_rank = None):
+ ctx = multiprocessing.get_context("fork")
+ queue = ctx.Queue()
+ with tempfile.TemporaryDirectory() as tmp:
+ init_file = os.path.join(tmp, "pg")
+ procs = [
+ ctx.Process(
+ target = _worker,
+ args = (rank, init_file, trainer_name, queue, shift_labels_on_rank),
+ )
+ for rank in range(_WORLD)
+ ]
+ for proc in procs:
+ proc.start()
+ results = dict(queue.get(timeout = 120) for _ in procs)
+ for proc in procs:
+ proc.join(timeout = 60)
+ for rank, result in results.items():
+ assert isinstance(result, dict), f"rank {rank} failed: {result}"
+ return results
+
+
+def _mean(values):
+ return sum(values) / len(values)
+
+
[email protected]("trainer_name", ["trl115", "trl022"])
+def test_multi_process_split_accuracy_matches_unsplit(trainer_name):
+ results = _run(trainer_name)
+ for result in results.values():
+ assert result["splits"] == 1 and result["n_chunks"] > 1
+ reference, split = result["reference"], result["split"]
+ assert len(split["mean_token_accuracy"]) == 1
+ assert _mean(split["mean_token_accuracy"]) == pytest.approx(
+ _mean(reference["mean_token_accuracy"]), rel = 1e-9
+ )
+ # Both ranks log the same gathered value, as TRL does.
+ assert results[0]["split"] == results[1]["split"]
+
+
+def test_multi_process_split_entropy_matches_unsplit_on_label_weighted_trl():
+ for result in _run("trl115").values():
+ reference, split = result["reference"], result["split"]
+ assert len(split["entropy"]) == 1
+ assert _mean(split["entropy"]) == pytest.approx(_mean(reference["entropy"]), rel = 1e-9)
+
+
+def test_multi_process_attention_entropy_left_unfolded():
+ # Older TRL averages per-rank entropy ratios; a token-weighted fold cannot reproduce that.
+ for result in _run("trl022").values():
+ assert len(result["split"]["entropy"]) == result["n_chunks"]
+
+
+def test_ranks_that_disagree_on_folding_neither_fold_nor_hang():
+ # The fold adds a collective; if one rank cannot fold, every rank must skip it (else the others hang).
+ results = _run("trl115", shift_labels_on_rank = 1)
+ for result in results.values():
+ assert result["splits"] == 1
+ assert len(result["split"]["mean_token_accuracy"]) == result["n_chunks"] > 1
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 5d06655d61d..d0029b77ebe 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -232,7 +232,17 @@ try:
"Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n"
"Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`"
)
+ _restore_nccl_debug = None
+ if os.environ.get("UNSLOTH_DEBUG_DISTRIBUTED"):
+ # unsloth_zoo's logging switches reset NCCL_DEBUG to WARN on import.
+ from .distributed_debug import keep_debug_nccl_env
+ _restore_nccl_debug = keep_debug_nccl_env()
+ del keep_debug_nccl_env
import unsloth_zoo
+
+ if _restore_nccl_debug is not None:
+ _restore_nccl_debug()
+ del _restore_nccl_debug
except PackageNotFoundError:
raise ImportError(
f"Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo` then retry!"
diff --git a/unsloth/distributed_debug.py b/unsloth/distributed_debug.py
index b539b7e2c1c..c4b71498554 100644
--- a/unsloth/distributed_debug.py
+++ b/unsloth/distributed_debug.py
@@ -4,7 +4,7 @@
import os
import sys
-__all__ = ["apply_debug_distributed_env"]
+__all__ = ["apply_debug_distributed_env", "keep_debug_nccl_env"]
# Read at process group / communicator creation, so they only need to precede init_process_group.
_DEBUG_DISTRIBUTED_ENV = (
@@ -20,6 +20,10 @@ _FR_BUFFER_NAMES = ("TORCH_FR_BUFFER_SIZE", "TORCH_NCCL_TRACE_BUFFER_SIZE")
_FR_BUFFER_SIZE = "2000"
+def _enabled(environ):
+ return environ.get("UNSLOTH_DEBUG_DISTRIBUTED", "0").lower() in ("1", "true", "yes", "on")
+
+
def apply_debug_distributed_env(environ = None):
"""Under UNSLOTH_DEBUG_DISTRIBUTED=1, enable NCCL / c10d hang diagnostics.
@@ -28,7 +32,7 @@ def apply_debug_distributed_env(environ = None):
TORCH_NCCL_TRACE_BUFFER_SIZE=0 is not overridden. Returns the names that were set.
"""
environ = os.environ if environ is None else environ
- if environ.get("UNSLOTH_DEBUG_DISTRIBUTED", "0").lower() not in ("1", "true", "yes", "on"):
+ if not _enabled(environ):
return []
dist = sys.modules.get("torch.distributed")
try:
@@ -56,3 +60,20 @@ def apply_debug_distributed_env(environ = None):
environ[name] = given[0] if given else _FR_BUFFER_SIZE
changed.append(name)
return changed
+
+
+def keep_debug_nccl_env(environ = None):
+ """Under UNSLOTH_DEBUG_DISTRIBUTED=1, snapshot NCCL_DEBUG and return a callable restoring it.
+
+ unsloth_zoo's import sets NCCL_DEBUG=WARN unconditionally when UNSLOTH_ENABLE_LOGGING=1 or
+ UNSLOTH_COMPILE_DEBUG=1, which would drop the INFO logs (or the user's own value) chosen here.
+ Off, or with NCCL_DEBUG unset, the callable does nothing.
+ """
+ environ = os.environ if environ is None else environ
+ saved = environ.get("NCCL_DEBUG") if _enabled(environ) else None
+
+ def restore():
+ if saved is not None and environ.get("NCCL_DEBUG") != saved:
+ environ["NCCL_DEBUG"] = saved
+
+ return restore
diff --git a/unsloth/utils/mfu.py b/unsloth/utils/mfu.py
index 4d4c2a5ace5..db13498ae3c 100644
--- a/unsloth/utils/mfu.py
+++ b/unsloth/utils/mfu.py
@@ -52,6 +52,7 @@ _PEAK_TFLOPS_16BIT = (
("H100 PCIE", 756.0),
("H100 NVL", 835.0),
("H100", 989.0),
+ ("H200 NVL", 835.5),
("H200", 989.0),
("GH200", 989.0),
("B200", 2250.0),
@@ -351,12 +352,15 @@ def _sequence_lengths(kwargs, args):
return attention_mask.sum(dim = -1)
position_ids = kwargs.get("position_ids")
if torch.is_tensor(position_ids) and position_ids.dim() == 2 and position_ids.shape[0] == 1:
- # Padding free packing: each document restarts its positions at 0.
+ # Padding free packing: each document restarts its positions at 0. Fixed shape [N] with
+ # zero padding (no nonzero / data dependent branch): no host sync, and fullgraph compiles.
position_ids = position_ids[0]
- starts = torch.nonzero(position_ids == 0).reshape(-1)
- if starts.numel() > 0:
- ends = torch.cat([starts[1:], starts.new_tensor([position_ids.numel()])])
- return ends - starts
+ starts = position_ids == 0
+ doc = starts.cumsum(0) - 1
+ lengths = torch.zeros_like(doc).scatter_add_(0, doc.clamp(min = 0), (doc >= 0).to(doc.dtype))
+ whole_row = torch.zeros_like(doc)
+ whole_row[0] = position_ids.numel()
+ return torch.where(starts.any(), lengths, whole_row)
if torch.is_tensor(input_ids) and input_ids.dim() >= 2:
return torch.full((input_ids.shape[0],), input_ids.shape[1], device = input_ids.device)
return None
diff --git a/unsloth/utils/token_budget.py b/unsloth/utils/token_budget.py
index 8fd49a54d69..4f8de419403 100644
--- a/unsloth/utils/token_budget.py
+++ b/unsloth/utils/token_budget.py
@@ -478,16 +478,24 @@ def _balance_dataloader(trainer, dataloader):
return True
-def _same_count_on_every_rank(count, accelerator):
+def _same_count_on_every_rank(
+ count,
+ accelerator,
+ flag = None,
+):
+ """True when every rank has the same count. With flag, returns (same, flag on every rank) from
+ the same collective, so a later collective guarded by flag runs on every rank or on none."""
if accelerator is None or getattr(accelerator, "num_processes", 1) <= 1:
- return True
+ return True if flag is None else (True, bool(flag))
import torch.distributed as dist
if not (dist.is_available() and dist.is_initialized()):
- return True
- bounds = torch.tensor([count, -count], dtype = torch.int64, device = accelerator.device)
+ return True if flag is None else (True, bool(flag))
+ values = [count, -count] + ([] if flag is None else [0 if flag else 1])
+ bounds = torch.tensor(values, dtype = torch.int64, device = accelerator.device)
dist.all_reduce(bounds, op = dist.ReduceOp.MAX)
- return int(bounds[0]) == -int(bounds[1])
+ same = int(bounds[0]) == -int(bounds[1])
+ return same if flag is None else (same, int(bounds[2]) == 0)
def _label_count(chunk):
@@ -559,12 +567,21 @@ def _is_prompt_learning(trainer, model):
return False
+def _is_multi_process(accelerator):
+ return accelerator is not None and int(getattr(accelerator, "num_processes", 1) or 1) > 1
+
+
def _can_merge_metrics(trainer, model, chunks, accelerator):
"""Fold per-chunk entries only when token weighting provably reproduces TRL's unsplit value:
- one process (TRL's cross-rank reductions differ per version), no prompt-learning PEFT (virtual
- token logits are counted differently per version), one label source, and known TRL metric code."""
- if accelerator is not None and int(getattr(accelerator, "num_processes", 1) or 1) > 1:
- return False
+ with several processes only under an initialized process group and without Liger (whose older
+ TRL paths average per-rank ratios), no prompt-learning PEFT (virtual token logits are counted
+ differently per version), one label source, and known TRL metric code."""
+ if _is_multi_process(accelerator):
+ import torch.distributed as dist
+ if not (dist.is_available() and dist.is_initialized()):
+ return False
+ if getattr(getattr(trainer, "args", None), "use_liger_kernel", False):
+ return False
if _is_prompt_learning(trainer, model):
return False
if any("shift_labels" in chunk and "labels" in chunk for chunk in chunks):
@@ -585,13 +602,44 @@ def _metric_lengths(trainer):
return metrics, lengths
-def _merge_chunk_metrics(trainer, metrics, lengths, chunks):
+def _global_chunk_weights(trainer, chunks, accelerator):
+ """Per-key chunk weights. With several processes TRL gathers each chunk's sums across ranks, so the
+ weight is the chunk's token count summed over ranks. Older TRL (attention style) averages per-rank
+ entropy ratios, which cannot be refolded, so entropy is left unfolded there."""
+ weights = {
+ key: [count(trainer, chunk) for chunk in chunks] for key, count in _RATIO_METRICS.items()
+ }
+ if not _is_multi_process(accelerator):
+ return weights
+ import torch.distributed as dist
+
+ # Same shape on every rank (chunk counts already agree), so the collective always matches.
+ local = torch.tensor(list(weights.values()), dtype = torch.int64, device = accelerator.device)
+ dist.all_reduce(local, op = dist.ReduceOp.SUM)
+ weights = {key: [int(w) for w in row] for key, row in zip(weights, local.tolist())}
+ if _metric_style(trainer) != "labels":
+ weights.pop("entropy", None)
+ return weights
+
+
+def _merge_chunk_metrics(
+ trainer,
+ metrics,
+ lengths,
+ chunks,
+ weights = None,
+):
"""Fold the per-chunk ratio entries back into one entry for the micro-batch, weighting each chunk by
the tokens TRL divided by, so the logged value matches an unsplit step."""
- keys = [key for key in _RATIO_METRICS if key in metrics and isinstance(metrics[key], list)]
+ if weights is None:
+ weights = {
+ key: [count(trainer, chunk) for chunk in chunks]
+ for key, count in _RATIO_METRICS.items()
+ }
+ keys = [key for key in weights if key in metrics and isinstance(metrics[key], list)]
keys = [key for key in keys if len(metrics[key]) - lengths.get(key, 0) == len(chunks)]
for key in keys:
- row = [_RATIO_METRICS[key](trainer, chunk) for chunk in chunks]
+ row = weights[key]
start = lengths.get(key, 0)
values = metrics[key][start:]
total = sum(row)
@@ -646,7 +694,18 @@ def install_token_budget(trainer):
_note_once(trainer, f"Unsloth: UNSLOTH_TOKEN_BUDGET not splitting: {reason}.")
return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)
chunks = split_batch_by_token_budget(inputs, budget)
- if not _same_count_on_every_rank(len(chunks), accelerator):
+ # Folding metrics across ranks adds a collective, so every rank must agree to fold.
+ metrics, lengths = (
+ _metric_lengths(trainer)
+ if len(chunks) > 1 and _can_merge_metrics(trainer, model, chunks, accelerator)
+ else (None, None)
+ )
+ same_count, merge = _same_count_on_every_rank(
+ len(chunks), accelerator, flag = metrics is not None
+ )
+ if not merge:
+ metrics, lengths = None, None
+ if not same_count:
# Each chunk runs compute_loss (TRL gathers metrics there) and its own backward:
# ranks with different counts would issue mismatched collectives and hang.
_note_once(
@@ -657,11 +716,6 @@ def install_token_budget(trainer):
if len(chunks) == 1:
return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)
trainer._unsloth_token_budget_splits += 1
- metrics, lengths = (
- _metric_lengths(trainer)
- if _can_merge_metrics(trainer, model, chunks, accelerator)
- else (None, None)
- )
total = None
for i, chunk in enumerate(chunks):
context = (
@@ -673,7 +727,8 @@ def install_token_budget(trainer):
loss = original_training_step(model, chunk, num_items_in_batch, *args, **kwargs)
total = loss if total is None else total + loss
if metrics is not None:
- _merge_chunk_metrics(trainer, metrics, lengths, chunks)
+ weights = _global_chunk_weights(trainer, chunks, accelerator)
+ _merge_chunk_metrics(trainer, metrics, lengths, chunks, weights)
return total
trainer.training_step = training_step