Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MFU logging under fullgraph, NCCL logs under distributed debug, token-budget metrics across ranks (#14207)
Daniel Han9f78f5e5803f
diff --git a/tests/test_debug_distributed_env.py b/tests/test_debug_distributed_env.pyindex a96b97b9c27..13bfb04307b 100644--- a/tests/test_debug_distributed_env.py+++ b/tests/test_debug_distributed_env.py@@ -83,3 +83,69 @@ def test_existing_process_group_leaves_the_environment_untouched(monkeypatch, caassert _module().apply_debug_distributed_env(env) == []assert env == {"UNSLOTH_DEBUG_DISTRIBUTED": "1"}assert "already exists" in capsys.readouterr().err++[email protected]("user_value", [None, "TRACE"])+def test_nccl_debug_survives_zoo_logging_switches(user_value):+ # unsloth_zoo sets NCCL_DEBUG=WARN on import under UNSLOTH_ENABLE_LOGGING / UNSLOTH_COMPILE_DEBUG.+ mod = _module()+ env = {"UNSLOTH_DEBUG_DISTRIBUTED": "1", "UNSLOTH_ENABLE_LOGGING": "1"}+ if user_value is not None:+ env["NCCL_DEBUG"] = user_value+ mod.apply_debug_distributed_env(env)+ restore = mod.keep_debug_nccl_env(env)+ env["NCCL_DEBUG"] = "WARN"+ restore()+ assert env["NCCL_DEBUG"] == (user_value or "INFO")+++def test_keep_nccl_debug_is_a_no_op_when_off():+ mod = _module()+ env = {"UNSLOTH_ENABLE_LOGGING": "1"}+ restore = mod.keep_debug_nccl_env(env)+ env["NCCL_DEBUG"] = "WARN"+ restore()+ assert env["NCCL_DEBUG"] == "WARN"+++def test_gpu_init_restores_nccl_debug_around_zoo_import():+ source = (_UNSLOTH / "_gpu_init.py").read_text(encoding = "utf-8")+ zoo = source.index("\n import unsloth_zoo\n")+ keep = source.rfind("keep_debug_nccl_env()", 0, zoo)+ restore = source.find("_restore_nccl_debug()", zoo)+ assert keep != -1 and restore != -1+ assert source.count("\n import unsloth_zoo\n") == 1+ assert restore < source.index("except PackageNotFoundError", zoo)++[email protected]("switch", ["UNSLOTH_ENABLE_LOGGING", "UNSLOTH_COMPILE_DEBUG"])+def test_real_zoo_import_keeps_nccl_info(switch):+ import os+ import subprocess+ import sys++ if importlib.util.find_spec("unsloth_zoo") is None:+ pytest.skip("unsloth_zoo not installed")+ code = (+ "import importlib.util, os, sys\n"+ "spec = importlib.util.spec_from_file_location('dd', sys.argv[1])\n"+ "m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n"+ "m.apply_debug_distributed_env()\n"+ "restore = m.keep_debug_nccl_env()\n"+ "try:\n"+ " import unsloth_zoo\n"+ "except BaseException:\n"+ " pass\n" # CPU hosts raise after the env block ran; only NCCL_DEBUG matters here.+ "restore()\n"+ "print('NCCL_DEBUG=' + os.environ.get('NCCL_DEBUG', ''))\n"+ )+ env = {k: v for k, v in os.environ.items() if k != "NCCL_DEBUG"}+ env.update({"UNSLOTH_DEBUG_DISTRIBUTED": "1", switch: "1"})+ out = subprocess.run(+ [sys.executable, "-c", code, str(_UNSLOTH / "distributed_debug.py")],+ env = env,+ capture_output = True,+ text = True,+ timeout = 600,+ ).stdout+ assert "NCCL_DEBUG=INFO" in out.splitlines()diff --git a/tests/test_mfu_logging.py b/tests/test_mfu_logging.pyindex 6a16a0f2928..331840de963 100644--- a/tests/test_mfu_logging.py+++ b/tests/test_mfu_logging.py@@ -99,7 +99,7 @@ def test_sequence_lengths_exclude_padding_and_split_packing():assert mfu._sequence_lengths({"attention_mask": mask}, ()).tolist() == [3, 2]packed = torch.tensor([[0, 1, 2, 0, 1, 0, 1, 2, 3]])lengths = mfu._sequence_lengths({"input_ids": packed, "position_ids": packed}, ())- assert lengths.tolist() == [3, 2, 4]+ assert [n for n in lengths.tolist() if n] == [3, 2, 4]ids = torch.zeros(2, 5, dtype = torch.long)assert mfu._sequence_lengths({"input_ids": ids}, ()).tolist() == [5, 5]@@ -571,3 +571,51 @@ def test_diffusion_and_data_parallel_report_nothing():callback = mfu.UnslothMFUCallback(peak_tflops = 1)callback.on_train_begin(SimpleNamespace(n_gpu = 1), _State(), None, model = model)assert not callback._enabled and callback._handle is None+++def test_position_id_packing_counts_under_fullgraph_compile():+ # Position-id-only padding free batches (TRL padding_free) must not add a data dependent+ # branch to a model compiled with fullgraph = True.+ from types import SimpleNamespace++ class Tiny(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(4, 4, bias = False)+ self.config = SimpleNamespace(num_hidden_layers = 1, num_attention_heads = 1, hidden_size = 4)++ def forward(+ self,+ input_ids,+ position_ids = None,+ ):+ return self.proj(torch.nn.functional.one_hot(input_ids, 4).float())++ torch._dynamo.reset()+ model = Tiny().train()+ callback = mfu.UnslothMFUCallback()+ callback.on_train_begin(SimpleNamespace(n_gpu = 1), None, None, model = model)+ try:+ compiled = torch.compile(model, backend = "eager", fullgraph = True)+ ids = torch.tensor([[0, 1, 2, 1, 2, 3, 0]])+ positions = torch.tensor([[0, 1, 2, 0, 1, 0, 1]])+ compiled(input_ids = ids, position_ids = positions).sum().backward()+ assert int(callback._tokens) == 7+ assert int(callback._pairs) == 6 + 3 + 3+ finally:+ callback.on_train_end(None, None, None)+ torch._dynamo.reset()+ # Leading tokens before the first restart are skipped; no restart counts the whole row.+ lead = mfu._sequence_lengths({"position_ids": torch.tensor([[3, 4, 0, 1]])}, ())+ assert [n for n in lead.tolist() if n] == [2]+ none = mfu._sequence_lengths({"position_ids": torch.tensor([[1, 2, 3]])}, ())+ assert [n for n in none.tolist() if n] == [3]+++def test_h200_nvl_uses_its_own_dense_peak(monkeypatch):+ # NVIDIA H200 datasheet: NVL 1,671 TFLOPS BF16 with sparsity -> 835.5 dense; SXM 1,979 -> 989.+ monkeypatch.delenv("UNSLOTH_PEAK_TFLOPS", raising = False)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ for name, peak in (("NVIDIA H200 NVL", 835.5), ("NVIDIA H200", 989.0)):+ monkeypatch.setattr(torch.cuda, "get_device_name", lambda device = None, n = name: n)+ assert mfu.get_peak_tflops(dtype = torch.bfloat16) == peak, namediff --git a/tests/test_token_budget_metrics_distributed.py b/tests/test_token_budget_metrics_distributed.pynew file mode 100644index 00000000000..352c53d450d--- /dev/null+++ b/tests/test_token_budget_metrics_distributed.py@@ -0,0 +1,224 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_TOKEN_BUDGET under several processes: TRL gathers each forward's metric sums across ranks,+so split-chunk entries must be folded with token counts summed over ranks to match an unsplit step.+Two CPU ranks over gloo."""++from __future__ import annotations++import contextlib+import importlib.util+import multiprocessing+import os+import sys+import tempfile+from collections import defaultdict+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+dist = pytest.importorskip("torch.distributed")++if not dist.is_available() or not dist.is_gloo_available() or sys.platform != "linux":+ pytest.skip("needs torch.distributed with gloo on Linux (fork)", allow_module_level = True)++_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "utils" / "token_budget.py"+_spec = importlib.util.spec_from_file_location("_unsloth_token_budget_dist", _PATH)+tb = importlib.util.module_from_spec(_spec)+sys.modules[_spec.name] = tb+_spec.loader.exec_module(tb)++_BUDGET = 36+_WORLD = 2+++class _Accelerator:+ distributed_type = "MULTI_GPU"+ num_processes = _WORLD+ device = torch.device("cpu")++ @contextlib.contextmanager+ def no_sync(self, model):+ yield++ def gather_sum(self, value):+ # accelerator.gather_for_metrics(x).sum() for a scalar.+ value = torch.as_tensor(value, dtype = torch.float64).clone()+ dist.all_reduce(value, op = dist.ReduceOp.SUM)+ return value+++class _TRL115Trainer:+ """TRL 1.15 SFTTrainer.compute_loss (sft_trainer.py:1638-1642): accuracy and entropy are global+ sums over valid labels divided by the global valid-label count."""++ num_virtual_tokens = 0++ def __init__(self):+ self._metrics = {"train": defaultdict(list), "eval": defaultdict(list)}+ self.accelerator = _Accelerator()+ self.args = type("Args", (), {"gradient_accumulation_steps": 1})()+ self.model_accepts_loss_kwargs = True+ self.label_smoother = None++ def compute_loss(self, inputs):+ ids = inputs["input_ids"]+ correct, entropy = (ids % 2 == 0).double(), ids.double()+ label_mask = inputs["labels"][..., 1:] != -100+ num_valid = self.accelerator.gather_sum(label_mask.sum())+ entropy_sum = self.accelerator.gather_sum((entropy[..., 1:] * label_mask).sum())+ num_correct = self.accelerator.gather_sum((correct[..., 1:] * label_mask).sum())+ self._metrics["train"]["entropy"].append(float(entropy_sum / num_valid))+ self._metrics["train"]["mean_token_accuracy"].append(float(num_correct / num_valid))++ def training_step(+ self,+ model,+ inputs,+ num_items_in_batch = None,+ ):+ self.compute_loss(inputs)+ return torch.tensor(1.0)+++class _TRL022Trainer(_TRL115Trainer):+ """TRL 0.22.2 (sft_trainer.py:1056-1061, 1099-1105): entropy is the mean of per-rank attention+ ratios (not refoldable), accuracy is global correct over global shifted labels."""++ def compute_loss(self, inputs):+ ids = inputs["input_ids"]+ correct, entropy = (ids % 2 == 0).double(), ids.double()+ attention_mask = inputs["attention_mask"].double()+ local_entropy = (entropy * attention_mask).sum() / attention_mask.sum()+ entropy = self.accelerator.gather_sum(local_entropy) / _WORLD+ self._metrics["train"]["entropy"].append(float(entropy))+ shift_labels = inputs["labels"][..., 1:]+ valid = shift_labels != -100+ correct_tokens = self.accelerator.gather_sum((correct[..., 1:].bool() & valid).sum())+ total_tokens = self.accelerator.gather_sum(valid.sum())+ self._metrics["train"]["mean_token_accuracy"].append(float(correct_tokens / total_tokens))+++_TRAINERS = {"trl115": _TRL115Trainer, "trl022": _TRL022Trainer}+++def _rank_batch(rank):+ # Unequal row lengths per rank, so chunks differ in token count within and across ranks.+ lengths = [[2, 3, 3, 12], [9, 2, 12, 5]][rank]+ width = 12+ gen = torch.Generator().manual_seed(rank)+ input_ids = torch.randint(1, 50, (len(lengths), width), generator = gen)+ attention_mask = torch.zeros(len(lengths), width, dtype = torch.long)+ for row, n in enumerate(lengths):+ attention_mask[row, :n] = 1+ labels = input_ids.masked_fill(attention_mask == 0, -100)+ labels[:, 0] = -100+ return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}+++def _worker(+ rank,+ init_file,+ trainer_name,+ queue,+ shift_labels_on_rank = None,+):+ try:+ dist.init_process_group(+ "gloo", init_method = f"file://{init_file}", rank = rank, world_size = _WORLD+ )+ trainer_cls = _TRAINERS[trainer_name]+ batch = _rank_batch(rank)+ if rank == shift_labels_on_rank:+ # A second label source makes only this rank decline to fold.+ batch["shift_labels"] = batch["labels"][..., 1:].contiguous()++ os.environ.pop("UNSLOTH_TOKEN_BALANCE", None)+ os.environ.pop("UNSLOTH_TOKEN_BUDGET", None)+ reference = trainer_cls()+ tb.install_token_budget(reference)+ reference.training_step(None, dict(batch), 10)++ os.environ["UNSLOTH_TOKEN_BUDGET"] = str(_BUDGET)+ trainer = trainer_cls()+ tb.install_token_budget(trainer)+ trainer.training_step(None, dict(batch), 10)+ n_chunks = len(tb.split_batch_by_token_budget(dict(batch), _BUDGET))+ queue.put(+ (+ rank,+ {+ "reference": {k: list(v) for k, v in reference._metrics["train"].items()},+ "split": {k: list(v) for k, v in trainer._metrics["train"].items()},+ "splits": trainer._unsloth_token_budget_splits,+ "n_chunks": n_chunks,+ },+ )+ )+ dist.destroy_process_group()+ except BaseException as error:+ queue.put((rank, repr(error)))+ raise+++def _run(trainer_name, shift_labels_on_rank = None):+ ctx = multiprocessing.get_context("fork")+ queue = ctx.Queue()+ with tempfile.TemporaryDirectory() as tmp:+ init_file = os.path.join(tmp, "pg")+ procs = [+ ctx.Process(+ target = _worker,+ args = (rank, init_file, trainer_name, queue, shift_labels_on_rank),+ )+ for rank in range(_WORLD)+ ]+ for proc in procs:+ proc.start()+ results = dict(queue.get(timeout = 120) for _ in procs)+ for proc in procs:+ proc.join(timeout = 60)+ for rank, result in results.items():+ assert isinstance(result, dict), f"rank {rank} failed: {result}"+ return results+++def _mean(values):+ return sum(values) / len(values)++[email protected]("trainer_name", ["trl115", "trl022"])+def test_multi_process_split_accuracy_matches_unsplit(trainer_name):+ results = _run(trainer_name)+ for result in results.values():+ assert result["splits"] == 1 and result["n_chunks"] > 1+ reference, split = result["reference"], result["split"]+ assert len(split["mean_token_accuracy"]) == 1+ assert _mean(split["mean_token_accuracy"]) == pytest.approx(+ _mean(reference["mean_token_accuracy"]), rel = 1e-9+ )+ # Both ranks log the same gathered value, as TRL does.+ assert results[0]["split"] == results[1]["split"]+++def test_multi_process_split_entropy_matches_unsplit_on_label_weighted_trl():+ for result in _run("trl115").values():+ reference, split = result["reference"], result["split"]+ assert len(split["entropy"]) == 1+ assert _mean(split["entropy"]) == pytest.approx(_mean(reference["entropy"]), rel = 1e-9)+++def test_multi_process_attention_entropy_left_unfolded():+ # Older TRL averages per-rank entropy ratios; a token-weighted fold cannot reproduce that.+ for result in _run("trl022").values():+ assert len(result["split"]["entropy"]) == result["n_chunks"]+++def test_ranks_that_disagree_on_folding_neither_fold_nor_hang():+ # The fold adds a collective; if one rank cannot fold, every rank must skip it (else the others hang).+ results = _run("trl115", shift_labels_on_rank = 1)+ for result in results.values():+ assert result["splits"] == 1+ assert len(result["split"]["mean_token_accuracy"]) == result["n_chunks"] > 1diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 5d06655d61d..d0029b77ebe 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -232,7 +232,17 @@ try:"Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n""Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`")+ _restore_nccl_debug = None+ if os.environ.get("UNSLOTH_DEBUG_DISTRIBUTED"):+ # unsloth_zoo's logging switches reset NCCL_DEBUG to WARN on import.+ from .distributed_debug import keep_debug_nccl_env+ _restore_nccl_debug = keep_debug_nccl_env()+ del keep_debug_nccl_envimport unsloth_zoo++ if _restore_nccl_debug is not None:+ _restore_nccl_debug()+ del _restore_nccl_debugexcept PackageNotFoundError:raise ImportError(f"Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo` then retry!"diff --git a/unsloth/distributed_debug.py b/unsloth/distributed_debug.pyindex b539b7e2c1c..c4b71498554 100644--- a/unsloth/distributed_debug.py+++ b/unsloth/distributed_debug.py@@ -4,7 +4,7 @@import osimport sys-__all__ = ["apply_debug_distributed_env"]+__all__ = ["apply_debug_distributed_env", "keep_debug_nccl_env"]# Read at process group / communicator creation, so they only need to precede init_process_group._DEBUG_DISTRIBUTED_ENV = (@@ -20,6 +20,10 @@ _FR_BUFFER_NAMES = ("TORCH_FR_BUFFER_SIZE", "TORCH_NCCL_TRACE_BUFFER_SIZE")_FR_BUFFER_SIZE = "2000"+def _enabled(environ):+ return environ.get("UNSLOTH_DEBUG_DISTRIBUTED", "0").lower() in ("1", "true", "yes", "on")++def apply_debug_distributed_env(environ = None):"""Under UNSLOTH_DEBUG_DISTRIBUTED=1, enable NCCL / c10d hang diagnostics.@@ -28,7 +32,7 @@ def apply_debug_distributed_env(environ = None):TORCH_NCCL_TRACE_BUFFER_SIZE=0 is not overridden. Returns the names that were set."""environ = os.environ if environ is None else environ- if environ.get("UNSLOTH_DEBUG_DISTRIBUTED", "0").lower() not in ("1", "true", "yes", "on"):+ if not _enabled(environ):return []dist = sys.modules.get("torch.distributed")try:@@ -56,3 +60,20 @@ def apply_debug_distributed_env(environ = None):environ[name] = given[0] if given else _FR_BUFFER_SIZEchanged.append(name)return changed+++def keep_debug_nccl_env(environ = None):+ """Under UNSLOTH_DEBUG_DISTRIBUTED=1, snapshot NCCL_DEBUG and return a callable restoring it.++ unsloth_zoo's import sets NCCL_DEBUG=WARN unconditionally when UNSLOTH_ENABLE_LOGGING=1 or+ UNSLOTH_COMPILE_DEBUG=1, which would drop the INFO logs (or the user's own value) chosen here.+ Off, or with NCCL_DEBUG unset, the callable does nothing.+ """+ environ = os.environ if environ is None else environ+ saved = environ.get("NCCL_DEBUG") if _enabled(environ) else None++ def restore():+ if saved is not None and environ.get("NCCL_DEBUG") != saved:+ environ["NCCL_DEBUG"] = saved++ return restorediff --git a/unsloth/utils/mfu.py b/unsloth/utils/mfu.pyindex 4d4c2a5ace5..db13498ae3c 100644--- a/unsloth/utils/mfu.py+++ b/unsloth/utils/mfu.py@@ -52,6 +52,7 @@ _PEAK_TFLOPS_16BIT = (("H100 PCIE", 756.0),("H100 NVL", 835.0),("H100", 989.0),+ ("H200 NVL", 835.5),("H200", 989.0),("GH200", 989.0),("B200", 2250.0),@@ -351,12 +352,15 @@ def _sequence_lengths(kwargs, args):return attention_mask.sum(dim = -1)position_ids = kwargs.get("position_ids")if torch.is_tensor(position_ids) and position_ids.dim() == 2 and position_ids.shape[0] == 1:- # Padding free packing: each document restarts its positions at 0.+ # Padding free packing: each document restarts its positions at 0. Fixed shape [N] with+ # zero padding (no nonzero / data dependent branch): no host sync, and fullgraph compiles.position_ids = position_ids[0]- starts = torch.nonzero(position_ids == 0).reshape(-1)- if starts.numel() > 0:- ends = torch.cat([starts[1:], starts.new_tensor([position_ids.numel()])])- return ends - starts+ starts = position_ids == 0+ doc = starts.cumsum(0) - 1+ lengths = torch.zeros_like(doc).scatter_add_(0, doc.clamp(min = 0), (doc >= 0).to(doc.dtype))+ whole_row = torch.zeros_like(doc)+ whole_row[0] = position_ids.numel()+ return torch.where(starts.any(), lengths, whole_row)if torch.is_tensor(input_ids) and input_ids.dim() >= 2:return torch.full((input_ids.shape[0],), input_ids.shape[1], device = input_ids.device)return Nonediff --git a/unsloth/utils/token_budget.py b/unsloth/utils/token_budget.pyindex 8fd49a54d69..4f8de419403 100644--- a/unsloth/utils/token_budget.py+++ b/unsloth/utils/token_budget.py@@ -478,16 +478,24 @@ def _balance_dataloader(trainer, dataloader):return True-def _same_count_on_every_rank(count, accelerator):+def _same_count_on_every_rank(+ count,+ accelerator,+ flag = None,+):+ """True when every rank has the same count. With flag, returns (same, flag on every rank) from+ the same collective, so a later collective guarded by flag runs on every rank or on none."""if accelerator is None or getattr(accelerator, "num_processes", 1) <= 1:- return True+ return True if flag is None else (True, bool(flag))import torch.distributed as distif not (dist.is_available() and dist.is_initialized()):- return True- bounds = torch.tensor([count, -count], dtype = torch.int64, device = accelerator.device)+ return True if flag is None else (True, bool(flag))+ values = [count, -count] + ([] if flag is None else [0 if flag else 1])+ bounds = torch.tensor(values, dtype = torch.int64, device = accelerator.device)dist.all_reduce(bounds, op = dist.ReduceOp.MAX)- return int(bounds[0]) == -int(bounds[1])+ same = int(bounds[0]) == -int(bounds[1])+ return same if flag is None else (same, int(bounds[2]) == 0)def _label_count(chunk):@@ -559,12 +567,21 @@ def _is_prompt_learning(trainer, model):return False+def _is_multi_process(accelerator):+ return accelerator is not None and int(getattr(accelerator, "num_processes", 1) or 1) > 1++def _can_merge_metrics(trainer, model, chunks, accelerator):"""Fold per-chunk entries only when token weighting provably reproduces TRL's unsplit value:- one process (TRL's cross-rank reductions differ per version), no prompt-learning PEFT (virtual- token logits are counted differently per version), one label source, and known TRL metric code."""- if accelerator is not None and int(getattr(accelerator, "num_processes", 1) or 1) > 1:- return False+ with several processes only under an initialized process group and without Liger (whose older+ TRL paths average per-rank ratios), no prompt-learning PEFT (virtual token logits are counted+ differently per version), one label source, and known TRL metric code."""+ if _is_multi_process(accelerator):+ import torch.distributed as dist+ if not (dist.is_available() and dist.is_initialized()):+ return False+ if getattr(getattr(trainer, "args", None), "use_liger_kernel", False):+ return Falseif _is_prompt_learning(trainer, model):return Falseif any("shift_labels" in chunk and "labels" in chunk for chunk in chunks):@@ -585,13 +602,44 @@ def _metric_lengths(trainer):return metrics, lengths-def _merge_chunk_metrics(trainer, metrics, lengths, chunks):+def _global_chunk_weights(trainer, chunks, accelerator):+ """Per-key chunk weights. With several processes TRL gathers each chunk's sums across ranks, so the+ weight is the chunk's token count summed over ranks. Older TRL (attention style) averages per-rank+ entropy ratios, which cannot be refolded, so entropy is left unfolded there."""+ weights = {+ key: [count(trainer, chunk) for chunk in chunks] for key, count in _RATIO_METRICS.items()+ }+ if not _is_multi_process(accelerator):+ return weights+ import torch.distributed as dist++ # Same shape on every rank (chunk counts already agree), so the collective always matches.+ local = torch.tensor(list(weights.values()), dtype = torch.int64, device = accelerator.device)+ dist.all_reduce(local, op = dist.ReduceOp.SUM)+ weights = {key: [int(w) for w in row] for key, row in zip(weights, local.tolist())}+ if _metric_style(trainer) != "labels":+ weights.pop("entropy", None)+ return weights+++def _merge_chunk_metrics(+ trainer,+ metrics,+ lengths,+ chunks,+ weights = None,+):"""Fold the per-chunk ratio entries back into one entry for the micro-batch, weighting each chunk bythe tokens TRL divided by, so the logged value matches an unsplit step."""- keys = [key for key in _RATIO_METRICS if key in metrics and isinstance(metrics[key], list)]+ if weights is None:+ weights = {+ key: [count(trainer, chunk) for chunk in chunks]+ for key, count in _RATIO_METRICS.items()+ }+ keys = [key for key in weights if key in metrics and isinstance(metrics[key], list)]keys = [key for key in keys if len(metrics[key]) - lengths.get(key, 0) == len(chunks)]for key in keys:- row = [_RATIO_METRICS[key](trainer, chunk) for chunk in chunks]+ row = weights[key]start = lengths.get(key, 0)values = metrics[key][start:]total = sum(row)@@ -646,7 +694,18 @@ def install_token_budget(trainer):_note_once(trainer, f"Unsloth: UNSLOTH_TOKEN_BUDGET not splitting: {reason}.")return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)chunks = split_batch_by_token_budget(inputs, budget)- if not _same_count_on_every_rank(len(chunks), accelerator):+ # Folding metrics across ranks adds a collective, so every rank must agree to fold.+ metrics, lengths = (+ _metric_lengths(trainer)+ if len(chunks) > 1 and _can_merge_metrics(trainer, model, chunks, accelerator)+ else (None, None)+ )+ same_count, merge = _same_count_on_every_rank(+ len(chunks), accelerator, flag = metrics is not None+ )+ if not merge:+ metrics, lengths = None, None+ if not same_count:# Each chunk runs compute_loss (TRL gathers metrics there) and its own backward:# ranks with different counts would issue mismatched collectives and hang._note_once(@@ -657,11 +716,6 @@ def install_token_budget(trainer):if len(chunks) == 1:return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)trainer._unsloth_token_budget_splits += 1- metrics, lengths = (- _metric_lengths(trainer)- if _can_merge_metrics(trainer, model, chunks, accelerator)- else (None, None)- )total = Nonefor i, chunk in enumerate(chunks):context = (@@ -673,7 +727,8 @@ def install_token_budget(trainer):loss = original_training_step(model, chunk, num_items_in_batch, *args, **kwargs)total = loss if total is None else total + lossif metrics is not None:- _merge_chunk_metrics(trainer, metrics, lengths, chunks)+ weights = _global_chunk_weights(trainer, chunks, accelerator)+ _merge_chunk_metrics(trainer, metrics, lengths, chunks, weights)return totaltrainer.training_step = training_step