Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2: free-memory fit check, MFU without recompute (HFU separately), nccl2 warning on small GPUs (#15158)
Daniel Handeed5b9397d5
diff --git a/tests/test_fsdp2_native_nccl2_warning.py b/tests/test_fsdp2_native_nccl2_warning.pynew file mode 100644index 00000000000..55e2117abf2--- /dev/null+++ b/tests/test_fsdp2_native_nccl2_warning.py@@ -0,0 +1,102 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native FSDP2 warns once when the process group runs torch 2.14's nccl2 on small GPUs (CPU only:+the process group and the CUDA device queries are faked)."""++import os+import sys+import types++import pytest+import torch++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402++native = _load_native()+++class ProcessGroupNCCL2:+ pass+++class ProcessGroupNCCL:+ pass+++def _dist(+ impl,+ name = "nccl",+ world = 2,+):+ group = types.SimpleNamespace(_get_backend = lambda device: impl())+ return types.SimpleNamespace(+ distributed_c10d = types.SimpleNamespace(_get_default_group = lambda: group),+ get_backend = lambda: name,+ get_world_size = lambda: world,+ )+++def gpus(monkeypatch):+ state = {"count": 2, "total": 15 << 30}+ monkeypatch.setattr(torch.cuda, "device_count", lambda: state["count"])+ monkeypatch.setattr(+ torch.cuda,+ "get_device_properties",+ lambda device: types.SimpleNamespace(total_memory = state["total"]),+ )+ monkeypatch.setattr(native, "_NCCL2_WARNED", False)+ return state+++DEV = torch.device("cuda", 0)+++def test_warns_once_for_nccl2_on_a_small_gpu(gpus, capsys):+ native._warn_nccl2_memory(_dist(ProcessGroupNCCL2), DEV)+ native._warn_nccl2_memory(_dist(ProcessGroupNCCL2), DEV)+ assert capsys.readouterr().out.count("nccl2 backend") == 1+++def test_env_redirect_is_detected_from_the_backend_object(gpus, capsys):+ # TORCH_DIST_USE_NCCL2=1: get_backend() still says "nccl"; the backend object is nccl2.+ native._warn_nccl2_memory(_dist(ProcessGroupNCCL2, name = "nccl"), DEV)+ assert "nccl2 backend" in capsys.readouterr().out+++ "case",+ ["plain_nccl", "large_gpu", "one_visible_gpu", "one_rank", "cpu"],+)+def test_no_warning(gpus, capsys, case):+ impl, world, device = ProcessGroupNCCL2, 2, DEV+ if case == "plain_nccl":+ impl = ProcessGroupNCCL+ elif case == "large_gpu":+ gpus["total"] = 80 << 30+ elif case == "one_visible_gpu":+ gpus["count"] = 1+ elif case == "one_rank":+ world = 1+ elif case == "cpu":+ device = torch.device("cpu")+ native._warn_nccl2_memory(_dist(impl, world = world), device)+ assert capsys.readouterr().out == ""+++def test_name_fallback_when_backend_object_is_unavailable(gpus, capsys, monkeypatch):+ def broken(device):+ raise RuntimeError("no backend for device")++ group = types.SimpleNamespace(_get_backend = broken)+ dist = _dist(ProcessGroupNCCL)+ dist.distributed_c10d = types.SimpleNamespace(_get_default_group = lambda: group)+ monkeypatch.setenv("TORCH_DIST_USE_NCCL2", "1")+ assert native._uses_nccl2(dist, DEV) is True+ dist.get_backend = lambda: "nccl2"+ monkeypatch.delenv("TORCH_DIST_USE_NCCL2")+ assert native._uses_nccl2(dist, DEV) is True+ dist.get_backend = lambda: "gloo"+ assert native._uses_nccl2(dist, DEV) is Falsediff --git a/tests/test_fsdp2_perf_free_memory_gloo.py b/tests/test_fsdp2_perf_free_memory_gloo.pynew file mode 100644index 00000000000..de9aeea1db5--- /dev/null+++ b/tests/test_fsdp2_perf_free_memory_gloo.py@@ -0,0 +1,117 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The FSDP2 fit layout checks the memory a rank can still use, not only the GPU's total.++mem_get_info is faked per rank on a gloo CPU world of 2: with plenty free on both ranks the decision+is the total-memory one (fit); when one rank's GPU is crowded (CUDA context, NCCL, another process)+that rank does not fit and both ranks agree on the sharded layout."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = ""++import importlib # noqa: E402+import sys # noqa: E402+import types # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_layout_agree_gloo import REPO_ID, _ids, _init, _spawn # noqa: E402+from test_fsdp2_native_units import _build, _load_native # noqa: E402+++def _worker(rank, world, port, out, free_by_rank):+ dist = _init(rank, world, port)+ try:+ native = _load_native()+ perf = importlib.import_module("unsloth.models.fsdp2_perf")+ perf._device_bytes = lambda device = None: 16 << 30 # a 16 GiB GPU on every rank+ free = free_by_rank[rank]+ perf._device_free_bytes = lambda device = None: free # faked mem_get_info + cache+ local = perf.plan_fsdp2_layout(_build(REPO_ID, "AutoModelForCausalLM")).as_dict()+ model = _build(REPO_ID, "AutoModelForCausalLM")+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32))+ layout = model._unsloth_fsdp2_layout+ mine = {+ "local_fits": local["fits"],+ "fits": layout["fits"],+ "reshard_after_forward": layout["reshard_after_forward"],+ "unit_reshards": [u._unsloth_reshard_after_forward for u in model._unsloth_fsdp2_units],+ "reasons": local["reasons"],+ }+ everyone = [None] * world+ dist.all_gather_object(everyone, mine)+ strip = lambda e: {k: v for k, v in e.items() if k not in ("local_fits", "reasons")}+ consistent = all(strip(e) == strip(everyone[0]) for e in everyone)+ step = None+ if consistent:+ ids = _ids(rank)+ model(input_ids = ids, labels = ids, use_cache = False).loss.backward()+ step = "ok"+ if rank == 0:+ out.put({"ranks": everyone, "consistent": consistent, "step": step})+ except Exception as exc:+ import traceback+ out.put({"error": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"})+ finally:+ dist.destroy_process_group()+++def test_plenty_free_keeps_the_total_memory_decision():+ pytest.importorskip("transformers")+ (r,) = _spawn(_worker, [15 << 30, 15 << 30])+ assert "error" not in r, r["error"]+ assert [x["local_fits"] for x in r["ranks"]] == [True, True], r+ assert all(x["fits"] is True for x in r["ranks"]), r+ assert r["consistent"] and r["step"] == "ok", r+++def test_one_crowded_rank_shards_every_rank():+ pytest.importorskip("transformers")+ # Rank 1: a few KiB left (the tiny model is a few MiB), so model > 0.5 x usable there.+ (r,) = _spawn(_worker, [15 << 30, 1 << 10])+ assert "error" not in r, r["error"]+ ranks = r["ranks"]+ assert [x["local_fits"] for x in ranks] == [True, False], r+ assert any("usable on this GPU" in reason for reason in ranks[1]["reasons"]), r+ assert all(x["fits"] is False for x in ranks), r+ assert all(x["reshard_after_forward"] is True for x in ranks), r+ assert r["consistent"] and r["step"] == "ok", r+++def test_device_free_bytes_reads_mem_get_info_plus_cache(monkeypatch):+ perf = importlib.import_module("unsloth.models.fsdp2_perf")+ fake = types.SimpleNamespace(+ mem_get_info = lambda device = None: (3 << 30, 16 << 30),+ memory_reserved = lambda device = None: 2 << 30,+ memory_allocated = lambda device = None: 1 << 30,+ current_device = lambda: 0,+ )+ monkeypatch.setattr(perf.torch, "cuda", fake)+ assert perf._device_free_bytes("cuda:0") == (3 << 30) + (1 << 30)+ assert perf._device_free_bytes("cpu") is None+ fake.mem_get_info = lambda device = None: (_ for _ in ()).throw(RuntimeError("no driver"))+ assert perf._device_free_bytes("cuda:0") is None+++def test_raised_fit_fraction_keeps_an_idle_card_fit(monkeypatch):+ # UNSLOTH_FSDP2_FIT_FRACTION = 0.6 and an idle card (85% usable): the free-memory rule scales with+ # the fit fraction, so a model at 55% of the card stays in the fit layout.+ pytest.importorskip("transformers")+ perf = importlib.import_module("unsloth.models.fsdp2_perf")+ model = _build(REPO_ID, "AutoModelForCausalLM")+ probe = perf.plan_fsdp2_layout(model)+ model_bytes = probe.frozen_bytes + probe.trainable_bytes+ device = int(model_bytes / 0.55)+ monkeypatch.setenv("UNSLOTH_FSDP2_FIT_FRACTION", "0.6")+ monkeypatch.delenv("UNSLOTH_FSDP2_FREE_FIT_FRACTION", raising = False)+ monkeypatch.setattr(perf, "_device_bytes", lambda device_ = None: device)+ monkeypatch.setattr(perf, "_device_free_bytes", lambda device_ = None: int(device * 0.85))+ assert perf.plan_fsdp2_layout(model).fits is Truediff --git a/tests/test_mfu_logging.py b/tests/test_mfu_logging.pyindex 331840de963..215a48de02d 100644--- a/tests/test_mfu_logging.py+++ b/tests/test_mfu_logging.py@@ -619,3 +619,58 @@ def test_h200_nvl_uses_its_own_dense_peak(monkeypatch):for name, peak in (("NVIDIA H200 NVL", 835.5), ("NVIDIA H200", 989.0)):monkeypatch.setattr(torch.cuda, "get_device_name", lambda device = None, n = name: n)assert mfu.get_peak_tflops(dtype = torch.bfloat16) == peak, name+++def test_checkpointing_splits_mfu_from_hfu(tmp_path, monkeypatch):+ """train_mfu leaves out the forward that checkpointing recomputes; train_hfu charges it."""+ from transformers import Trainer, TrainingArguments++ monkeypatch.setenv("UNSLOTH_PEAK_TFLOPS", "1")+ rows = [+ {"input_ids": [1, 2, 3, 4], "attention_mask": [1, 1, 1, 0], "labels": [1, 2, 3, -100]},+ {"input_ids": [5, 6, 7, 8], "attention_mask": [1, 1, 0, 0], "labels": [5, 6, -100, -100]},+ ]+ dataset = [{k: torch.tensor(v) for k, v in row.items()} for row in rows]++ def run(checkpointing):+ trainer = Trainer(+ model = _model(),+ args = TrainingArguments(+ output_dir = str(tmp_path),+ per_device_train_batch_size = 2,+ max_steps = 1,+ logging_steps = 1,+ report_to = [],+ use_cpu = True,+ save_strategy = "no",+ gradient_checkpointing = checkpointing,+ ),+ train_dataset = dataset,+ )+ mfu.add_mfu_callback(trainer)+ trainer.train()+ return [entry for entry in trainer.state.log_history if "loss" in entry][0]++ entry = run(True)+ # Per step: 5 tokens (lengths 3 and 2), 9 causal pairs, full head on every token.+ pair = 2 * 2 * 4 * (4 + 4)+ model_flops = 5 * 6 * LAYERS + 5 * 6 * HEAD + 9 * 3 * pair+ hw_flops = 5 * 8 * LAYERS + 5 * 6 * HEAD + 9 * 4 * pair+ assert entry["train_hfu"] / entry["train_mfu"] == pytest.approx(hw_flops / model_flops)+ assert entry["train_mfu"] == pytest.approx(entry["train_tflops_per_gpu"])+ assert entry["train_hfu"] == pytest.approx(entry["train_hw_tflops_per_gpu"])+ plain = run(False)+ assert plain["train_hfu"] == pytest.approx(plain["train_mfu"])+++def test_blackwell_dense_peaks(monkeypatch):+ monkeypatch.delenv("UNSLOTH_PEAK_TFLOPS", raising = False)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ for name, peak in (+ ("NVIDIA B200", 2250.0),+ ("NVIDIA B300 SXM6 AC", 2250.0),+ ("NVIDIA GB200", 2500.0),+ ("NVIDIA GB300", 2500.0),+ ):+ monkeypatch.setattr(torch.cuda, "get_device_name", lambda device = None, n = name: n)+ assert mfu.get_peak_tflops(dtype = torch.bfloat16) == peak, namediff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex d5040f71b0a..9ac65420db9 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -745,6 +745,53 @@ def _ensure_process_group(return dist+_NCCL2_WARNED = False+# GPUs at or below this size are where nccl2's extra per-rank memory is worth a warning.+_NCCL2_WARN_MAX_BYTES = 24 * 2**30+++def _uses_nccl2(dist, device):+ """torch 2.14's nccl2 backend; TORCH_DIST_USE_NCCL2=1 still reports "nccl", so check the object."""+ try:+ group = dist.distributed_c10d._get_default_group()+ impl = type(group._get_backend(device)).__name__+ return impl == "ProcessGroupNCCL2"+ except Exception:+ pass+ try:+ name = str(dist.get_backend()).lower()+ except Exception:+ return False+ return name == "nccl2" or (+ name == "nccl" and os.environ.get("TORCH_DIST_USE_NCCL2", "").strip() == "1"+ )+++def _warn_nccl2_memory(dist, device):+ """Under nccl2 every rank seeing GPU 0 makes a CUDA context there (torch 2.14.1, 2 x T4: 102 MiB+ per rank, plain nccl none), so small GPU 0 runs out first."""+ global _NCCL2_WARNED+ if _NCCL2_WARNED or device.type != "cuda":+ return+ try:+ if dist.get_world_size() < 2 or torch.cuda.device_count() < 2:+ return+ if torch.cuda.get_device_properties(device).total_memory > _NCCL2_WARN_MAX_BYTES:+ return+ except Exception:+ return+ if not _uses_nccl2(dist, device):+ return+ _NCCL2_WARNED = True+ _print_rank0(+ "Unsloth: the process group uses the nccl2 backend (backend = 'nccl2' or "+ "TORCH_DIST_USE_NCCL2=1). Under nccl2 every other rank also builds a CUDA context on GPU 0 "+ "(about 0.1 GiB each on 2 x T4, more on larger GPUs and grows with the number of ranks), so "+ "GPU 0 has less memory than the others. On GPUs of 24 GB or less prefer the default 'nccl' "+ "backend (unset TORCH_DIST_USE_NCCL2), or launch each rank with only its own GPU visible."+ )++def _default_mesh(device, dist):from torch.distributed.device_mesh import init_device_meshreturn init_device_mesh(device.type, (dist.get_world_size(),))@@ -1382,6 +1429,7 @@ def shard_model(model, config: Optional[FSDP2Config] = None):dist = _ensure_process_group(device, config.backend, config.timeout)if device.type == "cuda":torch.cuda.set_device(device)+ _warn_nccl2_memory(dist, device)mesh = config.mesh if config.mesh is not None else _default_mesh(device, dist)_sync_trainable_from_first_rank(model, dist, device, config.mesh)param_dtype = _resolve_param_dtype(model, config)diff --git a/unsloth/models/fsdp2_perf.py b/unsloth/models/fsdp2_perf.pyindex 5122b78a404..e450e1a48bf 100644--- a/unsloth/models/fsdp2_perf.py+++ b/unsloth/models/fsdp2_perf.py@@ -23,10 +23,12 @@ shard, and gradients plus optimizer state are sharded.Env overrides (read only when native FSDP2 is used): ``UNSLOTH_FSDP2_REPLICATE_FROZEN`` = auto | 1 | 0,``UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD`` = auto | 1 | 0, ``UNSLOTH_FSDP2_PREFETCH`` = 1 | 0,-``UNSLOTH_FSDP2_FIT_FRACTION`` (default 0.4 of one device's memory).+``UNSLOTH_FSDP2_FIT_FRACTION`` (default 0.4 of one device's memory), ``UNSLOTH_FSDP2_FREE_FIT_FRACTION``+(default max(0.5, fit fraction / 0.8) of free + cached + resident model memory, read after process group init)."""from dataclasses import dataclass, field+from typing import Optionalimport osimport torch@@ -79,6 +81,33 @@ def _device_bytes(device = None):return 0+def _device_free_bytes(device = None):+ """``mem_get_info`` free plus torch's unallocated cache; None off CUDA or if unreadable."""+ if device is not None and torch.device(device).type != "cuda":+ return None+ try:+ if device is None:+ device = torch.cuda.current_device()+ free, _ = torch.cuda.mem_get_info(device)+ cached = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)+ return int(free) + max(int(cached), 0)+ except Exception:+ return None+++def _resident_bytes(params, device):+ """Bytes of `params` already stored on `device`."""+ if device is None:+ return 0+ target = torch.device(device)+ if target.type == "cuda" and target.index is None:+ try:+ target = torch.device("cuda", torch.cuda.current_device())+ except Exception:+ return 0+ return _param_bytes(p for p in params if p.device == target)++@dataclassclass FSDP2Layout:replicate_frozen: bool = False@@ -89,6 +118,7 @@ class FSDP2Layout:frozen_bytes: int = 0trainable_bytes: int = 0device_bytes: int = 0+ available_bytes: Optional[int] = Nonereasons: list = field(default_factory = list)def as_dict(self):@@ -101,6 +131,9 @@ class FSDP2Layout:"frozen_gib": round(self.frozen_bytes / 2**30, 3),"trainable_gib": round(self.trainable_bytes / 2**30, 3),"device_gib": round(self.device_bytes / 2**30, 1),+ "available_gib": None+ if self.available_bytes is None+ else round(self.available_bytes / 2**30, 2),"reasons": list(self.reasons),}@@ -131,6 +164,15 @@ def plan_fsdp2_layout(fraction = float(os.environ.get("UNSLOTH_FSDP2_FIT_FRACTION", "0.4"))except ValueError:fraction = 0.4+ # Scales with `fraction`, so the free-memory rule binds only once > 20% of the card is in use.+ default_free_fraction = max(0.5, fraction / 0.8)+ try:+ free_fraction = float(+ os.environ.get("UNSLOTH_FSDP2_FREE_FIT_FRACTION", str(default_free_fraction))+ )+ except ValueError:+ free_fraction = default_free_fraction+ model_bytes = layout.frozen_bytes + layout.trainable_bytes# "Fits" = the whole model (what DDP keeps on every GPU) uses at most `fraction` of one device,# leaving the rest for activations, gradients and optimizer state.# CPU offload asks for memory, not speed: keep the plain sharded layout.@@ -143,10 +185,21 @@ def plan_fsdp2_layout(layout.fits = (not cpu_offloadand layout.device_bytes > 0- and (layout.frozen_bytes + layout.trainable_bytes) <= fraction * layout.device_bytes+ and model_bytes <= fraction * layout.device_bytes)+ if layout.fits:+ # Crowded GPUs shard; ranks AND their decision, so one crowded rank shards everyone.+ free = _device_free_bytes(device)+ if free is not None:+ layout.available_bytes = free + _resident_bytes(params, device)+ if model_bytes > free_fraction * layout.available_bytes:+ layout.fits = False+ layout.reasons.append(+ f"fits=False: only {layout.available_bytes / 2**30:.2f} GiB usable on this GPU "+ f"(free + cached + resident model), model needs <= {free_fraction} of it"+ )layout.reasons.append(- f"fits={layout.fits} (model {(layout.frozen_bytes + layout.trainable_bytes) / 2**30:.2f} GiB vs {fraction} x {layout.device_bytes / 2**30:.1f} GiB)"+ f"fits={layout.fits} (model {model_bytes / 2**30:.2f} GiB vs {fraction} x {layout.device_bytes / 2**30:.1f} GiB)")replicate = replicate_frozendiff --git a/unsloth/utils/mfu.py b/unsloth/utils/mfu.pyindex db13498ae3c..ada666b1e58 100644--- a/unsloth/utils/mfu.py+++ b/unsloth/utils/mfu.py@@ -4,18 +4,23 @@"""Opt-in achieved TFLOPs and MFU logging with LoRA-aware FLOP counts.Turn on with `UNSLOTH_LOG_MFU=1` before building the trainer, or call `add_mfu_callback(trainer)`.-Each training log then gains `train_tflops_per_gpu` and, when the GPU peak is known,-`train_mfu`. Set `UNSLOTH_PEAK_TFLOPS` to the per GPU dense peak to override the table-(for example a measured MAMF number).+Each training log then gains `train_tflops_per_gpu` and `train_hw_tflops_per_gpu` and, when the+GPU peak is known, `train_mfu` and `train_hfu`. Set `UNSLOTH_PEAK_TFLOPS` to the per GPU dense+peak to override the table (for example a measured MAMF number).++`train_mfu` excludes the checkpointing recompute (PaLM MFU); `train_hfu` includes it. They were+one value (HFU) before the split; that value is now `train_hfu` / `train_hw_tflops_per_gpu`.Model FLOPs per non padding token, summed over every linear layer with n = in * out weights:forward 2n every linear, frozen or trainablebackward input grads 2n every linear (gradients flow through frozen weights to LoRA)backward weight grads 2n only linears whose weight requires gradrecompute 2n every linear except the output head, when checkpointing+ (hardware FLOPs / HFU only)Attention adds 4 * layers * heads * head_dim FLOPs per causal (query, key) pair in the forward(QK^T and AV), L * (L + 1) / 2 pairs for a sequence of length L (L * L when every attention-layer is bidirectional), times 3 for forward plus backward and times 4 with checkpointing. The+layer is bidirectional), times 3 for forward plus backward (times 4 with checkpointing in the+hardware count). Theoutput head is charged only for the last `logits_to_keep` positions when the forward passes it.Only the active copy of a PEFT `modules_to_save` wrapper and the active LoRA adapters arecounted. Experts held as `nn.Linear` in an `experts` ModuleList count top_k / num_experts of@@ -55,6 +60,11 @@ _PEAK_TFLOPS_16BIT = (("H200 NVL", 835.5),("H200", 989.0),("GH200", 989.0),+ # Dense = half NVIDIA's sparse figure (nvidia.com/en-us/data-center/hgx, gb200-nvl72).+ # GB* first: the B200 / B300 keys would match inside their names.+ ("GB300", 2500.0),+ ("GB200", 2500.0),+ ("B300", 2250.0),("B200", 2250.0),("A100", 312.0),("L40S", 362.05),@@ -382,7 +392,8 @@ def _unsupported_setup(class UnslothMFUCallback(TrainerCallback):- """Adds `train_tflops_per_gpu` and `train_mfu` to training logs. Prefer `add_mfu_callback`."""+ """Adds `train_tflops_per_gpu` / `train_mfu` (model FLOPs) and `train_hw_tflops_per_gpu` /+ `train_hfu` (recompute included) to training logs. Prefer `add_mfu_callback`."""def __init__(self,@@ -395,6 +406,8 @@ class UnslothMFUCallback(TrainerCallback):self._linear_per_token = 0self._head_per_token = 0self._attention_per_pair = 0+ self._hw_linear_per_token = 0+ self._hw_attention_per_pair = 0self._bidirectional = Falseself._num_devices = 1self._devices = []@@ -467,11 +480,15 @@ class UnslothMFUCallback(TrainerCallback):checkpointing = self.gradient_checkpointingif checkpointing is None:checkpointing = _gradient_checkpointing_on(model, args)- total, _, self._head_per_token = _linear_flops(model, checkpointing)- self._linear_per_token = total - self._head_per_token+ total, breakdown, self._head_per_token = _linear_flops(model, checkpointing)+ self._hw_linear_per_token = total - self._head_per_token+ self._linear_per_token = self._hw_linear_per_token - breakdown["recompute"]self._bidirectional = _all_attention_bidirectional(model)self._unsloth_head_window = _runs_unsloth_head_window(model)self._attention_per_pair = (+ count_attention_flops_per_pair(config, False) if config is not None else 0+ )+ self._hw_attention_per_pair = (count_attention_flops_per_pair(config, checkpointing) if config is not None else 0)devices = sorted(@@ -551,16 +568,15 @@ class UnslothMFUCallback(TrainerCallback):self._reset()if tokens <= 0 or elapsed <= 0:return- flops = (- tokens * self._linear_per_token- + head_tokens * self._head_per_token- + pairs * self._attention_per_pair- )- flops = flops / self._num_devices- tflops = flops / elapsed / 1e12- extra = {"train_tflops_per_gpu": tflops}+ head = head_tokens * self._head_per_token+ flops = tokens * self._linear_per_token + head + pairs * self._attention_per_pair+ hw_flops = tokens * self._hw_linear_per_token + head + pairs * self._hw_attention_per_pair+ tflops = flops / self._num_devices / elapsed / 1e12+ hw_tflops = hw_flops / self._num_devices / elapsed / 1e12+ extra = {"train_tflops_per_gpu": tflops, "train_hw_tflops_per_gpu": hw_tflops}if self.peak_tflops:extra["train_mfu"] = tflops / self.peak_tflops+ extra["train_hfu"] = hw_tflops / self.peak_tflopslogs.update(extra)# Trainer.log appended a copy of logs to log_history before calling the callbacks.if state.log_history and state.log_history[-1].get("step") == state.global_step: