Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Divide by gradient accumulation for forwards that take **kwargs but return a mean loss (#11898)

Daniel Hanee495e4cd0a5
diff --git a/tests/test_accepts_loss_kwargs_remote_mean_loss.py b/tests/test_accepts_loss_kwargs_remote_mean_loss.py
new file mode 100644
index 00000000000..3b36255efd5
--- /dev/null
+++ b/tests/test_accepts_loss_kwargs_remote_mean_loss.py
@@ -0,0 +1,621 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""**kwargs forward returning its own CrossEntropyLoss mean must not count as consuming
+num_items_in_batch (NemotronH, GA=4: logged loss 4.22 vs eval 0.84). Helpers loaded by AST, CPU only."""
+
+import ast
+import inspect
+import os
+import pathlib
+import re
+import sys
+import textwrap
+import types
+
+import pytest
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss
+
+_UTILS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_utils.py"
+_NAMES = {
+ "_unsloth_compile_cache_leaves",
+ "_forward_is_unsloth_compiled",
+ "_find_concrete_accepts_loss_kwargs",
+ "_shadow_accepts_loss_kwargs",
+ "_forward_ignores_num_items_in_batch",
+ "_instance_accepts_loss_kwargs",
+ "_ce_calls_all_mean",
+ "_CE_PARAMS",
+ "_is_const",
+ "_DEFAULT_CE_NAMESPACE",
+ "_resolve_ce_callee",
+ "_dotted_name",
+ "_scan_ce_calls",
+ "_GUESSED_LOSS_KWARGS",
+ "_loss_kwargs_chain",
+ "_clear_guessed_accepts_loss_kwargs",
+ "_is_guess",
+ "_TORCH_CE",
+ "apply_accepts_loss_kwargs_fix",
+}
+
+
+def _load():
+ src = _UTILS.read_text(encoding = "utf-8")
+ tree = ast.parse(src)
+ keep = []
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name in _NAMES:
+ keep.append(node)
+ elif isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id in _NAMES for t in node.targets
+ ):
+ keep.append(node)
+ ns = {"ast": ast, "re": re, "inspect": inspect, "os": os, "textwrap": textwrap, "torch": torch}
+ exec(compile(ast.Module(body = keep, type_ignores = []), str(_UTILS), "exec"), ns)
+ return ns
+
+
+# On disk so inspect.getsource works, like a real remote module.
+_MODELS = '''
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss
+
+
+class Backbone(nn.Module):
+ def forward(self, input_ids, **kwargs):
+ return input_ids
+
+
+class NemotronHForCausalLM(nn.Module):
+ """Remote code: **kwargs only for generate, loss is a plain mean."""
+ def __init__(self):
+ super().__init__()
+ self.model = Backbone()
+
+ def forward(self, input_ids=None, labels=None, **kwargs): # for now we need this for generation
+ logits = torch.zeros(1)
+ loss = None
+ if labels is not None:
+ loss_fct = CrossEntropyLoss()
+ loss = loss_fct(logits.view(-1, 1), labels.view(-1))
+ return loss
+
+
+class NativeForCausalLM(nn.Module):
+ """transformers-native: kwargs reach self.loss_function, which divides by num_items_in_batch."""
+ def __init__(self):
+ super().__init__()
+ self.model = Backbone()
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ logits = torch.zeros(1)
+ return self.loss_function(logits, labels, 1, **kwargs)
+
+
+class HandRolledForCausalLM(nn.Module):
+ """Own cross_entropy that does honour num_items_in_batch."""
+ def __init__(self):
+ super().__init__()
+ self.model = Backbone()
+
+ def forward(self, input_ids=None, labels=None, num_items_in_batch=None, **kwargs):
+ logits = torch.zeros(1)
+ loss = torch.nn.functional.cross_entropy(logits, labels, reduction="sum")
+ return loss / num_items_in_batch
+
+
+class SumLossForCausalLM(nn.Module):
+ """Remote code whose objective is a summed loss: dividing by GA would change it."""
+ def __init__(self):
+ super().__init__()
+ self.model = Backbone()
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ logits = torch.zeros(1)
+ loss_fct = CrossEntropyLoss(reduction="sum")
+ return loss_fct(logits.view(-1, 1), labels.view(-1))
+
+
+class ExplicitMeanForCausalLM(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = Backbone()
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ logits = torch.zeros(1)
+ return torch.nn.functional.cross_entropy(logits, labels, reduction="mean")
+
+
+class InstanceDeclaredForCausalLM(nn.Module):
+ """Remote code that declares loss-kwargs support on the instance in __init__."""
+ def __init__(self):
+ super().__init__()
+ self.model = Backbone()
+ self.accepts_loss_kwargs = True
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ logits = torch.zeros(1)
+ loss_fct = CrossEntropyLoss()
+ return loss_fct(logits.view(-1, 1), labels.view(-1))
+
+
+class LossModuleForCausalLM(nn.Module):
+ """Builds its CrossEntropyLoss once in __init__ and calls it from forward."""
+ def __init__(self, reduction="mean"):
+ super().__init__()
+ self.model = Backbone()
+ self.loss_fct = CrossEntropyLoss(reduction=reduction)
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ logits = torch.zeros(1)
+ return self.loss_fct(logits.view(-1, 1), labels.view(-1))
+
+
+class QWenLMHeadModel(nn.Module):
+ """Remote causal LM named *LMHeadModel rather than *CausalLM."""
+ def __init__(self):
+ super().__init__()
+ self.transformer = Backbone()
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ logits = torch.zeros(1)
+ loss_fct = CrossEntropyLoss()
+ return loss_fct(logits.view(-1, 1), labels.view(-1))
+
+
+class NoKwargsForCausalLM(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = Backbone()
+
+ def forward(self, input_ids=None, labels=None):
+ loss_fct = CrossEntropyLoss()
+ return loss_fct(input_ids, labels)
+'''
+
+
+_PRETRAINED_MODELS = """
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from transformers import GenerationMixin, PretrainedConfig, PreTrainedModel
+
+
+class RemoteMeanLossConfig(PretrainedConfig):
+ model_type = "remote_mean_loss_for_ga_test"
+
+
+class RemoteMeanLossForCausalLM(PreTrainedModel, GenerationMixin):
+ config_class = RemoteMeanLossConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.proj = nn.Linear(4, 4)
+ self.post_init()
+
+ def _init_weights(self, module):
+ pass
+
+ def forward(self, input_ids=None, labels=None, **kwargs): # for now we need this for generation
+ logits = self.proj(torch.zeros(1, 4))
+ loss = None
+ if labels is not None:
+ loss_fct = CrossEntropyLoss()
+ loss = loss_fct(logits, labels.view(-1))
+ return loss
+"""
+
+
+def _models(
+ tmp_path,
+ source = _MODELS,
+ name = "remote_models_for_ga_test",
+):
+ p = tmp_path / f"{name}.py"
+ p.write_text(source, encoding = "utf-8")
+ sys.path.insert(0, str(tmp_path))
+ try:
+ import importlib
+ mod = importlib.import_module(name)
+ return importlib.reload(mod)
+ finally:
+ sys.path.remove(str(tmp_path))
+
+
+class _PeftLike(nn.Module):
+ def __init__(self, inner):
+ super().__init__()
+ self.base_model = types.SimpleNamespace(model = inner)
+ self.base_model.base_model = None
+
+ def forward(self, *args, **kwargs):
+ return None
+
+
+def test_remote_mean_loss_forward_is_not_a_loss_kwargs_consumer(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ model = mods.NemotronHForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert getattr(model, "accepts_loss_kwargs", None) is False
+
+
+def test_remote_mean_loss_under_a_peft_wrapper(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ inner = mods.NemotronHForCausalLM()
+ wrapped = _PeftLike(inner)
+ ns["apply_accepts_loss_kwargs_fix"](wrapped)
+ assert getattr(inner, "accepts_loss_kwargs", None) is False
+ assert getattr(wrapped, "accepts_loss_kwargs", None) is False
+
+
+def test_consumers_keep_the_hf_default(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ for cls in (
+ mods.NativeForCausalLM,
+ mods.HandRolledForCausalLM,
+ mods.NoKwargsForCausalLM,
+ mods.SumLossForCausalLM,
+ ):
+ model = cls()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert not hasattr(model, "accepts_loss_kwargs"), cls.__name__
+
+
+def test_explicit_class_attribute_still_wins(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+
+ class Declared(mods.NemotronHForCausalLM):
+ accepts_loss_kwargs = True
+
+ model = Declared()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert model.accepts_loss_kwargs is True
+
+
+def test_remote_mean_loss_under_a_real_peft_model(tmp_path):
+ peft = pytest.importorskip("peft")
+ ns = _load()
+ mods = _models(tmp_path, _PRETRAINED_MODELS, "remote_pretrained_for_ga_test")
+ inner = mods.RemoteMeanLossForCausalLM(mods.RemoteMeanLossConfig())
+ wrapped = peft.get_peft_model(
+ inner, peft.LoraConfig(r = 2, target_modules = ["proj"], task_type = "CAUSAL_LM")
+ )
+ assert "CausalLM" in type(wrapped).__name__
+ ns["apply_accepts_loss_kwargs_fix"](wrapped)
+ assert getattr(wrapped.get_base_model(), "accepts_loss_kwargs", None) is False
+ assert getattr(wrapped, "accepts_loss_kwargs", None) is False
+
+
+def test_explicit_mean_reduction_is_still_a_mean_loss(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ model = mods.ExplicitMeanForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert getattr(model, "accepts_loss_kwargs", None) is False
+
+
+def test_instance_declaration_is_kept(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ model = mods.InstanceDeclaredForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert model.accepts_loss_kwargs is True
+ wrapped = _PeftLike(model)
+ ns["apply_accepts_loss_kwargs_fix"](wrapped)
+ assert model.accepts_loss_kwargs is True
+
+
+def test_repeated_calls_keep_the_first_answer(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ model = mods.NemotronHForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ ns["apply_accepts_loss_kwargs_fix"](_PeftLike(model))
+ assert model.accepts_loss_kwargs is False
+
+
+ "call",
+ [
+ 'CrossEntropyLoss(None, None, -100, None, "sum")(logits, labels)',
+ 'torch.nn.functional.cross_entropy(logits, labels, None, None, -100, None, "none")',
+ "CrossEntropyLoss(size_average=False)(logits, labels)",
+ "CrossEntropyLoss(reduce=False)(logits, labels)",
+ "CrossEntropyLoss(*loss_args)(logits, labels)",
+ "CrossEntropyLoss(None, False)(logits, labels)",
+ "CrossEntropyLoss(None, None, -100, False)(logits, labels)",
+ "torch.nn.functional.cross_entropy(logits, labels, None, False)",
+ "torch.nn.functional.cross_entropy(logits, labels, None, None, -100, False)",
+ ],
+)
+def test_non_mean_reductions_in_any_spelling_keep_the_hf_default(call):
+ ns = _load()
+ source = f"def forward(self, logits, labels, **kwargs):\n return {call}\n"
+ assert ns["_ce_calls_all_mean"](source) is False
+
+
+ "call",
+ [
+ "CrossEntropyLoss()(logits, labels)",
+ "CrossEntropyLoss(None, None, -100)(logits, labels)",
+ 'CrossEntropyLoss(ignore_index=-100, reduction="mean")(logits, labels)',
+ "torch.nn.functional.cross_entropy(logits, labels, None, None, -100)",
+ "CrossEntropyLoss(size_average=None, reduce=True)(logits, labels)",
+ ],
+)
+def test_mean_reductions_in_any_spelling_count_as_a_mean(call):
+ ns = _load()
+ source = f"def forward(self, logits, labels, **kwargs):\n return {call}\n"
+ assert ns["_ce_calls_all_mean"](source) is True
+
+
+def test_a_mention_without_a_call_is_not_a_mean_loss():
+ ns = _load()
+ source = (
+ "def forward(self, logits, labels, **kwargs):\n"
+ ' """Unlike CrossEntropyLoss() this returns a sum."""\n'
+ " return (logits - labels).abs().sum()\n"
+ )
+ assert ns["_ce_calls_all_mean"](source) is False
+
+
+def test_a_model_helper_named_cross_entropy_is_not_the_torch_op():
+ ns = _load()
+ source = (
+ "def forward(self, logits, labels, **kwargs):\n"
+ " return self.loss_helpers.cross_entropy(logits, labels)\n"
+ )
+ assert ns["_ce_calls_all_mean"](source) is False
+ for torch_spelling in (
+ "torch.nn.functional.cross_entropy(logits, labels)",
+ "F.cross_entropy(logits, labels)",
+ "nn.CrossEntropyLoss()(logits, labels)",
+ "CrossEntropyLoss()(logits, labels)",
+ ):
+ source = f"def forward(self, logits, labels, **kwargs):\n return {torch_spelling}\n"
+ assert ns["_ce_calls_all_mean"](source) is True, torch_spelling
+
+
+def test_a_wrapper_added_after_load_gets_the_load_time_answer(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ inner = mods.NemotronHForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](inner)
+
+ class UserWrapper(nn.Module):
+ def __init__(self, model):
+ super().__init__()
+ self.model = model
+
+ def forward(self, *args, **kwargs):
+ return self.model(*args, **kwargs)
+
+ wrapped = UserWrapper(inner)
+ ns["apply_accepts_loss_kwargs_fix"](wrapped)
+ assert wrapped.accepts_loss_kwargs is False
+ assert inner.accepts_loss_kwargs is False
+
+
+def test_the_instance_forward_is_the_one_inspected(tmp_path):
+ import functools
+
+ ns = _load()
+ mods = _models(tmp_path)
+
+ model = mods.NemotronHForCausalLM()
+
+ def consuming_forward(
+ input_ids = None,
+ labels = None,
+ num_items_in_batch = None,
+ **kwargs,
+ ):
+ return None
+
+ model.forward = consuming_forward
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert not hasattr(model, "accepts_loss_kwargs")
+
+ # An accelerate-style hook (partial + update_wrapper) still resolves to the original mean-loss forward.
+ hooked = mods.NemotronHForCausalLM()
+ old_forward = hooked.forward
+
+ def new_forward(module, *args, **kwargs):
+ return old_forward(*args, **kwargs)
+
+ hooked.forward = functools.update_wrapper(functools.partial(new_forward, hooked), old_forward)
+ ns["apply_accepts_loss_kwargs_fix"](hooked)
+ assert hooked.accepts_loss_kwargs is False
+
+
+def test_a_loss_module_built_in_init_counts_by_its_reduction(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ model = mods.LossModuleForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert model.accepts_loss_kwargs is False
+ summed = mods.LossModuleForCausalLM(reduction = "sum")
+ ns["apply_accepts_loss_kwargs_fix"](summed)
+ assert not hasattr(summed, "accepts_loss_kwargs")
+
+
+def test_an_lm_head_model_is_a_causal_lm(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ model = mods.QWenLMHeadModel()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert model.accepts_loss_kwargs is False
+
+
+def test_a_forward_replaced_after_load_is_checked_again(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ model = mods.NemotronHForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert model.accepts_loss_kwargs is False
+
+ def consuming_forward(
+ input_ids = None,
+ labels = None,
+ num_items_in_batch = None,
+ **kwargs,
+ ):
+ return None
+
+ model.forward = consuming_forward
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert not hasattr(model, "accepts_loss_kwargs")
+
+
+def test_an_assignment_after_the_guess_wins(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path)
+ model = mods.NemotronHForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert model.accepts_loss_kwargs is False
+ model.accepts_loss_kwargs = True
+ ns["apply_accepts_loss_kwargs_fix"](_PeftLike(model))
+ assert model.accepts_loss_kwargs is True
+
+
+_OWN_HELPER_MODEL = """
+import torch
+from torch import nn
+
+
+def cross_entropy(logits, labels):
+ return torch.nn.functional.cross_entropy(logits, labels, reduction="sum")
+
+
+class OwnHelperForCausalLM(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Identity()
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ logits = torch.zeros(1, 2)
+ return cross_entropy(logits, labels)
+"""
+
+
+def test_a_bare_name_bound_to_a_model_helper_is_not_the_torch_op(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path, _OWN_HELPER_MODEL, "own_helper_for_ga_test")
+ model = mods.OwnHelperForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert not hasattr(model, "accepts_loss_kwargs")
+
+
+_LOCAL_HELPER_MODEL = """
+import torch
+from torch import nn
+
+
+class LocalHelperForCausalLM(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Identity()
+
+ def summed_loss(self, logits, labels):
+ return torch.nn.functional.cross_entropy(logits, labels, reduction="sum")
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ cross_entropy = self.summed_loss
+ logits = torch.zeros(1, 2)
+ return cross_entropy(logits, labels)
+"""
+
+
+def test_a_bare_name_bound_inside_forward_is_not_the_torch_op(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path, _LOCAL_HELPER_MODEL, "local_helper_for_ga_test")
+ model = mods.LocalHelperForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert not hasattr(model, "accepts_loss_kwargs")
+
+
+_OWN_F_MODEL = """
+import torch
+from torch import nn
+
+
+class _Helpers:
+ @staticmethod
+ def cross_entropy(logits, labels):
+ return torch.nn.functional.cross_entropy(logits, labels, reduction="sum")
+
+
+F = _Helpers()
+
+
+class OwnFForCausalLM(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Identity()
+
+ def forward(self, input_ids=None, labels=None, **kwargs):
+ logits = torch.zeros(1, 2)
+ return F.cross_entropy(logits, labels)
+"""
+
+
+def test_a_qualifier_bound_to_a_model_helper_is_not_the_torch_op(tmp_path):
+ ns = _load()
+ mods = _models(tmp_path, _OWN_F_MODEL, "own_f_for_ga_test")
+ model = mods.OwnFForCausalLM()
+ ns["apply_accepts_loss_kwargs_fix"](model)
+ assert not hasattr(model, "accepts_loss_kwargs")
+
+
+def test_a_guess_on_the_peft_causal_head_is_cleared_when_forward_changes(tmp_path):
+ peft = pytest.importorskip("peft")
+ ns = _load()
+ # base_model_prefix = "model", as in real checkpoints, so .base_model walks past the causal head.
+ source = _PRETRAINED_MODELS.replace(
+ " config_class = RemoteMeanLossConfig\n",
+ ' config_class = RemoteMeanLossConfig\n base_model_prefix = "model"\n',
+ ).replace(
+ " self.proj = nn.Linear(4, 4)\n",
+ " self.proj = nn.Linear(4, 4)\n self.model = nn.Linear(4, 4)\n",
+ )
+ mods = _models(tmp_path, source, "remote_prefixed_for_ga_test")
+ inner = mods.RemoteMeanLossForCausalLM(mods.RemoteMeanLossConfig())
+ assert inner.base_model is not inner
+ ns["apply_accepts_loss_kwargs_fix"](inner)
+ assert inner.accepts_loss_kwargs is False
+ wrapped = peft.get_peft_model(
+ inner, peft.LoraConfig(r = 2, target_modules = ["proj"], task_type = "CAUSAL_LM")
+ )
+
+ def consuming_forward(
+ input_ids = None,
+ labels = None,
+ num_items_in_batch = None,
+ **kwargs,
+ ):
+ return None
+
+ wrapped.get_base_model().forward = consuming_forward
+ ns["apply_accepts_loss_kwargs_fix"](wrapped)
+ assert "accepts_loss_kwargs" not in wrapped.get_base_model().__dict__
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index af78c2cb240..89c9b26ae29 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -115,6 +115,7 @@ from platform import system as platform_system
platform_system = platform_system()
import numpy as np
+import ast
import contextlib
import copy
import re
@@ -4249,8 +4250,16 @@ def _find_concrete_accepts_loss_kwargs(model):
return None, "no explicit accepts_loss_kwargs on any wrapper level"
-def _shadow_accepts_loss_kwargs(model, value):
+_GUESSED_LOSS_KWARGS = "_unsloth_guessed_accepts_loss_kwargs"
+
+
+def _shadow_accepts_loss_kwargs(
+ model,
+ value,
+ guessed = False,
+):
# Set the attribute at every wrapper level so HF's hasattr check resolves wherever accelerator or peft unwrap lands.
+ # guessed marks values the source heuristic wrote, so a later call re-checks them instead of trusting them.
seen = set()
m = model
for _ in range(8):
@@ -4259,6 +4268,10 @@ def _shadow_accepts_loss_kwargs(model, value):
seen.add(id(m))
try:
setattr(m, "accepts_loss_kwargs", value)
+ if guessed:
+ m.__dict__[_GUESSED_LOSS_KWARGS] = value
+ else:
+ m.__dict__.pop(_GUESSED_LOSS_KWARGS, None)
except Exception:
pass
nxt = getattr(m, "base_model", None)
@@ -4277,11 +4290,236 @@ def apply_accepts_loss_kwargs_fix(model):
value, reason = _find_concrete_accepts_loss_kwargs(model)
if value is None:
+ declared = _instance_accepts_loss_kwargs(model)
+ if declared is not None:
+ # Set by the model itself: keep it, and carry it onto any wrapper added since.
+ _shadow_accepts_loss_kwargs(model, declared)
+ return f"{declared} (instance accepts_loss_kwargs)"
+ # Re-check an earlier guess: forward may have been replaced since.
+ _clear_guessed_accepts_loss_kwargs(model)
+ causal_lm = _forward_ignores_num_items_in_batch(model)
+ if causal_lm is not None:
+ _shadow_accepts_loss_kwargs(model, False, guessed = True)
+ # transformers 5 reads the flag off get_base_model(), which the walk above can skip under PEFT.
+ _shadow_accepts_loss_kwargs(causal_lm, False, guessed = True)
+ return "False (forward takes **kwargs but computes its own mean loss)"
return f"default (signature inspection, {reason})"
_shadow_accepts_loss_kwargs(model, value)
return f"{value} ({reason})"
+def _loss_kwargs_chain(model):
+ seen = set()
+ m = model
+ for _ in range(8):
+ if m is None or id(m) in seen:
+ return
+ seen.add(id(m))
+ yield m
+ nxt = getattr(m, "base_model", None)
+ if nxt is None or nxt is m:
+ nxt = getattr(m, "model", None)
+ m = nxt
+
+
+def _is_guess(d):
+ # Still the heuristic's value; an assignment made since (e.g. by the user) is a declaration.
+ return _GUESSED_LOSS_KWARGS in d and d.get("accepts_loss_kwargs") == d[_GUESSED_LOSS_KWARGS]
+
+
+def _clear_guessed_accepts_loss_kwargs(model):
+ chain = list(_loss_kwargs_chain(model))
+ # Under PEFT the walk skips the causal head that get_base_model() returns (and transformers 5 reads).
+ try:
+ head = model.get_base_model() if hasattr(model, "get_base_model") else None
+ except Exception:
+ head = None
+ if head is not None and all(head is not m for m in chain):
+ chain.append(head)
+ for m in chain:
+ d = getattr(m, "__dict__", {})
+ if _is_guess(d):
+ d.pop("accepts_loss_kwargs", None)
+ d.pop(_GUESSED_LOSS_KWARGS, None)
+
+
+def _instance_accepts_loss_kwargs(model):
+ seen = set()
+ m = model
+ for _ in range(8):
+ if m is None or id(m) in seen:
+ return None
+ seen.add(id(m))
+ d = getattr(m, "__dict__", {})
+ value = None if _is_guess(d) else d.get("accepts_loss_kwargs", None)
+ if value is not None:
+ return value
+ nxt = getattr(m, "base_model", None)
+ if nxt is None or nxt is m:
+ nxt = getattr(m, "model", None)
+ m = nxt
+ return None
+
+
+_CE_PARAMS = {
+ "CrossEntropyLoss": (
+ "weight",
+ "size_average",
+ "ignore_index",
+ "reduce",
+ "reduction",
+ "label_smoothing",
+ ),
+ "cross_entropy": (
+ "input",
+ "target",
+ "weight",
+ "size_average",
+ "ignore_index",
+ "reduce",
+ "reduction",
+ "label_smoothing",
+ ),
+}
+
+
+def _dotted_name(node):
+ parts = []
+ while isinstance(node, ast.Attribute):
+ parts.append(node.attr)
+ node = node.value
+ if not isinstance(node, ast.Name):
+ return None
+ parts.append(node.id)
+ return ".".join(reversed(parts))
+
+
+def _is_const(node, allowed):
+ return isinstance(node, ast.Constant) and node.value in allowed
+
+
+def _ce_calls_all_mean(source, namespace = None):
+ all_mean, found = _scan_ce_calls(
+ source, _DEFAULT_CE_NAMESPACE if namespace is None else namespace
+ )
+ # A mention in a comment or docstring is not a call.
+ return all_mean and found
+
+
+_TORCH_CE = {
+ "CrossEntropyLoss": torch.nn.CrossEntropyLoss,
+ "cross_entropy": torch.nn.functional.cross_entropy,
+}
+
+
+# The usual imports, for callers that have source but no module to resolve it in.
+_DEFAULT_CE_NAMESPACE = {
+ "torch": torch,
+ "nn": torch.nn,
+ "F": torch.nn.functional,
+ "functional": torch.nn.functional,
+ **_TORCH_CE,
+}
+
+
+def _resolve_ce_callee(func, namespace):
+ dotted = _dotted_name(func)
+ if dotted is None or not namespace:
+ return None
+ head, *rest = dotted.split(".")
+ obj = namespace.get(head)
+ for attr in rest:
+ obj = getattr(obj, attr, None)
+ for name, target in _TORCH_CE.items():
+ if obj is target:
+ return name
+ return None
+
+
+def _scan_ce_calls(source, namespace = None):
+ # A sum / "none" reduction (modern or legacy, keyword or positional) is the model's own objective, not a micro-batch mean.
+ # Returns (every call is a mean, at least one call was seen).
+ try:
+ tree = ast.parse(textwrap.dedent(source))
+ except SyntaxError:
+ return False, False
+ found = False
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ # Only callees the forward's module binds to the PyTorch op; a model's own helper may reduce however it likes.
+ name = _resolve_ce_callee(node.func, namespace)
+ params = _CE_PARAMS.get(name)
+ if params is None:
+ continue
+ if len(node.args) > len(params) or any(isinstance(x, ast.Starred) for x in node.args):
+ return False, found
+ if any(kw.arg is None for kw in node.keywords):
+ return False, found
+ bound = dict(zip(params, node.args))
+ bound.update((kw.arg, kw.value) for kw in node.keywords)
+ if "reduction" in bound and not _is_const(bound["reduction"], ("mean",)):
+ return False, found
+ for legacy in ("size_average", "reduce"):
+ if legacy in bound and not _is_const(bound[legacy], (None, True)):
+ return False, found
+ found = True
+ return True, found
+
+
+def _forward_ignores_num_items_in_batch(model):
+ # HF treats **kwargs as consuming num_items_in_batch and skips 1/GA; remote code (NemotronH) keeps **kwargs for generate yet returns a CrossEntropyLoss mean, so loss + grads come out GA x too large.
+ m = model
+ try:
+ # PeftModelForCausalLM itself matches "CausalLM".
+ if hasattr(m, "get_base_model"):
+ m = m.get_base_model()
+ except Exception:
+ m = model
+ seen = set()
+ for _ in range(6):
+ if m is None or id(m) in seen:
+ return None
+ seen.add(id(m))
+ name = type(m).__name__
+ if "CausalLM" in name or "ForConditionalGeneration" in name or "LMHeadModel" in name:
+ break
+ nxt = getattr(m, "base_model", None)
+ if nxt is None or nxt is m:
+ nxt = getattr(m, "model", None)
+ m = nxt
+ else:
+ return None
+ # The instance forward is what Trainer inspects and calls (accelerate hooks and loaders replace it).
+ forward = getattr(m, "forward", None)
+ try:
+ forward = inspect.unwrap(forward) if forward is not None else None
+ except ValueError:
+ return None
+ try:
+ params = inspect.signature(forward).parameters.values()
+ source = inspect.getsource(forward)
+ except Exception:
+ return None
+ if not any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params):
+ return None
+ if "num_items_in_batch" in source or "loss_function" in source:
+ return None
+ used = [
+ sub
+ for name, sub in m.named_children()
+ if isinstance(sub, torch.nn.CrossEntropyLoss)
+ and re.search(rf"\bself\.{re.escape(name)}\s*\(", source)
+ ]
+ if any(sub.reduction != "mean" for sub in used):
+ return None
+ namespace = getattr(getattr(forward, "__func__", forward), "__globals__", None) or {}
+ all_mean, found = _scan_ce_calls(source, namespace)
+ if not all_mean or not (found or used):
+ return None
+ return m
+
+
def patch_tokenizer(model, tokenizer):
model, tokenizer = _patch_tokenizer(model, tokenizer)
if model is not None: