Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Divide by gradient accumulation for forwards that take **kwargs but return a mean loss (#11898)
Daniel Hanee495e4cd0a5
diff --git a/tests/test_accepts_loss_kwargs_remote_mean_loss.py b/tests/test_accepts_loss_kwargs_remote_mean_loss.pynew file mode 100644index 00000000000..3b36255efd5--- /dev/null+++ b/tests/test_accepts_loss_kwargs_remote_mean_loss.py@@ -0,0 +1,621 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.+"""**kwargs forward returning its own CrossEntropyLoss mean must not count as consuming+num_items_in_batch (NemotronH, GA=4: logged loss 4.22 vs eval 0.84). Helpers loaded by AST, CPU only."""++import ast+import inspect+import os+import pathlib+import re+import sys+import textwrap+import types++import pytest+import torch+from torch import nn+from torch.nn import CrossEntropyLoss++_UTILS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_utils.py"+_NAMES = {+ "_unsloth_compile_cache_leaves",+ "_forward_is_unsloth_compiled",+ "_find_concrete_accepts_loss_kwargs",+ "_shadow_accepts_loss_kwargs",+ "_forward_ignores_num_items_in_batch",+ "_instance_accepts_loss_kwargs",+ "_ce_calls_all_mean",+ "_CE_PARAMS",+ "_is_const",+ "_DEFAULT_CE_NAMESPACE",+ "_resolve_ce_callee",+ "_dotted_name",+ "_scan_ce_calls",+ "_GUESSED_LOSS_KWARGS",+ "_loss_kwargs_chain",+ "_clear_guessed_accepts_loss_kwargs",+ "_is_guess",+ "_TORCH_CE",+ "apply_accepts_loss_kwargs_fix",+}+++def _load():+ src = _UTILS.read_text(encoding = "utf-8")+ tree = ast.parse(src)+ keep = []+ for node in tree.body:+ if isinstance(node, ast.FunctionDef) and node.name in _NAMES:+ keep.append(node)+ elif isinstance(node, ast.Assign) and any(+ isinstance(t, ast.Name) and t.id in _NAMES for t in node.targets+ ):+ keep.append(node)+ ns = {"ast": ast, "re": re, "inspect": inspect, "os": os, "textwrap": textwrap, "torch": torch}+ exec(compile(ast.Module(body = keep, type_ignores = []), str(_UTILS), "exec"), ns)+ return ns+++# On disk so inspect.getsource works, like a real remote module.+_MODELS = '''+import torch+from torch import nn+from torch.nn import CrossEntropyLoss+++class Backbone(nn.Module):+ def forward(self, input_ids, **kwargs):+ return input_ids+++class NemotronHForCausalLM(nn.Module):+ """Remote code: **kwargs only for generate, loss is a plain mean."""+ def __init__(self):+ super().__init__()+ self.model = Backbone()++ def forward(self, input_ids=None, labels=None, **kwargs): # for now we need this for generation+ logits = torch.zeros(1)+ loss = None+ if labels is not None:+ loss_fct = CrossEntropyLoss()+ loss = loss_fct(logits.view(-1, 1), labels.view(-1))+ return loss+++class NativeForCausalLM(nn.Module):+ """transformers-native: kwargs reach self.loss_function, which divides by num_items_in_batch."""+ def __init__(self):+ super().__init__()+ self.model = Backbone()++ def forward(self, input_ids=None, labels=None, **kwargs):+ logits = torch.zeros(1)+ return self.loss_function(logits, labels, 1, **kwargs)+++class HandRolledForCausalLM(nn.Module):+ """Own cross_entropy that does honour num_items_in_batch."""+ def __init__(self):+ super().__init__()+ self.model = Backbone()++ def forward(self, input_ids=None, labels=None, num_items_in_batch=None, **kwargs):+ logits = torch.zeros(1)+ loss = torch.nn.functional.cross_entropy(logits, labels, reduction="sum")+ return loss / num_items_in_batch+++class SumLossForCausalLM(nn.Module):+ """Remote code whose objective is a summed loss: dividing by GA would change it."""+ def __init__(self):+ super().__init__()+ self.model = Backbone()++ def forward(self, input_ids=None, labels=None, **kwargs):+ logits = torch.zeros(1)+ loss_fct = CrossEntropyLoss(reduction="sum")+ return loss_fct(logits.view(-1, 1), labels.view(-1))+++class ExplicitMeanForCausalLM(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = Backbone()++ def forward(self, input_ids=None, labels=None, **kwargs):+ logits = torch.zeros(1)+ return torch.nn.functional.cross_entropy(logits, labels, reduction="mean")+++class InstanceDeclaredForCausalLM(nn.Module):+ """Remote code that declares loss-kwargs support on the instance in __init__."""+ def __init__(self):+ super().__init__()+ self.model = Backbone()+ self.accepts_loss_kwargs = True++ def forward(self, input_ids=None, labels=None, **kwargs):+ logits = torch.zeros(1)+ loss_fct = CrossEntropyLoss()+ return loss_fct(logits.view(-1, 1), labels.view(-1))+++class LossModuleForCausalLM(nn.Module):+ """Builds its CrossEntropyLoss once in __init__ and calls it from forward."""+ def __init__(self, reduction="mean"):+ super().__init__()+ self.model = Backbone()+ self.loss_fct = CrossEntropyLoss(reduction=reduction)++ def forward(self, input_ids=None, labels=None, **kwargs):+ logits = torch.zeros(1)+ return self.loss_fct(logits.view(-1, 1), labels.view(-1))+++class QWenLMHeadModel(nn.Module):+ """Remote causal LM named *LMHeadModel rather than *CausalLM."""+ def __init__(self):+ super().__init__()+ self.transformer = Backbone()++ def forward(self, input_ids=None, labels=None, **kwargs):+ logits = torch.zeros(1)+ loss_fct = CrossEntropyLoss()+ return loss_fct(logits.view(-1, 1), labels.view(-1))+++class NoKwargsForCausalLM(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = Backbone()++ def forward(self, input_ids=None, labels=None):+ loss_fct = CrossEntropyLoss()+ return loss_fct(input_ids, labels)+'''+++_PRETRAINED_MODELS = """+import torch+from torch import nn+from torch.nn import CrossEntropyLoss+from transformers import GenerationMixin, PretrainedConfig, PreTrainedModel+++class RemoteMeanLossConfig(PretrainedConfig):+ model_type = "remote_mean_loss_for_ga_test"+++class RemoteMeanLossForCausalLM(PreTrainedModel, GenerationMixin):+ config_class = RemoteMeanLossConfig++ def __init__(self, config):+ super().__init__(config)+ self.proj = nn.Linear(4, 4)+ self.post_init()++ def _init_weights(self, module):+ pass++ def forward(self, input_ids=None, labels=None, **kwargs): # for now we need this for generation+ logits = self.proj(torch.zeros(1, 4))+ loss = None+ if labels is not None:+ loss_fct = CrossEntropyLoss()+ loss = loss_fct(logits, labels.view(-1))+ return loss+"""+++def _models(+ tmp_path,+ source = _MODELS,+ name = "remote_models_for_ga_test",+):+ p = tmp_path / f"{name}.py"+ p.write_text(source, encoding = "utf-8")+ sys.path.insert(0, str(tmp_path))+ try:+ import importlib+ mod = importlib.import_module(name)+ return importlib.reload(mod)+ finally:+ sys.path.remove(str(tmp_path))+++class _PeftLike(nn.Module):+ def __init__(self, inner):+ super().__init__()+ self.base_model = types.SimpleNamespace(model = inner)+ self.base_model.base_model = None++ def forward(self, *args, **kwargs):+ return None+++def test_remote_mean_loss_forward_is_not_a_loss_kwargs_consumer(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ model = mods.NemotronHForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert getattr(model, "accepts_loss_kwargs", None) is False+++def test_remote_mean_loss_under_a_peft_wrapper(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ inner = mods.NemotronHForCausalLM()+ wrapped = _PeftLike(inner)+ ns["apply_accepts_loss_kwargs_fix"](wrapped)+ assert getattr(inner, "accepts_loss_kwargs", None) is False+ assert getattr(wrapped, "accepts_loss_kwargs", None) is False+++def test_consumers_keep_the_hf_default(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ for cls in (+ mods.NativeForCausalLM,+ mods.HandRolledForCausalLM,+ mods.NoKwargsForCausalLM,+ mods.SumLossForCausalLM,+ ):+ model = cls()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert not hasattr(model, "accepts_loss_kwargs"), cls.__name__+++def test_explicit_class_attribute_still_wins(tmp_path):+ ns = _load()+ mods = _models(tmp_path)++ class Declared(mods.NemotronHForCausalLM):+ accepts_loss_kwargs = True++ model = Declared()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert model.accepts_loss_kwargs is True+++def test_remote_mean_loss_under_a_real_peft_model(tmp_path):+ peft = pytest.importorskip("peft")+ ns = _load()+ mods = _models(tmp_path, _PRETRAINED_MODELS, "remote_pretrained_for_ga_test")+ inner = mods.RemoteMeanLossForCausalLM(mods.RemoteMeanLossConfig())+ wrapped = peft.get_peft_model(+ inner, peft.LoraConfig(r = 2, target_modules = ["proj"], task_type = "CAUSAL_LM")+ )+ assert "CausalLM" in type(wrapped).__name__+ ns["apply_accepts_loss_kwargs_fix"](wrapped)+ assert getattr(wrapped.get_base_model(), "accepts_loss_kwargs", None) is False+ assert getattr(wrapped, "accepts_loss_kwargs", None) is False+++def test_explicit_mean_reduction_is_still_a_mean_loss(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ model = mods.ExplicitMeanForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert getattr(model, "accepts_loss_kwargs", None) is False+++def test_instance_declaration_is_kept(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ model = mods.InstanceDeclaredForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert model.accepts_loss_kwargs is True+ wrapped = _PeftLike(model)+ ns["apply_accepts_loss_kwargs_fix"](wrapped)+ assert model.accepts_loss_kwargs is True+++def test_repeated_calls_keep_the_first_answer(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ model = mods.NemotronHForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ ns["apply_accepts_loss_kwargs_fix"](_PeftLike(model))+ assert model.accepts_loss_kwargs is False+++ "call",+ [+ 'CrossEntropyLoss(None, None, -100, None, "sum")(logits, labels)',+ 'torch.nn.functional.cross_entropy(logits, labels, None, None, -100, None, "none")',+ "CrossEntropyLoss(size_average=False)(logits, labels)",+ "CrossEntropyLoss(reduce=False)(logits, labels)",+ "CrossEntropyLoss(*loss_args)(logits, labels)",+ "CrossEntropyLoss(None, False)(logits, labels)",+ "CrossEntropyLoss(None, None, -100, False)(logits, labels)",+ "torch.nn.functional.cross_entropy(logits, labels, None, False)",+ "torch.nn.functional.cross_entropy(logits, labels, None, None, -100, False)",+ ],+)+def test_non_mean_reductions_in_any_spelling_keep_the_hf_default(call):+ ns = _load()+ source = f"def forward(self, logits, labels, **kwargs):\n return {call}\n"+ assert ns["_ce_calls_all_mean"](source) is False+++ "call",+ [+ "CrossEntropyLoss()(logits, labels)",+ "CrossEntropyLoss(None, None, -100)(logits, labels)",+ 'CrossEntropyLoss(ignore_index=-100, reduction="mean")(logits, labels)',+ "torch.nn.functional.cross_entropy(logits, labels, None, None, -100)",+ "CrossEntropyLoss(size_average=None, reduce=True)(logits, labels)",+ ],+)+def test_mean_reductions_in_any_spelling_count_as_a_mean(call):+ ns = _load()+ source = f"def forward(self, logits, labels, **kwargs):\n return {call}\n"+ assert ns["_ce_calls_all_mean"](source) is True+++def test_a_mention_without_a_call_is_not_a_mean_loss():+ ns = _load()+ source = (+ "def forward(self, logits, labels, **kwargs):\n"+ ' """Unlike CrossEntropyLoss() this returns a sum."""\n'+ " return (logits - labels).abs().sum()\n"+ )+ assert ns["_ce_calls_all_mean"](source) is False+++def test_a_model_helper_named_cross_entropy_is_not_the_torch_op():+ ns = _load()+ source = (+ "def forward(self, logits, labels, **kwargs):\n"+ " return self.loss_helpers.cross_entropy(logits, labels)\n"+ )+ assert ns["_ce_calls_all_mean"](source) is False+ for torch_spelling in (+ "torch.nn.functional.cross_entropy(logits, labels)",+ "F.cross_entropy(logits, labels)",+ "nn.CrossEntropyLoss()(logits, labels)",+ "CrossEntropyLoss()(logits, labels)",+ ):+ source = f"def forward(self, logits, labels, **kwargs):\n return {torch_spelling}\n"+ assert ns["_ce_calls_all_mean"](source) is True, torch_spelling+++def test_a_wrapper_added_after_load_gets_the_load_time_answer(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ inner = mods.NemotronHForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](inner)++ class UserWrapper(nn.Module):+ def __init__(self, model):+ super().__init__()+ self.model = model++ def forward(self, *args, **kwargs):+ return self.model(*args, **kwargs)++ wrapped = UserWrapper(inner)+ ns["apply_accepts_loss_kwargs_fix"](wrapped)+ assert wrapped.accepts_loss_kwargs is False+ assert inner.accepts_loss_kwargs is False+++def test_the_instance_forward_is_the_one_inspected(tmp_path):+ import functools++ ns = _load()+ mods = _models(tmp_path)++ model = mods.NemotronHForCausalLM()++ def consuming_forward(+ input_ids = None,+ labels = None,+ num_items_in_batch = None,+ **kwargs,+ ):+ return None++ model.forward = consuming_forward+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert not hasattr(model, "accepts_loss_kwargs")++ # An accelerate-style hook (partial + update_wrapper) still resolves to the original mean-loss forward.+ hooked = mods.NemotronHForCausalLM()+ old_forward = hooked.forward++ def new_forward(module, *args, **kwargs):+ return old_forward(*args, **kwargs)++ hooked.forward = functools.update_wrapper(functools.partial(new_forward, hooked), old_forward)+ ns["apply_accepts_loss_kwargs_fix"](hooked)+ assert hooked.accepts_loss_kwargs is False+++def test_a_loss_module_built_in_init_counts_by_its_reduction(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ model = mods.LossModuleForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert model.accepts_loss_kwargs is False+ summed = mods.LossModuleForCausalLM(reduction = "sum")+ ns["apply_accepts_loss_kwargs_fix"](summed)+ assert not hasattr(summed, "accepts_loss_kwargs")+++def test_an_lm_head_model_is_a_causal_lm(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ model = mods.QWenLMHeadModel()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert model.accepts_loss_kwargs is False+++def test_a_forward_replaced_after_load_is_checked_again(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ model = mods.NemotronHForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert model.accepts_loss_kwargs is False++ def consuming_forward(+ input_ids = None,+ labels = None,+ num_items_in_batch = None,+ **kwargs,+ ):+ return None++ model.forward = consuming_forward+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert not hasattr(model, "accepts_loss_kwargs")+++def test_an_assignment_after_the_guess_wins(tmp_path):+ ns = _load()+ mods = _models(tmp_path)+ model = mods.NemotronHForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert model.accepts_loss_kwargs is False+ model.accepts_loss_kwargs = True+ ns["apply_accepts_loss_kwargs_fix"](_PeftLike(model))+ assert model.accepts_loss_kwargs is True+++_OWN_HELPER_MODEL = """+import torch+from torch import nn+++def cross_entropy(logits, labels):+ return torch.nn.functional.cross_entropy(logits, labels, reduction="sum")+++class OwnHelperForCausalLM(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Identity()++ def forward(self, input_ids=None, labels=None, **kwargs):+ logits = torch.zeros(1, 2)+ return cross_entropy(logits, labels)+"""+++def test_a_bare_name_bound_to_a_model_helper_is_not_the_torch_op(tmp_path):+ ns = _load()+ mods = _models(tmp_path, _OWN_HELPER_MODEL, "own_helper_for_ga_test")+ model = mods.OwnHelperForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert not hasattr(model, "accepts_loss_kwargs")+++_LOCAL_HELPER_MODEL = """+import torch+from torch import nn+++class LocalHelperForCausalLM(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Identity()++ def summed_loss(self, logits, labels):+ return torch.nn.functional.cross_entropy(logits, labels, reduction="sum")++ def forward(self, input_ids=None, labels=None, **kwargs):+ cross_entropy = self.summed_loss+ logits = torch.zeros(1, 2)+ return cross_entropy(logits, labels)+"""+++def test_a_bare_name_bound_inside_forward_is_not_the_torch_op(tmp_path):+ ns = _load()+ mods = _models(tmp_path, _LOCAL_HELPER_MODEL, "local_helper_for_ga_test")+ model = mods.LocalHelperForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert not hasattr(model, "accepts_loss_kwargs")+++_OWN_F_MODEL = """+import torch+from torch import nn+++class _Helpers:+ @staticmethod+ def cross_entropy(logits, labels):+ return torch.nn.functional.cross_entropy(logits, labels, reduction="sum")+++F = _Helpers()+++class OwnFForCausalLM(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Identity()++ def forward(self, input_ids=None, labels=None, **kwargs):+ logits = torch.zeros(1, 2)+ return F.cross_entropy(logits, labels)+"""+++def test_a_qualifier_bound_to_a_model_helper_is_not_the_torch_op(tmp_path):+ ns = _load()+ mods = _models(tmp_path, _OWN_F_MODEL, "own_f_for_ga_test")+ model = mods.OwnFForCausalLM()+ ns["apply_accepts_loss_kwargs_fix"](model)+ assert not hasattr(model, "accepts_loss_kwargs")+++def test_a_guess_on_the_peft_causal_head_is_cleared_when_forward_changes(tmp_path):+ peft = pytest.importorskip("peft")+ ns = _load()+ # base_model_prefix = "model", as in real checkpoints, so .base_model walks past the causal head.+ source = _PRETRAINED_MODELS.replace(+ " config_class = RemoteMeanLossConfig\n",+ ' config_class = RemoteMeanLossConfig\n base_model_prefix = "model"\n',+ ).replace(+ " self.proj = nn.Linear(4, 4)\n",+ " self.proj = nn.Linear(4, 4)\n self.model = nn.Linear(4, 4)\n",+ )+ mods = _models(tmp_path, source, "remote_prefixed_for_ga_test")+ inner = mods.RemoteMeanLossForCausalLM(mods.RemoteMeanLossConfig())+ assert inner.base_model is not inner+ ns["apply_accepts_loss_kwargs_fix"](inner)+ assert inner.accepts_loss_kwargs is False+ wrapped = peft.get_peft_model(+ inner, peft.LoraConfig(r = 2, target_modules = ["proj"], task_type = "CAUSAL_LM")+ )++ def consuming_forward(+ input_ids = None,+ labels = None,+ num_items_in_batch = None,+ **kwargs,+ ):+ return None++ wrapped.get_base_model().forward = consuming_forward+ ns["apply_accepts_loss_kwargs_fix"](wrapped)+ assert "accepts_loss_kwargs" not in wrapped.get_base_model().__dict__diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex af78c2cb240..89c9b26ae29 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -115,6 +115,7 @@ from platform import system as platform_systemplatform_system = platform_system()import numpy as np+import astimport contextlibimport copyimport re@@ -4249,8 +4250,16 @@ def _find_concrete_accepts_loss_kwargs(model):return None, "no explicit accepts_loss_kwargs on any wrapper level"-def _shadow_accepts_loss_kwargs(model, value):+_GUESSED_LOSS_KWARGS = "_unsloth_guessed_accepts_loss_kwargs"+++def _shadow_accepts_loss_kwargs(+ model,+ value,+ guessed = False,+):# Set the attribute at every wrapper level so HF's hasattr check resolves wherever accelerator or peft unwrap lands.+ # guessed marks values the source heuristic wrote, so a later call re-checks them instead of trusting them.seen = set()m = modelfor _ in range(8):@@ -4259,6 +4268,10 @@ def _shadow_accepts_loss_kwargs(model, value):seen.add(id(m))try:setattr(m, "accepts_loss_kwargs", value)+ if guessed:+ m.__dict__[_GUESSED_LOSS_KWARGS] = value+ else:+ m.__dict__.pop(_GUESSED_LOSS_KWARGS, None)except Exception:passnxt = getattr(m, "base_model", None)@@ -4277,11 +4290,236 @@ def apply_accepts_loss_kwargs_fix(model):value, reason = _find_concrete_accepts_loss_kwargs(model)if value is None:+ declared = _instance_accepts_loss_kwargs(model)+ if declared is not None:+ # Set by the model itself: keep it, and carry it onto any wrapper added since.+ _shadow_accepts_loss_kwargs(model, declared)+ return f"{declared} (instance accepts_loss_kwargs)"+ # Re-check an earlier guess: forward may have been replaced since.+ _clear_guessed_accepts_loss_kwargs(model)+ causal_lm = _forward_ignores_num_items_in_batch(model)+ if causal_lm is not None:+ _shadow_accepts_loss_kwargs(model, False, guessed = True)+ # transformers 5 reads the flag off get_base_model(), which the walk above can skip under PEFT.+ _shadow_accepts_loss_kwargs(causal_lm, False, guessed = True)+ return "False (forward takes **kwargs but computes its own mean loss)"return f"default (signature inspection, {reason})"_shadow_accepts_loss_kwargs(model, value)return f"{value} ({reason})"+def _loss_kwargs_chain(model):+ seen = set()+ m = model+ for _ in range(8):+ if m is None or id(m) in seen:+ return+ seen.add(id(m))+ yield m+ nxt = getattr(m, "base_model", None)+ if nxt is None or nxt is m:+ nxt = getattr(m, "model", None)+ m = nxt+++def _is_guess(d):+ # Still the heuristic's value; an assignment made since (e.g. by the user) is a declaration.+ return _GUESSED_LOSS_KWARGS in d and d.get("accepts_loss_kwargs") == d[_GUESSED_LOSS_KWARGS]+++def _clear_guessed_accepts_loss_kwargs(model):+ chain = list(_loss_kwargs_chain(model))+ # Under PEFT the walk skips the causal head that get_base_model() returns (and transformers 5 reads).+ try:+ head = model.get_base_model() if hasattr(model, "get_base_model") else None+ except Exception:+ head = None+ if head is not None and all(head is not m for m in chain):+ chain.append(head)+ for m in chain:+ d = getattr(m, "__dict__", {})+ if _is_guess(d):+ d.pop("accepts_loss_kwargs", None)+ d.pop(_GUESSED_LOSS_KWARGS, None)+++def _instance_accepts_loss_kwargs(model):+ seen = set()+ m = model+ for _ in range(8):+ if m is None or id(m) in seen:+ return None+ seen.add(id(m))+ d = getattr(m, "__dict__", {})+ value = None if _is_guess(d) else d.get("accepts_loss_kwargs", None)+ if value is not None:+ return value+ nxt = getattr(m, "base_model", None)+ if nxt is None or nxt is m:+ nxt = getattr(m, "model", None)+ m = nxt+ return None+++_CE_PARAMS = {+ "CrossEntropyLoss": (+ "weight",+ "size_average",+ "ignore_index",+ "reduce",+ "reduction",+ "label_smoothing",+ ),+ "cross_entropy": (+ "input",+ "target",+ "weight",+ "size_average",+ "ignore_index",+ "reduce",+ "reduction",+ "label_smoothing",+ ),+}+++def _dotted_name(node):+ parts = []+ while isinstance(node, ast.Attribute):+ parts.append(node.attr)+ node = node.value+ if not isinstance(node, ast.Name):+ return None+ parts.append(node.id)+ return ".".join(reversed(parts))+++def _is_const(node, allowed):+ return isinstance(node, ast.Constant) and node.value in allowed+++def _ce_calls_all_mean(source, namespace = None):+ all_mean, found = _scan_ce_calls(+ source, _DEFAULT_CE_NAMESPACE if namespace is None else namespace+ )+ # A mention in a comment or docstring is not a call.+ return all_mean and found+++_TORCH_CE = {+ "CrossEntropyLoss": torch.nn.CrossEntropyLoss,+ "cross_entropy": torch.nn.functional.cross_entropy,+}+++# The usual imports, for callers that have source but no module to resolve it in.+_DEFAULT_CE_NAMESPACE = {+ "torch": torch,+ "nn": torch.nn,+ "F": torch.nn.functional,+ "functional": torch.nn.functional,+ **_TORCH_CE,+}+++def _resolve_ce_callee(func, namespace):+ dotted = _dotted_name(func)+ if dotted is None or not namespace:+ return None+ head, *rest = dotted.split(".")+ obj = namespace.get(head)+ for attr in rest:+ obj = getattr(obj, attr, None)+ for name, target in _TORCH_CE.items():+ if obj is target:+ return name+ return None+++def _scan_ce_calls(source, namespace = None):+ # A sum / "none" reduction (modern or legacy, keyword or positional) is the model's own objective, not a micro-batch mean.+ # Returns (every call is a mean, at least one call was seen).+ try:+ tree = ast.parse(textwrap.dedent(source))+ except SyntaxError:+ return False, False+ found = False+ for node in ast.walk(tree):+ if not isinstance(node, ast.Call):+ continue+ # Only callees the forward's module binds to the PyTorch op; a model's own helper may reduce however it likes.+ name = _resolve_ce_callee(node.func, namespace)+ params = _CE_PARAMS.get(name)+ if params is None:+ continue+ if len(node.args) > len(params) or any(isinstance(x, ast.Starred) for x in node.args):+ return False, found+ if any(kw.arg is None for kw in node.keywords):+ return False, found+ bound = dict(zip(params, node.args))+ bound.update((kw.arg, kw.value) for kw in node.keywords)+ if "reduction" in bound and not _is_const(bound["reduction"], ("mean",)):+ return False, found+ for legacy in ("size_average", "reduce"):+ if legacy in bound and not _is_const(bound[legacy], (None, True)):+ return False, found+ found = True+ return True, found+++def _forward_ignores_num_items_in_batch(model):+ # HF treats **kwargs as consuming num_items_in_batch and skips 1/GA; remote code (NemotronH) keeps **kwargs for generate yet returns a CrossEntropyLoss mean, so loss + grads come out GA x too large.+ m = model+ try:+ # PeftModelForCausalLM itself matches "CausalLM".+ if hasattr(m, "get_base_model"):+ m = m.get_base_model()+ except Exception:+ m = model+ seen = set()+ for _ in range(6):+ if m is None or id(m) in seen:+ return None+ seen.add(id(m))+ name = type(m).__name__+ if "CausalLM" in name or "ForConditionalGeneration" in name or "LMHeadModel" in name:+ break+ nxt = getattr(m, "base_model", None)+ if nxt is None or nxt is m:+ nxt = getattr(m, "model", None)+ m = nxt+ else:+ return None+ # The instance forward is what Trainer inspects and calls (accelerate hooks and loaders replace it).+ forward = getattr(m, "forward", None)+ try:+ forward = inspect.unwrap(forward) if forward is not None else None+ except ValueError:+ return None+ try:+ params = inspect.signature(forward).parameters.values()+ source = inspect.getsource(forward)+ except Exception:+ return None+ if not any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params):+ return None+ if "num_items_in_batch" in source or "loss_function" in source:+ return None+ used = [+ sub+ for name, sub in m.named_children()+ if isinstance(sub, torch.nn.CrossEntropyLoss)+ and re.search(rf"\bself\.{re.escape(name)}\s*\(", source)+ ]+ if any(sub.reduction != "mean" for sub in used):+ return None+ namespace = getattr(getattr(forward, "__func__", forward), "__globals__", None) or {}+ all_mean, found = _scan_ce_calls(source, namespace)+ if not all_mean or not (found or used):+ return None+ return m++def patch_tokenizer(model, tokenizer):model, tokenizer = _patch_tokenizer(model, tokenizer)if model is not None: