Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep a user's UNSLOTH_RETURN_LOGITS=1 after for_inference() on FastModel (#13406)
Daniel Han4b1259043d4c
diff --git a/tests/test_padding_free_signature_gate.py b/tests/test_padding_free_signature_gate.pyindex d3213ed80de..6d4699b60fa 100644--- a/tests/test_padding_free_signature_gate.py+++ b/tests/test_padding_free_signature_gate.py@@ -29,7 +29,7 @@ except ImportError:def _return_logits_unset(monkeypatch):"""UNSLOTH_RETURN_LOGITS=1 blocks packing and padding-free outright, so with it set thegate never runs and these tests see neither the refusal nor the metadata hook. Product- paths set it process-wide (for_inference, the eval prediction_step, zoo's compiler), and+ paths set it process-wide (the eval prediction_step, zoo's compiler), andan earlier test in the same xdist worker leaving it at "1" failed a different subset ofthis file on each run. Tests that want it set still say so with monkeypatch.setenv."""monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)diff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.pyindex e964c906f12..78ed72863a4 100644--- a/tests/test_return_logits_survives_for_training.py+++ b/tests/test_return_logits_survives_for_training.py@@ -1,7 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""A user's UNSLOTH_RETURN_LOGITS=1 must survive trainer.train() (#3071): for_training() may only undo-for_inference()'s forced "1". Methods are lifted out with ``ast`` so this runs without ``import unsloth``.+"""A user's UNSLOTH_RETURN_LOGITS=1 must survive trainer.train() (#3071): mode switches never write the flag+beyond defaulting it to "0". Methods are lifted out with ``ast`` so this runs without ``import unsloth``."""from __future__ import annotations@@ -12,7 +12,6 @@ import functoolsimport inspectimport osimport sys-import textwrapimport typesfrom pathlib import Path@@ -20,7 +19,6 @@ import pytestMODELS = Path(__file__).resolve().parents[1] / "unsloth" / "models"FLAG = "UNSLOTH_RETURN_LOGITS"-MARK = "_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE"class _Namespace(dict):@@ -108,42 +106,33 @@ def fbm(monkeypatch):monkeypatch.setitem(sys.modules, "unsloth_zoo", zoo)monkeypatch.setitem(sys.modules, "unsloth_zoo.training_utils", utils)monkeypatch.delenv(FLAG, raising = False)- monkeypatch.delenv(MARK, raising = False)return _fast_base_model()-def test_for_training_keeps_a_user_set_flag(fbm, monkeypatch):- monkeypatch.setenv(FLAG, "1")- fbm.for_training(_Model())- assert os.environ[FLAG] == "1"+def _cycle(fbm, model):# The trainer's own cycle: for_training in __init__, for_inference, for_training in train().- model = _Model()fbm.for_training(model)fbm.for_inference(model)fbm.for_training(model)- assert os.environ[FLAG] == "1"- assert MARK not in os.environ[email protected]("before", [None, "0"])-def test_for_training_undoes_the_flag_for_inference_forced(fbm, monkeypatch, before):- # Default training must go back to the fused CE path after generation / GRPO rollouts.- if before is not None:- monkeypatch.setenv(FLAG, before)[email protected]("user_flag", ["0", "1"])+def test_mode_switches_keep_a_user_set_flag(fbm, monkeypatch, user_flag):+ monkeypatch.setenv(FLAG, user_flag)model = _Model()+ _cycle(fbm, model)fbm.for_inference(model)- assert os.environ[FLAG] == "1"- fbm.for_inference(model) # repeated calls (every GRPO step) keep the first saved value- fbm.for_training(model)- assert os.environ[FLAG] == "0"- assert MARK not in os.environ- fbm.for_training(model)- assert os.environ[FLAG] == "0"+ assert os.environ[FLAG] == user_flag-def test_for_training_still_defaults_an_unset_flag_to_zero(fbm):- fbm.for_training(_Model())- assert os.environ[FLAG] == "0"+def test_flag_set_after_for_inference_is_kept(fbm):+ # model.generate() / for_inference() first, then the flag, then train(): the residual #3071 path.+ model = _Model()+ fbm.for_inference(model)+ assert FLAG not in os.environ+ os.environ[FLAG] = "1"+ fbm.for_training(model)+ assert os.environ[FLAG] == "1"def _fast_llama_for_training():@@ -165,69 +154,46 @@ def _fast_llama_for_training():@pytest.mark.parametrize("before", [None, "0", "1"])-def test_fast_language_model_for_training_undoes_generate(fbm, monkeypatch, before):- # FastModel-backed models (Qwen3.5, Gemma 3/4) run FastBaseModel.for_inference() in generate().+def test_fast_language_model_after_fastmodel_generate(fbm, monkeypatch, before):+ # FastModel-backed models (Qwen3.5, Gemma 3/4) run FastBaseModel.for_inference() in generate(),+ # then FastLanguageModel.for_training(): the flag is whatever the user left, "0" when unset.if before is not None:monkeypatch.setenv(FLAG, before)model = _Model()fbm.for_inference(model)- assert os.environ[FLAG] == "1"_fast_llama_for_training()(model)- assert os.environ[FLAG] == (before or "0")- assert MARK not in os.environ---def test_fast_language_model_for_training_leaves_a_user_flag(fbm, monkeypatch):- monkeypatch.setenv(FLAG, "1")- _fast_llama_for_training()(_Model())+ assert os.environ.get(FLAG, "0") == (before or "0")+ os.environ[FLAG] = "1"+ _fast_llama_for_training()(model)assert os.environ[FLAG] == "1"-def _trailing_for_inference_block():- tree = ast.parse((MODELS / "rl.py").read_text(encoding = "utf-8"))- template = next(- n.value.value- for n in tree.body- if isinstance(n, ast.Assign)- and any(getattr(t, "id", None) == "RLTrainer_replacement" for t in n.targets)- )- lines = template.splitlines()- start = next(i for i, line in enumerate(lines) if 'hasattr(model, "for_inference")' in line)- block = [lines[start]]- indent = len(lines[start]) - len(lines[start].lstrip())- for line in lines[start + 1 :]:- if line.strip() and len(line) - len(line.lstrip()) <= indent:- break- block.append(line)- # The template is filled with str.format: a literal brace in the block breaks every trainer.- return textwrap.dedent("\n".join(block)).format()---def test_trainer_init_leaves_the_flag_for_train(fbm, monkeypatch):+def test_unset_flag_defaults_to_zero_for_training(fbm):+ # Default training keeps the fused CE path after generation / GRPO rollouts.model = _Model()- model.for_inference = lambda: fbm.for_inference(model)- monkeypatch.setenv(FLAG, "0")- exec(_trailing_for_inference_block(), {"os": os, "model": model})- assert model.training is False- assert os.environ[FLAG] == "0"- assert MARK not in os.environ- os.environ[FLAG] = "1"+ fbm.for_inference(model)fbm.for_training(model)- assert os.environ[FLAG] == "1"+ assert os.environ[FLAG] == "0"+ fbm.for_inference(model)+ assert os.environ[FLAG] == "0"-def _prepare_for_training_mode():+def _template():tree = ast.parse((MODELS / "rl.py").read_text(encoding = "utf-8"))- template = next(+ return next(n.value.valuefor n in tree.bodyif isinstance(n, ast.Assign)and any(getattr(t, "id", None) == "RLTrainer_replacement" for t in n.targets))- lines = template.splitlines()+++def _prepare_for_training_mode():+ lines = _template().splitlines()start = lines.index("def prepare_for_training_mode(f):")end = lines.index(" return wrapper", start)- namespace = _Namespace(os = os, functools = __import__("functools"))+ namespace = _Namespace(os = os, functools = functools)+ # The template is filled with str.format: a literal brace in the block breaks every trainer.exec("\n".join(lines[start : end + 1]).format(), namespace)return namespace["prepare_for_training_mode"]@@ -255,23 +221,14 @@ def test_flag_set_between_two_train_calls_is_kept(fbm):assert seen == ["0", "1"]-def _logits_check():+def test_generated_trainer_init_never_writes_the_flag():+ # compute_metrics used to force "1" here, leaking into training and the next trainer; the eval+ # prediction_step forces it per call instead.tree = ast.parse((MODELS / "rl.py").read_text(encoding = "utf-8"))- return next(- n.value.value- for n in ast.walk(tree)- if isinstance(n, ast.Assign)- and any(getattr(t, "id", None) == "logits_check" for t in n.targets)+ patcher = next(+ n+ for n in tree.body+ if isinstance(n, ast.FunctionDef) and n.name == "_patch_trl_rl_trainers_impl")--[email protected]("user_flag", [None, "1"])-def test_compute_metrics_auto_flag_does_not_outlive_the_trainer_init(fbm, monkeypatch, user_flag):- # compute_metrics forces "1" while building the trainer; training (and the next trainer's- # padding-free gate) must see the user's value, not a leaked "1".- if user_flag is not None:- monkeypatch.setenv(FLAG, user_flag)- exec(_logits_check(), {"os": os, "compute_metrics": object()})- assert os.environ[FLAG] == "1"- fbm.for_training(_Model())- assert os.environ[FLAG] == (user_flag or "0")+ assert FLAG not in ast.unparse(patcher)+ assert FLAG not in _template()diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex f2b89953182..8dc484b9bbb 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -4758,10 +4758,6 @@ class FastLlamaModel:disable_use_cache(model)except ImportError:pass- # Undo only the UNSLOTH_RETURN_LOGITS=1 that FastBaseModel.for_inference() forced in generate().- _return_logits = os.environ.pop("_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE", None)- if _return_logits is not None:- os.environ["UNSLOTH_RETURN_LOGITS"] = _return_logitsreturn modeldiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 5c162c31628..3991e81211c 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -496,12 +496,7 @@ def prepare_for_training_mode(f):# Restore previous mode when possibleif hasattr(self, 'model') and hasattr(self.model, "for_inference"):if _was_training is False:- # Mode only: a UNSLOTH_RETURN_LOGITS=1 set before the next train() must survive (#3071).- _logits_env = [(_k, os.environ.get(_k)) for _k in ("UNSLOTH_RETURN_LOGITS", "_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE")]self.model.for_inference()- for _k, _v in _logits_env:- if _v is None: os.environ.pop(_k, None)- else: os.environ[_k] = _velif _was_training is True and hasattr(self.model, "for_training"):self.model.for_training(use_gradient_checkpointing=use_gc)# Reset gradient checkpointing buffers to free memory while staying ready for next run@@ -608,12 +603,7 @@ class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}):model.for_training(use_gradient_checkpointing=_use_gc)super().__init__({RLTrainer_call_args}{RLTrainer_kwargs})if "model" in locals() and hasattr(model, "for_inference"):- # Mode switch only: a UNSLOTH_RETURN_LOGITS=1 set between here and train() must survive (#3071).- _unsloth_logits_env = [(_unsloth_k, os.environ.get(_unsloth_k)) for _unsloth_k in ("UNSLOTH_RETURN_LOGITS", "_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE")]model.for_inference()- for _unsloth_k, _unsloth_v in _unsloth_logits_env:- if _unsloth_v is None: os.environ.pop(_unsloth_k, None)- else: os.environ[_unsloth_k] = _unsloth_v{RLTrainer_post}pass'''@@ -2517,17 +2507,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):extra_args += eval_changesif "model" in call_args:- logits_check = (- "_output_logits = False\n"- "if locals().get('compute_metrics', None) is not None: _output_logits = True\n"- "if locals().get('preprocess_logits_for_metrics', None) is not None: _output_logits = True\n"- "if _output_logits:\n"- # Recorded like for_inference()'s "1" so for_training() undoes it and only a user's "1" persists (#3071).- " _unsloth_rl = os.environ.get('UNSLOTH_RETURN_LOGITS', '0')\n"- " if _unsloth_rl != '1': os.environ['_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE'] = _unsloth_rl\n"- " os.environ['UNSLOTH_RETURN_LOGITS'] = '1'\n"- )- extra_args += logits_check+ # No UNSLOTH_RETURN_LOGITS=1 for compute_metrics here: prediction_step forces it per eval call; here it leaked into training (#3071).warnings_issued_check = ("if model is not None:\n"" _warnings_issued = getattr(model, 'warnings_issued', None)\n"diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 5ce46cf0e0f..5c73b74d805 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -5061,13 +5061,9 @@ class FastBaseModel:except ImportError:pass- # Returning hidden states must be disabled for GRPO, and returning logits enabled.+ # Returning hidden states must be disabled for GRPO. UNSLOTH_RETURN_LOGITS is left alone: label-free forwards+ # return logits anyway, and a "1" forced here is indistinguishable from a user's later "1" (#3071).os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0"- # Remember what the forced "1" replaced, for for_training() to put back (#3071).- _return_logits = os.environ.get("UNSLOTH_RETURN_LOGITS", "0")- if _return_logits != "1":- os.environ["_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE"] = _return_logits- os.environ["UNSLOTH_RETURN_LOGITS"] = "1"if torch_compiler_set_stance is not None:torch_compiler_set_stance(stance = "default", skip_guard_eval_unsafe = False)return model@@ -5122,11 +5118,8 @@ class FastBaseModel:except ImportError:pass- # Undo only for_inference()'s "1": forcing "0" here wiped a user's UNSLOTH_RETURN_LOGITS=1 (#3071).- _return_logits = os.environ.pop("_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE", None)- if _return_logits is None:- _return_logits = os.environ.get("UNSLOTH_RETURN_LOGITS", "0")- os.environ["UNSLOTH_RETURN_LOGITS"] = _return_logits+ # Never force "0": that wiped a user's UNSLOTH_RETURN_LOGITS=1 (#3071).+ os.environ.setdefault("UNSLOTH_RETURN_LOGITS", "0")if torch_compiler_set_stance is not None:torch_compiler_set_stance(stance = "default", skip_guard_eval_unsafe = False)return model