Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep a user's UNSLOTH_RETURN_LOGITS=1 after for_inference() on FastModel (#13406)

Daniel Han4b1259043d4c
diff --git a/tests/test_padding_free_signature_gate.py b/tests/test_padding_free_signature_gate.py
index d3213ed80de..6d4699b60fa 100644
--- a/tests/test_padding_free_signature_gate.py
+++ b/tests/test_padding_free_signature_gate.py
@@ -29,7 +29,7 @@ except ImportError:
def _return_logits_unset(monkeypatch):
"""UNSLOTH_RETURN_LOGITS=1 blocks packing and padding-free outright, so with it set the
gate never runs and these tests see neither the refusal nor the metadata hook. Product
- paths set it process-wide (for_inference, the eval prediction_step, zoo's compiler), and
+ paths set it process-wide (the eval prediction_step, zoo's compiler), and
an earlier test in the same xdist worker leaving it at "1" failed a different subset of
this file on each run. Tests that want it set still say so with monkeypatch.setenv."""
monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)
diff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.py
index e964c906f12..78ed72863a4 100644
--- a/tests/test_return_logits_survives_for_training.py
+++ b/tests/test_return_logits_survives_for_training.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""A user's UNSLOTH_RETURN_LOGITS=1 must survive trainer.train() (#3071): for_training() may only undo
-for_inference()'s forced "1". Methods are lifted out with ``ast`` so this runs without ``import unsloth``.
+"""A user's UNSLOTH_RETURN_LOGITS=1 must survive trainer.train() (#3071): mode switches never write the flag
+beyond defaulting it to "0". Methods are lifted out with ``ast`` so this runs without ``import unsloth``.
"""
from __future__ import annotations
@@ -12,7 +12,6 @@ import functools
import inspect
import os
import sys
-import textwrap
import types
from pathlib import Path
@@ -20,7 +19,6 @@ import pytest
MODELS = Path(__file__).resolve().parents[1] / "unsloth" / "models"
FLAG = "UNSLOTH_RETURN_LOGITS"
-MARK = "_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE"
class _Namespace(dict):
@@ -108,42 +106,33 @@ def fbm(monkeypatch):
monkeypatch.setitem(sys.modules, "unsloth_zoo", zoo)
monkeypatch.setitem(sys.modules, "unsloth_zoo.training_utils", utils)
monkeypatch.delenv(FLAG, raising = False)
- monkeypatch.delenv(MARK, raising = False)
return _fast_base_model()
-def test_for_training_keeps_a_user_set_flag(fbm, monkeypatch):
- monkeypatch.setenv(FLAG, "1")
- fbm.for_training(_Model())
- assert os.environ[FLAG] == "1"
+def _cycle(fbm, model):
# The trainer's own cycle: for_training in __init__, for_inference, for_training in train().
- model = _Model()
fbm.for_training(model)
fbm.for_inference(model)
fbm.for_training(model)
- assert os.environ[FLAG] == "1"
- assert MARK not in os.environ
[email protected]("before", [None, "0"])
-def test_for_training_undoes_the_flag_for_inference_forced(fbm, monkeypatch, before):
- # Default training must go back to the fused CE path after generation / GRPO rollouts.
- if before is not None:
- monkeypatch.setenv(FLAG, before)
[email protected]("user_flag", ["0", "1"])
+def test_mode_switches_keep_a_user_set_flag(fbm, monkeypatch, user_flag):
+ monkeypatch.setenv(FLAG, user_flag)
model = _Model()
+ _cycle(fbm, model)
fbm.for_inference(model)
- assert os.environ[FLAG] == "1"
- fbm.for_inference(model) # repeated calls (every GRPO step) keep the first saved value
- fbm.for_training(model)
- assert os.environ[FLAG] == "0"
- assert MARK not in os.environ
- fbm.for_training(model)
- assert os.environ[FLAG] == "0"
+ assert os.environ[FLAG] == user_flag
-def test_for_training_still_defaults_an_unset_flag_to_zero(fbm):
- fbm.for_training(_Model())
- assert os.environ[FLAG] == "0"
+def test_flag_set_after_for_inference_is_kept(fbm):
+ # model.generate() / for_inference() first, then the flag, then train(): the residual #3071 path.
+ model = _Model()
+ fbm.for_inference(model)
+ assert FLAG not in os.environ
+ os.environ[FLAG] = "1"
+ fbm.for_training(model)
+ assert os.environ[FLAG] == "1"
def _fast_llama_for_training():
@@ -165,69 +154,46 @@ def _fast_llama_for_training():
@pytest.mark.parametrize("before", [None, "0", "1"])
-def test_fast_language_model_for_training_undoes_generate(fbm, monkeypatch, before):
- # FastModel-backed models (Qwen3.5, Gemma 3/4) run FastBaseModel.for_inference() in generate().
+def test_fast_language_model_after_fastmodel_generate(fbm, monkeypatch, before):
+ # FastModel-backed models (Qwen3.5, Gemma 3/4) run FastBaseModel.for_inference() in generate(),
+ # then FastLanguageModel.for_training(): the flag is whatever the user left, "0" when unset.
if before is not None:
monkeypatch.setenv(FLAG, before)
model = _Model()
fbm.for_inference(model)
- assert os.environ[FLAG] == "1"
_fast_llama_for_training()(model)
- assert os.environ[FLAG] == (before or "0")
- assert MARK not in os.environ
-
-
-def test_fast_language_model_for_training_leaves_a_user_flag(fbm, monkeypatch):
- monkeypatch.setenv(FLAG, "1")
- _fast_llama_for_training()(_Model())
+ assert os.environ.get(FLAG, "0") == (before or "0")
+ os.environ[FLAG] = "1"
+ _fast_llama_for_training()(model)
assert os.environ[FLAG] == "1"
-def _trailing_for_inference_block():
- tree = ast.parse((MODELS / "rl.py").read_text(encoding = "utf-8"))
- template = next(
- n.value.value
- for n in tree.body
- if isinstance(n, ast.Assign)
- and any(getattr(t, "id", None) == "RLTrainer_replacement" for t in n.targets)
- )
- lines = template.splitlines()
- start = next(i for i, line in enumerate(lines) if 'hasattr(model, "for_inference")' in line)
- block = [lines[start]]
- indent = len(lines[start]) - len(lines[start].lstrip())
- for line in lines[start + 1 :]:
- if line.strip() and len(line) - len(line.lstrip()) <= indent:
- break
- block.append(line)
- # The template is filled with str.format: a literal brace in the block breaks every trainer.
- return textwrap.dedent("\n".join(block)).format()
-
-
-def test_trainer_init_leaves_the_flag_for_train(fbm, monkeypatch):
+def test_unset_flag_defaults_to_zero_for_training(fbm):
+ # Default training keeps the fused CE path after generation / GRPO rollouts.
model = _Model()
- model.for_inference = lambda: fbm.for_inference(model)
- monkeypatch.setenv(FLAG, "0")
- exec(_trailing_for_inference_block(), {"os": os, "model": model})
- assert model.training is False
- assert os.environ[FLAG] == "0"
- assert MARK not in os.environ
- os.environ[FLAG] = "1"
+ fbm.for_inference(model)
fbm.for_training(model)
- assert os.environ[FLAG] == "1"
+ assert os.environ[FLAG] == "0"
+ fbm.for_inference(model)
+ assert os.environ[FLAG] == "0"
-def _prepare_for_training_mode():
+def _template():
tree = ast.parse((MODELS / "rl.py").read_text(encoding = "utf-8"))
- template = next(
+ return next(
n.value.value
for n in tree.body
if isinstance(n, ast.Assign)
and any(getattr(t, "id", None) == "RLTrainer_replacement" for t in n.targets)
)
- lines = template.splitlines()
+
+
+def _prepare_for_training_mode():
+ lines = _template().splitlines()
start = lines.index("def prepare_for_training_mode(f):")
end = lines.index(" return wrapper", start)
- namespace = _Namespace(os = os, functools = __import__("functools"))
+ namespace = _Namespace(os = os, functools = functools)
+ # The template is filled with str.format: a literal brace in the block breaks every trainer.
exec("\n".join(lines[start : end + 1]).format(), namespace)
return namespace["prepare_for_training_mode"]
@@ -255,23 +221,14 @@ def test_flag_set_between_two_train_calls_is_kept(fbm):
assert seen == ["0", "1"]
-def _logits_check():
+def test_generated_trainer_init_never_writes_the_flag():
+ # compute_metrics used to force "1" here, leaking into training and the next trainer; the eval
+ # prediction_step forces it per call instead.
tree = ast.parse((MODELS / "rl.py").read_text(encoding = "utf-8"))
- return next(
- n.value.value
- for n in ast.walk(tree)
- if isinstance(n, ast.Assign)
- and any(getattr(t, "id", None) == "logits_check" for t in n.targets)
+ patcher = next(
+ n
+ for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == "_patch_trl_rl_trainers_impl"
)
-
-
[email protected]("user_flag", [None, "1"])
-def test_compute_metrics_auto_flag_does_not_outlive_the_trainer_init(fbm, monkeypatch, user_flag):
- # compute_metrics forces "1" while building the trainer; training (and the next trainer's
- # padding-free gate) must see the user's value, not a leaked "1".
- if user_flag is not None:
- monkeypatch.setenv(FLAG, user_flag)
- exec(_logits_check(), {"os": os, "compute_metrics": object()})
- assert os.environ[FLAG] == "1"
- fbm.for_training(_Model())
- assert os.environ[FLAG] == (user_flag or "0")
+ assert FLAG not in ast.unparse(patcher)
+ assert FLAG not in _template()
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index f2b89953182..8dc484b9bbb 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -4758,10 +4758,6 @@ class FastLlamaModel:
disable_use_cache(model)
except ImportError:
pass
- # Undo only the UNSLOTH_RETURN_LOGITS=1 that FastBaseModel.for_inference() forced in generate().
- _return_logits = os.environ.pop("_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE", None)
- if _return_logits is not None:
- os.environ["UNSLOTH_RETURN_LOGITS"] = _return_logits
return model
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 5c162c31628..3991e81211c 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -496,12 +496,7 @@ def prepare_for_training_mode(f):
# Restore previous mode when possible
if hasattr(self, 'model') and hasattr(self.model, "for_inference"):
if _was_training is False:
- # Mode only: a UNSLOTH_RETURN_LOGITS=1 set before the next train() must survive (#3071).
- _logits_env = [(_k, os.environ.get(_k)) for _k in ("UNSLOTH_RETURN_LOGITS", "_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE")]
self.model.for_inference()
- for _k, _v in _logits_env:
- if _v is None: os.environ.pop(_k, None)
- else: os.environ[_k] = _v
elif _was_training is True and hasattr(self.model, "for_training"):
self.model.for_training(use_gradient_checkpointing=use_gc)
# Reset gradient checkpointing buffers to free memory while staying ready for next run
@@ -608,12 +603,7 @@ class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}):
model.for_training(use_gradient_checkpointing=_use_gc)
super().__init__({RLTrainer_call_args}{RLTrainer_kwargs})
if "model" in locals() and hasattr(model, "for_inference"):
- # Mode switch only: a UNSLOTH_RETURN_LOGITS=1 set between here and train() must survive (#3071).
- _unsloth_logits_env = [(_unsloth_k, os.environ.get(_unsloth_k)) for _unsloth_k in ("UNSLOTH_RETURN_LOGITS", "_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE")]
model.for_inference()
- for _unsloth_k, _unsloth_v in _unsloth_logits_env:
- if _unsloth_v is None: os.environ.pop(_unsloth_k, None)
- else: os.environ[_unsloth_k] = _unsloth_v
{RLTrainer_post}
pass
'''
@@ -2517,17 +2507,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
extra_args += eval_changes
if "model" in call_args:
- logits_check = (
- "_output_logits = False\n"
- "if locals().get('compute_metrics', None) is not None: _output_logits = True\n"
- "if locals().get('preprocess_logits_for_metrics', None) is not None: _output_logits = True\n"
- "if _output_logits:\n"
- # Recorded like for_inference()'s "1" so for_training() undoes it and only a user's "1" persists (#3071).
- " _unsloth_rl = os.environ.get('UNSLOTH_RETURN_LOGITS', '0')\n"
- " if _unsloth_rl != '1': os.environ['_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE'] = _unsloth_rl\n"
- " os.environ['UNSLOTH_RETURN_LOGITS'] = '1'\n"
- )
- extra_args += logits_check
+ # No UNSLOTH_RETURN_LOGITS=1 for compute_metrics here: prediction_step forces it per eval call; here it leaked into training (#3071).
warnings_issued_check = (
"if model is not None:\n"
" _warnings_issued = getattr(model, 'warnings_issued', None)\n"
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 5ce46cf0e0f..5c73b74d805 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -5061,13 +5061,9 @@ class FastBaseModel:
except ImportError:
pass
- # Returning hidden states must be disabled for GRPO, and returning logits enabled.
+ # Returning hidden states must be disabled for GRPO. UNSLOTH_RETURN_LOGITS is left alone: label-free forwards
+ # return logits anyway, and a "1" forced here is indistinguishable from a user's later "1" (#3071).
os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0"
- # Remember what the forced "1" replaced, for for_training() to put back (#3071).
- _return_logits = os.environ.get("UNSLOTH_RETURN_LOGITS", "0")
- if _return_logits != "1":
- os.environ["_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE"] = _return_logits
- os.environ["UNSLOTH_RETURN_LOGITS"] = "1"
if torch_compiler_set_stance is not None:
torch_compiler_set_stance(stance = "default", skip_guard_eval_unsafe = False)
return model
@@ -5122,11 +5118,8 @@ class FastBaseModel:
except ImportError:
pass
- # Undo only for_inference()'s "1": forcing "0" here wiped a user's UNSLOTH_RETURN_LOGITS=1 (#3071).
- _return_logits = os.environ.pop("_UNSLOTH_RETURN_LOGITS_BEFORE_INFERENCE", None)
- if _return_logits is None:
- _return_logits = os.environ.get("UNSLOTH_RETURN_LOGITS", "0")
- os.environ["UNSLOTH_RETURN_LOGITS"] = _return_logits
+ # Never force "0": that wiped a user's UNSLOTH_RETURN_LOGITS=1 (#3071).
+ os.environ.setdefault("UNSLOTH_RETURN_LOGITS", "0")
if torch_compiler_set_stance is not None:
torch_compiler_set_stance(stance = "default", skip_guard_eval_unsafe = False)
return model