Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(grpo): autocast with DEVICE_TYPE_TORCH instead of a pinned "cuda" (#11461)
li-lizhe5f2136a111ef
diff --git a/tests/test_grpo_autocast_disabled.py b/tests/test_grpo_autocast_disabled.pyindex ebcb7694156..9a5b16c2a3d 100644--- a/tests/test_grpo_autocast_disabled.py+++ b/tests/test_grpo_autocast_disabled.py@@ -43,12 +43,14 @@ REPO_ROOT = Path(__file__).resolve().parents[1]sys.path.insert(0, str(REPO_ROOT))RL_REPLACEMENTS = REPO_ROOT / "unsloth" / "models" / "rl_replacements.py"+RL_PY = REPO_ROOT / "unsloth" / "models" / "rl.py"SRC = RL_REPLACEMENTS.read_text(encoding = "utf-8")# ---- the premise ---------------------------------------------------------#-# Every check drives torch.amp.autocast(device_type = "cuda"), and on a CPU runner torch hands back a no-op instead.+# Every check drives torch.amp.autocast(device_type = DEVICE_TYPE_TORCH), pinned to "cuda" in the namespaces below,+# and on a CPU runner torch hands back a no-op instead.# Claiming the device is present is what lets these run anywhere; only torch's dispatch decisions are needed.class _pretend_cuda:"""torch.cuda answering as a card without bfloat16, or with it."""@@ -116,6 +118,8 @@ def _namespace(env):"nullcontext": nullcontext,"self": type("Trainer", (), {})(),"seen": [],+ # The generated trainer gets this from rl.py's preamble, so the header resolves it there too.+ "DEVICE_TYPE_TORCH": "cuda",}namespace["os"].environ = envexec(_autocast_helper_source(), namespace)@@ -154,6 +158,25 @@ def test_the_injected_snippet_only_autocasts_when_asked(precision, has_bf16, expassert namespace["seen"] == [expect_enabled]+def test_the_generated_trainer_imports_the_device_type_it_autocasts_with():+ """getsource inlines these bodies but not this file's imports, so the name has+ to come from the template rl.py splices in, or the generated cache raises."""+ preamble = "from unsloth_zoo.device_type import DEVICE_TYPE, DEVICE_TYPE_TORCH"+ assert preamble in RL_PY.read_text(encoding = "utf-8"), "rl.py's trainer template must import it"+++def test_no_autocast_call_pins_the_device_type_to_cuda():+ """A device_type that does not match the accelerator is inert, not loud+ (pytorch#165730): a literal "cuda" drops autocast on every other one, so GRPO+ ran its forward in float32 against the dtype _unsloth_grpo_autocast latched.+ DEVICE_TYPE_TORCH is "npu" on Ascend, "xpu" on Intel, "cuda" on CUDA and ROCm."""+ calls = re.findall(r"torch\.amp\.autocast\((?:[^()]|\([^()]*\))*\)", SRC)+ pinning = [c for c in calls if re.search(r"device_type\s*=\s*[\"']cuda[\"']", c)]+ assert pinning == [], pinning+ typed = [c for c in calls if "device_type" in c]+ assert typed, "expected the autocast call sites to still choose a device_type"++# ---- _get_per_token_logps and friends, which run as ordinary code --------diff --git a/tests/test_grpo_autocast_per_trainer.py b/tests/test_grpo_autocast_per_trainer.pyindex 44dcef2587b..f280c935e4d 100644--- a/tests/test_grpo_autocast_per_trainer.py+++ b/tests/test_grpo_autocast_per_trainer.py@@ -209,6 +209,8 @@ def _generate(trainer, env, has_bf16):"nullcontext": nullcontext,"self": trainer,"seen": [],+ # The generated trainer gets this from rl.py's preamble, so the header resolves it there too.+ "DEVICE_TYPE_TORCH": "cuda",}helpers = _autocast_helper_source()if helpers:diff --git a/tests/test_grpo_hidden_states_signal.py b/tests/test_grpo_hidden_states_signal.pyindex 4f21950566c..e6a3faf8ef1 100644--- a/tests/test_grpo_hidden_states_signal.py+++ b/tests/test_grpo_hidden_states_signal.py@@ -316,6 +316,8 @@ def _run_padded_loop(*, returns_hidden_states, signal):"chunked_hidden_states_selective_log_softmax": _ZOO_HIDDEN,"chunked_selective_log_softmax": _ZOO_RAW,"device_synchronize": lambda *a, **k: None,+ # The generated trainer gets this from rl.py's preamble, so the block resolves it here too.+ "DEVICE_TYPE_TORCH": "cuda","_get_inference_mode_context_manager": lambda _model: contextlib.nullcontext(),"model": model,"unwrapped_model": model,diff --git a/tests/test_grpo_packed_raw_logits_nograd.py b/tests/test_grpo_packed_raw_logits_nograd.pyindex b551d0369d8..396370fa69c 100644--- a/tests/test_grpo_packed_raw_logits_nograd.py+++ b/tests/test_grpo_packed_raw_logits_nograd.py@@ -315,6 +315,8 @@ def _run_packed_block(hidden_states = False, model = None):"calculate_pad_tokens_in_prompt": HELPERS["calculate_pad_tokens_in_prompt"],"_get_inference_mode_context_manager": lambda _model: torch.no_grad(),"device_synchronize": lambda *args, **kwargs: None,+ # The generated trainer gets this from rl.py's preamble, so the block resolves it here too.+ "DEVICE_TYPE_TORCH": "cuda","UNSLOTH_ENABLE_LOGGING": False,"UNSLOTH_GRPO_SEQ_PACKING_ON": True,"UNSLOTH_ZOO_HAS_MASKED_COL_GUARD": True,diff --git a/tests/test_grpo_padded_raw_logits.py b/tests/test_grpo_padded_raw_logits.pyindex 5bda19dd2de..c0c122c7a88 100644--- a/tests/test_grpo_padded_raw_logits.py+++ b/tests/test_grpo_padded_raw_logits.py@@ -296,6 +296,8 @@ def _build_namespace("chunked_hidden_states_selective_log_softmax": _HELPER_HIDDEN,"chunked_selective_log_softmax": _HELPER_RAW,"device_synchronize": lambda *a, **k: None,+ # The generated trainer gets this from rl.py's preamble, so the block resolves it here too.+ "DEVICE_TYPE_TORCH": "cuda","_get_inference_mode_context_manager": lambda _model: contextlib.nullcontext(),"model": stub,"unwrapped_model": stub,diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex b227bc83958..af772d7fe8d 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -353,7 +353,7 @@ from torch.nn import functional as Fimport inspectfrom transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling as TransformersDataCollatorForLanguageModelingfrom transformers.training_args import ParallelMode-from unsloth_zoo.device_type import DEVICE_TYPE, device_synchronize+from unsloth_zoo.device_type import DEVICE_TYPE, DEVICE_TYPE_TORCH, device_synchronize# Wrap trainer with padding to right and enable training modeimport functoolsdiff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex e3c52618e67..d65c61f0bd5 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -46,7 +46,6 @@ import importlib.utilfrom ..device_type import (is_hip,get_device_type,- DEVICE_TYPE,DEVICE_TYPE_TORCH,DEVICE_COUNT,ALLOW_PREQUANTIZED_MODELS,@@ -909,7 +908,7 @@ def _unsloth_grpo_autocast(self):return self._autocast_enabled, self._autocast_dtype-def _unsloth_grpo_autocast_kwargs(self, device_type = "cuda"):+def _unsloth_grpo_autocast_kwargs(self, device_type = DEVICE_TYPE_TORCH):"""torch.amp.autocast kwargs for GRPO generation."""enabled, dtype = _unsloth_grpo_autocast(self)if not getattr(self, "_autocast_force_float32", False) and torch.is_autocast_enabled(@@ -1162,7 +1161,8 @@ def grpo_trainer__prepare_inputs(function_name, function):function = function.replace("with torch.inference_mode():","with torch.inference_mode(), "- "torch.amp.autocast(device_type = 'cuda', **_unsloth_grpo_autocast_kwargs(self)):",+ "torch.amp.autocast(device_type = DEVICE_TYPE_TORCH, "+ "**_unsloth_grpo_autocast_kwargs(self)):",)function = function.replace("self.accelerator.unwrap_model(self.model)",@@ -1854,7 +1854,7 @@ def grpo_trainer__get_per_token_logps(function_name, function):os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"with torch.amp.autocast(- device_type = DEVICE_TYPE,+ device_type = DEVICE_TYPE_TORCH,dtype = self._autocast_dtype,enabled = getattr(self, "_autocast_enabled", True),):@@ -2097,7 +2097,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):def _pg_run_forward(_pg_layout = _pg_layout, _pg_chunks = _pg_chunks):with _get_inference_mode_context_manager(model):with torch.amp.autocast(- device_type = "cuda",+ device_type = DEVICE_TYPE_TORCH,dtype = self._autocast_dtype,enabled = getattr(self, "_autocast_enabled", True),):@@ -2218,7 +2218,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):_pk_ctgt = (_pk_nz_idx[1:, 1] >= _pk_cstart[_pk_nz_idx[1:, 0]]) & _pk_withinwith _get_inference_mode_context_manager(model):with torch.amp.autocast(- device_type = "cuda",+ device_type = DEVICE_TYPE_TORCH,dtype = self._autocast_dtype,enabled = getattr(self, "_autocast_enabled", True),):@@ -2283,7 +2283,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):_pk_ref = torch.zeros_like(_pk_result)with _get_inference_mode_context_manager(model):with torch.amp.autocast(- device_type = "cuda",+ device_type = DEVICE_TYPE_TORCH,dtype = self._autocast_dtype,enabled = getattr(self, "_autocast_enabled", True),):@@ -2458,7 +2458,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):vision_chunk,) in zipped_inputs:with torch.amp.autocast(- device_type = "cuda",+ device_type = DEVICE_TYPE_TORCH,dtype = self._autocast_dtype,enabled = getattr(self, "_autocast_enabled", True),):