Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix: release the device cache through the active accelerator on the vLLM resume/teardown paths (#13438)
li-lizhe23711635c1f9
diff --git a/tests/test_device_helpers.py b/tests/test_device_helpers.pyindex 282432079ec..600d2498980 100644--- a/tests/test_device_helpers.py+++ b/tests/test_device_helpers.py@@ -1,3 +1,4 @@+import astimport importlib.utilimport sysimport types@@ -177,6 +178,62 @@ def test_mlx_helpers_do_not_require_torch(monkeypatch):assert device_type.get_current_device() == 0+def _function_source(path, name):+ source = path.read_text(encoding = "utf-8")+ for node in ast.parse(source).body:+ if isinstance(node, ast.FunctionDef) and node.name == name:+ return ast.get_source_segment(source, node)+ raise AssertionError(f"{name} not found in {path}")+++class _SleepingLLM:+ def __init__(self, enable_sleep_mode):+ self.slept = []+ model_config = types.SimpleNamespace(enable_sleep_mode = enable_sleep_mode)+ self.llm_engine = types.SimpleNamespace(+ vllm_config = types.SimpleNamespace(model_config = model_config)+ )++ def sleep(self, level):+ self.slept.append(level)+++def _resume_helper(device_count, flushes):+ namespace = {+ "DEVICE_COUNT": device_count,+ "clean_gpu_cache": lambda: flushes.append(1),+ }+ source = _function_source(+ REPO_ROOT / "unsloth" / "models" / "rl.py", "_maybe_prepare_vllm_for_resume"+ )+ exec(compile(source, "rl.py", "exec"), namespace)+ return namespace["_maybe_prepare_vllm_for_resume"]++[email protected]("device_count", [1, 4])+def test_vllm_resume_sleeps_and_flushes_on_any_accelerator(device_count):+ # Resolved without torch.cuda: on NPU / XPU torch.cuda.is_available() is False.+ flushes = []+ helper = _resume_helper(device_count, flushes)+ llm = _SleepingLLM(enable_sleep_mode = True)+ helper(types.SimpleNamespace(llm = llm))+ assert llm.slept == [1] and len(flushes) == 3+ # The engine can also hang off the model; no sleep mode means nothing to release.+ engine = _SleepingLLM(enable_sleep_mode = True)+ helper(types.SimpleNamespace(llm = None, model = types.SimpleNamespace(vllm_engine = engine)))+ assert engine.slept == [1] and len(flushes) == 6+ idle = _SleepingLLM(enable_sleep_mode = False)+ helper(types.SimpleNamespace(llm = idle))+ assert idle.slept == [] and len(flushes) == 6+++def test_vllm_resume_is_a_noop_without_an_accelerator():+ flushes = []+ llm = _SleepingLLM(enable_sleep_mode = True)+ _resume_helper(0, flushes)(types.SimpleNamespace(llm = llm))+ assert llm.slept == [] and flushes == []++def test_model_call_sites_use_shared_cache_dispatch():llama_source = (REPO_ROOT / "unsloth" / "models" / "llama.py").read_text(encoding = "utf-8")vision_source = (REPO_ROOT / "unsloth" / "models" / "vision.py").read_text(encoding = "utf-8")@@ -189,6 +246,12 @@ def test_model_call_sites_use_shared_cache_dispatch():q_galore_source = (REPO_ROOT / "unsloth" / "optimizers" / "q_galore_adamw.py").read_text(encoding = "utf-8")+ resume_source = _function_source(+ REPO_ROOT / "unsloth" / "models" / "rl.py", "_maybe_prepare_vllm_for_resume"+ )+ synthetic_source = (REPO_ROOT / "unsloth" / "dataprep" / "synthetic.py").read_text(+ encoding = "utf-8"+ )assert "torch.xpu.empty_cache()" not in llama_sourceassert "torch.xpu.empty_cache()" not in vision_source@@ -210,6 +273,15 @@ def test_model_call_sites_use_shared_cache_dispatch():assert "clean_gpu_cache()" in loader_utils_sourceassert "torch.cuda.synchronize()" not in q_galore_sourceassert "device_synchronize()" in q_galore_source+ # The vLLM resume / teardown releases were keyed off CUDA too, so on NPU / XPU the+ # sleep-and-free path returned before flushing and the freed blocks stayed reserved.+ # Scoped to the resume helper: rl.py keeps CUDA-only paths (optimizer parking) elsewhere.+ assert "torch.cuda.is_available()" not in resume_source+ assert "DEVICE_COUNT < 1" in resume_source+ assert "torch.cuda.empty_cache()" not in resume_source+ assert "clean_gpu_cache()" in resume_source+ assert "torch.cuda.empty_cache()" not in synthetic_source+ assert "clean_gpu_cache()" in synthetic_sourceNPU_PROPERTIES = types.SimpleNamespace(diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.pyindex 45bb05445dd..c68f41782c7 100644--- a/unsloth/dataprep/synthetic.py+++ b/unsloth/dataprep/synthetic.py@@ -19,6 +19,7 @@ from collections import dequeimport timeimport osimport importlib.util as _importlib_util+from ..device_type import clean_gpu_cachedef _hf_transfer_importable() -> bool:@@ -491,7 +492,7 @@ class SyntheticDataKit:except Exception as kill_e:print(f"Error during forceful kill: {kill_e}")for _ in range(10):- torch.cuda.empty_cache()+ clean_gpu_cache()gc.collect()if hasattr(self, "_delete_vllm"):diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 6f9e194ec16..ad4175dfa15 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -35,7 +35,7 @@ from unsloth_zoo.compiler import create_new_functionfrom unsloth_zoo.log import loggerfrom unsloth_zoo.logging_utils import PatchRLStatisticsfrom unsloth_zoo.rl_replacements import RL_REPLACEMENTS-from ..device_type import DEVICE_TYPE+from ..device_type import DEVICE_COUNT, DEVICE_TYPE, clean_gpu_cachefrom .rl_replacements import (_unsloth_get_model_config,_unsloth_text_configs,@@ -141,7 +141,7 @@ def vLLMSamplingParams(**kwargs):def _maybe_prepare_vllm_for_resume(trainer):- if not torch.cuda.is_available():+ if DEVICE_COUNT < 1:returnllm = getattr(trainer, "llm", None)@@ -167,7 +167,7 @@ def _maybe_prepare_vllm_for_resume(trainer):for _ in range(3):gc.collect()- torch.cuda.empty_cache()+ clean_gpu_cache()def _patch_resume_from_checkpoint_memory(trainer_class):