Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix: release the device cache through the active accelerator on the vLLM resume/teardown paths (#13438)

li-lizhe23711635c1f9
diff --git a/tests/test_device_helpers.py b/tests/test_device_helpers.py
index 282432079ec..600d2498980 100644
--- a/tests/test_device_helpers.py
+++ b/tests/test_device_helpers.py
@@ -1,3 +1,4 @@
+import ast
import importlib.util
import sys
import types
@@ -177,6 +178,62 @@ def test_mlx_helpers_do_not_require_torch(monkeypatch):
assert device_type.get_current_device() == 0
+def _function_source(path, name):
+ source = path.read_text(encoding = "utf-8")
+ for node in ast.parse(source).body:
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ return ast.get_source_segment(source, node)
+ raise AssertionError(f"{name} not found in {path}")
+
+
+class _SleepingLLM:
+ def __init__(self, enable_sleep_mode):
+ self.slept = []
+ model_config = types.SimpleNamespace(enable_sleep_mode = enable_sleep_mode)
+ self.llm_engine = types.SimpleNamespace(
+ vllm_config = types.SimpleNamespace(model_config = model_config)
+ )
+
+ def sleep(self, level):
+ self.slept.append(level)
+
+
+def _resume_helper(device_count, flushes):
+ namespace = {
+ "DEVICE_COUNT": device_count,
+ "clean_gpu_cache": lambda: flushes.append(1),
+ }
+ source = _function_source(
+ REPO_ROOT / "unsloth" / "models" / "rl.py", "_maybe_prepare_vllm_for_resume"
+ )
+ exec(compile(source, "rl.py", "exec"), namespace)
+ return namespace["_maybe_prepare_vllm_for_resume"]
+
+
[email protected]("device_count", [1, 4])
+def test_vllm_resume_sleeps_and_flushes_on_any_accelerator(device_count):
+ # Resolved without torch.cuda: on NPU / XPU torch.cuda.is_available() is False.
+ flushes = []
+ helper = _resume_helper(device_count, flushes)
+ llm = _SleepingLLM(enable_sleep_mode = True)
+ helper(types.SimpleNamespace(llm = llm))
+ assert llm.slept == [1] and len(flushes) == 3
+ # The engine can also hang off the model; no sleep mode means nothing to release.
+ engine = _SleepingLLM(enable_sleep_mode = True)
+ helper(types.SimpleNamespace(llm = None, model = types.SimpleNamespace(vllm_engine = engine)))
+ assert engine.slept == [1] and len(flushes) == 6
+ idle = _SleepingLLM(enable_sleep_mode = False)
+ helper(types.SimpleNamespace(llm = idle))
+ assert idle.slept == [] and len(flushes) == 6
+
+
+def test_vllm_resume_is_a_noop_without_an_accelerator():
+ flushes = []
+ llm = _SleepingLLM(enable_sleep_mode = True)
+ _resume_helper(0, flushes)(types.SimpleNamespace(llm = llm))
+ assert llm.slept == [] and flushes == []
+
+
def test_model_call_sites_use_shared_cache_dispatch():
llama_source = (REPO_ROOT / "unsloth" / "models" / "llama.py").read_text(encoding = "utf-8")
vision_source = (REPO_ROOT / "unsloth" / "models" / "vision.py").read_text(encoding = "utf-8")
@@ -189,6 +246,12 @@ def test_model_call_sites_use_shared_cache_dispatch():
q_galore_source = (REPO_ROOT / "unsloth" / "optimizers" / "q_galore_adamw.py").read_text(
encoding = "utf-8"
)
+ resume_source = _function_source(
+ REPO_ROOT / "unsloth" / "models" / "rl.py", "_maybe_prepare_vllm_for_resume"
+ )
+ synthetic_source = (REPO_ROOT / "unsloth" / "dataprep" / "synthetic.py").read_text(
+ encoding = "utf-8"
+ )
assert "torch.xpu.empty_cache()" not in llama_source
assert "torch.xpu.empty_cache()" not in vision_source
@@ -210,6 +273,15 @@ def test_model_call_sites_use_shared_cache_dispatch():
assert "clean_gpu_cache()" in loader_utils_source
assert "torch.cuda.synchronize()" not in q_galore_source
assert "device_synchronize()" in q_galore_source
+ # The vLLM resume / teardown releases were keyed off CUDA too, so on NPU / XPU the
+ # sleep-and-free path returned before flushing and the freed blocks stayed reserved.
+ # Scoped to the resume helper: rl.py keeps CUDA-only paths (optimizer parking) elsewhere.
+ assert "torch.cuda.is_available()" not in resume_source
+ assert "DEVICE_COUNT < 1" in resume_source
+ assert "torch.cuda.empty_cache()" not in resume_source
+ assert "clean_gpu_cache()" in resume_source
+ assert "torch.cuda.empty_cache()" not in synthetic_source
+ assert "clean_gpu_cache()" in synthetic_source
NPU_PROPERTIES = types.SimpleNamespace(
diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py
index 45bb05445dd..c68f41782c7 100644
--- a/unsloth/dataprep/synthetic.py
+++ b/unsloth/dataprep/synthetic.py
@@ -19,6 +19,7 @@ from collections import deque
import time
import os
import importlib.util as _importlib_util
+from ..device_type import clean_gpu_cache
def _hf_transfer_importable() -> bool:
@@ -491,7 +492,7 @@ class SyntheticDataKit:
except Exception as kill_e:
print(f"Error during forceful kill: {kill_e}")
for _ in range(10):
- torch.cuda.empty_cache()
+ clean_gpu_cache()
gc.collect()
if hasattr(self, "_delete_vllm"):
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 6f9e194ec16..ad4175dfa15 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -35,7 +35,7 @@ from unsloth_zoo.compiler import create_new_function
from unsloth_zoo.log import logger
from unsloth_zoo.logging_utils import PatchRLStatistics
from unsloth_zoo.rl_replacements import RL_REPLACEMENTS
-from ..device_type import DEVICE_TYPE
+from ..device_type import DEVICE_COUNT, DEVICE_TYPE, clean_gpu_cache
from .rl_replacements import (
_unsloth_get_model_config,
_unsloth_text_configs,
@@ -141,7 +141,7 @@ def vLLMSamplingParams(**kwargs):
def _maybe_prepare_vllm_for_resume(trainer):
- if not torch.cuda.is_available():
+ if DEVICE_COUNT < 1:
return
llm = getattr(trainer, "llm", None)
@@ -167,7 +167,7 @@ def _maybe_prepare_vllm_for_resume(trainer):
for _ in range(3):
gc.collect()
- torch.cuda.empty_cache()
+ clean_gpu_cache()
def _patch_resume_from_checkpoint_memory(trainer_class):