Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MLX: drain GPU work before clearing the cache (#9921)
Long Yixing6f87da3d4a36
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 76bae9bbb46..a5a5f21ee4b 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -8,6 +8,7 @@ instead of torch/transformers for model loading and generation.import jsonimport osimport re+import sysimport threadingfrom contextlib import contextmanagerfrom typing import Optional, Generator@@ -732,6 +733,46 @@ def _kv_quant_probe(language_model, entries, bits):_restore_mlx_rng_key(rng_key)+# A no-argument mx.synchronize() waits on the default stream, which generation does not+# use. mlx-vlm moves the symbol between layouts, and speculative decoding owns its own.+_GENERATION_STREAM_MODULES = (+ "mlx_lm.generate",+ "mlx_vlm.generate",+ "mlx_vlm.generate.dispatch",+ "mlx_vlm.generate.ar",+ "mlx_vlm.speculative.common",+)+++def _drain_generation_streams(mx):+ """Best effort: every caller is a cleanup path whose old body could not fail.++ At the mlx-vlm pin floor generation_stream is a plain mx.new_stream, which raises+ when synchronized off its creating thread (mlx made command encoders thread local+ in 0.31.2). Draining is a margin on top of the clear, never a precondition, so a+ drain we cannot perform is skipped.+ """++ synchronize = getattr(mx, "synchronize", None)+ if not callable(synchronize):+ return+ drained = []+ for name in _GENERATION_STREAM_MODULES:+ try:+ stream = getattr(sys.modules.get(name), "generation_stream", None)+ # 0.6.x aliases one object across every mlx_vlm.generate name.+ if stream is None or any(stream is seen for seen in drained):+ continue+ drained.append(stream)+ synchronize(stream)+ except Exception as error:+ logger.debug("MLX stream drain skipped for %s: %s", name, error)+ try:+ synchronize()+ except Exception as error:+ logger.debug("MLX default stream drain skipped: %s", error)++def _kv_quant_eligibility(model,is_vlm,@@ -766,6 +807,7 @@ def _kv_quant_eligibility(entries.clear()del entries+ _drain_generation_streams(mx)mx.clear_cache()if failure is not None:return "refused", f"this model's KV cache cannot be quantized: {failure}", True@@ -1989,6 +2031,7 @@ class MLXInferenceBackend:self.active_model_name = Noneself._clear_prompt_cache()gc.collect()+ _drain_generation_streams(mx)mx.clear_cache()if mx.metal.is_available() and self._memory_limits_applied and not self.models:@@ -2861,4 +2904,5 @@ class MLXInferenceBackend:import gcgc.collect()+ _drain_generation_streams(mx)mx.clear_cache()diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex f8e7908fec7..3cf90bcdc0b 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -1,5 +1,6 @@# SPDX-License-Identifier: AGPL-3.0-only+import astimport asyncioimport contextlibimport copy@@ -31,6 +32,10 @@ class _DummyMX:def device_info():return {"max_recommended_working_set_size": 1024}+ @staticmethod+ def synchronize(_stream = None):+ return None+class _DummyTokenizer:pass@@ -51,6 +56,7 @@ def _install_fake_mlx(monkeypatch):mlx_core.metal = _DummyMetal()mlx_core.set_wired_limit = _DummyMX.set_wired_limitmlx_core.device_info = _DummyMX.device_info+ mlx_core.synchronize = _DummyMX.synchronizemlx_utils.tree_unflatten = dictmlx_pkg.core = mlx_coremlx_pkg.utils = mlx_utils@@ -3287,6 +3293,144 @@ def test_rng_capture_stays_quiet_when_the_state_cannot_be_read(monkeypatch):assert warnings == []+_BACKEND_SOURCE = Path(__file__).resolve().parents[1] / "core" / "inference" / "mlx_inference.py"+_BACKEND_TREE = ast.parse(_BACKEND_SOURCE.read_text(encoding = "utf-8"))+++def _is_call(+ node,+ name,+ receiver = "mx",+):+ if not (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)):+ return False+ func = node.value.func+ if isinstance(func, ast.Attribute):+ return func.attr == name and isinstance(func.value, ast.Name) and func.value.id == receiver+ return isinstance(func, ast.Name) and func.id == name+++def _clear_cache_sites():+ for node in ast.walk(_BACKEND_TREE):+ for field in ("body", "orelse", "finalbody"):+ block = getattr(node, field, None)+ if not isinstance(block, list):+ continue+ for index, statement in enumerate(block):+ if _is_call(statement, "clear_cache"):+ yield statement.lineno, (block[index - 1] if index else None)+++def test_backend_has_mlx_cache_clear_sites():+ assert list(_clear_cache_sites()), "no mx.clear_cache() found -- test is stale"+++def test_every_cache_clear_drains_gpu_work_first():+ """MLX does not pin a dropped output array, so every clear must drain first."""+ unguarded = [+ line+ for line, previous in _clear_cache_sites()+ if not _is_call(previous, "_drain_generation_streams", receiver = None)+ ]+ assert not unguarded, (+ f"mx.clear_cache() at line(s) {unguarded} is not immediately preceded by "+ "_drain_generation_streams()"+ )+++def test_the_drain_covers_both_generation_stream_modules():+ """Draining only the default stream would miss the stream generation actually uses."""+ assigned = [+ node+ for node in ast.walk(_BACKEND_TREE)+ if isinstance(node, ast.Assign)+ and any(getattr(t, "id", None) == "_GENERATION_STREAM_MODULES" for t in node.targets)+ ]+ assert len(assigned) == 1+ names = {element.value for element in assigned[0].value.elts}+ assert names == {+ "mlx_lm.generate",+ "mlx_vlm.generate",+ "mlx_vlm.generate.dispatch",+ "mlx_vlm.generate.ar",+ # A second stream since 0.6.0, never wired-limited.+ "mlx_vlm.speculative.common",+ }+++def _fake_stream_module(monkeypatch, name, stream):+ monkeypatch.setitem(sys.modules, name, types.SimpleNamespace(generation_stream = stream))+++def _recording_mx(monkeypatch, synchronize = None):+ from core.inference import mlx_inference++ events = []+ mx = types.SimpleNamespace(+ synchronize = synchronize+ or (+ lambda stream = None: events.append(+ f"synchronize:{'default' if stream is None else stream}"+ )+ ),+ )+ for name in mlx_inference._GENERATION_STREAM_MODULES:+ monkeypatch.delitem(sys.modules, name, raising = False)+ return mlx_inference, mx, events+++def test_a_stream_that_cannot_be_drained_does_not_stop_the_caller(monkeypatch):+ """A plain mx.new_stream raises when synchronized off its creating thread."""++ def synchronize(stream = None):+ if stream == "foreign":+ raise RuntimeError("There is no Stream(gpu, 0) in current thread.")+ events.append(f"synchronize:{'default' if stream is None else stream}")++ mlx_inference, mx, events = _recording_mx(monkeypatch, synchronize)+ _fake_stream_module(monkeypatch, "mlx_lm.generate", "foreign")++ mlx_inference._drain_generation_streams(mx)++ assert events == ["synchronize:default"]+++def test_a_module_getattr_that_raises_does_not_stop_the_caller(monkeypatch):+ mlx_inference, mx, events = _recording_mx(monkeypatch)+ module = types.ModuleType("mlx_vlm.generate")+ module.__getattr__ = lambda name: (_ for _ in ()).throw(RuntimeError(name))+ monkeypatch.setitem(sys.modules, "mlx_vlm.generate", module)++ mlx_inference._drain_generation_streams(mx)++ assert events == ["synchronize:default"]+++def test_one_stream_shared_by_several_modules_is_drained_once(monkeypatch):+ """0.6.x defines the stream once and re-exports it from every candidate name."""+ mlx_inference, mx, events = _recording_mx(monkeypatch)+ for name in ("mlx_vlm.generate", "mlx_vlm.generate.dispatch", "mlx_vlm.generate.ar"):+ _fake_stream_module(monkeypatch, name, "shared")++ mlx_inference._drain_generation_streams(mx)++ assert events == ["synchronize:shared", "synchronize:default"]+++def test_the_speculative_decoding_stream_is_drained_too(monkeypatch):+ mlx_inference, mx, events = _recording_mx(monkeypatch)+ _fake_stream_module(monkeypatch, "mlx_vlm.speculative.common", "speculative")++ mlx_inference._drain_generation_streams(mx)++ assert events == ["synchronize:speculative", "synchronize:default"]+++def test_a_runtime_without_synchronize_is_not_an_error():+ from core.inference import mlx_inference+ mlx_inference._drain_generation_streams(types.SimpleNamespace())++# Llama 3.1 carries the second beside its scaled window, mlx-lm's Kimi Linear the third.@pytest.mark.parametrize("name", ["n_ctx", "original_max_position_embeddings", "model_max_length"])def test_mlx_native_context_length_reads_every_spelling_and_every_config(name):diff --git a/tests/python/test_mlx_public_trainer_api.py b/tests/python/test_mlx_public_trainer_api.pyindex 6059e730b2a..7b52d1290d8 100644--- a/tests/python/test_mlx_public_trainer_api.py+++ b/tests/python/test_mlx_public_trainer_api.py@@ -190,6 +190,177 @@ def test_mlx_clear_gpu_memory_uses_metal_fallback(monkeypatch):assert called == ["metal"]+_GENERATION_STREAM_MODULES = (+ "mlx_lm.generate",+ "mlx_vlm.generate",+ "mlx_vlm.generate.dispatch",+ "mlx_vlm.generate.ar",+ # A second stream since mlx-vlm 0.6.0, never wired-limited.+ "mlx_vlm.speculative.common",+)+++def _stub_generation_streams(monkeypatch, *names):+ """Generation runs on mlx-lm's / mlx-vlm's own thread-local stream, not the default."""+ for name in _GENERATION_STREAM_MODULES:+ if name in names:+ monkeypatch.setitem(+ sys.modules,+ name,+ types.SimpleNamespace(generation_stream = name),+ )+ else:+ monkeypatch.delitem(sys.modules, name, raising = False)++[email protected]("shape", ["core", "metal"])+def test_mlx_clear_gpu_memory_drains_gpu_work_before_clearing(monkeypatch, shape):+ """MLX does not pin a dropped output array, so drain before clearing."""+ unsloth = _import_mlx_unsloth()+ import mlx.core as mx++ events = []+ monkeypatch.setattr(+ mx,+ "synchronize",+ lambda stream = None: events.append(f"synchronize:{'default' if stream is None else stream}"),+ )+ clear = lambda: events.append("clear_cache")+ if shape == "core":+ monkeypatch.setattr(mx, "clear_cache", clear)+ else:+ # Older MLX releases expose cache clearing under mx.metal.clear_cache.+ monkeypatch.delattr(mx, "clear_cache", raising = False)+ metal = getattr(mx, "metal", None) or type("Metal", (), {})()+ monkeypatch.setattr(mx, "metal", metal, raising = False)+ monkeypatch.setattr(metal, "clear_cache", clear, raising = False)+ _stub_generation_streams(+ monkeypatch,+ "mlx_lm.generate",+ "mlx_vlm.generate.dispatch",+ )++ unsloth.clear_gpu_memory()++ assert events == [+ "synchronize:mlx_lm.generate",+ "synchronize:mlx_vlm.generate.dispatch",+ "synchronize:default",+ "clear_cache",+ ]+++def test_mlx_clear_gpu_memory_drains_only_the_streams_that_exist(monkeypatch):+ unsloth = _import_mlx_unsloth()+ import mlx.core as mx++ events = []+ monkeypatch.setattr(+ mx,+ "synchronize",+ lambda stream = None: events.append(f"synchronize:{'default' if stream is None else stream}"),+ )+ monkeypatch.setattr(mx, "clear_cache", lambda: events.append("clear_cache"))+ _stub_generation_streams(monkeypatch, "mlx_lm.generate")++ unsloth.clear_gpu_memory()++ assert events == ["synchronize:mlx_lm.generate", "synchronize:default", "clear_cache"]+++def test_mlx_clear_gpu_memory_is_a_noop_without_cache_clearing(monkeypatch):+ unsloth = _import_mlx_unsloth()+ import mlx.core as mx++ events = []+ monkeypatch.setattr(mx, "synchronize", lambda *a, **k: events.append("synchronize"))+ monkeypatch.delattr(mx, "clear_cache", raising = False)+ monkeypatch.setattr(mx, "metal", type("Metal", (), {})(), raising = False)++ unsloth.clear_gpu_memory()++ assert events == []+++def _recording_synchronize(+ monkeypatch,+ mx,+ events,+ failing = (),+):+ def synchronize(stream = None):+ if stream in failing:+ raise RuntimeError("There is no Stream(gpu, 0) in current thread.")+ events.append(f"synchronize:{'default' if stream is None else stream}")++ monkeypatch.setattr(mx, "synchronize", synchronize)+ monkeypatch.setattr(mx, "clear_cache", lambda: events.append("clear_cache"))+++def test_mlx_clear_gpu_memory_still_clears_when_a_stream_cannot_be_drained(monkeypatch):+ """empty_cache() routes here from finally arms, and a foreign stream raises."""+ unsloth = _import_mlx_unsloth()+ import mlx.core as mx++ events = []+ _recording_synchronize(monkeypatch, mx, events, failing = ("mlx_lm.generate",))+ _stub_generation_streams(monkeypatch, "mlx_lm.generate")++ unsloth.clear_gpu_memory()++ assert events == ["synchronize:default", "clear_cache"]+++def test_mlx_clear_gpu_memory_drains_a_shared_stream_once(monkeypatch):+ """mlx-vlm 0.6.x defines the stream once and re-exports it from every candidate."""+ unsloth = _import_mlx_unsloth()+ import mlx.core as mx++ events = []+ _recording_synchronize(monkeypatch, mx, events)+ shared = types.SimpleNamespace(generation_stream = "shared")+ for name in _GENERATION_STREAM_MODULES:+ if name.startswith("mlx_vlm.generate"):+ monkeypatch.setitem(sys.modules, name, shared)+ else:+ monkeypatch.delitem(sys.modules, name, raising = False)++ unsloth.clear_gpu_memory()++ assert events == ["synchronize:shared", "synchronize:default", "clear_cache"]+++def test_mlx_clear_gpu_memory_drains_the_speculative_stream(monkeypatch):+ unsloth = _import_mlx_unsloth()+ import mlx.core as mx++ events = []+ _recording_synchronize(monkeypatch, mx, events)+ _stub_generation_streams(monkeypatch, "mlx_vlm.speculative.common")++ unsloth.clear_gpu_memory()++ assert events == [+ "synchronize:mlx_vlm.speculative.common",+ "synchronize:default",+ "clear_cache",+ ]+++def test_mlx_clear_gpu_memory_still_clears_without_synchronize(monkeypatch):+ unsloth = _import_mlx_unsloth()+ import mlx.core as mx++ events = []+ monkeypatch.setattr(mx, "clear_cache", lambda: events.append("clear_cache"))+ monkeypatch.delattr(mx, "synchronize", raising = False)+ _stub_generation_streams(monkeypatch, "mlx_lm.generate")++ unsloth.clear_gpu_memory()++ assert events == ["clear_cache"]++def test_mlx_training_arguments_preserve_explicit_epoch_training():"""Epoch-based configs should not inherit the MLX max_steps default."""unsloth = _import_mlx_unsloth()diff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex e71dbb0d265..12d1afeed3b 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -356,6 +356,35 @@ if _IS_MLX:if clear_cache is None and hasattr(mx, "metal"):clear_cache = getattr(mx.metal, "clear_cache", None)if callable(clear_cache):+ # MLX pins buffers a live command buffer reads, but not a dropped output array.+ # Generation runs on its own streams, which a no-argument mx.synchronize()+ # would not wait on. Best effort: this helper is torch.cuda.empty_cache() on+ # MLX, called from finally arms on any thread, and synchronizing a stream+ # bound on another thread raises (mlx 0.31.2 made encoders thread local).+ _synchronize = getattr(mx, "synchronize", None)+ if callable(_synchronize):+ _drained = []+ for _module in (+ "mlx_lm.generate",+ "mlx_vlm.generate",+ "mlx_vlm.generate.dispatch",+ "mlx_vlm.generate.ar",+ # Speculative decoding owns a second stream wired_limit never sees.+ "mlx_vlm.speculative.common",+ ):+ try:+ _stream = getattr(sys.modules.get(_module), "generation_stream", None)+ # 0.6.x aliases one object across every mlx_vlm.generate name.+ if _stream is None or any(_stream is _seen for _seen in _drained):+ continue+ _drained.append(_stream)+ _synchronize(_stream)+ except Exception:+ continue+ try:+ _synchronize()+ except Exception:+ passclear_cache()def _patch_mlx_torch_cuda_compat_api():