Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MLX: drain GPU work before clearing the cache (#9921)

Long Yixing6f87da3d4a36
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 76bae9bbb46..a5a5f21ee4b 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -8,6 +8,7 @@ instead of torch/transformers for model loading and generation.
import json
import os
import re
+import sys
import threading
from contextlib import contextmanager
from typing import Optional, Generator
@@ -732,6 +733,46 @@ def _kv_quant_probe(language_model, entries, bits):
_restore_mlx_rng_key(rng_key)
+# A no-argument mx.synchronize() waits on the default stream, which generation does not
+# use. mlx-vlm moves the symbol between layouts, and speculative decoding owns its own.
+_GENERATION_STREAM_MODULES = (
+ "mlx_lm.generate",
+ "mlx_vlm.generate",
+ "mlx_vlm.generate.dispatch",
+ "mlx_vlm.generate.ar",
+ "mlx_vlm.speculative.common",
+)
+
+
+def _drain_generation_streams(mx):
+ """Best effort: every caller is a cleanup path whose old body could not fail.
+
+ At the mlx-vlm pin floor generation_stream is a plain mx.new_stream, which raises
+ when synchronized off its creating thread (mlx made command encoders thread local
+ in 0.31.2). Draining is a margin on top of the clear, never a precondition, so a
+ drain we cannot perform is skipped.
+ """
+
+ synchronize = getattr(mx, "synchronize", None)
+ if not callable(synchronize):
+ return
+ drained = []
+ for name in _GENERATION_STREAM_MODULES:
+ try:
+ stream = getattr(sys.modules.get(name), "generation_stream", None)
+ # 0.6.x aliases one object across every mlx_vlm.generate name.
+ if stream is None or any(stream is seen for seen in drained):
+ continue
+ drained.append(stream)
+ synchronize(stream)
+ except Exception as error:
+ logger.debug("MLX stream drain skipped for %s: %s", name, error)
+ try:
+ synchronize()
+ except Exception as error:
+ logger.debug("MLX default stream drain skipped: %s", error)
+
+
def _kv_quant_eligibility(
model,
is_vlm,
@@ -766,6 +807,7 @@ def _kv_quant_eligibility(
entries.clear()
del entries
+ _drain_generation_streams(mx)
mx.clear_cache()
if failure is not None:
return "refused", f"this model's KV cache cannot be quantized: {failure}", True
@@ -1989,6 +2031,7 @@ class MLXInferenceBackend:
self.active_model_name = None
self._clear_prompt_cache()
gc.collect()
+ _drain_generation_streams(mx)
mx.clear_cache()
if mx.metal.is_available() and self._memory_limits_applied and not self.models:
@@ -2861,4 +2904,5 @@ class MLXInferenceBackend:
import gc
gc.collect()
+ _drain_generation_streams(mx)
mx.clear_cache()
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index f8e7908fec7..3cf90bcdc0b 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
+import ast
import asyncio
import contextlib
import copy
@@ -31,6 +32,10 @@ class _DummyMX:
def device_info():
return {"max_recommended_working_set_size": 1024}
+ @staticmethod
+ def synchronize(_stream = None):
+ return None
+
class _DummyTokenizer:
pass
@@ -51,6 +56,7 @@ def _install_fake_mlx(monkeypatch):
mlx_core.metal = _DummyMetal()
mlx_core.set_wired_limit = _DummyMX.set_wired_limit
mlx_core.device_info = _DummyMX.device_info
+ mlx_core.synchronize = _DummyMX.synchronize
mlx_utils.tree_unflatten = dict
mlx_pkg.core = mlx_core
mlx_pkg.utils = mlx_utils
@@ -3287,6 +3293,144 @@ def test_rng_capture_stays_quiet_when_the_state_cannot_be_read(monkeypatch):
assert warnings == []
+_BACKEND_SOURCE = Path(__file__).resolve().parents[1] / "core" / "inference" / "mlx_inference.py"
+_BACKEND_TREE = ast.parse(_BACKEND_SOURCE.read_text(encoding = "utf-8"))
+
+
+def _is_call(
+ node,
+ name,
+ receiver = "mx",
+):
+ if not (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)):
+ return False
+ func = node.value.func
+ if isinstance(func, ast.Attribute):
+ return func.attr == name and isinstance(func.value, ast.Name) and func.value.id == receiver
+ return isinstance(func, ast.Name) and func.id == name
+
+
+def _clear_cache_sites():
+ for node in ast.walk(_BACKEND_TREE):
+ for field in ("body", "orelse", "finalbody"):
+ block = getattr(node, field, None)
+ if not isinstance(block, list):
+ continue
+ for index, statement in enumerate(block):
+ if _is_call(statement, "clear_cache"):
+ yield statement.lineno, (block[index - 1] if index else None)
+
+
+def test_backend_has_mlx_cache_clear_sites():
+ assert list(_clear_cache_sites()), "no mx.clear_cache() found -- test is stale"
+
+
+def test_every_cache_clear_drains_gpu_work_first():
+ """MLX does not pin a dropped output array, so every clear must drain first."""
+ unguarded = [
+ line
+ for line, previous in _clear_cache_sites()
+ if not _is_call(previous, "_drain_generation_streams", receiver = None)
+ ]
+ assert not unguarded, (
+ f"mx.clear_cache() at line(s) {unguarded} is not immediately preceded by "
+ "_drain_generation_streams()"
+ )
+
+
+def test_the_drain_covers_both_generation_stream_modules():
+ """Draining only the default stream would miss the stream generation actually uses."""
+ assigned = [
+ node
+ for node in ast.walk(_BACKEND_TREE)
+ if isinstance(node, ast.Assign)
+ and any(getattr(t, "id", None) == "_GENERATION_STREAM_MODULES" for t in node.targets)
+ ]
+ assert len(assigned) == 1
+ names = {element.value for element in assigned[0].value.elts}
+ assert names == {
+ "mlx_lm.generate",
+ "mlx_vlm.generate",
+ "mlx_vlm.generate.dispatch",
+ "mlx_vlm.generate.ar",
+ # A second stream since 0.6.0, never wired-limited.
+ "mlx_vlm.speculative.common",
+ }
+
+
+def _fake_stream_module(monkeypatch, name, stream):
+ monkeypatch.setitem(sys.modules, name, types.SimpleNamespace(generation_stream = stream))
+
+
+def _recording_mx(monkeypatch, synchronize = None):
+ from core.inference import mlx_inference
+
+ events = []
+ mx = types.SimpleNamespace(
+ synchronize = synchronize
+ or (
+ lambda stream = None: events.append(
+ f"synchronize:{'default' if stream is None else stream}"
+ )
+ ),
+ )
+ for name in mlx_inference._GENERATION_STREAM_MODULES:
+ monkeypatch.delitem(sys.modules, name, raising = False)
+ return mlx_inference, mx, events
+
+
+def test_a_stream_that_cannot_be_drained_does_not_stop_the_caller(monkeypatch):
+ """A plain mx.new_stream raises when synchronized off its creating thread."""
+
+ def synchronize(stream = None):
+ if stream == "foreign":
+ raise RuntimeError("There is no Stream(gpu, 0) in current thread.")
+ events.append(f"synchronize:{'default' if stream is None else stream}")
+
+ mlx_inference, mx, events = _recording_mx(monkeypatch, synchronize)
+ _fake_stream_module(monkeypatch, "mlx_lm.generate", "foreign")
+
+ mlx_inference._drain_generation_streams(mx)
+
+ assert events == ["synchronize:default"]
+
+
+def test_a_module_getattr_that_raises_does_not_stop_the_caller(monkeypatch):
+ mlx_inference, mx, events = _recording_mx(monkeypatch)
+ module = types.ModuleType("mlx_vlm.generate")
+ module.__getattr__ = lambda name: (_ for _ in ()).throw(RuntimeError(name))
+ monkeypatch.setitem(sys.modules, "mlx_vlm.generate", module)
+
+ mlx_inference._drain_generation_streams(mx)
+
+ assert events == ["synchronize:default"]
+
+
+def test_one_stream_shared_by_several_modules_is_drained_once(monkeypatch):
+ """0.6.x defines the stream once and re-exports it from every candidate name."""
+ mlx_inference, mx, events = _recording_mx(monkeypatch)
+ for name in ("mlx_vlm.generate", "mlx_vlm.generate.dispatch", "mlx_vlm.generate.ar"):
+ _fake_stream_module(monkeypatch, name, "shared")
+
+ mlx_inference._drain_generation_streams(mx)
+
+ assert events == ["synchronize:shared", "synchronize:default"]
+
+
+def test_the_speculative_decoding_stream_is_drained_too(monkeypatch):
+ mlx_inference, mx, events = _recording_mx(monkeypatch)
+ _fake_stream_module(monkeypatch, "mlx_vlm.speculative.common", "speculative")
+
+ mlx_inference._drain_generation_streams(mx)
+
+ assert events == ["synchronize:speculative", "synchronize:default"]
+
+
+def test_a_runtime_without_synchronize_is_not_an_error():
+ from core.inference import mlx_inference
+ mlx_inference._drain_generation_streams(types.SimpleNamespace())
+
+
# Llama 3.1 carries the second beside its scaled window, mlx-lm's Kimi Linear the third.
@pytest.mark.parametrize("name", ["n_ctx", "original_max_position_embeddings", "model_max_length"])
def test_mlx_native_context_length_reads_every_spelling_and_every_config(name):
diff --git a/tests/python/test_mlx_public_trainer_api.py b/tests/python/test_mlx_public_trainer_api.py
index 6059e730b2a..7b52d1290d8 100644
--- a/tests/python/test_mlx_public_trainer_api.py
+++ b/tests/python/test_mlx_public_trainer_api.py
@@ -190,6 +190,177 @@ def test_mlx_clear_gpu_memory_uses_metal_fallback(monkeypatch):
assert called == ["metal"]
+_GENERATION_STREAM_MODULES = (
+ "mlx_lm.generate",
+ "mlx_vlm.generate",
+ "mlx_vlm.generate.dispatch",
+ "mlx_vlm.generate.ar",
+ # A second stream since mlx-vlm 0.6.0, never wired-limited.
+ "mlx_vlm.speculative.common",
+)
+
+
+def _stub_generation_streams(monkeypatch, *names):
+ """Generation runs on mlx-lm's / mlx-vlm's own thread-local stream, not the default."""
+ for name in _GENERATION_STREAM_MODULES:
+ if name in names:
+ monkeypatch.setitem(
+ sys.modules,
+ name,
+ types.SimpleNamespace(generation_stream = name),
+ )
+ else:
+ monkeypatch.delitem(sys.modules, name, raising = False)
+
+
[email protected]("shape", ["core", "metal"])
+def test_mlx_clear_gpu_memory_drains_gpu_work_before_clearing(monkeypatch, shape):
+ """MLX does not pin a dropped output array, so drain before clearing."""
+ unsloth = _import_mlx_unsloth()
+ import mlx.core as mx
+
+ events = []
+ monkeypatch.setattr(
+ mx,
+ "synchronize",
+ lambda stream = None: events.append(f"synchronize:{'default' if stream is None else stream}"),
+ )
+ clear = lambda: events.append("clear_cache")
+ if shape == "core":
+ monkeypatch.setattr(mx, "clear_cache", clear)
+ else:
+ # Older MLX releases expose cache clearing under mx.metal.clear_cache.
+ monkeypatch.delattr(mx, "clear_cache", raising = False)
+ metal = getattr(mx, "metal", None) or type("Metal", (), {})()
+ monkeypatch.setattr(mx, "metal", metal, raising = False)
+ monkeypatch.setattr(metal, "clear_cache", clear, raising = False)
+ _stub_generation_streams(
+ monkeypatch,
+ "mlx_lm.generate",
+ "mlx_vlm.generate.dispatch",
+ )
+
+ unsloth.clear_gpu_memory()
+
+ assert events == [
+ "synchronize:mlx_lm.generate",
+ "synchronize:mlx_vlm.generate.dispatch",
+ "synchronize:default",
+ "clear_cache",
+ ]
+
+
+def test_mlx_clear_gpu_memory_drains_only_the_streams_that_exist(monkeypatch):
+ unsloth = _import_mlx_unsloth()
+ import mlx.core as mx
+
+ events = []
+ monkeypatch.setattr(
+ mx,
+ "synchronize",
+ lambda stream = None: events.append(f"synchronize:{'default' if stream is None else stream}"),
+ )
+ monkeypatch.setattr(mx, "clear_cache", lambda: events.append("clear_cache"))
+ _stub_generation_streams(monkeypatch, "mlx_lm.generate")
+
+ unsloth.clear_gpu_memory()
+
+ assert events == ["synchronize:mlx_lm.generate", "synchronize:default", "clear_cache"]
+
+
+def test_mlx_clear_gpu_memory_is_a_noop_without_cache_clearing(monkeypatch):
+ unsloth = _import_mlx_unsloth()
+ import mlx.core as mx
+
+ events = []
+ monkeypatch.setattr(mx, "synchronize", lambda *a, **k: events.append("synchronize"))
+ monkeypatch.delattr(mx, "clear_cache", raising = False)
+ monkeypatch.setattr(mx, "metal", type("Metal", (), {})(), raising = False)
+
+ unsloth.clear_gpu_memory()
+
+ assert events == []
+
+
+def _recording_synchronize(
+ monkeypatch,
+ mx,
+ events,
+ failing = (),
+):
+ def synchronize(stream = None):
+ if stream in failing:
+ raise RuntimeError("There is no Stream(gpu, 0) in current thread.")
+ events.append(f"synchronize:{'default' if stream is None else stream}")
+
+ monkeypatch.setattr(mx, "synchronize", synchronize)
+ monkeypatch.setattr(mx, "clear_cache", lambda: events.append("clear_cache"))
+
+
+def test_mlx_clear_gpu_memory_still_clears_when_a_stream_cannot_be_drained(monkeypatch):
+ """empty_cache() routes here from finally arms, and a foreign stream raises."""
+ unsloth = _import_mlx_unsloth()
+ import mlx.core as mx
+
+ events = []
+ _recording_synchronize(monkeypatch, mx, events, failing = ("mlx_lm.generate",))
+ _stub_generation_streams(monkeypatch, "mlx_lm.generate")
+
+ unsloth.clear_gpu_memory()
+
+ assert events == ["synchronize:default", "clear_cache"]
+
+
+def test_mlx_clear_gpu_memory_drains_a_shared_stream_once(monkeypatch):
+ """mlx-vlm 0.6.x defines the stream once and re-exports it from every candidate."""
+ unsloth = _import_mlx_unsloth()
+ import mlx.core as mx
+
+ events = []
+ _recording_synchronize(monkeypatch, mx, events)
+ shared = types.SimpleNamespace(generation_stream = "shared")
+ for name in _GENERATION_STREAM_MODULES:
+ if name.startswith("mlx_vlm.generate"):
+ monkeypatch.setitem(sys.modules, name, shared)
+ else:
+ monkeypatch.delitem(sys.modules, name, raising = False)
+
+ unsloth.clear_gpu_memory()
+
+ assert events == ["synchronize:shared", "synchronize:default", "clear_cache"]
+
+
+def test_mlx_clear_gpu_memory_drains_the_speculative_stream(monkeypatch):
+ unsloth = _import_mlx_unsloth()
+ import mlx.core as mx
+
+ events = []
+ _recording_synchronize(monkeypatch, mx, events)
+ _stub_generation_streams(monkeypatch, "mlx_vlm.speculative.common")
+
+ unsloth.clear_gpu_memory()
+
+ assert events == [
+ "synchronize:mlx_vlm.speculative.common",
+ "synchronize:default",
+ "clear_cache",
+ ]
+
+
+def test_mlx_clear_gpu_memory_still_clears_without_synchronize(monkeypatch):
+ unsloth = _import_mlx_unsloth()
+ import mlx.core as mx
+
+ events = []
+ monkeypatch.setattr(mx, "clear_cache", lambda: events.append("clear_cache"))
+ monkeypatch.delattr(mx, "synchronize", raising = False)
+ _stub_generation_streams(monkeypatch, "mlx_lm.generate")
+
+ unsloth.clear_gpu_memory()
+
+ assert events == ["clear_cache"]
+
+
def test_mlx_training_arguments_preserve_explicit_epoch_training():
"""Epoch-based configs should not inherit the MLX max_steps default."""
unsloth = _import_mlx_unsloth()
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index e71dbb0d265..12d1afeed3b 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -356,6 +356,35 @@ if _IS_MLX:
if clear_cache is None and hasattr(mx, "metal"):
clear_cache = getattr(mx.metal, "clear_cache", None)
if callable(clear_cache):
+ # MLX pins buffers a live command buffer reads, but not a dropped output array.
+ # Generation runs on its own streams, which a no-argument mx.synchronize()
+ # would not wait on. Best effort: this helper is torch.cuda.empty_cache() on
+ # MLX, called from finally arms on any thread, and synchronizing a stream
+ # bound on another thread raises (mlx 0.31.2 made encoders thread local).
+ _synchronize = getattr(mx, "synchronize", None)
+ if callable(_synchronize):
+ _drained = []
+ for _module in (
+ "mlx_lm.generate",
+ "mlx_vlm.generate",
+ "mlx_vlm.generate.dispatch",
+ "mlx_vlm.generate.ar",
+ # Speculative decoding owns a second stream wired_limit never sees.
+ "mlx_vlm.speculative.common",
+ ):
+ try:
+ _stream = getattr(sys.modules.get(_module), "generation_stream", None)
+ # 0.6.x aliases one object across every mlx_vlm.generate name.
+ if _stream is None or any(_stream is _seen for _seen in _drained):
+ continue
+ _drained.append(_stream)
+ _synchronize(_stream)
+ except Exception:
+ continue
+ try:
+ _synchronize()
+ except Exception:
+ pass
clear_cache()
def _patch_mlx_torch_cuda_compat_api():