Skip to content

The open-source AI voice studio. Clone, dictate, create.

fix(backend): release memory when unloading MLX models

JnyRoad19f8f5140800
diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py
index 142c430c..eb436c2b 100644
--- a/backend/backends/__init__.py
+++ b/backend/backends/__init__.py
@@ -566,6 +566,7 @@ def unload_model_by_config(config: ModelConfig) -> bool:
"""Unload a model given its config. Returns True if it was loaded, False otherwise."""
from . import get_tts_backend_for_engine
from ..services import tts, transcribe, llm as llm_service
+ from ..utils.cache import clear_voice_prompt_memory_cache
if config.engine == "whisper":
whisper_model = transcribe.get_whisper_model()
@@ -595,6 +596,7 @@ def unload_model_by_config(config: ModelConfig) -> bool:
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
if backend.is_loaded() and loaded_size == config.model_size:
backend.unload_model()
+ clear_voice_prompt_memory_cache()
return True
return False
@@ -602,6 +604,7 @@ def unload_model_by_config(config: ModelConfig) -> bool:
backend = get_tts_backend_for_engine(config.engine)
if backend.is_loaded():
backend.unload_model()
+ clear_voice_prompt_memory_cache()
return True
return False
diff --git a/backend/backends/base.py b/backend/backends/base.py
index a8dcb442..30b3a06c 100644
--- a/backend/backends/base.py
+++ b/backend/backends/base.py
@@ -224,6 +224,20 @@ def empty_device_cache(device: str) -> None:
torch.mps.empty_cache()
+def empty_mlx_cache() -> None:
+ """
+ Free cached memory in the MLX allocator.
+
+ MLX keeps freed array buffers in an internal pool for reuse instead of
+ returning them to the OS. Backends must call this after unloading an
+ MLX model, or the process's memory footprint never shrinks even though
+ the model object itself was dropped.
+ """
+ import mlx.core as mx
+
+ mx.clear_cache()
+
+
def manual_seed(seed: int, device: str) -> None:
"""
Set the random seed on both CPU and the active accelerator.
diff --git a/backend/backends/mlx_backend.py b/backend/backends/mlx_backend.py
index ce3cdc31..84b9b620 100644
--- a/backend/backends/mlx_backend.py
+++ b/backend/backends/mlx_backend.py
@@ -33,7 +33,12 @@ patch_huggingface_hub_offline()
ensure_original_qwen_config_cached()
from . import TTSBackend, STTBackend, LANGUAGE_CODE_TO_NAME, WHISPER_HF_REPOS
-from .base import is_model_cached, combine_voice_prompts as _combine_voice_prompts, model_load_progress
+from .base import (
+ is_model_cached,
+ combine_voice_prompts as _combine_voice_prompts,
+ model_load_progress,
+ empty_mlx_cache,
+)
from ..utils.cache import get_cache_key, get_cached_voice_prompt, cache_voice_prompt
@@ -149,6 +154,7 @@ class MLXTTSBackend:
del self.model
self.model = None
self._current_model_size = None
+ empty_mlx_cache()
logger.info("MLX TTS model unloaded")
async def create_voice_prompt(
@@ -381,6 +387,7 @@ class MLXSTTBackend:
if self.model is not None:
del self.model
self.model = None
+ empty_mlx_cache()
logger.info("MLX Whisper model unloaded")
async def transcribe(
diff --git a/backend/backends/qwen_llm_backend.py b/backend/backends/qwen_llm_backend.py
index a7b99632..11b8ca62 100644
--- a/backend/backends/qwen_llm_backend.py
+++ b/backend/backends/qwen_llm_backend.py
@@ -16,6 +16,7 @@ from .base import (
is_model_cached,
get_torch_device,
empty_device_cache,
+ empty_mlx_cache,
manual_seed,
model_load_progress,
)
@@ -258,6 +259,7 @@ class MLXQwenLLMBackend:
self.model = None
self.tokenizer = None
self._current_model_size = None
+ empty_mlx_cache()
logger.info("Qwen3 (MLX) unloaded")
async def generate(
diff --git a/backend/services/tts.py b/backend/services/tts.py
index d4f90ff3..9e7505f3 100644
--- a/backend/services/tts.py
+++ b/backend/services/tts.py
@@ -8,6 +8,7 @@ import io
import soundfile as sf
from ..backends import get_tts_backend, TTSBackend
+from ..utils.cache import clear_voice_prompt_memory_cache
def get_tts_model() -> TTSBackend:
@@ -24,6 +25,7 @@ def unload_tts_model():
"""Unload TTS model to free memory."""
backend = get_tts_backend()
backend.unload_model()
+ clear_voice_prompt_memory_cache()
def audio_to_wav_bytes(audio: np.ndarray, sample_rate: int) -> bytes:
diff --git a/backend/tests/test_mlx_unload_clears_cache.py b/backend/tests/test_mlx_unload_clears_cache.py
new file mode 100644
index 00000000..05075485
--- /dev/null
+++ b/backend/tests/test_mlx_unload_clears_cache.py
@@ -0,0 +1,82 @@
+"""
+Regression tests: unloading an MLX-backed model must release MLX's internal
+buffer cache, not just drop the Python reference.
+
+MLX keeps freed array buffers in an internal pool for reuse instead of
+returning them to the OS (see `mlx.core.clear_cache` / `get_cache_memory`).
+Before this fix, `unload_model()` on the MLX TTS/Whisper/LLM backends only
+did `del self.model; self.model = None`, so the process's memory footprint
+never shrank after "unload" (issue: resources stay held after first use on
+Apple Silicon). These tests assert each backend's `unload_model()` calls the
+shared `empty_mlx_cache()` helper, without requiring the real `mlx` package
+to be installed — `empty_mlx_cache` is monkeypatched, so its own lazy
+`import mlx.core` never executes here.
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+pytest.importorskip("torch")
+
+from backend.backends import mlx_backend, qwen_llm_backend
+
+
+def test_mlx_tts_backend_unload_clears_mlx_cache(monkeypatch):
+ """Unloading the MLX TTS backend must call empty_mlx_cache()."""
+ mock_clear = MagicMock()
+ monkeypatch.setattr(mlx_backend, "empty_mlx_cache", mock_clear)
+
+ backend = mlx_backend.MLXTTSBackend()
+ backend.model = MagicMock()
+ backend._current_model_size = "1.7B"
+
+ backend.unload_model()
+
+ assert backend.model is None
+ assert backend._current_model_size is None
+ mock_clear.assert_called_once()
+
+
+def test_mlx_stt_backend_unload_clears_mlx_cache(monkeypatch):
+ """Unloading the MLX Whisper backend must call empty_mlx_cache()."""
+ mock_clear = MagicMock()
+ monkeypatch.setattr(mlx_backend, "empty_mlx_cache", mock_clear)
+
+ backend = mlx_backend.MLXSTTBackend()
+ backend.model = MagicMock()
+
+ backend.unload_model()
+
+ assert backend.model is None
+ mock_clear.assert_called_once()
+
+
+def test_mlx_llm_backend_unload_clears_mlx_cache(monkeypatch):
+ """Unloading the MLX Qwen3 LLM backend must call empty_mlx_cache()."""
+ mock_clear = MagicMock()
+ monkeypatch.setattr(qwen_llm_backend, "empty_mlx_cache", mock_clear)
+
+ backend = qwen_llm_backend.MLXQwenLLMBackend()
+ backend.model = MagicMock()
+ backend.tokenizer = MagicMock()
+ backend._current_model_size = "4B"
+
+ backend.unload_model()
+
+ assert backend.model is None
+ assert backend.tokenizer is None
+ mock_clear.assert_called_once()
+
+
+def test_mlx_backends_do_not_clear_cache_when_already_unloaded(monkeypatch):
+ """Calling unload on an already-unloaded backend is a no-op (no spurious clear)."""
+ mock_clear = MagicMock()
+ monkeypatch.setattr(mlx_backend, "empty_mlx_cache", mock_clear)
+
+ backend = mlx_backend.MLXTTSBackend()
+ assert backend.model is None
+
+ backend.unload_model()
+
+ mock_clear.assert_not_called()
diff --git a/backend/tests/test_voice_prompt_cache_unload.py b/backend/tests/test_voice_prompt_cache_unload.py
new file mode 100644
index 00000000..ab09a013
--- /dev/null
+++ b/backend/tests/test_voice_prompt_cache_unload.py
@@ -0,0 +1,91 @@
+"""
+Regression tests: unloading a TTS model must also drop the in-memory
+voice-prompt cache.
+
+`backend/utils/cache.py` keeps a process-lifetime `_memory_cache` dict of
+voice-clone prompts (tensors or device-backed dicts produced by whichever
+TTS model created them). Before this fix, nothing ever cleared it on
+unload, so those prompts stayed referenced — and their memory held —
+indefinitely, even after the model that produced them was gone. Whisper
+and the LLM backends never produce voice prompts, so their unload paths
+must leave the cache alone.
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from backend import backends as backends_module
+from backend.backends import ModelConfig
+from backend.services import tts as tts_service
+from backend.utils import cache as cache_module
+
+
[email protected](autouse=True)
+def _reset_memory_cache():
+ """Isolate each test from the shared process-lifetime _memory_cache dict."""
+ cache_module._memory_cache.clear()
+ yield
+ cache_module._memory_cache.clear()
+
+
+def _populate_memory_cache():
+ cache_module._memory_cache["some-cache-key"] = {"ref_audio": "x.wav", "ref_text": "hi"}
+
+
+def test_clear_voice_prompt_memory_cache_leaves_disk_cache_alone(tmp_path, monkeypatch):
+ """Clearing the memory cache must not touch cached .prompt files on disk."""
+ monkeypatch.setattr(cache_module, "_get_cache_dir", lambda: tmp_path)
+ disk_file = tmp_path / "some-cache-key.prompt"
+ disk_file.write_bytes(b"fake torch.save payload")
+ _populate_memory_cache()
+
+ cache_module.clear_voice_prompt_memory_cache()
+
+ assert cache_module._memory_cache == {}
+ assert disk_file.exists()
+
+
+def test_unload_tts_model_clears_voice_prompt_memory_cache(monkeypatch):
+ """/models/unload (the legacy qwen-only endpoint) must clear the prompt cache."""
+ fake_backend = MagicMock()
+ monkeypatch.setattr(tts_service, "get_tts_backend", lambda: fake_backend)
+ _populate_memory_cache()
+
+ tts_service.unload_tts_model()
+
+ fake_backend.unload_model.assert_called_once()
+ assert cache_module._memory_cache == {}
+
+
+def test_unload_model_by_config_clears_cache_for_generic_tts_engine(monkeypatch):
+ """Unloading any non-qwen TTS engine (e.g. kokoro) must clear the prompt cache."""
+ fake_backend = MagicMock()
+ fake_backend.is_loaded.return_value = True
+ monkeypatch.setattr(backends_module, "get_tts_backend_for_engine", lambda engine: fake_backend)
+ _populate_memory_cache()
+
+ config = ModelConfig(model_name="kokoro", display_name="Kokoro", engine="kokoro", hf_repo_id="x/y")
+ was_loaded = backends_module.unload_model_by_config(config)
+
+ assert was_loaded is True
+ fake_backend.unload_model.assert_called_once()
+ assert cache_module._memory_cache == {}
+
+
+def test_unload_model_by_config_leaves_cache_alone_for_whisper(monkeypatch):
+ """Whisper never produces voice prompts, so unloading it must not touch the cache."""
+ fake_whisper = MagicMock()
+ fake_whisper.is_loaded.return_value = True
+ fake_whisper.model_size = "base"
+ monkeypatch.setattr("backend.services.transcribe.get_whisper_model", lambda: fake_whisper)
+ _populate_memory_cache()
+ cache_before = dict(cache_module._memory_cache)
+
+ config = ModelConfig(
+ model_name="whisper-base", display_name="Whisper Base", engine="whisper", hf_repo_id="x/y", model_size="base"
+ )
+ was_loaded = backends_module.unload_model_by_config(config)
+
+ assert was_loaded is True
+ assert cache_module._memory_cache == cache_before
diff --git a/backend/utils/cache.py b/backend/utils/cache.py
index dd4b9f83..4c9a0733 100644
--- a/backend/utils/cache.py
+++ b/backend/utils/cache.py
@@ -93,6 +93,19 @@ def cache_voice_prompt(
torch.save(voice_prompt, cache_file)
+def clear_voice_prompt_memory_cache() -> None:
+ """
+ Drop the in-memory voice prompt cache without touching the disk cache.
+
+ Backends call this when a TTS model unloads: the cached prompts (tensors
+ or device-backed dicts produced by that model) would otherwise keep
+ referencing memory forever, since nothing else ever clears this
+ process-lifetime dict. The disk cache is left alone, so the next
+ generation just reloads the prompt from disk instead of recomputing it.
+ """
+ _memory_cache.clear()
+
+
def clear_voice_prompt_cache() -> int:
"""
Clear all voice prompt caches (memory and disk).