Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: synchronise VAE decode and encode calls on Apple MPS (#14940)
Long Yixing5e5ffb91f2b6
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 27bdfc9996b..1826ed1d41a 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -107,6 +107,7 @@ from .diffusion_device import (diffusion_device_scope,diffusion_device_target_from_torch_device,install_frame_pad_fix,+ install_vae_sync,pin_cuda_ordinal,placed_cuda_ordinal,auto_cuda_ordinal_for_load,@@ -9777,6 +9778,7 @@ class DiffusionBackend:except Exception as exc: # noqa: BLE001 - keep the stock tiled decodelogger.warning("diffusion.vae_tiling: not installed: %s", exc)install_frame_pad_fix(pipe, target, logger = logger)+ install_vae_sync(pipe, target, logger = logger)# Before the speed optims so their decode compile lands inside the non-finite check; `off` keeps fp32.vae_fp16 = str(speed_mode or ""diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.pyindex 9bd17d2c9de..75364e5f681 100644--- a/studio/backend/core/inference/diffusion_device.py+++ b/studio/backend/core/inference/diffusion_device.py@@ -140,62 +140,39 @@ def install_frame_pad_fix(return patched-# Fraction of the device's recommended working set above which a decode starts synchronising.-DECODE_SYNC_FRACTION = 0.85---def install_decoder_sync(+def install_vae_sync(pipe: Any,target: DiffusionDeviceTarget,*,logger: Any = None,) -> bool:- """Cap the memory a video VAE decode holds on Metal, by synchronising once it is running out.-- Wan's VAE decodes one latent frame per call in a loop that never forces a commit, and Metal- cannot reuse a buffer until the work holding it completes, so intermediates accumulate until- the OS kills the process. Neither tiling (the growth is within one tile) nor torch's adaptive- commit bounds it.+ """Synchronise Metal after every VAE decoder / encoder call (one per tile or latent frame).- Fires per decoder call and only above the threshold, so a decode with room to spare pays only- the memory read; synchronising costs the pipelining, not the decode.-- ``torch.mps.recommended_max_memory()`` arrived in torch 2.5 while install.sh keeps an existing- venv's torch as far back as 2.4, so an unreadable budget falls back to synchronising every- call (measured to hold the same decode at 4.90 GiB for no wall-clock cost) rather than- failing the load or dropping the bound. Every probe is best-effort for the same reason.+ Metal cannot reuse a buffer until the queued work holding it completes, so an unsynchronised+ tile loop accumulates every call's intermediates; ``empty_cache`` cannot return them."""if target.device != "mps":return False- decoder = getattr(getattr(pipe, "vae", None), "decoder", None)- if not callable(getattr(decoder, "register_forward_hook", None)):+ vae = getattr(pipe, "vae", None)+ modules = [+ module+ for module in (getattr(vae, "decoder", None), getattr(vae, "encoder", None))+ if callable(getattr(module, "register_forward_hook", None))+ ]+ if not modules:return Falseimport torch- budget: Optional[float] = None- try:- budget = torch.mps.recommended_max_memory() * DECODE_SYNC_FRACTION- except Exception as exc: # noqa: BLE001 -- torch < 2.5 has no such reading- if logger is not None:- logger.info(- "video.decoder_sync: no memory reading (%s); synchronising every decode", exc- )-def _sync(_module, _args, _output) -> None:- if budget is not None:- try:- if torch.mps.driver_allocated_memory() < budget:- return- except Exception: # noqa: BLE001 -- an unreadable gauge syncs, the safe side- passtry:torch.mps.synchronize()except Exception: # noqa: BLE001 -- a decode is worth more than the boundpass- decoder.register_forward_hook(_sync)- if logger is not None and budget is not None:- logger.info("video.decoder_sync: decode synchronises above %.1f GiB", budget / 1024**3)+ for module in modules:+ module.register_forward_hook(_sync)+ if logger is not None:+ logger.info("diffusion.vae_sync: VAE synchronises after every decoder / encoder call")return Truediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 9754b54a24c..70808208d31 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -103,8 +103,8 @@ from .diffusion_device import (apply_diffusion_device_ordinal,diffusion_device_scope,force_float32_rope,- install_decoder_sync,install_rocm_vae_bf16_decode,+ install_vae_sync,pin_cuda_ordinal,placed_cuda_ordinal,auto_cuda_ordinal_for_load,@@ -9588,7 +9588,7 @@ class VideoBackend:if install_untiled_decode(pipe, fam.name, logger = logger):speed_optims += ("vae_untiled_when_fits",)# Wan's decode also grows within a single tile, which tiling alone cannot bound.- install_decoder_sync(pipe, target, logger = logger)+ install_vae_sync(pipe, target, logger = logger)if _video_denoiser_hooked(pipe):# Below the offload hooks (as the image backend), whether the speed layer planned the offload or a split# fell back to a hooked placement: compiled through them, each block splits at the hook'sdiff --git a/studio/backend/tests/test_diffusion_device.py b/studio/backend/tests/test_diffusion_device.pyindex 6902729e4cd..aa3a994c6a4 100644--- a/studio/backend/tests/test_diffusion_device.py+++ b/studio/backend/tests/test_diffusion_device.py@@ -503,7 +503,22 @@ def test_the_image_loader_installs_the_frame_pad_fix():assert "install_frame_pad_fix(pipe, target, logger = logger)" in src-# ── Pressure-gated decoder sync ───────────────────────────────────────+def test_the_image_loader_syncs_every_decoder_call():+ import ast+ import inspect+ import textwrap++ from core.inference import diffusion++ src = textwrap.dedent(+ inspect.getsource(inspect.unwrap(diffusion.DiffusionBackend.load_pipeline))+ )+ calls = [+ node+ for node in ast.walk(ast.parse(src))+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "install_vae_sync"+ ]+ assert len(calls) == 1def _target(device: str) -> dd.DiffusionDeviceTarget:@@ -539,137 +554,62 @@ class _FakeDecoder:return outputs-def _pipe_with(decoder) -> types.SimpleNamespace:- return types.SimpleNamespace(vae = types.SimpleNamespace(decoder = decoder))+def _pipe_with(decoder, encoder = None) -> types.SimpleNamespace:+ return types.SimpleNamespace(vae = types.SimpleNamespace(decoder = decoder, encoder = encoder))-def _mps_torch(used = 0, recommended = 100) -> types.ModuleType:- """torch whose mps backend counts synchronize() calls over a settable memory reading."""+def _mps_torch() -> types.ModuleType:torch = types.ModuleType("torch")torch.syncs = 0- torch.used = useddef _bump():torch.syncs += 1- torch.mps = types.SimpleNamespace(- synchronize = _bump,- recommended_max_memory = lambda: recommended,- driver_allocated_memory = lambda: torch.used,- )+ torch.mps = types.SimpleNamespace(synchronize = _bump)return torch@pytest.mark.parametrize("device", ["cuda", "xpu", "cpu"])-def test_decoder_sync_is_metal_only(monkeypatch, device):+def test_vae_sync_is_metal_only(monkeypatch, device):monkeypatch.setitem(sys.modules, "torch", _mps_torch())decoder = _FakeDecoder()- assert dd.install_decoder_sync(_pipe_with(decoder), _target(device)) is False+ assert dd.install_vae_sync(_pipe_with(decoder), _target(device)) is Falseassert decoder.hooks == []-def test_decoder_sync_idle_while_memory_is_plentiful(monkeypatch):- torch = _mps_torch(recommended = 100, used = 10)- monkeypatch.setitem(sys.modules, "torch", torch)- decoder = _FakeDecoder()- assert dd.install_decoder_sync(_pipe_with(decoder), _target("mps")) is True- decoder.decode(5)- assert torch.syncs == 0---def test_decoder_sync_runs_once_per_decoder_call_above_the_threshold(monkeypatch):- torch = _mps_torch(recommended = 100, used = 10)- monkeypatch.setitem(sys.modules, "torch", torch)- decoder = _FakeDecoder()- dd.install_decoder_sync(_pipe_with(decoder), _target("mps"))- decoder.decode(2)- assert torch.syncs == 0- torch.used = 100 * dd.DECODE_SYNC_FRACTION- decoder.decode(3)- assert torch.syncs == 3- torch.used = 10- decoder.decode(4)- assert torch.syncs == 3---def test_decoder_sync_threshold_scales_with_the_device(monkeypatch):- # The budget is a fraction of this device's working set, not a fixed byte count.- torch = _mps_torch(recommended = 200, used = 169)- monkeypatch.setitem(sys.modules, "torch", torch)- decoder = _FakeDecoder()- dd.install_decoder_sync(_pipe_with(decoder), _target("mps"))- decoder.decode(1)- assert torch.syncs == 0- torch.used = 170- decoder.decode(1)- assert torch.syncs == 1- assert dd.DECODE_SYNC_FRACTION == 0.85---def test_decoder_sync_preserves_the_decoder_output(monkeypatch):+def test_vae_sync_runs_after_every_decoder_call_and_keeps_its_output(monkeypatch):# An nn.Module forward hook that returns non-None REPLACES the output; this one must not.- torch = _mps_torch(recommended = 100, used = 100)- monkeypatch.setitem(sys.modules, "torch", torch)- decoder = _FakeDecoder()- dd.install_decoder_sync(_pipe_with(decoder), _target("mps"))- assert decoder.decode(2) == ["out0", "out1"]- assert torch.syncs == 2--[email protected]("pipe", [types.SimpleNamespace(), _pipe_with(None), _pipe_with(object())])-def test_decoder_sync_no_op_without_a_hookable_decoder(monkeypatch, pipe):- monkeypatch.setitem(sys.modules, "torch", _mps_torch())- assert dd.install_decoder_sync(pipe, _target("mps")) is False---def _mps_torch_without_recommended(used = 0) -> types.ModuleType:- """torch 2.4 has no mps working-set reading: only driver_allocated_memory and synchronize exist."""- torch = _mps_torch(used = used)- del torch.mps.recommended_max_memory- return torch---def test_decoder_sync_survives_a_torch_without_the_memory_reading(monkeypatch):- # install.sh keeps an existing venv's torch (>=2.4); recommended_max_memory is a 2.5 API.- torch = _mps_torch_without_recommended()+ torch = _mps_torch()monkeypatch.setitem(sys.modules, "torch", torch)decoder = _FakeDecoder()- assert dd.install_decoder_sync(_pipe_with(decoder), _target("mps")) is True- # No budget: sync every call rather than assume the decode is fine (Wan grew past 148 GiB).+ assert dd.install_vae_sync(_pipe_with(decoder), _target("mps")) is Trueassert decoder.decode(3) == ["out0", "out1", "out2"]assert torch.syncs == 3-def test_decoder_sync_survives_a_working_set_reading_that_raises(monkeypatch):+def test_vae_sync_runs_after_every_encoder_call_too(monkeypatch):torch = _mps_torch()-- def _boom():- raise RuntimeError("MPS backend is not available")-- torch.mps.recommended_max_memory = _boommonkeypatch.setitem(sys.modules, "torch", torch)- decoder = _FakeDecoder()- assert dd.install_decoder_sync(_pipe_with(decoder), _target("mps")) is True- decoder.decode(2)+ decoder, encoder = _FakeDecoder(), _FakeDecoder()+ assert dd.install_vae_sync(_pipe_with(decoder, encoder), _target("mps")) is True+ assert encoder.decode(2) == ["out0", "out1"]assert torch.syncs == 2+ decoder.decode(1)+ assert torch.syncs == 3+ lone = _FakeDecoder()+ assert dd.install_vae_sync(_pipe_with(object(), lone), _target("mps")) is True+ lone.decode(1)+ assert torch.syncs == 4-def test_decoder_sync_survives_a_gauge_that_raises_mid_decode(monkeypatch):- torch = _mps_torch(recommended = 100, used = 10)-- def _boom():- raise RuntimeError("driver reading unavailable")-- torch.mps.driver_allocated_memory = _boom- monkeypatch.setitem(sys.modules, "torch", torch)- decoder = _FakeDecoder()- dd.install_decoder_sync(_pipe_with(decoder), _target("mps"))- assert decoder.decode(2) == ["out0", "out1"]- assert torch.syncs == 2[email protected]("pipe", [types.SimpleNamespace(), _pipe_with(None), _pipe_with(object())])+def test_vae_sync_no_op_without_a_hookable_decoder(monkeypatch, pipe):+ monkeypatch.setitem(sys.modules, "torch", _mps_torch())+ assert dd.install_vae_sync(pipe, _target("mps")) is False-def test_decoder_sync_survives_a_synchronize_that_raises(monkeypatch):- torch = _mps_torch_without_recommended()+def test_vae_sync_survives_a_synchronize_that_raises(monkeypatch):+ torch = _mps_torch()def _boom():raise RuntimeError("Torch not compiled with MPS enabled")@@ -677,7 +617,7 @@ def test_decoder_sync_survives_a_synchronize_that_raises(monkeypatch):torch.mps.synchronize = _boommonkeypatch.setitem(sys.modules, "torch", torch)decoder = _FakeDecoder()- assert dd.install_decoder_sync(_pipe_with(decoder), _target("mps")) is True+ assert dd.install_vae_sync(_pipe_with(decoder), _target("mps")) is Trueassert decoder.decode(2) == ["out0", "out1"]diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 1c8210d26c6..b74b64b94d1 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -1789,16 +1789,15 @@ def _fits_in_memory_snapshot(device):@pytest.mark.parametrize("device,hooked", [("mps", 1), ("cuda", 0)])-def test_load_installs_the_pressure_gated_decoder_sync_on_mps(- fake_runtime, monkeypatch, device, hooked-):+def test_load_installs_the_vae_sync_on_mps(fake_runtime, monkeypatch, device, hooked):# Tiling alone does not bound a Wan decode on MPS; the load must arm the sync.torch = sys.modules["torch"]+ syncs = []monkeypatch.setattr(torch,device,types.SimpleNamespace(- synchronize = lambda: None,+ synchronize = lambda: syncs.append(1),recommended_max_memory = lambda: 64 * 1024**3,driver_allocated_memory = lambda: 0,),@@ -1824,7 +1823,11 @@ def test_load_installs_the_pressure_gated_decoder_sync_on_mps()backend = VideoBackend()backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")- assert len(backend._state.pipe.vae.decoder.hooks) == hooked+ hooks = backend._state.pipe.vae.decoder.hooks+ assert len(hooks) == hooked+ for hook in hooks:+ hook(None, (), None)+ assert len(syncs) == hookeddef test_video_dense_speed_defaults_to_compile_profile(fake_runtime):