Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: synchronise VAE decode and encode calls on Apple MPS (#14940)

Long Yixing5e5ffb91f2b6
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 27bdfc9996b..1826ed1d41a 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -107,6 +107,7 @@ from .diffusion_device import (
diffusion_device_scope,
diffusion_device_target_from_torch_device,
install_frame_pad_fix,
+ install_vae_sync,
pin_cuda_ordinal,
placed_cuda_ordinal,
auto_cuda_ordinal_for_load,
@@ -9777,6 +9778,7 @@ class DiffusionBackend:
except Exception as exc: # noqa: BLE001 - keep the stock tiled decode
logger.warning("diffusion.vae_tiling: not installed: %s", exc)
install_frame_pad_fix(pipe, target, logger = logger)
+ install_vae_sync(pipe, target, logger = logger)
# Before the speed optims so their decode compile lands inside the non-finite check; `off` keeps fp32.
vae_fp16 = str(
speed_mode or ""
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index 9bd17d2c9de..75364e5f681 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -140,62 +140,39 @@ def install_frame_pad_fix(
return patched
-# Fraction of the device's recommended working set above which a decode starts synchronising.
-DECODE_SYNC_FRACTION = 0.85
-
-
-def install_decoder_sync(
+def install_vae_sync(
pipe: Any,
target: DiffusionDeviceTarget,
*,
logger: Any = None,
) -> bool:
- """Cap the memory a video VAE decode holds on Metal, by synchronising once it is running out.
-
- Wan's VAE decodes one latent frame per call in a loop that never forces a commit, and Metal
- cannot reuse a buffer until the work holding it completes, so intermediates accumulate until
- the OS kills the process. Neither tiling (the growth is within one tile) nor torch's adaptive
- commit bounds it.
+ """Synchronise Metal after every VAE decoder / encoder call (one per tile or latent frame).
- Fires per decoder call and only above the threshold, so a decode with room to spare pays only
- the memory read; synchronising costs the pipelining, not the decode.
-
- ``torch.mps.recommended_max_memory()`` arrived in torch 2.5 while install.sh keeps an existing
- venv's torch as far back as 2.4, so an unreadable budget falls back to synchronising every
- call (measured to hold the same decode at 4.90 GiB for no wall-clock cost) rather than
- failing the load or dropping the bound. Every probe is best-effort for the same reason.
+ Metal cannot reuse a buffer until the queued work holding it completes, so an unsynchronised
+ tile loop accumulates every call's intermediates; ``empty_cache`` cannot return them.
"""
if target.device != "mps":
return False
- decoder = getattr(getattr(pipe, "vae", None), "decoder", None)
- if not callable(getattr(decoder, "register_forward_hook", None)):
+ vae = getattr(pipe, "vae", None)
+ modules = [
+ module
+ for module in (getattr(vae, "decoder", None), getattr(vae, "encoder", None))
+ if callable(getattr(module, "register_forward_hook", None))
+ ]
+ if not modules:
return False
import torch
- budget: Optional[float] = None
- try:
- budget = torch.mps.recommended_max_memory() * DECODE_SYNC_FRACTION
- except Exception as exc: # noqa: BLE001 -- torch < 2.5 has no such reading
- if logger is not None:
- logger.info(
- "video.decoder_sync: no memory reading (%s); synchronising every decode", exc
- )
-
def _sync(_module, _args, _output) -> None:
- if budget is not None:
- try:
- if torch.mps.driver_allocated_memory() < budget:
- return
- except Exception: # noqa: BLE001 -- an unreadable gauge syncs, the safe side
- pass
try:
torch.mps.synchronize()
except Exception: # noqa: BLE001 -- a decode is worth more than the bound
pass
- decoder.register_forward_hook(_sync)
- if logger is not None and budget is not None:
- logger.info("video.decoder_sync: decode synchronises above %.1f GiB", budget / 1024**3)
+ for module in modules:
+ module.register_forward_hook(_sync)
+ if logger is not None:
+ logger.info("diffusion.vae_sync: VAE synchronises after every decoder / encoder call")
return True
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 9754b54a24c..70808208d31 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -103,8 +103,8 @@ from .diffusion_device import (
apply_diffusion_device_ordinal,
diffusion_device_scope,
force_float32_rope,
- install_decoder_sync,
install_rocm_vae_bf16_decode,
+ install_vae_sync,
pin_cuda_ordinal,
placed_cuda_ordinal,
auto_cuda_ordinal_for_load,
@@ -9588,7 +9588,7 @@ class VideoBackend:
if install_untiled_decode(pipe, fam.name, logger = logger):
speed_optims += ("vae_untiled_when_fits",)
# Wan's decode also grows within a single tile, which tiling alone cannot bound.
- install_decoder_sync(pipe, target, logger = logger)
+ install_vae_sync(pipe, target, logger = logger)
if _video_denoiser_hooked(pipe):
# Below the offload hooks (as the image backend), whether the speed layer planned the offload or a split
# fell back to a hooked placement: compiled through them, each block splits at the hook's
diff --git a/studio/backend/tests/test_diffusion_device.py b/studio/backend/tests/test_diffusion_device.py
index 6902729e4cd..aa3a994c6a4 100644
--- a/studio/backend/tests/test_diffusion_device.py
+++ b/studio/backend/tests/test_diffusion_device.py
@@ -503,7 +503,22 @@ def test_the_image_loader_installs_the_frame_pad_fix():
assert "install_frame_pad_fix(pipe, target, logger = logger)" in src
-# ── Pressure-gated decoder sync ───────────────────────────────────────
+def test_the_image_loader_syncs_every_decoder_call():
+ import ast
+ import inspect
+ import textwrap
+
+ from core.inference import diffusion
+
+ src = textwrap.dedent(
+ inspect.getsource(inspect.unwrap(diffusion.DiffusionBackend.load_pipeline))
+ )
+ calls = [
+ node
+ for node in ast.walk(ast.parse(src))
+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "install_vae_sync"
+ ]
+ assert len(calls) == 1
def _target(device: str) -> dd.DiffusionDeviceTarget:
@@ -539,137 +554,62 @@ class _FakeDecoder:
return outputs
-def _pipe_with(decoder) -> types.SimpleNamespace:
- return types.SimpleNamespace(vae = types.SimpleNamespace(decoder = decoder))
+def _pipe_with(decoder, encoder = None) -> types.SimpleNamespace:
+ return types.SimpleNamespace(vae = types.SimpleNamespace(decoder = decoder, encoder = encoder))
-def _mps_torch(used = 0, recommended = 100) -> types.ModuleType:
- """torch whose mps backend counts synchronize() calls over a settable memory reading."""
+def _mps_torch() -> types.ModuleType:
torch = types.ModuleType("torch")
torch.syncs = 0
- torch.used = used
def _bump():
torch.syncs += 1
- torch.mps = types.SimpleNamespace(
- synchronize = _bump,
- recommended_max_memory = lambda: recommended,
- driver_allocated_memory = lambda: torch.used,
- )
+ torch.mps = types.SimpleNamespace(synchronize = _bump)
return torch
@pytest.mark.parametrize("device", ["cuda", "xpu", "cpu"])
-def test_decoder_sync_is_metal_only(monkeypatch, device):
+def test_vae_sync_is_metal_only(monkeypatch, device):
monkeypatch.setitem(sys.modules, "torch", _mps_torch())
decoder = _FakeDecoder()
- assert dd.install_decoder_sync(_pipe_with(decoder), _target(device)) is False
+ assert dd.install_vae_sync(_pipe_with(decoder), _target(device)) is False
assert decoder.hooks == []
-def test_decoder_sync_idle_while_memory_is_plentiful(monkeypatch):
- torch = _mps_torch(recommended = 100, used = 10)
- monkeypatch.setitem(sys.modules, "torch", torch)
- decoder = _FakeDecoder()
- assert dd.install_decoder_sync(_pipe_with(decoder), _target("mps")) is True
- decoder.decode(5)
- assert torch.syncs == 0
-
-
-def test_decoder_sync_runs_once_per_decoder_call_above_the_threshold(monkeypatch):
- torch = _mps_torch(recommended = 100, used = 10)
- monkeypatch.setitem(sys.modules, "torch", torch)
- decoder = _FakeDecoder()
- dd.install_decoder_sync(_pipe_with(decoder), _target("mps"))
- decoder.decode(2)
- assert torch.syncs == 0
- torch.used = 100 * dd.DECODE_SYNC_FRACTION
- decoder.decode(3)
- assert torch.syncs == 3
- torch.used = 10
- decoder.decode(4)
- assert torch.syncs == 3
-
-
-def test_decoder_sync_threshold_scales_with_the_device(monkeypatch):
- # The budget is a fraction of this device's working set, not a fixed byte count.
- torch = _mps_torch(recommended = 200, used = 169)
- monkeypatch.setitem(sys.modules, "torch", torch)
- decoder = _FakeDecoder()
- dd.install_decoder_sync(_pipe_with(decoder), _target("mps"))
- decoder.decode(1)
- assert torch.syncs == 0
- torch.used = 170
- decoder.decode(1)
- assert torch.syncs == 1
- assert dd.DECODE_SYNC_FRACTION == 0.85
-
-
-def test_decoder_sync_preserves_the_decoder_output(monkeypatch):
+def test_vae_sync_runs_after_every_decoder_call_and_keeps_its_output(monkeypatch):
# An nn.Module forward hook that returns non-None REPLACES the output; this one must not.
- torch = _mps_torch(recommended = 100, used = 100)
- monkeypatch.setitem(sys.modules, "torch", torch)
- decoder = _FakeDecoder()
- dd.install_decoder_sync(_pipe_with(decoder), _target("mps"))
- assert decoder.decode(2) == ["out0", "out1"]
- assert torch.syncs == 2
-
-
[email protected]("pipe", [types.SimpleNamespace(), _pipe_with(None), _pipe_with(object())])
-def test_decoder_sync_no_op_without_a_hookable_decoder(monkeypatch, pipe):
- monkeypatch.setitem(sys.modules, "torch", _mps_torch())
- assert dd.install_decoder_sync(pipe, _target("mps")) is False
-
-
-def _mps_torch_without_recommended(used = 0) -> types.ModuleType:
- """torch 2.4 has no mps working-set reading: only driver_allocated_memory and synchronize exist."""
- torch = _mps_torch(used = used)
- del torch.mps.recommended_max_memory
- return torch
-
-
-def test_decoder_sync_survives_a_torch_without_the_memory_reading(monkeypatch):
- # install.sh keeps an existing venv's torch (>=2.4); recommended_max_memory is a 2.5 API.
- torch = _mps_torch_without_recommended()
+ torch = _mps_torch()
monkeypatch.setitem(sys.modules, "torch", torch)
decoder = _FakeDecoder()
- assert dd.install_decoder_sync(_pipe_with(decoder), _target("mps")) is True
- # No budget: sync every call rather than assume the decode is fine (Wan grew past 148 GiB).
+ assert dd.install_vae_sync(_pipe_with(decoder), _target("mps")) is True
assert decoder.decode(3) == ["out0", "out1", "out2"]
assert torch.syncs == 3
-def test_decoder_sync_survives_a_working_set_reading_that_raises(monkeypatch):
+def test_vae_sync_runs_after_every_encoder_call_too(monkeypatch):
torch = _mps_torch()
-
- def _boom():
- raise RuntimeError("MPS backend is not available")
-
- torch.mps.recommended_max_memory = _boom
monkeypatch.setitem(sys.modules, "torch", torch)
- decoder = _FakeDecoder()
- assert dd.install_decoder_sync(_pipe_with(decoder), _target("mps")) is True
- decoder.decode(2)
+ decoder, encoder = _FakeDecoder(), _FakeDecoder()
+ assert dd.install_vae_sync(_pipe_with(decoder, encoder), _target("mps")) is True
+ assert encoder.decode(2) == ["out0", "out1"]
assert torch.syncs == 2
+ decoder.decode(1)
+ assert torch.syncs == 3
+ lone = _FakeDecoder()
+ assert dd.install_vae_sync(_pipe_with(object(), lone), _target("mps")) is True
+ lone.decode(1)
+ assert torch.syncs == 4
-def test_decoder_sync_survives_a_gauge_that_raises_mid_decode(monkeypatch):
- torch = _mps_torch(recommended = 100, used = 10)
-
- def _boom():
- raise RuntimeError("driver reading unavailable")
-
- torch.mps.driver_allocated_memory = _boom
- monkeypatch.setitem(sys.modules, "torch", torch)
- decoder = _FakeDecoder()
- dd.install_decoder_sync(_pipe_with(decoder), _target("mps"))
- assert decoder.decode(2) == ["out0", "out1"]
- assert torch.syncs == 2
[email protected]("pipe", [types.SimpleNamespace(), _pipe_with(None), _pipe_with(object())])
+def test_vae_sync_no_op_without_a_hookable_decoder(monkeypatch, pipe):
+ monkeypatch.setitem(sys.modules, "torch", _mps_torch())
+ assert dd.install_vae_sync(pipe, _target("mps")) is False
-def test_decoder_sync_survives_a_synchronize_that_raises(monkeypatch):
- torch = _mps_torch_without_recommended()
+def test_vae_sync_survives_a_synchronize_that_raises(monkeypatch):
+ torch = _mps_torch()
def _boom():
raise RuntimeError("Torch not compiled with MPS enabled")
@@ -677,7 +617,7 @@ def test_decoder_sync_survives_a_synchronize_that_raises(monkeypatch):
torch.mps.synchronize = _boom
monkeypatch.setitem(sys.modules, "torch", torch)
decoder = _FakeDecoder()
- assert dd.install_decoder_sync(_pipe_with(decoder), _target("mps")) is True
+ assert dd.install_vae_sync(_pipe_with(decoder), _target("mps")) is True
assert decoder.decode(2) == ["out0", "out1"]
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 1c8210d26c6..b74b64b94d1 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -1789,16 +1789,15 @@ def _fits_in_memory_snapshot(device):
@pytest.mark.parametrize("device,hooked", [("mps", 1), ("cuda", 0)])
-def test_load_installs_the_pressure_gated_decoder_sync_on_mps(
- fake_runtime, monkeypatch, device, hooked
-):
+def test_load_installs_the_vae_sync_on_mps(fake_runtime, monkeypatch, device, hooked):
# Tiling alone does not bound a Wan decode on MPS; the load must arm the sync.
torch = sys.modules["torch"]
+ syncs = []
monkeypatch.setattr(
torch,
device,
types.SimpleNamespace(
- synchronize = lambda: None,
+ synchronize = lambda: syncs.append(1),
recommended_max_memory = lambda: 64 * 1024**3,
driver_allocated_memory = lambda: 0,
),
@@ -1824,7 +1823,11 @@ def test_load_installs_the_pressure_gated_decoder_sync_on_mps(
)
backend = VideoBackend()
backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")
- assert len(backend._state.pipe.vae.decoder.hooks) == hooked
+ hooks = backend._state.pipe.vae.decoder.hooks
+ assert len(hooks) == hooked
+ for hook in hooks:
+ hook(None, (), None)
+ assert len(syncs) == hooked
def test_video_dense_speed_defaults_to_compile_profile(fake_runtime):