Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: leave MiniMax-H3's unpinned streamed blocks to diffusers' onload (#12753)
oobabooga56f451df1271
diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.pyindex 189a9f58ad0..7b74bf9764f 100644--- a/studio/backend/core/inference/video_minimax_h3_residency.py+++ b/studio/backend/core/inference/video_minimax_h3_residency.py@@ -503,9 +503,17 @@ def install_h3_stream_prefetch(top, blocks = h3_offload_groups(transformer)if not blocks:return 0- depth = h3_stream_prefetch_depth([group_payload_bytes(g) for g in blocks], prefetch_depth())from .diffusion_memory import _pinned_memory_capped+ if not _pinned_memory_capped() and any(getattr(g, "low_cpu_mem_usage", False) for g in blocks):+ # Unpinned host copies: the prefetcher would pin a fresh copy per onload, two blocks ahead (more host RAM, no speed-up).+ if logger is not None:+ logger.info(+ "video.h3_prefetch: left to diffusers' onload (the streamed blocks have unpinned host copies)"+ )+ return 0+ depth = h3_stream_prefetch_depth([group_payload_bytes(g) for g in blocks], prefetch_depth())+if _pinned_memory_capped():depth = 1 # each in-flight group is pinned on the fly; diffusers' own path holds two near a ~1 GiB cap# The prefetcher refuses groups whose onload_ is replaced: lift the outside-inference_mode wrappers (torchao v1diff --git a/studio/backend/tests/test_h3_stream_prefetch.py b/studio/backend/tests/test_h3_stream_prefetch.pyindex ca562c0d42f..097f05efcf9 100644--- a/studio/backend/tests/test_h3_stream_prefetch.py+++ b/studio/backend/tests/test_h3_stream_prefetch.py@@ -120,6 +120,7 @@ def _h3_streamed(prefetch = True,outside_inference = True,pin_top = True,+ unpinned = False,):"""What the H3 load does at a capped tier: stream groups with record_stream, pinned top group, prefetch."""import inspect@@ -142,6 +143,8 @@ def _h3_streamed(kwargs["record_stream"] = Trueif "non_blocking" in params:kwargs["non_blocking"] = True+ if unpinned:+ kwargs["low_cpu_mem_usage"] = Trueapply_group_offloading(net, **kwargs)if outside_inference:# what stream_prequantized_module does for torchao v1 int8: every group move outside inference_mode@@ -325,12 +328,32 @@ def test_top_pin_kill_switch_keeps_diffusers_top_group(monkeypatch):def test_capped_pinned_host_prefetches_one_ahead(monkeypatch):- """Windows / WSL2 pin each in-flight group on the fly under a ~1 GiB cap: one ahead, as diffusers holds."""+ """Windows / WSL2 pin each in-flight group on the fly under a ~1 GiB cap: one ahead, as diffusers holds. Their+ streamed blocks have unpinned host copies, which elsewhere keep diffusers' onload."""_cuda()import core.inference.diffusion_memory as dmfrom core.inference.diffusion_offload_prefetch import module_prefetchermonkeypatch.setattr(dm, "_pinned_memory_capped", lambda: True)net = _DiT().eval()- assert _h3_streamed(net, monkeypatch) == 1 + len(net.transformer_blocks)+ assert _h3_streamed(net, monkeypatch, unpinned = True) == 1 + len(net.transformer_blocks)assert module_prefetcher(net).depth == 1+++def test_unpinned_streamed_blocks_keep_diffusers_onload(monkeypatch):+ """A host whose pin budget refused the denoiser streams it from unpinned copies; the prefetcher would pin a fresh+ copy of every streamed tensor per onload, so diffusers' own onload keeps them."""+ _cuda()+ import core.inference.diffusion_memory as dm+ from core.inference.diffusion_offload_prefetch import module_prefetcher++ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: False)+ torch.manual_seed(0)+ net = _DiT().eval()+ ref = copy.deepcopy(net).cuda()+ assert _h3_streamed(net, monkeypatch, unpinned = True) == 0+ assert module_prefetcher(net) is None+ x = torch.randn(16, 64, device = "cuda")+ with torch.no_grad():+ for _ in range(2):+ assert torch.equal(net(x), ref(x))