Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fill the H3 conditioner's pinned arenas once, before registering (33% faster cold pin) (#16887)
Daniel Hanccc8a827d2c0
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex c2da4767502..02b2cd592ff 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -6929,9 +6929,14 @@ class _RegisteredHostBuffer:REGISTERED_SLAB_ENV = "UNSLOTH_DIFFUSION_REGISTERED_SLABS"-def registered_pinned_empty(nbytes: int) -> Optional[Any]:+def registered_pinned_empty(+ nbytes: int, fill: Optional[Callable[[Any], None]] = None+) -> Optional[Any]:"""Page-locked uint8 host tensor in registered memory, or None (use torch's pinned allocator).- Torch's allocator faults pages on one thread (~18 s for MiniMax-H3); ``zero_`` here faults them in parallel."""+ Torch's allocator faults pages on one thread (~18 s for MiniMax-H3); ``zero_`` here faults them in parallel.+ ``fill(buf)``, when given, writes every byte before registering, in place of the zero fill: the copy faults the+ pages, saving a full write pass (an 8 GB H3 conditioner pin: 3.8 -> 3.4 s hot, 6.1 -> 4.1 s cold, A100 VM).+ Its errors propagate."""if (os.environ.get(REGISTERED_SLAB_ENV) or "").strip().lower() in ("0", "off", "false", "no"):return Noneif int(nbytes) <= 0 or not _fast_pin_supported():@@ -6942,7 +6947,13 @@ def registered_pinned_empty(nbytes: int) -> Optional[Any]:try:holder = _RegisteredHostBuffer(int(nbytes))buf = torch.from_numpy(np.asarray(holder))- buf.zero_()+ except Exception: # noqa: BLE001+ return None+ if fill is not None:+ fill(buf)+ try:+ if fill is None:+ buf.zero_()holder.register()except Exception: # noqa: BLE001return Nonediff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.pyindex e2633975630..2a81aecf85b 100644--- a/studio/backend/core/inference/video_minimax_h3_te.py+++ b/studio/backend/core/inference/video_minimax_h3_te.py@@ -8,6 +8,7 @@ fall back to released bf16 encoder."""from __future__ import annotations+import functoolsfrom functools import lru_cachefrom typing import Any, Optional@@ -374,6 +375,9 @@ H3_TE_STREAMED_GB = 3.0# Power-of-two arenas: torch's pinned allocator rounds each allocation up (13% extra pinned one by one)._PIN_ARENA_BYTES = 1 << 31_PIN_ALIGN = 512+# Arena fills (copies plus zeroed tails) run on a pool, so they are parallel from any calling thread.+_PIN_COPY_THREADS = 8+_PIN_ZERO_CHUNK = 64 << 20def h3_te_stream_enabled() -> bool:@@ -483,15 +487,18 @@ def _round_up(value: int, multiple: int) -> int:return (value + multiple - 1) // multiple * multiple-def _pinned_arena(size: int) -> Any:+def _pinned_arena(size: int, fill: Any) -> Any:+ """A pinned uint8 arena of ``size`` bytes, every byte written by ``fill`` (before registering, where registered)."""import torchfrom .diffusion_memory import registered_pinned_empty- arena = registered_pinned_empty(size)+ arena = registered_pinned_empty(size, fill = fill)if arena is not None:return arena- return torch.empty(size, dtype = torch.uint8, pin_memory = True)+ arena = torch.empty(size, dtype = torch.uint8, pin_memory = True)+ fill(arena)+ return arenadef pin_module_in_place(@@ -519,22 +526,33 @@ def pin_module_in_place(if tensor.numel() == 0 or (_arena_factory is None and tensor.is_pinned()):continuetodo.append((tensor, owners))- # Largest first, first fit.+ # Largest first, first fit. Placed up front, so each arena is filled before it is registered.todo.sort(key = lambda item: item[0].numel() * item[0].element_size(), reverse = True)- arenas: list[list] = [] # [arena tensor, used bytes]+ arenas: list[list] = [] # [arena bytes, used bytes, [(tensor, owners, offset)]]pinned = 0for tensor, owners in todo:nbytes = tensor.numel() * tensor.element_size()need = _round_up(nbytes, _PIN_ALIGN)- slot = next((a for a in arenas if a[0].numel() - a[1] >= need), None)+ slot = next((a for a in arenas if a[0] - a[1] >= need), None)if slot is None:- size = max(int(arena_bytes), 1 << (need - 1).bit_length())- arena = _arena_factory(size) if _arena_factory is not None else _pinned_arena(size)- slot = [arena, 0]+ slot = [max(int(arena_bytes), 1 << (need - 1).bit_length()), 0, []]arenas.append(slot)- offset = slot[1]- view = slot[0][offset : offset + nbytes].view(tensor.dtype).view(tensor.shape)- view.copy_(tensor.detach())+ slot[2].append((tensor, owners, slot[1]))+ slot[1] += need+ pinned += nbytes+ placed = [] # (tensor, owners, pinned view)+ for size, _used, entries in arenas:+ fill = functools.partial(_fill_arena, entries = entries)+ if _arena_factory is not None:+ arena = _arena_factory(size)+ fill(arena)+ else:+ arena = _pinned_arena(size, fill)+ placed.extend(+ (tensor, owners, _arena_view(arena, tensor, offset))+ for tensor, owners, offset in entries+ )+ for tensor, owners, view in placed:if repoint:# Only when the old storage is not an mmap view (weights the loader already copied or cast).tensor.data = view@@ -546,11 +564,48 @@ def pin_module_in_place()for table, name, _is_param in owners:table[name] = replacement- slot[1] = offset + need- pinned += nbytesreturn pinned+def _arena_view(arena: Any, tensor: Any, offset: int) -> Any:+ nbytes = tensor.numel() * tensor.element_size()+ return arena[offset : offset + nbytes].view(tensor.dtype).view(tensor.shape)+++def _fill_arena(arena: Any, *, entries: list) -> None:+ """Copies each tensor to its offset and zeroes the rest, on a thread pool (copy_, zero_ and their page faults+ release the GIL). Writing every byte faults the arena's pages in the copy itself, with no separate zero pass."""+ jobs = [] # (destination, source tensor or None to zero)+ end = 0+ for tensor, _owners, offset in entries:+ if offset > end:+ jobs.append((arena[end:offset], None))+ jobs.append((_arena_view(arena, tensor, offset), tensor))+ end = offset + tensor.numel() * tensor.element_size()+ jobs.extend(+ (arena[start : start + _PIN_ZERO_CHUNK], None)+ for start in range(end, arena.numel(), _PIN_ZERO_CHUNK)+ )++ def run(job: tuple) -> None:+ dst, src = job+ if src is None:+ dst.zero_()+ else:+ dst.copy_(src.detach())++ workers = min(_PIN_COPY_THREADS, len(jobs))+ if workers <= 1:+ for job in jobs:+ run(job)+ return+ from concurrent.futures import ThreadPoolExecutor++ with ThreadPoolExecutor(max_workers = workers, thread_name_prefix = "unsloth-h3-te-pin") as pool:+ # list() re-raises the first failed copy, before any owner is replaced.+ list(pool.map(run, jobs))++def stream_h3_text_encoder(manager: Any,text_encoder: Any,diff --git a/studio/backend/tests/test_diffusion_registered_slabs.py b/studio/backend/tests/test_diffusion_registered_slabs.pyindex 4570d75f9b4..7ebc81948f7 100644--- a/studio/backend/tests/test_diffusion_registered_slabs.py+++ b/studio/backend/tests/test_diffusion_registered_slabs.py@@ -107,6 +107,50 @@ def test_h3_conditioner_pin_in_place_uses_registered_arenas(fake_registered):assert torch.equal(v, expect[k]), k+def test_h3_conditioner_pin_fills_each_arena_before_registering(fake_registered, monkeypatch):+ """The copies write the arena before it is registered (they fault its pages, no separate zero pass); the bytes+ no tensor uses are zeroed, as the zero fill left them."""+ from core.inference import video_minimax_h3_te as te++ zeroed = []+ real_zero = torch.Tensor.zero_+ monkeypatch.setattr(+ torch.Tensor, "zero_", lambda self: (zeroed.append(self.numel()), real_zero(self))[1]+ )+ m = torch.nn.Sequential(torch.nn.Linear(16, 32), torch.nn.Linear(32, 8))+ m.register_buffer("q", torch.arange(-32, 32, dtype = torch.int8).view(8, 8))+ expect = {k: v.detach().clone() for k, v in m.state_dict().items()}+ seen = {}++ def register(holder):+ seen[id(holder)] = holder._arr.copy()++ monkeypatch.setattr(_FakeHolder, "register", register)+ te.pin_module_in_place(m, arena_bytes = 1 << 14)+ assert len(fake_registered) == 1+ arr = fake_registered[0]._arr+ assert np.array_equal(seen[id(fake_registered[0])], arr), "registered before the copies landed"+ for k, v in m.state_dict().items():+ assert torch.equal(v, expect[k]), k+ unused = np.ones(arr.size, dtype = bool)+ base = arr.ctypes.data+ for v in m.state_dict().values():+ start = v.data_ptr() - base+ unused[start : start + v.numel() * v.element_size()] = False+ assert sum(zeroed) == unused.sum() > 0+ assert not arr[unused].any(), "a byte no tensor uses kept the dirty fill"+++def test_registered_fill_errors_propagate(fake_registered):+ def boom(buf):+ raise RuntimeError("copy failed")++ with pytest.raises(RuntimeError, match = "copy failed"):+ dm.registered_pinned_empty(4096, fill = boom)+ buf = dm.registered_pinned_empty(4096, fill = lambda b: b.fill_(3))+ assert bool((buf == 3).all()) and fake_registered[-1].registered_after_zero is False++@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")def test_real_registered_slab_is_pinned_and_uploads_exactly(monkeypatch):monkeypatch.delenv(dm.REGISTERED_SLAB_ENV, raising = False)diff --git a/studio/backend/tests/test_h3_low_vram_stream.py b/studio/backend/tests/test_h3_low_vram_stream.pyindex 72aaee7c994..9c57d3b0262 100644--- a/studio/backend/tests/test_h3_low_vram_stream.py+++ b/studio/backend/tests/test_h3_low_vram_stream.py@@ -333,3 +333,87 @@ def test_pinning_is_budgeted_on_the_whole_payload_not_one_arena(monkeypatch):assert not h3_te_pin_allowed(27 * 2**30)monkeypatch.setenv(dm.GROUP_OFFLOAD_PIN_ENV, "1")assert h3_te_pin_allowed(27 * 2**30)+++def test_pinning_in_place_copies_on_a_pool_with_the_same_placement(monkeypatch):+ """The copies run on a thread pool (a to_thread worker copies on one torch thread), with the serial placement:+ largest first, first fit, same offsets, same values, ties kept, every owner replaced."""+ import threading++ from core.inference import video_minimax_h3_te as te++ def make():+ torch.manual_seed(0)+ g = torch.Generator().manual_seed(0)+ m = torch.nn.Sequential(+ *[torch.nn.Linear(16 + 8 * i, 24, bias = i % 2 == 0) for i in range(6)]+ )+ m.register_buffer("q", torch.randint(-128, 127, (40, 8), dtype = torch.int8, generator = g))+ m.register_buffer("h", torch.randn(33, generator = g).to(torch.bfloat16))+ m.tied = torch.nn.Linear(24, 32, bias = False)+ m.tied.weight = m[2].weight+ m.register_buffer("q_alias", m.q)+ return m++ def run(m, threads):+ monkeypatch.setattr(te, "_PIN_COPY_THREADS", threads)+ arenas = []++ def factory(size):+ arenas.append(torch.full((size,), 0xA5, dtype = torch.uint8))+ return arenas[-1]++ names = []+ real_copy = torch.Tensor.copy_++ def spy(self, *a, **k):+ names.append(threading.current_thread().name)+ return real_copy(self, *a, **k)++ monkeypatch.setattr(torch.Tensor, "copy_", spy)+ try:+ n = te.pin_module_in_place(m, arena_bytes = 4096, _arena_factory = factory)+ finally:+ monkeypatch.setattr(torch.Tensor, "copy_", real_copy)+ return n, arenas, names++ m_serial, m_pool = make(), make()+ expect = {k: v.detach().clone() for k, v in m_pool.state_dict().items()}+ n1, a1, names1 = run(m_serial, 1)+ n2, a2, names2 = run(m_pool, 8)+ assert n1 == n2 > 0+ assert len(names2) == len(names1) > 1+ assert all(name.startswith("unsloth-h3-te-pin") for name in names2), names2+ assert not any(name.startswith("unsloth-h3-te-pin") for name in names1)+ # Same arenas, byte for byte (same offsets, same alignment padding).+ assert [a.numel() for a in a1] == [a.numel() for a in a2]+ assert all(torch.equal(x, y) for x, y in zip(a1, a2))+ for k, v in m_pool.state_dict().items():+ assert torch.equal(v, expect[k]), k+ assert v.untyped_storage().data_ptr() in {a.untyped_storage().data_ptr() for a in a2}, k+ assert m_pool.tied.weight is m_pool[2].weight and m_pool.q_alias is m_pool.q+ assert isinstance(m_pool[0].weight, torch.nn.Parameter) and not isinstance(+ m_pool.q, torch.nn.Parameter+ )+++def test_pinning_in_place_replaces_nothing_when_a_pooled_copy_fails(monkeypatch):+ from core.inference import video_minimax_h3_te as te++ m = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.Linear(8, 4))+ before = {k: v for k, v in m.state_dict(keep_vars = True).items()}+ real_copy = torch.Tensor.copy_+ calls = []++ def flaky(self, *a, **k):+ calls.append(1)+ if len(calls) == 2:+ raise RuntimeError("pinned host RAM refused")+ return real_copy(self, *a, **k)++ monkeypatch.setattr(torch.Tensor, "copy_", flaky)+ with pytest.raises(RuntimeError, match = "refused"):+ te.pin_module_in_place(m, _arena_factory = lambda n: torch.zeros(n, dtype = torch.uint8))+ monkeypatch.setattr(torch.Tensor, "copy_", real_copy)+ after = m.state_dict(keep_vars = True)+ assert all(after[k] is before[k] for k in before)