Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fill the H3 conditioner's pinned arenas once, before registering (33% faster cold pin) (#16887)

Daniel Hanccc8a827d2c0
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index c2da4767502..02b2cd592ff 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -6929,9 +6929,14 @@ class _RegisteredHostBuffer:
REGISTERED_SLAB_ENV = "UNSLOTH_DIFFUSION_REGISTERED_SLABS"
-def registered_pinned_empty(nbytes: int) -> Optional[Any]:
+def registered_pinned_empty(
+ nbytes: int, fill: Optional[Callable[[Any], None]] = None
+) -> Optional[Any]:
"""Page-locked uint8 host tensor in registered memory, or None (use torch's pinned allocator).
- Torch's allocator faults pages on one thread (~18 s for MiniMax-H3); ``zero_`` here faults them in parallel."""
+ Torch's allocator faults pages on one thread (~18 s for MiniMax-H3); ``zero_`` here faults them in parallel.
+ ``fill(buf)``, when given, writes every byte before registering, in place of the zero fill: the copy faults the
+ pages, saving a full write pass (an 8 GB H3 conditioner pin: 3.8 -> 3.4 s hot, 6.1 -> 4.1 s cold, A100 VM).
+ Its errors propagate."""
if (os.environ.get(REGISTERED_SLAB_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
return None
if int(nbytes) <= 0 or not _fast_pin_supported():
@@ -6942,7 +6947,13 @@ def registered_pinned_empty(nbytes: int) -> Optional[Any]:
try:
holder = _RegisteredHostBuffer(int(nbytes))
buf = torch.from_numpy(np.asarray(holder))
- buf.zero_()
+ except Exception: # noqa: BLE001
+ return None
+ if fill is not None:
+ fill(buf)
+ try:
+ if fill is None:
+ buf.zero_()
holder.register()
except Exception: # noqa: BLE001
return None
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index e2633975630..2a81aecf85b 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -8,6 +8,7 @@ fall back to released bf16 encoder."""
from __future__ import annotations
+import functools
from functools import lru_cache
from typing import Any, Optional
@@ -374,6 +375,9 @@ H3_TE_STREAMED_GB = 3.0
# Power-of-two arenas: torch's pinned allocator rounds each allocation up (13% extra pinned one by one).
_PIN_ARENA_BYTES = 1 << 31
_PIN_ALIGN = 512
+# Arena fills (copies plus zeroed tails) run on a pool, so they are parallel from any calling thread.
+_PIN_COPY_THREADS = 8
+_PIN_ZERO_CHUNK = 64 << 20
def h3_te_stream_enabled() -> bool:
@@ -483,15 +487,18 @@ def _round_up(value: int, multiple: int) -> int:
return (value + multiple - 1) // multiple * multiple
-def _pinned_arena(size: int) -> Any:
+def _pinned_arena(size: int, fill: Any) -> Any:
+ """A pinned uint8 arena of ``size`` bytes, every byte written by ``fill`` (before registering, where registered)."""
import torch
from .diffusion_memory import registered_pinned_empty
- arena = registered_pinned_empty(size)
+ arena = registered_pinned_empty(size, fill = fill)
if arena is not None:
return arena
- return torch.empty(size, dtype = torch.uint8, pin_memory = True)
+ arena = torch.empty(size, dtype = torch.uint8, pin_memory = True)
+ fill(arena)
+ return arena
def pin_module_in_place(
@@ -519,22 +526,33 @@ def pin_module_in_place(
if tensor.numel() == 0 or (_arena_factory is None and tensor.is_pinned()):
continue
todo.append((tensor, owners))
- # Largest first, first fit.
+ # Largest first, first fit. Placed up front, so each arena is filled before it is registered.
todo.sort(key = lambda item: item[0].numel() * item[0].element_size(), reverse = True)
- arenas: list[list] = [] # [arena tensor, used bytes]
+ arenas: list[list] = [] # [arena bytes, used bytes, [(tensor, owners, offset)]]
pinned = 0
for tensor, owners in todo:
nbytes = tensor.numel() * tensor.element_size()
need = _round_up(nbytes, _PIN_ALIGN)
- slot = next((a for a in arenas if a[0].numel() - a[1] >= need), None)
+ slot = next((a for a in arenas if a[0] - a[1] >= need), None)
if slot is None:
- size = max(int(arena_bytes), 1 << (need - 1).bit_length())
- arena = _arena_factory(size) if _arena_factory is not None else _pinned_arena(size)
- slot = [arena, 0]
+ slot = [max(int(arena_bytes), 1 << (need - 1).bit_length()), 0, []]
arenas.append(slot)
- offset = slot[1]
- view = slot[0][offset : offset + nbytes].view(tensor.dtype).view(tensor.shape)
- view.copy_(tensor.detach())
+ slot[2].append((tensor, owners, slot[1]))
+ slot[1] += need
+ pinned += nbytes
+ placed = [] # (tensor, owners, pinned view)
+ for size, _used, entries in arenas:
+ fill = functools.partial(_fill_arena, entries = entries)
+ if _arena_factory is not None:
+ arena = _arena_factory(size)
+ fill(arena)
+ else:
+ arena = _pinned_arena(size, fill)
+ placed.extend(
+ (tensor, owners, _arena_view(arena, tensor, offset))
+ for tensor, owners, offset in entries
+ )
+ for tensor, owners, view in placed:
if repoint:
# Only when the old storage is not an mmap view (weights the loader already copied or cast).
tensor.data = view
@@ -546,11 +564,48 @@ def pin_module_in_place(
)
for table, name, _is_param in owners:
table[name] = replacement
- slot[1] = offset + need
- pinned += nbytes
return pinned
+def _arena_view(arena: Any, tensor: Any, offset: int) -> Any:
+ nbytes = tensor.numel() * tensor.element_size()
+ return arena[offset : offset + nbytes].view(tensor.dtype).view(tensor.shape)
+
+
+def _fill_arena(arena: Any, *, entries: list) -> None:
+ """Copies each tensor to its offset and zeroes the rest, on a thread pool (copy_, zero_ and their page faults
+ release the GIL). Writing every byte faults the arena's pages in the copy itself, with no separate zero pass."""
+ jobs = [] # (destination, source tensor or None to zero)
+ end = 0
+ for tensor, _owners, offset in entries:
+ if offset > end:
+ jobs.append((arena[end:offset], None))
+ jobs.append((_arena_view(arena, tensor, offset), tensor))
+ end = offset + tensor.numel() * tensor.element_size()
+ jobs.extend(
+ (arena[start : start + _PIN_ZERO_CHUNK], None)
+ for start in range(end, arena.numel(), _PIN_ZERO_CHUNK)
+ )
+
+ def run(job: tuple) -> None:
+ dst, src = job
+ if src is None:
+ dst.zero_()
+ else:
+ dst.copy_(src.detach())
+
+ workers = min(_PIN_COPY_THREADS, len(jobs))
+ if workers <= 1:
+ for job in jobs:
+ run(job)
+ return
+ from concurrent.futures import ThreadPoolExecutor
+
+ with ThreadPoolExecutor(max_workers = workers, thread_name_prefix = "unsloth-h3-te-pin") as pool:
+ # list() re-raises the first failed copy, before any owner is replaced.
+ list(pool.map(run, jobs))
+
+
def stream_h3_text_encoder(
manager: Any,
text_encoder: Any,
diff --git a/studio/backend/tests/test_diffusion_registered_slabs.py b/studio/backend/tests/test_diffusion_registered_slabs.py
index 4570d75f9b4..7ebc81948f7 100644
--- a/studio/backend/tests/test_diffusion_registered_slabs.py
+++ b/studio/backend/tests/test_diffusion_registered_slabs.py
@@ -107,6 +107,50 @@ def test_h3_conditioner_pin_in_place_uses_registered_arenas(fake_registered):
assert torch.equal(v, expect[k]), k
+def test_h3_conditioner_pin_fills_each_arena_before_registering(fake_registered, monkeypatch):
+ """The copies write the arena before it is registered (they fault its pages, no separate zero pass); the bytes
+ no tensor uses are zeroed, as the zero fill left them."""
+ from core.inference import video_minimax_h3_te as te
+
+ zeroed = []
+ real_zero = torch.Tensor.zero_
+ monkeypatch.setattr(
+ torch.Tensor, "zero_", lambda self: (zeroed.append(self.numel()), real_zero(self))[1]
+ )
+ m = torch.nn.Sequential(torch.nn.Linear(16, 32), torch.nn.Linear(32, 8))
+ m.register_buffer("q", torch.arange(-32, 32, dtype = torch.int8).view(8, 8))
+ expect = {k: v.detach().clone() for k, v in m.state_dict().items()}
+ seen = {}
+
+ def register(holder):
+ seen[id(holder)] = holder._arr.copy()
+
+ monkeypatch.setattr(_FakeHolder, "register", register)
+ te.pin_module_in_place(m, arena_bytes = 1 << 14)
+ assert len(fake_registered) == 1
+ arr = fake_registered[0]._arr
+ assert np.array_equal(seen[id(fake_registered[0])], arr), "registered before the copies landed"
+ for k, v in m.state_dict().items():
+ assert torch.equal(v, expect[k]), k
+ unused = np.ones(arr.size, dtype = bool)
+ base = arr.ctypes.data
+ for v in m.state_dict().values():
+ start = v.data_ptr() - base
+ unused[start : start + v.numel() * v.element_size()] = False
+ assert sum(zeroed) == unused.sum() > 0
+ assert not arr[unused].any(), "a byte no tensor uses kept the dirty fill"
+
+
+def test_registered_fill_errors_propagate(fake_registered):
+ def boom(buf):
+ raise RuntimeError("copy failed")
+
+ with pytest.raises(RuntimeError, match = "copy failed"):
+ dm.registered_pinned_empty(4096, fill = boom)
+ buf = dm.registered_pinned_empty(4096, fill = lambda b: b.fill_(3))
+ assert bool((buf == 3).all()) and fake_registered[-1].registered_after_zero is False
+
+
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
def test_real_registered_slab_is_pinned_and_uploads_exactly(monkeypatch):
monkeypatch.delenv(dm.REGISTERED_SLAB_ENV, raising = False)
diff --git a/studio/backend/tests/test_h3_low_vram_stream.py b/studio/backend/tests/test_h3_low_vram_stream.py
index 72aaee7c994..9c57d3b0262 100644
--- a/studio/backend/tests/test_h3_low_vram_stream.py
+++ b/studio/backend/tests/test_h3_low_vram_stream.py
@@ -333,3 +333,87 @@ def test_pinning_is_budgeted_on_the_whole_payload_not_one_arena(monkeypatch):
assert not h3_te_pin_allowed(27 * 2**30)
monkeypatch.setenv(dm.GROUP_OFFLOAD_PIN_ENV, "1")
assert h3_te_pin_allowed(27 * 2**30)
+
+
+def test_pinning_in_place_copies_on_a_pool_with_the_same_placement(monkeypatch):
+ """The copies run on a thread pool (a to_thread worker copies on one torch thread), with the serial placement:
+ largest first, first fit, same offsets, same values, ties kept, every owner replaced."""
+ import threading
+
+ from core.inference import video_minimax_h3_te as te
+
+ def make():
+ torch.manual_seed(0)
+ g = torch.Generator().manual_seed(0)
+ m = torch.nn.Sequential(
+ *[torch.nn.Linear(16 + 8 * i, 24, bias = i % 2 == 0) for i in range(6)]
+ )
+ m.register_buffer("q", torch.randint(-128, 127, (40, 8), dtype = torch.int8, generator = g))
+ m.register_buffer("h", torch.randn(33, generator = g).to(torch.bfloat16))
+ m.tied = torch.nn.Linear(24, 32, bias = False)
+ m.tied.weight = m[2].weight
+ m.register_buffer("q_alias", m.q)
+ return m
+
+ def run(m, threads):
+ monkeypatch.setattr(te, "_PIN_COPY_THREADS", threads)
+ arenas = []
+
+ def factory(size):
+ arenas.append(torch.full((size,), 0xA5, dtype = torch.uint8))
+ return arenas[-1]
+
+ names = []
+ real_copy = torch.Tensor.copy_
+
+ def spy(self, *a, **k):
+ names.append(threading.current_thread().name)
+ return real_copy(self, *a, **k)
+
+ monkeypatch.setattr(torch.Tensor, "copy_", spy)
+ try:
+ n = te.pin_module_in_place(m, arena_bytes = 4096, _arena_factory = factory)
+ finally:
+ monkeypatch.setattr(torch.Tensor, "copy_", real_copy)
+ return n, arenas, names
+
+ m_serial, m_pool = make(), make()
+ expect = {k: v.detach().clone() for k, v in m_pool.state_dict().items()}
+ n1, a1, names1 = run(m_serial, 1)
+ n2, a2, names2 = run(m_pool, 8)
+ assert n1 == n2 > 0
+ assert len(names2) == len(names1) > 1
+ assert all(name.startswith("unsloth-h3-te-pin") for name in names2), names2
+ assert not any(name.startswith("unsloth-h3-te-pin") for name in names1)
+ # Same arenas, byte for byte (same offsets, same alignment padding).
+ assert [a.numel() for a in a1] == [a.numel() for a in a2]
+ assert all(torch.equal(x, y) for x, y in zip(a1, a2))
+ for k, v in m_pool.state_dict().items():
+ assert torch.equal(v, expect[k]), k
+ assert v.untyped_storage().data_ptr() in {a.untyped_storage().data_ptr() for a in a2}, k
+ assert m_pool.tied.weight is m_pool[2].weight and m_pool.q_alias is m_pool.q
+ assert isinstance(m_pool[0].weight, torch.nn.Parameter) and not isinstance(
+ m_pool.q, torch.nn.Parameter
+ )
+
+
+def test_pinning_in_place_replaces_nothing_when_a_pooled_copy_fails(monkeypatch):
+ from core.inference import video_minimax_h3_te as te
+
+ m = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.Linear(8, 4))
+ before = {k: v for k, v in m.state_dict(keep_vars = True).items()}
+ real_copy = torch.Tensor.copy_
+ calls = []
+
+ def flaky(self, *a, **k):
+ calls.append(1)
+ if len(calls) == 2:
+ raise RuntimeError("pinned host RAM refused")
+ return real_copy(self, *a, **k)
+
+ monkeypatch.setattr(torch.Tensor, "copy_", flaky)
+ with pytest.raises(RuntimeError, match = "refused"):
+ te.pin_module_in_place(m, _arena_factory = lambda n: torch.zeros(n, dtype = torch.uint8))
+ monkeypatch.setattr(torch.Tensor, "copy_", real_copy)
+ after = m.state_dict(keep_vars = True)
+ assert all(after[k] is before[k] for k in before)