Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio small-host route: prefetch the streamed text encoder and fuse the int8 dequant (T4 FLUX.1-schnell 5.19 to 4.49 s per step, pixel-identical) (#12532)

Daniel Han0c6ef28c5f5b
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index e5561317c44..95f8e7ad9b1 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -3681,6 +3681,9 @@ def _apply_group_offload(
_pin_vision_embedding_device(module)
if te_room > 0:
te_room -= _keep_groups_resident(module, te_room, onload, logger)
+ if getattr(pipe, "_unsloth_small_host", None):
+ from .diffusion_small_host import install_encoder_prefetch
+ install_encoder_prefetch(module, onload, logger)
except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the load
if not stream_transformer and installed == 0:
# Resident encoder here would OOM; fall back to model offload, which rejects partial hooks.
diff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.py
index 1b9a4824222..657e51bb496 100644
--- a/studio/backend/core/inference/diffusion_small_host.py
+++ b/studio/backend/core/inference/diffusion_small_host.py
@@ -307,7 +307,12 @@ def _int8_linear_class():
self.bias = None if bias is None else torch.nn.Parameter(bias, requires_grad = False)
def forward(self, x):
- w = self.qweight.to(x.dtype) * self.scale.to(x.dtype)
+ scale = self.scale
+ if scale.dtype == x.dtype:
+ # int8 * float promotes: cast + scale in one pass, bit-identical (int8 -> fp16/fp32 is exact).
+ w = torch.mul(self.qweight, scale)
+ else:
+ w = self.qweight.to(x.dtype) * scale.to(x.dtype)
return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype))
def extra_repr(self) -> str:
@@ -479,3 +484,318 @@ def mark(pipe: Any, info: dict) -> None:
def engaged_on(pipe: Any) -> Optional[dict]:
return getattr(pipe, SMALL_HOST_ATTR, None)
+
+
+ENCODER_PREFETCH_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_PREFETCH"
+ENCODER_PREFETCH_ATTR = "_unsloth_encoder_prefetch"
+# Pinned staging ring: the only host bytes this adds.
+_STAGE_SLOT_BYTES = 32 * _MIB
+_STAGE_SLOTS = 4
+# Device bytes copied ahead: a few groups (T5-XXL MLP weight 80 MiB, Qwen2.5-VL 130 MiB).
+_PREFETCH_MAX_BYTES = 384 * _MIB
+_PREFETCH_MIN_BYTES = 128 * _MIB
+
+
+def encoder_prefetch_disabled() -> bool:
+ return _env(ENCODER_PREFETCH_ENV) in ("0", "off", "false", "no")
+
+
+def _group_tensors(group: Any) -> list:
+ """(tensor, host source) pairs of a diffusers offload group, in the order diffusers onloads them."""
+ out: list = []
+ seen: set[int] = set()
+ cpu = getattr(group, "cpu_param_dict", None) or {}
+ for t in (
+ [p for m in group.modules for p in m.parameters()]
+ + [b for m in group.modules for b in m.buffers()]
+ + list(getattr(group, "parameters", None) or [])
+ + list(getattr(group, "buffers", None) or [])
+ ):
+ if id(t) in seen:
+ continue
+ seen.add(id(t))
+ out.append((t, cpu.get(t, t)))
+ return out
+
+
+class _EncoderPrefetcher:
+ """Copies a memory-mapped encoder's offload groups ahead of its forward, in the previous forward's order, on a
+ worker thread through a pinned ring (diffusers pins each group on the calling thread). Off-order groups and worker
+ errors fall back to a synchronous copy."""
+
+ def __init__(self, module: Any, groups: list, device: Any):
+ import threading
+
+ import torch
+
+ self.module = module
+ self.groups = groups
+ dev = torch.device(device)
+ if dev.type == "cuda" and dev.index is None:
+ dev = torch.device("cuda", torch.cuda.current_device())
+ self.device = dev
+ self.stream = None # side stream, created on the first copy
+ self.compute = None # the stream the encoder runs on, read when a forward begins
+ self.order: list = [] # group ids in the last forward's onload order
+ self.seen: list = []
+ self.active = False
+ self.cond = threading.Condition()
+ self.ready: dict = {} # id(group) -> (device tensors, event)
+ self.inflight = 0
+ self.budget = _PREFETCH_MIN_BYTES
+ self.pos = 0
+ self.stop = False
+ self.worker = None
+ self.error: Optional[BaseException] = None
+ self.stage: list = []
+ self.stage_events: list = []
+ self.slot = 0
+ self.stats = {"prefetched": 0, "sync": 0, "passes": 0}
+ self.by_id = {id(g): g for g in groups}
+
+ def _side_stream(self) -> Any:
+ import torch
+ if self.stream is None:
+ self.stream = torch.cuda.Stream(device = self.device)
+ return self.stream
+
+ def _ensure_stage(self) -> None:
+ import torch
+ if not self.stage:
+ # inference_mode is thread-local: the worker cannot write inference tensors made on the forward thread.
+ with torch.inference_mode(False):
+ self.stage = [
+ torch.empty(_STAGE_SLOT_BYTES, dtype = torch.uint8, pin_memory = True)
+ for _ in range(_STAGE_SLOTS)
+ ]
+ self.stage_events = [None] * _STAGE_SLOTS
+
+ def _copy_group(self, group: Any) -> tuple:
+ """Device copies of ``group``'s host tensors, issued on the side stream. Returns (tensors, event, bytes)."""
+ import torch
+
+ self._ensure_stage()
+ stream = self._side_stream()
+ compute = self.compute or torch.cuda.current_stream(self.device)
+ moved: list = []
+ nbytes = 0
+ with torch.cuda.stream(stream):
+ for _t, src in _group_tensors(group):
+ n = int(src.numel()) * int(src.element_size())
+ if src.device.type != "cpu" or n == 0 or not src.is_contiguous() or src.is_pinned():
+ with torch.cuda.stream(compute):
+ moved.append(
+ src.to(
+ self.device,
+ non_blocking = src.device.type != "cpu" or src.is_pinned(),
+ )
+ )
+ continue
+ nbytes += n
+ # Compute-stream pool (a side-stream block stays cached away from the denoise); the copy waits for
+ # work queued there, covering the last user of a freed block.
+ with torch.cuda.stream(compute):
+ dst = torch.empty(src.shape, dtype = src.dtype, device = self.device)
+ queued = torch.cuda.Event()
+ queued.record(compute)
+ stream.wait_event(queued)
+ sb = src.reshape(-1).view(torch.uint8)
+ db = dst.reshape(-1).view(torch.uint8)
+ off = 0
+ while off < n:
+ k = min(_STAGE_SLOT_BYTES, n - off)
+ i = self.slot
+ self.slot = (i + 1) % _STAGE_SLOTS
+ ev = self.stage_events[i]
+ if ev is not None:
+ ev.synchronize()
+ buf = self.stage[i][:k]
+ buf.copy_(sb[off : off + k])
+ db[off : off + k].copy_(buf, non_blocking = True)
+ ev = torch.cuda.Event()
+ ev.record(stream)
+ self.stage_events[i] = ev
+ off += k
+ # A dropped entry (halt, exception) frees dst before the copy lands.
+ dst.record_stream(stream)
+ moved.append(dst)
+ done = torch.cuda.Event()
+ done.record(stream)
+ return moved, done, nbytes
+
+ def _attach(self, group: Any, moved: list, done: Any) -> None:
+ import torch
+
+ cur = torch.cuda.current_stream(self.device)
+ cur.wait_event(done)
+ if getattr(group, "stream", None) is not None:
+ # diffusers fences its own-stream prefetch only in the next group's onload_, which this replaces.
+ cur.wait_stream(group.stream)
+ for (t, _src), dev in zip(_group_tensors(group), moved):
+ t.data = dev
+ dev.record_stream(cur)
+
+ def _run(self, order: list) -> None:
+ import torch
+ try:
+ if self.device.type == "cuda":
+ torch.cuda.set_device(self.device)
+ for gid in order:
+ with self.cond:
+ # A group the forward onloads twice waits for its first copy to be taken.
+ while not self.stop and (
+ (self.inflight > 0 and self.inflight >= self.budget) or gid in self.ready
+ ):
+ self.cond.wait(0.05)
+ if self.stop:
+ return
+ group = self.by_id.get(gid)
+ if group is None:
+ continue
+ moved, done, n = self._copy_group(group)
+ with self.cond:
+ if self.stop:
+ return
+ self.ready[gid] = (moved, done, n)
+ self.inflight += n
+ self.cond.notify_all()
+ except BaseException as exc: # noqa: BLE001 - every group falls back to a synchronous copy
+ with self.cond:
+ self.error = exc
+ self.cond.notify_all()
+
+ def _halt(self) -> None:
+ with self.cond:
+ self.stop = True
+ self.cond.notify_all()
+ if self.worker is not None:
+ self.worker.join()
+ self.worker = None
+ with self.cond:
+ self.ready.clear()
+ self.inflight = 0
+
+ def begin(self) -> None:
+ import threading
+
+ import torch
+
+ self._halt()
+ self.stop = False
+ self.error = None
+ self.seen = []
+ self.pos = 0
+ self.active = True
+ self.stats["passes"] += 1
+ self.compute = (
+ torch.cuda.current_stream(self.device) if self.device.type == "cuda" else None
+ )
+ if not self.order:
+ return # first forward: synchronous copies, and it records the order
+ try:
+ free, _total = torch.cuda.mem_get_info(self.device)
+ self.budget = max(_PREFETCH_MIN_BYTES, min(_PREFETCH_MAX_BYTES, int(free) // 8))
+ except Exception: # noqa: BLE001
+ self.budget = _PREFETCH_MIN_BYTES
+ self._order_after_compute()
+ self.worker = threading.Thread(
+ target = self._run, args = (list(self.order),), daemon = True, name = "unsloth-encoder-prefetch"
+ )
+ self.worker.start()
+
+ def _order_after_compute(self) -> None:
+ """The worker's copies must not overtake work the compute stream already queued (the prompt's input ids)."""
+ import torch
+ self._side_stream().wait_stream(torch.cuda.current_stream(self.device))
+
+ def end(self) -> None:
+ if not self.active:
+ return
+ self.active = False
+ self._halt()
+ if self.seen:
+ self.order = list(self.seen)
+
+ def onload(self, group: Any) -> None:
+ gid = id(group)
+ if self.active:
+ self.seen.append(gid)
+ entry = None
+ if self.worker is not None:
+ expected = self.order[self.pos] if self.pos < len(self.order) else None
+ if expected == gid:
+ self.pos += 1
+ with self.cond:
+ while gid not in self.ready and self.error is None and self.worker.is_alive():
+ self.cond.wait(0.05)
+ entry = self.ready.pop(gid, None)
+ if entry is not None:
+ self.inflight -= entry[2]
+ self.cond.notify_all()
+ else:
+ # Off the recorded order: stop prefetching for this forward; the next one re-records the order.
+ self._halt()
+ if entry is None:
+ moved, done, _n = self._copy_group(group)
+ self.stats["sync"] += 1
+ else:
+ moved, done, _n = entry
+ self.stats["prefetched"] += 1
+ self._attach(group, moved, done)
+
+
+def install_encoder_prefetch(
+ module: Any,
+ device: Any,
+ logger: Any = None,
+) -> int:
+ """Swap each streamed offload group's ``onload_`` of a small-host text encoder for the prefetching copy. Groups made
+ resident are left alone. Returns the number of groups covered (0: unchanged, e.g. kill switch or no CUDA)."""
+ if encoder_prefetch_disabled():
+ return 0
+ try:
+ import torch
+
+ if torch.device(device).type != "cuda" or not torch.cuda.is_available():
+ return 0
+ from .diffusion_memory import GROUP_OFFLOAD_PIN_ENV, _offload_groups
+
+ if _env(GROUP_OFFLOAD_PIN_ENV) in ("0", "off", "false", "no"):
+ return 0 # the ring is pinned memory: the user's "pin nothing" override wins
+
+ groups = [
+ g
+ for g in _offload_groups(module)
+ if not getattr(g, "_unsloth_resident", False)
+ and getattr(g, "stream", None) is not None
+ and not getattr(g, "offload_to_disk_path", None)
+ ]
+ if not groups:
+ return 0
+ pf = _EncoderPrefetcher(module, groups, device)
+ pf._ensure_stage() # a pinned allocation failing here keeps diffusers' onload, not a failed render
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+ for group in groups:
+
+ def onload_(
+ *_a: Any,
+ _g: Any = group,
+ **_k: Any,
+ ) -> None:
+ pf.onload(_g)
+
+ group.onload_ = disable(onload_) if callable(disable) else onload_
+ module.register_forward_pre_hook(lambda *_a, **_k: pf.begin())
+ module.register_forward_hook(lambda *_a, **_k: pf.end(), always_call = True)
+ setattr(module, ENCODER_PREFETCH_ATTR, pf)
+ if logger is not None:
+ logger.info(
+ "diffusion.small_host: %s streams %d offload groups through a prefetching copy",
+ type(module).__name__,
+ len(groups),
+ )
+ return len(groups)
+ except Exception as exc: # noqa: BLE001 - keep diffusers' own onload
+ if logger is not None:
+ logger.warning("diffusion.small_host: encoder prefetch unavailable (%s)", exc)
+ return 0
diff --git a/studio/backend/tests/test_diffusion_small_host_prefetch.py b/studio/backend/tests/test_diffusion_small_host_prefetch.py
new file mode 100644
index 00000000000..9d4c7729522
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_small_host_prefetch.py
@@ -0,0 +1,509 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Small-host route speed fixes: the streamed text encoder's prefetching copy and the fused int8 dequant.
+
+The prefetch order / budget / fallback logic runs on CPU with fake groups; the end-to-end check (diffusers leaf-level
+group offload + layerwise casting, the small-host encoder setup) needs CUDA and is skipped without it.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+import types
+
+import pytest
+import torch
+
+import core.inference.diffusion_small_host as sh
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ monkeypatch.delenv(sh.ENCODER_PREFETCH_ENV, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", raising = False)
+
+
+# ------------------------------------------------------------------------------------------------ fused int8 dequant
[email protected]("dtype", [torch.float32, torch.float16])
+def test_int8_dequant_is_one_pass_and_bit_identical(dtype):
+ from torch.utils._python_dispatch import TorchDispatchMode
+
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(256, 512).to(torch.bfloat16)
+ # the same per-row absmax quantisation quantize_int8_weight_ applies (this layer is below its size threshold)
+ w = lin.weight.detach().float()
+ scale = w.abs().amax(dim = 1, keepdim = True).clamp_min(1e-12) / 127.0
+ q = torch.round(w / scale).clamp_(-127, 127).to(torch.int8)
+ layer = sh.int8_linear_class()(q, scale.to(dtype), lin.bias.detach().to(dtype), 256, 512)
+ x = torch.randn(4, 256, dtype = dtype)
+
+ class Ops(TorchDispatchMode):
+ def __init__(self):
+ super().__init__()
+ self.ops = []
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types_,
+ args = (),
+ kwargs = None,
+ ):
+ self.ops.append(str(func))
+ return func(*args, **(kwargs or {}))
+
+ with Ops() as rec:
+ out = layer(x)
+ # the int8 weight is read once: no separate cast of the full weight before the scale multiply
+ passes = [o for o in rec.ops if "_to_copy" in o or "mul" in o]
+ assert passes == ["aten.mul.Tensor"], rec.ops
+ want = torch.nn.functional.linear(x, q.to(dtype) * scale.to(dtype), lin.bias.detach().to(dtype))
+ assert torch.equal(out, want)
+
+
+def test_int8_dequant_other_input_dtype_still_casts():
+ q = torch.randint(-127, 128, (8, 4), dtype = torch.int8)
+ scale = torch.rand(8, 1, dtype = torch.float16)
+ layer = sh.int8_linear_class()(q, scale, None, 4, 8)
+ x = torch.randn(2, 4, dtype = torch.float32)
+ out = layer(x)
+ assert out.dtype == torch.float32
+ assert torch.equal(out, torch.nn.functional.linear(x, q.float() * scale.float()))
+
+
+# ------------------------------------------------------------------------------------------------ prefetch logic (CPU)
+class _FakeGroup:
+ def __init__(self, name: str, nbytes: int):
+ self.name = name
+ self.nbytes = nbytes
+
+
+class _FakePrefetcher(sh._EncoderPrefetcher):
+ """The real ordering / budget / fallback logic with the CUDA copy replaced by a recorded fake."""
+
+ def __init__(
+ self,
+ groups,
+ *,
+ fail_at = None,
+ delay = 0.0,
+ ):
+ super().__init__(module = None, groups = groups, device = "cpu")
+ self.copies: list = []
+ self.attached: list = []
+ self.fail_at = fail_at
+ self.delay = delay
+ self.peak_inflight = 0
+ self.lock = threading.Lock()
+
+ def _copy_group(self, group):
+ if self.delay:
+ time.sleep(self.delay)
+ with self.lock:
+ if (
+ self.fail_at is not None
+ and group.name == self.fail_at
+ and threading.current_thread() is not threading.main_thread()
+ ):
+ raise RuntimeError("copy failed")
+ self.copies.append((group.name, threading.current_thread() is threading.main_thread()))
+ with self.cond:
+ self.peak_inflight = max(self.peak_inflight, self.inflight + group.nbytes)
+ return [group.name], None, group.nbytes
+
+ def _attach(self, group, moved, done):
+ self.attached.append(moved[0])
+
+ def _order_after_compute(self):
+ return None
+
+
+def _forward(pf, groups):
+ pf.begin()
+ try:
+ for g in groups:
+ pf.onload(g)
+ finally:
+ pf.end()
+
+
+def test_prefetch_records_the_order_then_copies_ahead_on_a_worker():
+ groups = [_FakeGroup(f"g{i}", 10) for i in range(12)]
+ pf = _FakePrefetcher(groups)
+ _forward(pf, groups)
+ # first forward: no order yet, every group copied on the calling thread
+ assert (pf.stats["prefetched"], pf.stats["sync"], pf.stats["passes"]) == (0, 12, 1)
+ assert all(main for _n, main in pf.copies)
+ pf.copies.clear()
+ _forward(pf, groups)
+ assert pf.stats["prefetched"] == 12 and pf.stats["sync"] == 12
+ assert [n for n, _ in pf.copies] == [g.name for g in groups]
+ assert not any(main for _n, main in pf.copies)
+ assert pf.attached[-12:] == [g.name for g in groups]
+ assert pf.worker is None and pf.inflight == 0 and not pf.ready
+
+
+def test_prefetch_stays_within_the_byte_budget():
+ groups = [_FakeGroup(f"g{i}", sh._PREFETCH_MIN_BYTES // 4) for i in range(16)]
+ pf = _FakePrefetcher(groups)
+ _forward(pf, groups)
+
+ pf.begin()
+ try:
+ for g in groups:
+ time.sleep(0.01) # a slow consumer: the worker must wait for the budget, not run away
+ pf.onload(g)
+ finally:
+ pf.end()
+ assert pf.stats["prefetched"] == 16
+ # at most one group past the budget is ever in flight
+ assert pf.peak_inflight <= pf.budget + sh._PREFETCH_MIN_BYTES // 4
+
+
+def test_prefetch_one_oversized_group_still_progresses():
+ groups = [_FakeGroup("big", sh._PREFETCH_MAX_BYTES * 4), _FakeGroup("small", 1)]
+ pf = _FakePrefetcher(groups)
+ _forward(pf, groups)
+ _forward(pf, groups)
+ assert pf.stats["prefetched"] == 2
+
+
+def test_prefetch_off_order_falls_back_and_rerecords():
+ groups = [_FakeGroup(f"g{i}", 10) for i in range(6)]
+ pf = _FakePrefetcher(groups)
+ _forward(pf, groups)
+ swapped = groups[:2] + [groups[3], groups[2]] + groups[4:]
+ _forward(pf, swapped)
+ # g0, g1 prefetched; g3 is off the recorded order, so it and everything after it copy synchronously
+ assert pf.stats["prefetched"] == 2
+ assert pf.attached[-6:] == [g.name for g in swapped]
+ assert pf.order == [id(g) for g in swapped]
+ _forward(pf, swapped)
+ assert pf.stats["prefetched"] == 2 + 6
+
+
+def test_prefetch_worker_error_falls_back_to_synchronous_copies():
+ groups = [_FakeGroup(f"g{i}", 10) for i in range(5)]
+ pf = _FakePrefetcher(groups, fail_at = "g2")
+ _forward(pf, groups)
+ _forward(pf, groups)
+ assert pf.attached[-5:] == [g.name for g in groups]
+ assert pf.stats["prefetched"] == 2 and pf.stats["sync"] == 5 + 3
+
+
+def test_prefetch_exception_in_forward_stops_the_worker():
+ groups = [_FakeGroup(f"g{i}", 10) for i in range(5)]
+ pf = _FakePrefetcher(groups, delay = 0.01)
+ _forward(pf, groups)
+ pf.begin()
+ pf.onload(groups[0])
+ pf.end() # the forward hook runs with always_call=True on an exception
+ assert pf.worker is None and not pf.ready and pf.inflight == 0
+
+
+def test_install_is_off_on_cpu_and_with_the_kill_switch(monkeypatch):
+ m = torch.nn.Linear(4, 4)
+ assert sh.install_encoder_prefetch(m, "cpu") == 0
+ monkeypatch.setenv(sh.ENCODER_PREFETCH_ENV, "0")
+ assert sh.encoder_prefetch_disabled()
+ assert sh.install_encoder_prefetch(m, "cuda") == 0
+
+
+def test_group_offload_installs_prefetch_only_on_small_host_pipes(monkeypatch):
+ pytest.importorskip("diffusers")
+ import core.inference.diffusion_memory as dm
+
+ calls: list = []
+ monkeypatch.setattr(
+ sh,
+ "install_encoder_prefetch",
+ lambda module, device, logger = None: calls.append(module) or 1,
+ )
+
+ def _pipe(small_host: bool):
+ te = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.Linear(8, 8))
+ tr = torch.nn.Sequential(torch.nn.Linear(8, 8))
+ pipe = types.SimpleNamespace(
+ transformer = tr, text_encoder = te, components = {"transformer": tr, "text_encoder": te}
+ )
+ if small_host:
+ pipe._unsloth_small_host = {
+ "components": {"text_encoder": "memory-mapped, layerwise cast (1 MiB)"}
+ }
+ return pipe, te
+
+ pipe, te = _pipe(False)
+ assert dm._apply_group_offload(pipe, "cpu", None, stream_text_encoders = True)
+ assert calls == []
+ pipe, te = _pipe(True)
+ assert dm._apply_group_offload(pipe, "cpu", None, stream_text_encoders = True)
+ assert calls == [te]
+
+
+# ------------------------------------------------------------------------------------------------ end to end (CUDA)
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_prefetched_encoder_is_bit_identical_to_diffusers_stream(tmp_path):
+ from diffusers.hooks import apply_group_offloading
+
+ def build(path = None):
+ torch.manual_seed(0)
+
+ class Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.norm = torch.nn.LayerNorm(256)
+ self.wi = torch.nn.Linear(256, 1024, bias = False)
+ self.wo = torch.nn.Linear(1024, 256, bias = False)
+
+ def forward(self, x):
+ return x + self.wo(torch.relu(self.wi(self.norm(x))))
+
+ class Enc(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed = torch.nn.Embedding(1000, 256)
+ self.blocks = torch.nn.ModuleList([Block() for _ in range(6)])
+ self.final = torch.nn.LayerNorm(256)
+
+ def forward(self, ids):
+ h = self.embed(ids)
+ for b in self.blocks:
+ h = b(h)
+ return self.final(h)
+
+ enc = Enc().to(torch.bfloat16).eval()
+ sh.prepare_streamed_encoder_(enc, torch.float16)
+ if path is not None:
+ from safetensors.torch import load_file, save_file
+
+ save_file({k: v.contiguous() for k, v in enc.state_dict().items()}, str(path))
+ loaded = load_file(str(path))
+ for name, p in enc.named_parameters():
+ p.data = loaded[name]
+ apply_group_offloading(
+ enc,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "leaf_level",
+ use_stream = True,
+ non_blocking = True,
+ record_stream = True,
+ low_cpu_mem_usage = True,
+ )
+ return enc
+
+ ref_enc = build()
+ # the small-host load: the encoder's weights view a safetensors file mapping
+ enc = build(tmp_path / "enc.safetensors")
+
+ n = sh.install_encoder_prefetch(enc, "cuda")
+ assert n > 0
+ pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)
+ with torch.no_grad():
+ for i in range(4):
+ ids = torch.randint(0, 1000, (1, 16 + i), device = "cuda")
+ want = ref_enc(ids)
+ got = enc(ids)
+ torch.cuda.synchronize()
+ assert torch.equal(got, want)
+ # the first forward records the order; every later one copies every group ahead on the worker
+ assert pf.error is None
+ assert pf.stats["prefetched"] == 3 * len(pf.order) and pf.stats["sync"] == len(pf.order)
+ # weights are back on their host storage between forwards
+ assert enc.blocks[0].wi.weight.device.type == "cpu"
+ assert enc.blocks[0].wi.weight.dtype == torch.bfloat16
+
+
+def _mapped_streamed(enc, path):
+ """The small-host setup: weights view a safetensors file mapping, diffusers leaf-level stream offload."""
+ pytest.importorskip("diffusers")
+ from diffusers.hooks import apply_group_offloading
+ from safetensors.torch import load_file, save_file
+
+ save_file({k: v.contiguous() for k, v in enc.state_dict().items()}, str(path))
+ loaded = load_file(str(path))
+ for name, p in enc.named_parameters():
+ p.data = loaded[name]
+ apply_group_offloading(
+ enc,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "leaf_level",
+ use_stream = True,
+ non_blocking = True,
+ record_stream = True,
+ low_cpu_mem_usage = True,
+ )
+ return enc
+
+
+def _small_linear_stack():
+ torch.manual_seed(0)
+ return (
+ torch.nn.Sequential(*[torch.nn.Linear(64, 64) for _ in range(3)]).to(torch.bfloat16).eval()
+ )
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_pin_opt_out_keeps_diffusers_onload(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")
+ enc = _mapped_streamed(_small_linear_stack(), tmp_path / "enc.safetensors")
+ assert sh.install_encoder_prefetch(enc, "cuda") == 0
+ assert getattr(enc, sh.ENCODER_PREFETCH_ATTR, None) is None
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_failed_ring_pin_keeps_diffusers_onload(tmp_path, monkeypatch):
+ enc = _mapped_streamed(_small_linear_stack(), tmp_path / "enc.safetensors")
+ real_empty = torch.empty
+
+ def empty(*a, **k):
+ if k.get("pin_memory"):
+ raise RuntimeError("CUDA error: out of memory")
+ return real_empty(*a, **k)
+
+ monkeypatch.setattr(torch, "empty", empty)
+ assert sh.install_encoder_prefetch(enc, "cuda") == 0
+ monkeypatch.setattr(torch, "empty", real_empty)
+ assert getattr(enc, sh.ENCODER_PREFETCH_ATTR, None) is None
+ ref = _small_linear_stack().cuda()
+ x = torch.randn(2, 64, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ assert torch.equal(enc(x), ref(x))
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_prefetch_runs_under_inference_mode(tmp_path):
+ # Studio renders under torch.inference_mode(); the worker thread is outside it and must still fill the ring.
+ def build():
+ torch.manual_seed(0)
+ return (
+ torch.nn.Sequential(*[torch.nn.Linear(512, 512) for _ in range(6)])
+ .to(torch.bfloat16)
+ .eval()
+ )
+
+ ref = build().cuda()
+ enc = _mapped_streamed(build(), tmp_path / "enc.safetensors")
+ assert sh.install_encoder_prefetch(enc, "cuda") > 0
+ pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)
+ with torch.inference_mode():
+ for i in range(4):
+ x = torch.randn(2 + i, 512, dtype = torch.bfloat16, device = "cuda")
+ assert torch.equal(enc(x), ref(x))
+ assert pf.error is None
+ assert pf.stats["prefetched"] == 3 * len(pf.order)
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
[email protected]("drop", ["exception", "prefix"])
+def test_dropped_prefetch_never_lands_in_a_reused_block(tmp_path, drop):
+ # Groups copied ahead but never onloaded are freed while their copy may still be queued on the side stream.
+ H = 4096
+
+ class Enc(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.head = torch.nn.ModuleList([torch.nn.Linear(H, H, bias = False) for _ in range(2)])
+ self.tail = torch.nn.ModuleList([torch.nn.Linear(H, H, bias = False) for _ in range(6)])
+
+ def forward(
+ self,
+ x,
+ tail = True,
+ boom = False,
+ ):
+ for m in self.head:
+ x = m(x)
+ if boom:
+ raise RuntimeError("boom")
+ for m in self.tail if tail else ():
+ x = m(x)
+ return x
+
+ enc = Enc().to(torch.bfloat16).eval()
+ with torch.no_grad():
+ for p in enc.parameters():
+ p.fill_(1.0)
+ enc = _mapped_streamed(enc, tmp_path / "enc.safetensors")
+ assert sh.install_encoder_prefetch(enc, "cuda") > 0
+ x = torch.zeros(1, H, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ enc(x) # records the full order
+ for _ in range(10):
+ try:
+ enc(x, tail = False, boom = drop == "exception")
+ except RuntimeError:
+ pass
+ outs = [torch.empty(H, H, dtype = torch.bfloat16, device = "cuda") for _ in range(6)]
+ for o in outs:
+ o.fill_(-2.0)
+ torch.cuda.synchronize()
+ assert all(bool((o == -2.0).all()) for o in outs)
+ del outs
+ enc(x) # back to the full order
+ torch.cuda.synchronize()
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_released_resident_groups_are_fenced():
+ # release_resident_groups hands resident encoder groups back to diffusers' onload_, whose copy runs on diffusers'
+ # stream next to groups this prefetcher onloads.
+ pytest.importorskip("diffusers")
+ import copy
+
+ import core.inference.diffusion_memory as dm
+
+ H, F = 4096, 16384
+
+ class Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.norm = torch.nn.LayerNorm(H)
+ self.wi = torch.nn.Linear(H, F, bias = False)
+ self.wo = torch.nn.Linear(F, H, bias = False)
+
+ def forward(self, x):
+ return x + self.wo(torch.relu(self.wi(self.norm(x))))
+
+ class Enc(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed = torch.nn.Embedding(1000, H)
+ self.blocks = torch.nn.ModuleList([Block() for _ in range(3)])
+ self.final = torch.nn.LayerNorm(H)
+
+ def forward(self, ids):
+ h = self.embed(ids)
+ for b in self.blocks:
+ h = b(h)
+ return self.final(h)
+
+ torch.manual_seed(0)
+ enc = Enc().half().eval()
+ for p in enc.parameters():
+ p.data.normal_(0, 0.02)
+ ref = copy.deepcopy(enc).cuda()
+ tr = torch.nn.Sequential(torch.nn.Linear(8, 8))
+ pipe = types.SimpleNamespace(
+ transformer = tr, text_encoder = enc, components = {"transformer": tr, "text_encoder": enc}
+ )
+ pipe._unsloth_small_host = {"components": {"text_encoder": "memory-mapped"}}
+ assert dm._apply_group_offload(
+ pipe, "cuda", None, stream_text_encoders = True, resident_text_encoder_mib = 140
+ )
+ pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)
+ with torch.no_grad():
+ for _ in range(2):
+ enc(torch.randint(0, 1000, (4, 512), device = "cuda"))
+ assert dm.release_resident_groups(pipe, 100, None) is not None
+ for _ in range(10):
+ ids = torch.randint(0, 1000, (4, 512), device = "cuda")
+ want = ref(ids)
+ torch.cuda.synchronize()
+ got = enc(ids)
+ torch.cuda.synchronize()
+ assert torch.equal(got, want)
+ assert pf.error is None and pf.stats["prefetched"] > 0