Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio small-host route: prefetch the streamed text encoder and fuse the int8 dequant (T4 FLUX.1-schnell 5.19 to 4.49 s per step, pixel-identical) (#12532)
Daniel Han0c6ef28c5f5b
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex e5561317c44..95f8e7ad9b1 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -3681,6 +3681,9 @@ def _apply_group_offload(_pin_vision_embedding_device(module)if te_room > 0:te_room -= _keep_groups_resident(module, te_room, onload, logger)+ if getattr(pipe, "_unsloth_small_host", None):+ from .diffusion_small_host import install_encoder_prefetch+ install_encoder_prefetch(module, onload, logger)except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the loadif not stream_transformer and installed == 0:# Resident encoder here would OOM; fall back to model offload, which rejects partial hooks.diff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.pyindex 1b9a4824222..657e51bb496 100644--- a/studio/backend/core/inference/diffusion_small_host.py+++ b/studio/backend/core/inference/diffusion_small_host.py@@ -307,7 +307,12 @@ def _int8_linear_class():self.bias = None if bias is None else torch.nn.Parameter(bias, requires_grad = False)def forward(self, x):- w = self.qweight.to(x.dtype) * self.scale.to(x.dtype)+ scale = self.scale+ if scale.dtype == x.dtype:+ # int8 * float promotes: cast + scale in one pass, bit-identical (int8 -> fp16/fp32 is exact).+ w = torch.mul(self.qweight, scale)+ else:+ w = self.qweight.to(x.dtype) * scale.to(x.dtype)return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype))def extra_repr(self) -> str:@@ -479,3 +484,318 @@ def mark(pipe: Any, info: dict) -> None:def engaged_on(pipe: Any) -> Optional[dict]:return getattr(pipe, SMALL_HOST_ATTR, None)+++ENCODER_PREFETCH_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_PREFETCH"+ENCODER_PREFETCH_ATTR = "_unsloth_encoder_prefetch"+# Pinned staging ring: the only host bytes this adds.+_STAGE_SLOT_BYTES = 32 * _MIB+_STAGE_SLOTS = 4+# Device bytes copied ahead: a few groups (T5-XXL MLP weight 80 MiB, Qwen2.5-VL 130 MiB).+_PREFETCH_MAX_BYTES = 384 * _MIB+_PREFETCH_MIN_BYTES = 128 * _MIB+++def encoder_prefetch_disabled() -> bool:+ return _env(ENCODER_PREFETCH_ENV) in ("0", "off", "false", "no")+++def _group_tensors(group: Any) -> list:+ """(tensor, host source) pairs of a diffusers offload group, in the order diffusers onloads them."""+ out: list = []+ seen: set[int] = set()+ cpu = getattr(group, "cpu_param_dict", None) or {}+ for t in (+ [p for m in group.modules for p in m.parameters()]+ + [b for m in group.modules for b in m.buffers()]+ + list(getattr(group, "parameters", None) or [])+ + list(getattr(group, "buffers", None) or [])+ ):+ if id(t) in seen:+ continue+ seen.add(id(t))+ out.append((t, cpu.get(t, t)))+ return out+++class _EncoderPrefetcher:+ """Copies a memory-mapped encoder's offload groups ahead of its forward, in the previous forward's order, on a+ worker thread through a pinned ring (diffusers pins each group on the calling thread). Off-order groups and worker+ errors fall back to a synchronous copy."""++ def __init__(self, module: Any, groups: list, device: Any):+ import threading++ import torch++ self.module = module+ self.groups = groups+ dev = torch.device(device)+ if dev.type == "cuda" and dev.index is None:+ dev = torch.device("cuda", torch.cuda.current_device())+ self.device = dev+ self.stream = None # side stream, created on the first copy+ self.compute = None # the stream the encoder runs on, read when a forward begins+ self.order: list = [] # group ids in the last forward's onload order+ self.seen: list = []+ self.active = False+ self.cond = threading.Condition()+ self.ready: dict = {} # id(group) -> (device tensors, event)+ self.inflight = 0+ self.budget = _PREFETCH_MIN_BYTES+ self.pos = 0+ self.stop = False+ self.worker = None+ self.error: Optional[BaseException] = None+ self.stage: list = []+ self.stage_events: list = []+ self.slot = 0+ self.stats = {"prefetched": 0, "sync": 0, "passes": 0}+ self.by_id = {id(g): g for g in groups}++ def _side_stream(self) -> Any:+ import torch+ if self.stream is None:+ self.stream = torch.cuda.Stream(device = self.device)+ return self.stream++ def _ensure_stage(self) -> None:+ import torch+ if not self.stage:+ # inference_mode is thread-local: the worker cannot write inference tensors made on the forward thread.+ with torch.inference_mode(False):+ self.stage = [+ torch.empty(_STAGE_SLOT_BYTES, dtype = torch.uint8, pin_memory = True)+ for _ in range(_STAGE_SLOTS)+ ]+ self.stage_events = [None] * _STAGE_SLOTS++ def _copy_group(self, group: Any) -> tuple:+ """Device copies of ``group``'s host tensors, issued on the side stream. Returns (tensors, event, bytes)."""+ import torch++ self._ensure_stage()+ stream = self._side_stream()+ compute = self.compute or torch.cuda.current_stream(self.device)+ moved: list = []+ nbytes = 0+ with torch.cuda.stream(stream):+ for _t, src in _group_tensors(group):+ n = int(src.numel()) * int(src.element_size())+ if src.device.type != "cpu" or n == 0 or not src.is_contiguous() or src.is_pinned():+ with torch.cuda.stream(compute):+ moved.append(+ src.to(+ self.device,+ non_blocking = src.device.type != "cpu" or src.is_pinned(),+ )+ )+ continue+ nbytes += n+ # Compute-stream pool (a side-stream block stays cached away from the denoise); the copy waits for+ # work queued there, covering the last user of a freed block.+ with torch.cuda.stream(compute):+ dst = torch.empty(src.shape, dtype = src.dtype, device = self.device)+ queued = torch.cuda.Event()+ queued.record(compute)+ stream.wait_event(queued)+ sb = src.reshape(-1).view(torch.uint8)+ db = dst.reshape(-1).view(torch.uint8)+ off = 0+ while off < n:+ k = min(_STAGE_SLOT_BYTES, n - off)+ i = self.slot+ self.slot = (i + 1) % _STAGE_SLOTS+ ev = self.stage_events[i]+ if ev is not None:+ ev.synchronize()+ buf = self.stage[i][:k]+ buf.copy_(sb[off : off + k])+ db[off : off + k].copy_(buf, non_blocking = True)+ ev = torch.cuda.Event()+ ev.record(stream)+ self.stage_events[i] = ev+ off += k+ # A dropped entry (halt, exception) frees dst before the copy lands.+ dst.record_stream(stream)+ moved.append(dst)+ done = torch.cuda.Event()+ done.record(stream)+ return moved, done, nbytes++ def _attach(self, group: Any, moved: list, done: Any) -> None:+ import torch++ cur = torch.cuda.current_stream(self.device)+ cur.wait_event(done)+ if getattr(group, "stream", None) is not None:+ # diffusers fences its own-stream prefetch only in the next group's onload_, which this replaces.+ cur.wait_stream(group.stream)+ for (t, _src), dev in zip(_group_tensors(group), moved):+ t.data = dev+ dev.record_stream(cur)++ def _run(self, order: list) -> None:+ import torch+ try:+ if self.device.type == "cuda":+ torch.cuda.set_device(self.device)+ for gid in order:+ with self.cond:+ # A group the forward onloads twice waits for its first copy to be taken.+ while not self.stop and (+ (self.inflight > 0 and self.inflight >= self.budget) or gid in self.ready+ ):+ self.cond.wait(0.05)+ if self.stop:+ return+ group = self.by_id.get(gid)+ if group is None:+ continue+ moved, done, n = self._copy_group(group)+ with self.cond:+ if self.stop:+ return+ self.ready[gid] = (moved, done, n)+ self.inflight += n+ self.cond.notify_all()+ except BaseException as exc: # noqa: BLE001 - every group falls back to a synchronous copy+ with self.cond:+ self.error = exc+ self.cond.notify_all()++ def _halt(self) -> None:+ with self.cond:+ self.stop = True+ self.cond.notify_all()+ if self.worker is not None:+ self.worker.join()+ self.worker = None+ with self.cond:+ self.ready.clear()+ self.inflight = 0++ def begin(self) -> None:+ import threading++ import torch++ self._halt()+ self.stop = False+ self.error = None+ self.seen = []+ self.pos = 0+ self.active = True+ self.stats["passes"] += 1+ self.compute = (+ torch.cuda.current_stream(self.device) if self.device.type == "cuda" else None+ )+ if not self.order:+ return # first forward: synchronous copies, and it records the order+ try:+ free, _total = torch.cuda.mem_get_info(self.device)+ self.budget = max(_PREFETCH_MIN_BYTES, min(_PREFETCH_MAX_BYTES, int(free) // 8))+ except Exception: # noqa: BLE001+ self.budget = _PREFETCH_MIN_BYTES+ self._order_after_compute()+ self.worker = threading.Thread(+ target = self._run, args = (list(self.order),), daemon = True, name = "unsloth-encoder-prefetch"+ )+ self.worker.start()++ def _order_after_compute(self) -> None:+ """The worker's copies must not overtake work the compute stream already queued (the prompt's input ids)."""+ import torch+ self._side_stream().wait_stream(torch.cuda.current_stream(self.device))++ def end(self) -> None:+ if not self.active:+ return+ self.active = False+ self._halt()+ if self.seen:+ self.order = list(self.seen)++ def onload(self, group: Any) -> None:+ gid = id(group)+ if self.active:+ self.seen.append(gid)+ entry = None+ if self.worker is not None:+ expected = self.order[self.pos] if self.pos < len(self.order) else None+ if expected == gid:+ self.pos += 1+ with self.cond:+ while gid not in self.ready and self.error is None and self.worker.is_alive():+ self.cond.wait(0.05)+ entry = self.ready.pop(gid, None)+ if entry is not None:+ self.inflight -= entry[2]+ self.cond.notify_all()+ else:+ # Off the recorded order: stop prefetching for this forward; the next one re-records the order.+ self._halt()+ if entry is None:+ moved, done, _n = self._copy_group(group)+ self.stats["sync"] += 1+ else:+ moved, done, _n = entry+ self.stats["prefetched"] += 1+ self._attach(group, moved, done)+++def install_encoder_prefetch(+ module: Any,+ device: Any,+ logger: Any = None,+) -> int:+ """Swap each streamed offload group's ``onload_`` of a small-host text encoder for the prefetching copy. Groups made+ resident are left alone. Returns the number of groups covered (0: unchanged, e.g. kill switch or no CUDA)."""+ if encoder_prefetch_disabled():+ return 0+ try:+ import torch++ if torch.device(device).type != "cuda" or not torch.cuda.is_available():+ return 0+ from .diffusion_memory import GROUP_OFFLOAD_PIN_ENV, _offload_groups++ if _env(GROUP_OFFLOAD_PIN_ENV) in ("0", "off", "false", "no"):+ return 0 # the ring is pinned memory: the user's "pin nothing" override wins++ groups = [+ g+ for g in _offload_groups(module)+ if not getattr(g, "_unsloth_resident", False)+ and getattr(g, "stream", None) is not None+ and not getattr(g, "offload_to_disk_path", None)+ ]+ if not groups:+ return 0+ pf = _EncoderPrefetcher(module, groups, device)+ pf._ensure_stage() # a pinned allocation failing here keeps diffusers' onload, not a failed render+ disable = getattr(getattr(torch, "compiler", None), "disable", None)+ for group in groups:++ def onload_(+ *_a: Any,+ _g: Any = group,+ **_k: Any,+ ) -> None:+ pf.onload(_g)++ group.onload_ = disable(onload_) if callable(disable) else onload_+ module.register_forward_pre_hook(lambda *_a, **_k: pf.begin())+ module.register_forward_hook(lambda *_a, **_k: pf.end(), always_call = True)+ setattr(module, ENCODER_PREFETCH_ATTR, pf)+ if logger is not None:+ logger.info(+ "diffusion.small_host: %s streams %d offload groups through a prefetching copy",+ type(module).__name__,+ len(groups),+ )+ return len(groups)+ except Exception as exc: # noqa: BLE001 - keep diffusers' own onload+ if logger is not None:+ logger.warning("diffusion.small_host: encoder prefetch unavailable (%s)", exc)+ return 0diff --git a/studio/backend/tests/test_diffusion_small_host_prefetch.py b/studio/backend/tests/test_diffusion_small_host_prefetch.pynew file mode 100644index 00000000000..9d4c7729522--- /dev/null+++ b/studio/backend/tests/test_diffusion_small_host_prefetch.py@@ -0,0 +1,509 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Small-host route speed fixes: the streamed text encoder's prefetching copy and the fused int8 dequant.++The prefetch order / budget / fallback logic runs on CPU with fake groups; the end-to-end check (diffusers leaf-level+group offload + layerwise casting, the small-host encoder setup) needs CUDA and is skipped without it.+"""++from __future__ import annotations++import threading+import time+import types++import pytest+import torch++import core.inference.diffusion_small_host as sh++[email protected](autouse = True)+def _clean_env(monkeypatch):+ monkeypatch.delenv(sh.ENCODER_PREFETCH_ENV, raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", raising = False)+++# ------------------------------------------------------------------------------------------------ fused int8 dequant[email protected]("dtype", [torch.float32, torch.float16])+def test_int8_dequant_is_one_pass_and_bit_identical(dtype):+ from torch.utils._python_dispatch import TorchDispatchMode++ torch.manual_seed(0)+ lin = torch.nn.Linear(256, 512).to(torch.bfloat16)+ # the same per-row absmax quantisation quantize_int8_weight_ applies (this layer is below its size threshold)+ w = lin.weight.detach().float()+ scale = w.abs().amax(dim = 1, keepdim = True).clamp_min(1e-12) / 127.0+ q = torch.round(w / scale).clamp_(-127, 127).to(torch.int8)+ layer = sh.int8_linear_class()(q, scale.to(dtype), lin.bias.detach().to(dtype), 256, 512)+ x = torch.randn(4, 256, dtype = dtype)++ class Ops(TorchDispatchMode):+ def __init__(self):+ super().__init__()+ self.ops = []++ def __torch_dispatch__(+ self,+ func,+ types_,+ args = (),+ kwargs = None,+ ):+ self.ops.append(str(func))+ return func(*args, **(kwargs or {}))++ with Ops() as rec:+ out = layer(x)+ # the int8 weight is read once: no separate cast of the full weight before the scale multiply+ passes = [o for o in rec.ops if "_to_copy" in o or "mul" in o]+ assert passes == ["aten.mul.Tensor"], rec.ops+ want = torch.nn.functional.linear(x, q.to(dtype) * scale.to(dtype), lin.bias.detach().to(dtype))+ assert torch.equal(out, want)+++def test_int8_dequant_other_input_dtype_still_casts():+ q = torch.randint(-127, 128, (8, 4), dtype = torch.int8)+ scale = torch.rand(8, 1, dtype = torch.float16)+ layer = sh.int8_linear_class()(q, scale, None, 4, 8)+ x = torch.randn(2, 4, dtype = torch.float32)+ out = layer(x)+ assert out.dtype == torch.float32+ assert torch.equal(out, torch.nn.functional.linear(x, q.float() * scale.float()))+++# ------------------------------------------------------------------------------------------------ prefetch logic (CPU)+class _FakeGroup:+ def __init__(self, name: str, nbytes: int):+ self.name = name+ self.nbytes = nbytes+++class _FakePrefetcher(sh._EncoderPrefetcher):+ """The real ordering / budget / fallback logic with the CUDA copy replaced by a recorded fake."""++ def __init__(+ self,+ groups,+ *,+ fail_at = None,+ delay = 0.0,+ ):+ super().__init__(module = None, groups = groups, device = "cpu")+ self.copies: list = []+ self.attached: list = []+ self.fail_at = fail_at+ self.delay = delay+ self.peak_inflight = 0+ self.lock = threading.Lock()++ def _copy_group(self, group):+ if self.delay:+ time.sleep(self.delay)+ with self.lock:+ if (+ self.fail_at is not None+ and group.name == self.fail_at+ and threading.current_thread() is not threading.main_thread()+ ):+ raise RuntimeError("copy failed")+ self.copies.append((group.name, threading.current_thread() is threading.main_thread()))+ with self.cond:+ self.peak_inflight = max(self.peak_inflight, self.inflight + group.nbytes)+ return [group.name], None, group.nbytes++ def _attach(self, group, moved, done):+ self.attached.append(moved[0])++ def _order_after_compute(self):+ return None+++def _forward(pf, groups):+ pf.begin()+ try:+ for g in groups:+ pf.onload(g)+ finally:+ pf.end()+++def test_prefetch_records_the_order_then_copies_ahead_on_a_worker():+ groups = [_FakeGroup(f"g{i}", 10) for i in range(12)]+ pf = _FakePrefetcher(groups)+ _forward(pf, groups)+ # first forward: no order yet, every group copied on the calling thread+ assert (pf.stats["prefetched"], pf.stats["sync"], pf.stats["passes"]) == (0, 12, 1)+ assert all(main for _n, main in pf.copies)+ pf.copies.clear()+ _forward(pf, groups)+ assert pf.stats["prefetched"] == 12 and pf.stats["sync"] == 12+ assert [n for n, _ in pf.copies] == [g.name for g in groups]+ assert not any(main for _n, main in pf.copies)+ assert pf.attached[-12:] == [g.name for g in groups]+ assert pf.worker is None and pf.inflight == 0 and not pf.ready+++def test_prefetch_stays_within_the_byte_budget():+ groups = [_FakeGroup(f"g{i}", sh._PREFETCH_MIN_BYTES // 4) for i in range(16)]+ pf = _FakePrefetcher(groups)+ _forward(pf, groups)++ pf.begin()+ try:+ for g in groups:+ time.sleep(0.01) # a slow consumer: the worker must wait for the budget, not run away+ pf.onload(g)+ finally:+ pf.end()+ assert pf.stats["prefetched"] == 16+ # at most one group past the budget is ever in flight+ assert pf.peak_inflight <= pf.budget + sh._PREFETCH_MIN_BYTES // 4+++def test_prefetch_one_oversized_group_still_progresses():+ groups = [_FakeGroup("big", sh._PREFETCH_MAX_BYTES * 4), _FakeGroup("small", 1)]+ pf = _FakePrefetcher(groups)+ _forward(pf, groups)+ _forward(pf, groups)+ assert pf.stats["prefetched"] == 2+++def test_prefetch_off_order_falls_back_and_rerecords():+ groups = [_FakeGroup(f"g{i}", 10) for i in range(6)]+ pf = _FakePrefetcher(groups)+ _forward(pf, groups)+ swapped = groups[:2] + [groups[3], groups[2]] + groups[4:]+ _forward(pf, swapped)+ # g0, g1 prefetched; g3 is off the recorded order, so it and everything after it copy synchronously+ assert pf.stats["prefetched"] == 2+ assert pf.attached[-6:] == [g.name for g in swapped]+ assert pf.order == [id(g) for g in swapped]+ _forward(pf, swapped)+ assert pf.stats["prefetched"] == 2 + 6+++def test_prefetch_worker_error_falls_back_to_synchronous_copies():+ groups = [_FakeGroup(f"g{i}", 10) for i in range(5)]+ pf = _FakePrefetcher(groups, fail_at = "g2")+ _forward(pf, groups)+ _forward(pf, groups)+ assert pf.attached[-5:] == [g.name for g in groups]+ assert pf.stats["prefetched"] == 2 and pf.stats["sync"] == 5 + 3+++def test_prefetch_exception_in_forward_stops_the_worker():+ groups = [_FakeGroup(f"g{i}", 10) for i in range(5)]+ pf = _FakePrefetcher(groups, delay = 0.01)+ _forward(pf, groups)+ pf.begin()+ pf.onload(groups[0])+ pf.end() # the forward hook runs with always_call=True on an exception+ assert pf.worker is None and not pf.ready and pf.inflight == 0+++def test_install_is_off_on_cpu_and_with_the_kill_switch(monkeypatch):+ m = torch.nn.Linear(4, 4)+ assert sh.install_encoder_prefetch(m, "cpu") == 0+ monkeypatch.setenv(sh.ENCODER_PREFETCH_ENV, "0")+ assert sh.encoder_prefetch_disabled()+ assert sh.install_encoder_prefetch(m, "cuda") == 0+++def test_group_offload_installs_prefetch_only_on_small_host_pipes(monkeypatch):+ pytest.importorskip("diffusers")+ import core.inference.diffusion_memory as dm++ calls: list = []+ monkeypatch.setattr(+ sh,+ "install_encoder_prefetch",+ lambda module, device, logger = None: calls.append(module) or 1,+ )++ def _pipe(small_host: bool):+ te = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.Linear(8, 8))+ tr = torch.nn.Sequential(torch.nn.Linear(8, 8))+ pipe = types.SimpleNamespace(+ transformer = tr, text_encoder = te, components = {"transformer": tr, "text_encoder": te}+ )+ if small_host:+ pipe._unsloth_small_host = {+ "components": {"text_encoder": "memory-mapped, layerwise cast (1 MiB)"}+ }+ return pipe, te++ pipe, te = _pipe(False)+ assert dm._apply_group_offload(pipe, "cpu", None, stream_text_encoders = True)+ assert calls == []+ pipe, te = _pipe(True)+ assert dm._apply_group_offload(pipe, "cpu", None, stream_text_encoders = True)+ assert calls == [te]+++# ------------------------------------------------------------------------------------------------ end to end (CUDA)[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_prefetched_encoder_is_bit_identical_to_diffusers_stream(tmp_path):+ from diffusers.hooks import apply_group_offloading++ def build(path = None):+ torch.manual_seed(0)++ class Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.norm = torch.nn.LayerNorm(256)+ self.wi = torch.nn.Linear(256, 1024, bias = False)+ self.wo = torch.nn.Linear(1024, 256, bias = False)++ def forward(self, x):+ return x + self.wo(torch.relu(self.wi(self.norm(x))))++ class Enc(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.embed = torch.nn.Embedding(1000, 256)+ self.blocks = torch.nn.ModuleList([Block() for _ in range(6)])+ self.final = torch.nn.LayerNorm(256)++ def forward(self, ids):+ h = self.embed(ids)+ for b in self.blocks:+ h = b(h)+ return self.final(h)++ enc = Enc().to(torch.bfloat16).eval()+ sh.prepare_streamed_encoder_(enc, torch.float16)+ if path is not None:+ from safetensors.torch import load_file, save_file++ save_file({k: v.contiguous() for k, v in enc.state_dict().items()}, str(path))+ loaded = load_file(str(path))+ for name, p in enc.named_parameters():+ p.data = loaded[name]+ apply_group_offloading(+ enc,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "leaf_level",+ use_stream = True,+ non_blocking = True,+ record_stream = True,+ low_cpu_mem_usage = True,+ )+ return enc++ ref_enc = build()+ # the small-host load: the encoder's weights view a safetensors file mapping+ enc = build(tmp_path / "enc.safetensors")++ n = sh.install_encoder_prefetch(enc, "cuda")+ assert n > 0+ pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)+ with torch.no_grad():+ for i in range(4):+ ids = torch.randint(0, 1000, (1, 16 + i), device = "cuda")+ want = ref_enc(ids)+ got = enc(ids)+ torch.cuda.synchronize()+ assert torch.equal(got, want)+ # the first forward records the order; every later one copies every group ahead on the worker+ assert pf.error is None+ assert pf.stats["prefetched"] == 3 * len(pf.order) and pf.stats["sync"] == len(pf.order)+ # weights are back on their host storage between forwards+ assert enc.blocks[0].wi.weight.device.type == "cpu"+ assert enc.blocks[0].wi.weight.dtype == torch.bfloat16+++def _mapped_streamed(enc, path):+ """The small-host setup: weights view a safetensors file mapping, diffusers leaf-level stream offload."""+ pytest.importorskip("diffusers")+ from diffusers.hooks import apply_group_offloading+ from safetensors.torch import load_file, save_file++ save_file({k: v.contiguous() for k, v in enc.state_dict().items()}, str(path))+ loaded = load_file(str(path))+ for name, p in enc.named_parameters():+ p.data = loaded[name]+ apply_group_offloading(+ enc,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "leaf_level",+ use_stream = True,+ non_blocking = True,+ record_stream = True,+ low_cpu_mem_usage = True,+ )+ return enc+++def _small_linear_stack():+ torch.manual_seed(0)+ return (+ torch.nn.Sequential(*[torch.nn.Linear(64, 64) for _ in range(3)]).to(torch.bfloat16).eval()+ )++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_pin_opt_out_keeps_diffusers_onload(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")+ enc = _mapped_streamed(_small_linear_stack(), tmp_path / "enc.safetensors")+ assert sh.install_encoder_prefetch(enc, "cuda") == 0+ assert getattr(enc, sh.ENCODER_PREFETCH_ATTR, None) is None++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_failed_ring_pin_keeps_diffusers_onload(tmp_path, monkeypatch):+ enc = _mapped_streamed(_small_linear_stack(), tmp_path / "enc.safetensors")+ real_empty = torch.empty++ def empty(*a, **k):+ if k.get("pin_memory"):+ raise RuntimeError("CUDA error: out of memory")+ return real_empty(*a, **k)++ monkeypatch.setattr(torch, "empty", empty)+ assert sh.install_encoder_prefetch(enc, "cuda") == 0+ monkeypatch.setattr(torch, "empty", real_empty)+ assert getattr(enc, sh.ENCODER_PREFETCH_ATTR, None) is None+ ref = _small_linear_stack().cuda()+ x = torch.randn(2, 64, dtype = torch.bfloat16, device = "cuda")+ with torch.no_grad():+ assert torch.equal(enc(x), ref(x))++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_prefetch_runs_under_inference_mode(tmp_path):+ # Studio renders under torch.inference_mode(); the worker thread is outside it and must still fill the ring.+ def build():+ torch.manual_seed(0)+ return (+ torch.nn.Sequential(*[torch.nn.Linear(512, 512) for _ in range(6)])+ .to(torch.bfloat16)+ .eval()+ )++ ref = build().cuda()+ enc = _mapped_streamed(build(), tmp_path / "enc.safetensors")+ assert sh.install_encoder_prefetch(enc, "cuda") > 0+ pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)+ with torch.inference_mode():+ for i in range(4):+ x = torch.randn(2 + i, 512, dtype = torch.bfloat16, device = "cuda")+ assert torch.equal(enc(x), ref(x))+ assert pf.error is None+ assert pf.stats["prefetched"] == 3 * len(pf.order)++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")[email protected]("drop", ["exception", "prefix"])+def test_dropped_prefetch_never_lands_in_a_reused_block(tmp_path, drop):+ # Groups copied ahead but never onloaded are freed while their copy may still be queued on the side stream.+ H = 4096++ class Enc(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.head = torch.nn.ModuleList([torch.nn.Linear(H, H, bias = False) for _ in range(2)])+ self.tail = torch.nn.ModuleList([torch.nn.Linear(H, H, bias = False) for _ in range(6)])++ def forward(+ self,+ x,+ tail = True,+ boom = False,+ ):+ for m in self.head:+ x = m(x)+ if boom:+ raise RuntimeError("boom")+ for m in self.tail if tail else ():+ x = m(x)+ return x++ enc = Enc().to(torch.bfloat16).eval()+ with torch.no_grad():+ for p in enc.parameters():+ p.fill_(1.0)+ enc = _mapped_streamed(enc, tmp_path / "enc.safetensors")+ assert sh.install_encoder_prefetch(enc, "cuda") > 0+ x = torch.zeros(1, H, dtype = torch.bfloat16, device = "cuda")+ with torch.no_grad():+ enc(x) # records the full order+ for _ in range(10):+ try:+ enc(x, tail = False, boom = drop == "exception")+ except RuntimeError:+ pass+ outs = [torch.empty(H, H, dtype = torch.bfloat16, device = "cuda") for _ in range(6)]+ for o in outs:+ o.fill_(-2.0)+ torch.cuda.synchronize()+ assert all(bool((o == -2.0).all()) for o in outs)+ del outs+ enc(x) # back to the full order+ torch.cuda.synchronize()++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_released_resident_groups_are_fenced():+ # release_resident_groups hands resident encoder groups back to diffusers' onload_, whose copy runs on diffusers'+ # stream next to groups this prefetcher onloads.+ pytest.importorskip("diffusers")+ import copy++ import core.inference.diffusion_memory as dm++ H, F = 4096, 16384++ class Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.norm = torch.nn.LayerNorm(H)+ self.wi = torch.nn.Linear(H, F, bias = False)+ self.wo = torch.nn.Linear(F, H, bias = False)++ def forward(self, x):+ return x + self.wo(torch.relu(self.wi(self.norm(x))))++ class Enc(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.embed = torch.nn.Embedding(1000, H)+ self.blocks = torch.nn.ModuleList([Block() for _ in range(3)])+ self.final = torch.nn.LayerNorm(H)++ def forward(self, ids):+ h = self.embed(ids)+ for b in self.blocks:+ h = b(h)+ return self.final(h)++ torch.manual_seed(0)+ enc = Enc().half().eval()+ for p in enc.parameters():+ p.data.normal_(0, 0.02)+ ref = copy.deepcopy(enc).cuda()+ tr = torch.nn.Sequential(torch.nn.Linear(8, 8))+ pipe = types.SimpleNamespace(+ transformer = tr, text_encoder = enc, components = {"transformer": tr, "text_encoder": enc}+ )+ pipe._unsloth_small_host = {"components": {"text_encoder": "memory-mapped"}}+ assert dm._apply_group_offload(+ pipe, "cuda", None, stream_text_encoders = True, resident_text_encoder_mib = 140+ )+ pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)+ with torch.no_grad():+ for _ in range(2):+ enc(torch.randint(0, 1000, (4, 512), device = "cuda"))+ assert dm.release_resident_groups(pipe, 100, None) is not None+ for _ in range(10):+ ids = torch.randint(0, 1000, (4, 512), device = "cuda")+ want = ref(ids)+ torch.cuda.synchronize()+ got = enc(ids)+ torch.cuda.synchronize()+ assert torch.equal(got, want)+ assert pf.error is None and pf.stats["prefetched"] > 0