Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Prefetch order, budget and fallback are CPU-tested with fake groups; end-to-end needs CUDA."""
from __future__ import annotations
import threading
import time
import types
import pytest
import torch
import core.inference.diffusion_small_host as sh
@pytest.fixture(autouse = True)
def _clean_env(monkeypatch):
monkeypatch.delenv(sh.ENCODER_PREFETCH_ENV, raising = False)
monkeypatch.delenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", raising = False)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16])
def test_int8_dequant_is_one_pass_and_bit_identical(dtype):
from torch.utils._python_dispatch import TorchDispatchMode
torch.manual_seed(0)
lin = torch.nn.Linear(256, 512).to(torch.bfloat16)
# same per-row absmax as quantize_int8_weight_ (this layer is below its size threshold)
w = lin.weight.detach().float()
scale = w.abs().amax(dim = 1, keepdim = True).clamp_min(1e-12) / 127.0
q = torch.round(w / scale).clamp_(-127, 127).to(torch.int8)
layer = sh.int8_linear_class()(q, scale.to(dtype), lin.bias.detach().to(dtype), 256, 512)
x = torch.randn(4, 256, dtype = dtype)
class Ops(TorchDispatchMode):
def __init__(self):
super().__init__()
self.ops = []
def __torch_dispatch__(
self,
func,
types_,
args = (),
kwargs = None,
):
self.ops.append(str(func))
return func(*args, **(kwargs or {}))
with Ops() as rec:
out = layer(x)
passes = [o for o in rec.ops if "_to_copy" in o or "mul" in o]
assert passes == ["aten.mul.Tensor"], rec.ops
want = torch.nn.functional.linear(x, q.to(dtype) * scale.to(dtype), lin.bias.detach().to(dtype))
assert torch.equal(out, want)
def test_int8_dequant_other_input_dtype_still_casts():
q = torch.randint(-127, 128, (8, 4), dtype = torch.int8)
scale = torch.rand(8, 1, dtype = torch.float16)
layer = sh.int8_linear_class()(q, scale, None, 4, 8)
x = torch.randn(2, 4, dtype = torch.float32)
out = layer(x)
assert out.dtype == torch.float32
assert torch.equal(out, torch.nn.functional.linear(x, q.float() * scale.float()))
class _FakeGroup:
def __init__(self, name: str, nbytes: int):
self.name = name
self.nbytes = nbytes
class _FakePrefetcher(sh._EncoderPrefetcher):
"""The real ordering / budget / fallback logic with the CUDA copy replaced by a recorded fake."""
def __init__(
self,
groups,
*,
fail_at = None,
delay = 0.0,
):
super().__init__(module = None, groups = groups, device = "cpu")
self.copies: list = []
self.attached: list = []
self.fail_at = fail_at
self.delay = delay
self.peak_inflight = 0
self.lock = threading.Lock()
def _copy_group(self, group):
if self.delay:
time.sleep(self.delay)
with self.lock:
if (
self.fail_at is not None
and group.name == self.fail_at
and threading.current_thread() is not threading.main_thread()
):
raise RuntimeError("copy failed")
self.copies.append((group.name, threading.current_thread() is threading.main_thread()))
if threading.current_thread() is not threading.main_thread():
with self.cond:
self.peak_inflight = max(self.peak_inflight, self.inflight + group.nbytes)
return [group.name], None, group.nbytes
def _group_bytes(self, group):
return group.nbytes
def _attach(self, group, moved, done):
self.attached.append(moved[0])
def _order_after_compute(self):
return None
def _forward(pf, groups):
pf.begin()
try:
for g in groups:
pf.onload(g)
finally:
pf.end()
def test_prefetch_records_the_order_then_copies_ahead_on_a_worker():
groups = [_FakeGroup(f"g{i}", 10) for i in range(12)]
pf = _FakePrefetcher(groups)
_forward(pf, groups)
# first forward has no recorded order, so every group copies on the calling thread
assert (pf.stats["prefetched"], pf.stats["sync"], pf.stats["passes"]) == (0, 12, 1)
assert all(main for _n, main in pf.copies)
pf.copies.clear()
_forward(pf, groups)
assert pf.stats["prefetched"] == 12 and pf.stats["sync"] == 12
assert [n for n, _ in pf.copies] == [g.name for g in groups]
assert not any(main for _n, main in pf.copies)
assert pf.attached[-12:] == [g.name for g in groups]
assert pf.worker is None and pf.inflight == 0 and not pf.ready
def test_prefetch_stays_within_the_byte_budget():
groups = [_FakeGroup(f"g{i}", sh._PREFETCH_MIN_BYTES // 4) for i in range(16)]
pf = _FakePrefetcher(groups)
_forward(pf, groups)
pf.begin()
try:
for g in groups:
time.sleep(0.01)
pf.onload(g)
finally:
pf.end()
assert pf.stats["prefetched"] == 16
assert pf.peak_inflight <= pf.budget + sh._PREFETCH_MIN_BYTES // 4
def test_prefetch_with_no_budgeted_room_starts_no_worker(monkeypatch):
"""#14816 r9: a zero budget copies every group synchronously instead of one group ahead."""
monkeypatch.setattr(sh, "_prefetch_budget_bytes", lambda device: 0)
groups = [_FakeGroup(f"g{i}", 10) for i in range(6)]
pf = _FakePrefetcher(groups)
_forward(pf, groups)
pf.copies.clear()
pf.begin()
try:
assert pf.worker is None
for g in groups:
pf.onload(g)
finally:
pf.end()
assert pf.stats["prefetched"] == 0 and pf.stats["sync"] == 12
assert all(main for _n, main in pf.copies) and pf.inflight == 0 and not pf.ready
assert pf.order == [id(g) for g in groups]
def test_prefetch_one_oversized_group_still_progresses():
groups = [_FakeGroup("big", sh._PREFETCH_MAX_BYTES * 4), _FakeGroup("small", 1)]
pf = _FakePrefetcher(groups)
_forward(pf, groups)
_forward(pf, groups)
assert pf.stats["prefetched"] == 2
def test_lowered_budget_copies_a_group_over_the_window_synchronously(monkeypatch):
"""Codex #14816 r10: under a lowered Budget a whole next group larger than the window would eat the reserve."""
monkeypatch.setattr(sh, "_prefetch_budget_bytes", lambda device: 256 * sh._MIB)
monkeypatch.setattr(sh, "_prefetch_budget_lowered", lambda device: True, raising = False)
big = 1536 * sh._MIB
groups = [
_FakeGroup("a", big),
_FakeGroup("s0", 100 * sh._MIB),
_FakeGroup("b", big),
_FakeGroup("s1", 100 * sh._MIB),
_FakeGroup("s2", 100 * sh._MIB),
]
pf = _FakePrefetcher(groups)
_forward(pf, groups)
pf.copies.clear()
_forward(pf, groups)
on_worker = [n for n, main in pf.copies if not main]
assert "a" not in on_worker and "b" not in on_worker
assert sorted(on_worker) == ["s0", "s1", "s2"]
assert pf.peak_inflight <= 256 * sh._MIB
assert pf.attached[-5:] == [g.name for g in groups]
assert pf.stats["prefetched"] == 3 and not pf.skipped and pf.inflight == 0
def test_lowered_budget_sync_copy_and_worker_never_share_a_staging_slot(monkeypatch):
"""#14816: a skipped group copies on the forward thread while the worker stages the next one. Without one copy
lock the worker wraps the ring and overwrites a staging slot the forward thread wrote but has not enqueued."""
import contextlib
class Event:
def record(self, stream):
pass
def synchronize(self):
pass
class Stream:
def wait_event(self, event):
pass
def wait_stream(self, stream):
pass
stream = Stream()
main = threading.main_thread()
wrote = threading.Event() # forward thread wrote slot 0 and has not enqueued it
progressed = threading.Event() # worker finished its copy, or is blocked on the copy lock
big, small = types.SimpleNamespace(), types.SimpleNamespace()
big.source = torch.full((40,), 1, dtype = torch.uint8)
small.source = torch.full((32,), 2, dtype = torch.uint8)
class Lock:
def __init__(self):
self.inner = threading.Lock()
def __enter__(self):
if threading.current_thread() is not main:
progressed.set()
self.inner.acquire()
def __exit__(self, *exc):
self.inner.release()
class Prefetcher(sh._EncoderPrefetcher):
def __init__(self):
super().__init__(None, [big, small], "cpu")
self.stage = [torch.empty(8, dtype = torch.uint8) for _ in range(4)]
self.stage_events = [None] * 4
self.stream = stream
self.copy_lock = Lock()
self.result = {}
def _group_bytes(self, group):
if group is small:
assert wrote.wait(10)
return super()._group_bytes(group)
def _copy_group(self, group):
out = super()._copy_group(group)
if group is small:
progressed.set()
return out
def _attach(self, group, moved, done):
self.result[id(group)] = moved[0].clone()
pf = Prefetcher()
pf.order = [id(big), id(small)]
copy = torch.Tensor.copy_
paused = []
def copy_(dst, src, *args, **kwargs):
out = copy(dst, src, *args, **kwargs)
if (
not paused
and threading.current_thread() is main
and dst.data_ptr() == pf.stage[0].data_ptr()
):
paused.append(True)
wrote.set()
assert progressed.wait(10)
return out
for obj, name, value in [
(sh, "_STAGE_SLOT_BYTES", 8),
(sh, "_STAGE_SLOTS", 4),
(sh, "_group_tensors", lambda g: [(g.source, g.source)]),
(sh, "_prefetch_budget_bytes", lambda dev: 32),
(sh, "_prefetch_budget_lowered", lambda dev: True),
(torch.cuda, "current_stream", lambda *a: stream),
(torch.cuda, "stream", lambda *a: contextlib.nullcontext()),
(torch.cuda, "Event", Event),
(torch.Tensor, "record_stream", lambda *a: None),
(torch.Tensor, "copy_", copy_),
]:
monkeypatch.setattr(obj, name, value, raising = False)
pf.begin()
try:
pf.onload(big)
pf.onload(small)
finally:
pf.end()
assert paused and pf.stats == {"prefetched": 1, "sync": 1, "passes": 1}
assert torch.equal(pf.result[id(big)], big.source)
assert torch.equal(pf.result[id(small)], small.source)
def test_default_budget_still_prefetches_an_oversized_group(monkeypatch):
monkeypatch.setattr(sh, "_prefetch_budget_bytes", lambda device: 256 * sh._MIB)
monkeypatch.setattr(sh, "_prefetch_budget_lowered", lambda device: False, raising = False)
groups = [_FakeGroup("a", 1536 * sh._MIB), _FakeGroup("s0", 1)]
pf = _FakePrefetcher(groups)
_forward(pf, groups)
_forward(pf, groups)
assert pf.stats["prefetched"] == 2
def test_prefetch_off_order_falls_back_and_rerecords():
groups = [_FakeGroup(f"g{i}", 10) for i in range(6)]
pf = _FakePrefetcher(groups)
_forward(pf, groups)
swapped = groups[:2] + [groups[3], groups[2]] + groups[4:]
_forward(pf, swapped)
# g3 is off the recorded order, so it and everything after copy synchronously
assert pf.stats["prefetched"] == 2
assert pf.attached[-6:] == [g.name for g in swapped]
assert pf.order == [id(g) for g in swapped]
_forward(pf, swapped)
assert pf.stats["prefetched"] == 2 + 6
def test_prefetch_worker_error_falls_back_to_synchronous_copies():
groups = [_FakeGroup(f"g{i}", 10) for i in range(5)]
pf = _FakePrefetcher(groups, fail_at = "g2")
_forward(pf, groups)
_forward(pf, groups)
assert pf.attached[-5:] == [g.name for g in groups]
assert pf.stats["prefetched"] == 2 and pf.stats["sync"] == 5 + 3
def test_prefetch_exception_in_forward_stops_the_worker():
groups = [_FakeGroup(f"g{i}", 10) for i in range(5)]
pf = _FakePrefetcher(groups, delay = 0.01)
_forward(pf, groups)
pf.begin()
pf.onload(groups[0])
pf.end()
assert pf.worker is None and not pf.ready and pf.inflight == 0
def test_install_is_off_on_cpu_and_with_the_kill_switch(monkeypatch):
m = torch.nn.Linear(4, 4)
assert sh.install_encoder_prefetch(m, "cpu") == 0
monkeypatch.setenv(sh.ENCODER_PREFETCH_ENV, "0")
assert sh.encoder_prefetch_disabled()
assert sh.install_encoder_prefetch(m, "cuda") == 0
def test_group_offload_installs_prefetch_only_on_small_host_pipes(monkeypatch):
pytest.importorskip("diffusers")
import core.inference.diffusion_memory as dm
calls: list = []
monkeypatch.setattr(
sh,
"install_encoder_prefetch",
lambda module, device, logger = None: calls.append(module) or 1,
)
def _pipe(small_host: bool):
te = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.Linear(8, 8))
tr = torch.nn.Sequential(torch.nn.Linear(8, 8))
pipe = types.SimpleNamespace(
transformer = tr, text_encoder = te, components = {"transformer": tr, "text_encoder": te}
)
if small_host:
pipe._unsloth_small_host = {
"components": {"text_encoder": "memory-mapped, layerwise cast (1 MiB)"}
}
return pipe, te
pipe, te = _pipe(False)
assert dm._apply_group_offload(pipe, "cpu", None, stream_text_encoders = True)
assert calls == []
pipe, te = _pipe(True)
assert dm._apply_group_offload(pipe, "cpu", None, stream_text_encoders = True)
assert calls == [te]
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
def test_prefetched_encoder_is_bit_identical_to_diffusers_stream(tmp_path):
from diffusers.hooks import apply_group_offloading
def build(path = None):
torch.manual_seed(0)
class Block(torch.nn.Module):
def __init__(self):
super().__init__()
self.norm = torch.nn.LayerNorm(256)
self.wi = torch.nn.Linear(256, 1024, bias = False)
self.wo = torch.nn.Linear(1024, 256, bias = False)
def forward(self, x):
return x + self.wo(torch.relu(self.wi(self.norm(x))))
class Enc(torch.nn.Module):
def __init__(self):
super().__init__()
self.embed = torch.nn.Embedding(1000, 256)
self.blocks = torch.nn.ModuleList([Block() for _ in range(6)])
self.final = torch.nn.LayerNorm(256)
def forward(self, ids):
h = self.embed(ids)
for b in self.blocks:
h = b(h)
return self.final(h)
enc = Enc().to(torch.bfloat16).eval()
sh.prepare_streamed_encoder_(enc, torch.float16)
if path is not None:
from safetensors.torch import load_file, save_file
save_file({k: v.contiguous() for k, v in enc.state_dict().items()}, str(path))
loaded = load_file(str(path))
for name, p in enc.named_parameters():
p.data = loaded[name]
apply_group_offloading(
enc,
onload_device = torch.device("cuda"),
offload_device = torch.device("cpu"),
offload_type = "leaf_level",
use_stream = True,
non_blocking = True,
record_stream = True,
low_cpu_mem_usage = True,
)
return enc
ref_enc = build()
enc = build(tmp_path / "enc.safetensors")
n = sh.install_encoder_prefetch(enc, "cuda")
assert n > 0
pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)
with torch.no_grad():
for i in range(4):
ids = torch.randint(0, 1000, (1, 16 + i), device = "cuda")
want = ref_enc(ids)
got = enc(ids)
torch.cuda.synchronize()
assert torch.equal(got, want)
assert pf.error is None
assert pf.stats["prefetched"] == 3 * len(pf.order) and pf.stats["sync"] == len(pf.order)
assert enc.blocks[0].wi.weight.device.type == "cpu"
assert enc.blocks[0].wi.weight.dtype == torch.bfloat16
def _mapped_streamed(enc, path):
"""The small-host setup: weights view a safetensors file mapping, diffusers leaf-level stream offload."""
pytest.importorskip("diffusers")
from diffusers.hooks import apply_group_offloading
from safetensors.torch import load_file, save_file
save_file({k: v.contiguous() for k, v in enc.state_dict().items()}, str(path))
loaded = load_file(str(path))
for name, p in enc.named_parameters():
p.data = loaded[name]
apply_group_offloading(
enc,
onload_device = torch.device("cuda"),
offload_device = torch.device("cpu"),
offload_type = "leaf_level",
use_stream = True,
non_blocking = True,
record_stream = True,
low_cpu_mem_usage = True,
)
return enc
def _small_linear_stack():
torch.manual_seed(0)
return (
torch.nn.Sequential(*[torch.nn.Linear(64, 64) for _ in range(3)]).to(torch.bfloat16).eval()
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
def test_pin_opt_out_keeps_diffusers_onload(tmp_path, monkeypatch):
monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")
enc = _mapped_streamed(_small_linear_stack(), tmp_path / "enc.safetensors")
assert sh.install_encoder_prefetch(enc, "cuda") == 0
assert getattr(enc, sh.ENCODER_PREFETCH_ATTR, None) is None
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
def test_failed_ring_pin_keeps_diffusers_onload(tmp_path, monkeypatch):
enc = _mapped_streamed(_small_linear_stack(), tmp_path / "enc.safetensors")
real_empty = torch.empty
def empty(*a, **k):
if k.get("pin_memory"):
raise RuntimeError("CUDA error: out of memory")
return real_empty(*a, **k)
monkeypatch.setattr(torch, "empty", empty)
assert sh.install_encoder_prefetch(enc, "cuda") == 0
monkeypatch.setattr(torch, "empty", real_empty)
assert getattr(enc, sh.ENCODER_PREFETCH_ATTR, None) is None
ref = _small_linear_stack().cuda()
x = torch.randn(2, 64, dtype = torch.bfloat16, device = "cuda")
with torch.no_grad():
assert torch.equal(enc(x), ref(x))
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
def test_prefetch_runs_under_inference_mode(tmp_path):
# Studio renders under inference_mode; the worker thread is outside it and must still fill the ring
def build():
torch.manual_seed(0)
return (
torch.nn.Sequential(*[torch.nn.Linear(512, 512) for _ in range(6)])
.to(torch.bfloat16)
.eval()
)
ref = build().cuda()
enc = _mapped_streamed(build(), tmp_path / "enc.safetensors")
assert sh.install_encoder_prefetch(enc, "cuda") > 0
pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)
with torch.inference_mode():
for i in range(4):
x = torch.randn(2 + i, 512, dtype = torch.bfloat16, device = "cuda")
assert torch.equal(enc(x), ref(x))
assert pf.error is None
assert pf.stats["prefetched"] == 3 * len(pf.order)
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
@pytest.mark.parametrize("drop", ["exception", "prefix"])
def test_dropped_prefetch_never_lands_in_a_reused_block(tmp_path, drop):
# copied-ahead groups never onloaded are freed while their copy may still be queued
H = 4096
class Enc(torch.nn.Module):
def __init__(self):
super().__init__()
self.head = torch.nn.ModuleList([torch.nn.Linear(H, H, bias = False) for _ in range(2)])
self.tail = torch.nn.ModuleList([torch.nn.Linear(H, H, bias = False) for _ in range(6)])
def forward(
self,
x,
tail = True,
boom = False,
):
for m in self.head:
x = m(x)
if boom:
raise RuntimeError("boom")
for m in self.tail if tail else ():
x = m(x)
return x
enc = Enc().to(torch.bfloat16).eval()
with torch.no_grad():
for p in enc.parameters():
p.fill_(1.0)
enc = _mapped_streamed(enc, tmp_path / "enc.safetensors")
assert sh.install_encoder_prefetch(enc, "cuda") > 0
x = torch.zeros(1, H, dtype = torch.bfloat16, device = "cuda")
with torch.no_grad():
enc(x)
for _ in range(10):
try:
enc(x, tail = False, boom = drop == "exception")
except RuntimeError:
pass
outs = [torch.empty(H, H, dtype = torch.bfloat16, device = "cuda") for _ in range(6)]
for o in outs:
o.fill_(-2.0)
torch.cuda.synchronize()
assert all(bool((o == -2.0).all()) for o in outs)
del outs
enc(x)
torch.cuda.synchronize()
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
def test_released_resident_groups_are_fenced():
# resident groups go back to diffusers' onload_, whose copy runs on diffusers' own stream
pytest.importorskip("diffusers")
import copy
import core.inference.diffusion_memory as dm
H, F = 4096, 16384
class Block(torch.nn.Module):
def __init__(self):
super().__init__()
self.norm = torch.nn.LayerNorm(H)
self.wi = torch.nn.Linear(H, F, bias = False)
self.wo = torch.nn.Linear(F, H, bias = False)
def forward(self, x):
return x + self.wo(torch.relu(self.wi(self.norm(x))))
class Enc(torch.nn.Module):
def __init__(self):
super().__init__()
self.embed = torch.nn.Embedding(1000, H)
self.blocks = torch.nn.ModuleList([Block() for _ in range(3)])
self.final = torch.nn.LayerNorm(H)
def forward(self, ids):
h = self.embed(ids)
for b in self.blocks:
h = b(h)
return self.final(h)
torch.manual_seed(0)
enc = Enc().half().eval()
for p in enc.parameters():
p.data.normal_(0, 0.02)
ref = copy.deepcopy(enc).cuda()
tr = torch.nn.Sequential(torch.nn.Linear(8, 8))
pipe = types.SimpleNamespace(
transformer = tr, text_encoder = enc, components = {"transformer": tr, "text_encoder": enc}
)
pipe._unsloth_small_host = {"components": {"text_encoder": "memory-mapped"}}
assert dm._apply_group_offload(
pipe, "cuda", None, stream_text_encoders = True, resident_text_encoder_mib = 140
)
pf = getattr(enc, sh.ENCODER_PREFETCH_ATTR)
with torch.no_grad():
for _ in range(2):
enc(torch.randint(0, 1000, (4, 512), device = "cuda"))
assert dm.release_resident_groups(pipe, 100, None) is not None
for _ in range(10):
ids = torch.randint(0, 1000, (4, 512), device = "cuda")
want = ref(ids)
torch.cuda.synchronize()
got = enc(ids)
torch.cuda.synchronize()
assert torch.equal(got, want)
assert pf.error is None and pf.stats["prefetched"] > 0