Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Byte-neutral image-load I/O. Off: UNSLOTH_DIFFUSION_PREFETCH=0, UNSLOTH_DIFFUSION_FAST_UPLOAD=0."""
from __future__ import annotations
import contextlib
import json
import os
import re
import threading
from typing import Any, Iterable, Iterator, Optional, Sequence
PREFETCH_ENV = "UNSLOTH_DIFFUSION_PREFETCH"
# "0": every slice is read whole, its page-cache-resident chunks included.
PREFETCH_SKIP_CACHED_ENV = "UNSLOTH_DIFFUSION_PREFETCH_SKIP_CACHED"
FAST_UPLOAD_ENV = "UNSLOTH_DIFFUSION_FAST_UPLOAD"
_SWITCH_OFF = ("0", "off", "false", "no")
_PREFETCH_THREADS = 8
_PREFETCH_READ_BYTES = 64 << 20
_PREFETCH_MIN_BYTES = 256 << 20
_PREFETCH_THREAD_PREFIX = "unsloth-image-prefetch"
_UPLOAD_THREADS = 8
_UPLOAD_CHUNK_BYTES = 16 << 20
_UPLOAD_BUFFERS = 16
_UPLOAD_MIN_TENSOR_BYTES = 1 << 20
_UPLOAD_MIN_TOTAL_BYTES = 256 << 20
# variant=None: from_pretrained never opens *.fp16.safetensors-style twins; variant="fp16" opens only those.
_DEFAULT_WEIGHT_RE = re.compile(
r"^(?:diffusion_pytorch_model|model)(?:-\d{5}-of-\d{5})?\.safetensors$"
)
_FP16_WEIGHT_RE = re.compile(r"^(?:diffusion_pytorch_model|model)\.fp16\.safetensors$")
_DENOISER_COMPONENTS = ("transformer", "unet")
def _switch_on(env: str) -> bool:
return (os.environ.get(env) or "").strip().lower() not in _SWITCH_OFF
def prefetch_enabled() -> bool:
return _switch_on(PREFETCH_ENV)
def fast_upload_enabled() -> bool:
return _switch_on(FAST_UPLOAD_ENV)
class LoadPrefetch:
"""Handle on a running prefetch: ``stop`` ends the reads at the next request boundary."""
def __init__(
self, files: list[str], need: int, threads: list[threading.Thread], stop: threading.Event
):
self.files = files
self.need = need
self.threads = threads
self._stop = stop
def stop(self) -> None:
self._stop.set()
def join(self, timeout: Optional[float] = None) -> bool:
for thread in self.threads:
thread.join(timeout)
return not any(thread.is_alive() for thread in self.threads)
def _uncached_bytes(path: str) -> int:
from .video_ltx2 import _uncached_bytes as uncached
return uncached(path)
_LSB = bytes(b & 1 for b in range(256)) # mincore defines only bit 0 of each byte
_LIBC: list = []
def _mincore() -> Any:
if not _LIBC:
import ctypes
libc = ctypes.CDLL(None, use_errno = True)
libc.mincore.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
_LIBC.append(libc.mincore)
return _LIBC[0]
def _chunk_resident(mapped: Any, start: int, length: int) -> bool:
"""True when every page of the chunk is in the page cache; False when it cannot tell."""
import ctypes
import mmap
try:
page = mmap.PAGESIZE
first = start - start % page
span = start + length - first
pages = (span + page - 1) // page
mincore = _mincore()
anchor = ctypes.c_char.from_buffer(mapped, first)
try:
vector = (ctypes.c_ubyte * pages)()
if mincore(ctypes.addressof(anchor), ctypes.c_size_t(span), vector) != 0:
return False
return b"\x00" not in bytes(vector).translate(_LSB)
finally:
del anchor
except Exception: # noqa: BLE001 - unknown residency: read it
return False
def _residency_map(path: str) -> Any:
"""A mapping for ``mincore``, or None (non-Linux, switch off, unmappable)."""
import sys
if not sys.platform.startswith("linux") or not _switch_on(PREFETCH_SKIP_CACHED_ENV):
return None
try:
import mmap
with open(path, "rb") as handle:
return mmap.mmap(handle.fileno(), 0, access = mmap.ACCESS_COPY)
except Exception: # noqa: BLE001
return None
def _available_host_mib() -> Optional[int]:
try:
from .diffusion_memory import _available_system_memory_mib
return _available_system_memory_mib()
except Exception: # noqa: BLE001
return None
def _on_rotational_disk(path: str) -> bool:
"""True only when sysfs says the file's block device spins: parallel slice reads would seek-thrash it."""
try:
st = os.stat(path)
base = f"/sys/dev/block/{os.major(st.st_dev)}:{os.minor(st.st_dev)}"
for queue in (os.path.join(base, "queue"), os.path.join(base, "..", "queue")):
flag = os.path.join(queue, "rotational")
if os.path.isfile(flag):
with open(flag, encoding = "utf-8") as handle:
return handle.read().strip() == "1"
except Exception: # noqa: BLE001 - unknown (non-Linux, overlay, network): not rotational
pass
return False
def _slices(size: int, parts: int) -> list[tuple[int, int]]:
"""``parts`` contiguous [start, end) ranges covering ``size``, aligned to 1 MiB."""
align = 1 << 20
step = -(-size // parts)
step = -(-step // align) * align
return [(start, min(size, start + step)) for start in range(0, size, step)]
def start_prefetch(
paths: Sequence[str],
*,
logger: Any = None,
threads: int = _PREFETCH_THREADS,
min_bytes: int = _PREFETCH_MIN_BYTES,
) -> Optional[LoadPrefetch]:
"""Prefetches uncached paths in order, skipping when uncached bytes exceed half of host RAM; never raises."""
if not prefetch_enabled():
return None
try:
seen: set[str] = set()
files: list[str] = []
for path in paths:
if not path:
continue
real = os.path.realpath(path)
if real in seen or not os.path.isfile(real):
continue
seen.add(real)
files.append(real)
pending = []
need = 0
for path in files:
missing = _uncached_bytes(path)
if missing > 0:
pending.append(path)
need += missing
if not pending or need < min_bytes:
return None
if any(_on_rotational_disk(path) for path in pending):
if logger is not None:
logger.info("diffusion.prefetch: skipped, the weights are on a rotational disk")
return None
available = _available_host_mib()
threads = max(1, int(threads))
largest = max(os.path.getsize(path) for path in pending)
buffer_bytes = max(1 << 20, min(_PREFETCH_READ_BYTES, -(-largest // threads)))
if available is not None:
# each worker's read buffer counts too: fewer workers before none
room = (int(available) << 20) // 2 - need
threads = max(1, min(threads, room // buffer_bytes))
if available is None or need + threads * buffer_bytes > (int(available) << 20) // 2:
if logger is not None:
logger.info(
"diffusion.prefetch: skipped, %.1f GiB uncached against %s MiB available host RAM",
need / 2**30,
available,
)
return None
# Thread i reads slice i of every file, files in load order, so the first file is warm first.
plan: list[list[tuple[str, int, int]]] = [[] for _ in range(threads)]
for path in pending:
for index, (start, end) in enumerate(_slices(os.path.getsize(path), threads)):
plan[index].append((path, start, end))
except Exception as exc: # noqa: BLE001 - a prefetch is only a hint
if logger is not None:
logger.debug("diffusion.prefetch: skipped (%r)", exc)
return None
stop = threading.Event()
def _worker(jobs: list[tuple[str, int, int]]) -> None:
buffer = bytearray(buffer_bytes)
view = memoryview(buffer)
try:
for path, start, end in jobs:
if stop.is_set():
return
mapped = _residency_map(path)
try:
with open(path, "rb", buffering = 0) as handle:
at = start
while at < end and not stop.is_set():
want = min(buffer_bytes, end - at)
if mapped is not None and _chunk_resident(mapped, at, want):
at += want
continue
handle.seek(at)
got = handle.readinto(view[:want])
if not got:
break
at += got
finally:
if mapped is not None:
mapped.close()
except Exception: # noqa: BLE001 - a prefetch is only a hint
pass
workers = []
for index, jobs in enumerate(plan):
if not jobs:
continue
thread = threading.Thread(
target = _worker, args = (jobs,), name = f"{_PREFETCH_THREAD_PREFIX}-{index}", daemon = True
)
thread.start()
workers.append(thread)
if logger is not None:
logger.info(
"diffusion.prefetch: warming %.1f GiB of %d file(s) on %d threads",
need / 2**30,
len(pending),
len(workers),
)
return LoadPrefetch(pending, need, workers, stop)
def stop_prefetch(handle: Optional[LoadPrefetch]) -> None:
if handle is not None:
try:
handle.stop()
except Exception: # noqa: BLE001
pass
def _snapshot_dir(
base: Optional[str], base_local_dir: Optional[str], cache_dir: Optional[str]
) -> Optional[str]:
if base_local_dir and os.path.isdir(base_local_dir):
return base_local_dir
if base and os.path.isdir(base):
return base
if not base:
return None
try:
from huggingface_hub import try_to_load_from_cache
for root in (cache_dir, None) if cache_dir else (None,):
hit = try_to_load_from_cache(base, "model_index.json", cache_dir = root)
if isinstance(hit, str):
return os.path.dirname(hit)
except Exception: # noqa: BLE001 - uncached (the load downloads it) or an unreadable cache
return None
return None
def pipeline_component_files(
snapshot: Optional[str],
*,
skip_denoiser: bool,
skip_components: Iterable[str] = (),
variant: Optional[str] = None,
) -> list[str]:
"""The weight files ``from_pretrained`` reads from a local diffusers snapshot: every component
``model_index.json`` names, text encoders first, then the VAE, then the denoiser. Cache only.
``variant="fp16"`` names the ``.fp16`` twins instead."""
if not snapshot:
return []
try:
with open(os.path.join(snapshot, "model_index.json"), "r", encoding = "utf-8") as handle:
index = json.load(handle)
except Exception: # noqa: BLE001
return []
components = [
name
for name, value in index.items()
if not name.startswith("_")
and isinstance(value, (list, tuple))
and len(value) == 2
and value[0]
]
def _rank(name: str) -> int:
if name.startswith("text_encoder"):
return 0
if name in _DENOISER_COMPONENTS:
return 2
return 1
skip = set(skip_components or ())
files: list[str] = []
for name in sorted(components, key = lambda n: (_rank(n), n)):
if skip_denoiser and name in _DENOISER_COMPONENTS:
continue
if name in skip:
continue
folder = os.path.join(snapshot, name)
if not os.path.isdir(folder):
continue
try:
entries = sorted(os.listdir(folder))
except OSError:
continue
pattern = _FP16_WEIGHT_RE if variant == "fp16" else _DEFAULT_WEIGHT_RE
for entry in entries:
if pattern.match(entry):
path = os.path.join(folder, entry)
if os.path.isfile(path):
files.append(path)
return files
def te_precast_components(fam: Any, base: str, te_quant_mode: Any, target: Any) -> frozenset:
"""Encoders a hosted pre-cast checkpoint replaces; a runtime cast still reads the dense shards."""
try:
from .diffusion_te_prequant import te_prequant_sources_for_base
return frozenset(
te_prequant_sources_for_base(fam, base, te_quant_mode = te_quant_mode, target = target)
)
except Exception: # noqa: BLE001 - a prefetch is only a hint
return frozenset()
def start_load_prefetch(
fam: Any,
base: Optional[str],
*,
base_local_dir: Optional[str] = None,
prequant_scheme: Optional[str] = None,
prequant_path_override: Optional[str] = None,
prequant_base_repo: Optional[str] = None,
text_encoders_replaced: Iterable[str] = (),
cache_dir: Optional[str] = None,
logger: Any = None,
variant: Optional[str] = None,
) -> Optional[LoadPrefetch]:
"""Prefetch what a diffusers pipeline load of ``base`` will read. Never raises."""
if not prefetch_enabled():
return None
try:
paths: list[str] = []
seeded = False
if prequant_scheme:
from .diffusion_denoiser_prequant import denoiser_prequant_source
source = denoiser_prequant_source(
fam,
prequant_scheme,
base_repo = prequant_base_repo or base,
path_override = prequant_path_override,
# Only a cached file is prefetched, so its header is judged from the cache, never the Hub.
local_files_only = True,
cache_dir = cache_dir,
)
if source is not None:
location = None
if getattr(source, "kind", None) == "repo":
from .diffusion_prequant import cached_checkpoint_path
location = cached_checkpoint_path(source, cache_dir = cache_dir)
else:
location = getattr(source, "location", None)
if location and os.path.isfile(str(location)):
paths.append(str(location))
seeded = True
paths += pipeline_component_files(
_snapshot_dir(base, base_local_dir, cache_dir),
skip_denoiser = seeded,
skip_components = text_encoders_replaced,
variant = variant,
)
return start_prefetch(paths, logger = logger)
except Exception as exc: # noqa: BLE001 - a prefetch is only a hint
if logger is not None:
logger.debug("diffusion.prefetch: skipped (%r)", exc)
return None
def _upload_candidates(modules: Sequence[Any], device: Any) -> list[Any]:
"""Plain, dense, contiguous CPU parameters and buffers worth a ring slot, each tensor once."""
import torch
out: list[Any] = []
seen: set[int] = set()
for module in modules:
if module is None or not hasattr(module, "parameters"):
continue
tensors = list(module.parameters()) + list(module.buffers())
for tensor in tensors:
if id(tensor) in seen:
continue
seen.add(id(tensor))
if type(tensor) not in (torch.Tensor, torch.nn.Parameter):
continue # subclasses (torchao) keep their own .to
if tensor.device.type != "cpu" or tensor.is_sparse or tensor.is_quantized:
continue
if not tensor.is_contiguous() or tensor.storage_offset() < 0:
continue
if tensor.numel() * tensor.element_size() < _UPLOAD_MIN_TENSOR_BYTES:
continue
out.append(tensor)
return out
def _wrapper_subclass_parts(tensor: Any) -> Optional[tuple[list[str], Any, list[Any]]]:
"""``(inner names, flatten context, inner tensors)`` of a CPU torchao-style wrapper subclass with plain dense
contiguous CPU inner tensors; else None. Its own ``.to`` copies inside ``__torch_dispatch__``, bypassing the ring."""
import torch
try:
from torch.utils._python_dispatch import is_traceable_wrapper_subclass
except ImportError: # noqa: BLE001 - an older torch keeps the subclass's own .to
return None
try:
if not is_traceable_wrapper_subclass(tensor) or tensor.device.type != "cpu":
return None
if not callable(getattr(type(tensor), "__tensor_unflatten__", None)):
return None
names, ctx = tensor.__tensor_flatten__()
inner = [getattr(tensor, name) for name in names]
except Exception: # noqa: BLE001
return None
# A context that records a device (torchao's ``device`` attribute) would come back still naming the host.
values = (
ctx.values() if isinstance(ctx, dict) else ctx if isinstance(ctx, (list, tuple)) else (ctx,)
)
if (isinstance(ctx, dict) and "device" in ctx) or any(
isinstance(v, torch.device) for v in values
):
return None
for part in inner:
if type(part) is not torch.Tensor or part.device.type != "cpu":
return None
if (
part.is_sparse
or part.is_quantized
or not part.is_contiguous()
or part.storage_offset() < 0
):
return None
return list(names), ctx, inner
def _subclass_candidates(
modules: Sequence[Any], plain_seen: set[int]
) -> list[tuple[Any, list[str], Any, list[Any]]]:
out: list[tuple[Any, list[str], Any, list[Any]]] = []
seen: set[int] = set()
for module in modules:
if module is None or not hasattr(module, "parameters"):
continue
for tensor in list(module.parameters()) + list(module.buffers()):
if id(tensor) in seen or id(tensor) in plain_seen:
continue
seen.add(id(tensor))
parts = _wrapper_subclass_parts(tensor)
if parts is not None:
out.append((tensor, *parts))
return out
def _ring_copy(sources: list[Any], device: Any) -> list[Any]:
"""Device copies of ``sources`` (byte-identical), read through a pinned ring on worker threads."""
from concurrent.futures import ThreadPoolExecutor
import torch
targets = [torch.empty_like(src, device = device) for src in sources]
jobs = []
for src, dst in zip(sources, targets):
nbytes = src.numel() * src.element_size()
src8 = src.detach().reshape(-1).view(torch.uint8)
dst8 = dst.view(-1).view(torch.uint8)
for offset in range(0, nbytes, _UPLOAD_CHUNK_BYTES):
jobs.append((src8, dst8, offset, min(_UPLOAD_CHUNK_BYTES, nbytes - offset)))
if not jobs:
return targets
buffers = max(1, min(_UPLOAD_BUFFERS, len(jobs)))
staging = [
torch.empty(_UPLOAD_CHUNK_BYTES, dtype = torch.uint8, pin_memory = True) for _ in range(buffers)
]
done: list[Any] = [None] * buffers
stream = torch.cuda.Stream(device = device)
def _fill(slot: int, src8: Any, offset: int, size: int) -> None:
event = done[slot]
if event is not None:
event.synchronize()
staging[slot][:size].copy_(src8[offset : offset + size])
try:
with ThreadPoolExecutor(
max_workers = _UPLOAD_THREADS, thread_name_prefix = "unsloth-upload"
) as pool:
pending = {}
for index in range(buffers):
src8, _dst8, offset, size = jobs[index]
pending[index] = pool.submit(_fill, index, src8, offset, size)
with torch.cuda.stream(stream):
for index, (src8, dst8, offset, size) in enumerate(jobs):
slot = index % buffers
pending.pop(index).result()
dst8[offset : offset + size].copy_(staging[slot][:size], non_blocking = True)
event = torch.cuda.Event()
event.record(stream)
done[slot] = event
following = index + buffers
if following < len(jobs):
nsrc8, _ndst8, noffset, nsize = jobs[following]
pending[following] = pool.submit(_fill, slot, nsrc8, noffset, nsize)
stream.synchronize()
finally:
try:
stream.synchronize()
except Exception: # noqa: BLE001
pass
del staging
try:
torch._C._host_emptyCache()
except Exception: # noqa: BLE001 - older torch keeps the ring cached
pass
# The default stream must not read the targets before the side stream wrote them.
torch.cuda.current_stream(device).wait_stream(stream)
return targets
def _same_device(a: Any, b: Any) -> bool:
import torch
a, b = torch.device(a), torch.device(b)
if a.type != b.type:
return False
if a.type != "cuda":
return a == b
index_a = a.index if a.index is not None else torch.cuda.current_device()
index_b = b.index if b.index is not None else torch.cuda.current_device()
return index_a == index_b
@contextlib.contextmanager
def fast_upload(
modules: Sequence[Any],
device: Any,
*,
logger: Any = None,
) -> Iterator[int]:
"""Inside the block, Tensor.to from CPU goes through a ring upload; placement itself is unchanged."""
staged: dict[int, tuple[Any, Any]] = {}
mode = None
try:
if fast_upload_enabled():
import torch
from .host_pinning import host_pinning_allowed
target = torch.device(device)
# The ring is pinned memory: an integrated GPU takes the stock ``.to`` instead.
if target.type == "cuda" and torch.cuda.is_available() and host_pinning_allowed(target):
if target.index is None:
target = torch.device("cuda", torch.cuda.current_device())
sources = _upload_candidates(modules, target)
subclasses = _subclass_candidates(modules, {id(t) for t in sources})
known = {id(t) for t in sources}
for _tensor, _names, _ctx, inner in subclasses:
for part in inner:
if id(part) not in known:
known.add(id(part))
sources.append(part)
total = sum(t.numel() * t.element_size() for t in sources)
if sources and total >= _UPLOAD_MIN_TOTAL_BYTES:
with torch.no_grad():
copies = _ring_copy(sources, target)
staged = {id(src): (src, dst) for src, dst in zip(sources, copies)}
wrapped = {
id(tensor): (tensor, names, ctx, inner)
for tensor, names, ctx, inner in subclasses
}
mode = _StagedTo(staged, target, wrapped)
if logger is not None:
logger.info(
"diffusion.fast_upload: staged %.1f GiB in %d tensor(s) onto %s%s",
total / 2**30,
len(sources),
target,
f" ({len(wrapped)} quantized weight(s))" if wrapped else "",
)
except Exception as exc: # noqa: BLE001 - the stock .to() still runs inside the block
if logger is not None:
logger.warning("diffusion.fast_upload: falling back to the stock copy (%s)", exc)
staged, mode = {}, None
try:
import torch
torch.cuda.empty_cache()
except Exception: # noqa: BLE001
pass
if mode is None:
yield 0
return
with mode:
yield len(staged)
staged.clear()
def _staged_to_class():
from torch.overrides import TorchFunctionMode
class _StagedToMode(TorchFunctionMode):
"""Answers ``Tensor.to`` for a staged source with its device copy; every other call runs as usual."""
def __init__(
self,
staged: dict[int, tuple[Any, Any]],
device: Any,
wrapped: Optional[dict[int, tuple[Any, list[str], Any, list[Any]]]] = None,
):
super().__init__()
self.staged = staged
self.device = device
self.wrapped = wrapped or {}
def _wrapped_copy(self, tensor: Any, device: Any, dtype: Any) -> Any:
hit = self.wrapped.get(id(tensor))
if hit is None or hit[0] is not tensor:
return None
if not _same_device(device, self.device) or (
dtype is not None and dtype != tensor.dtype
):
return None
_tensor, names, ctx, inner = hit
parts = {}
for name, part in zip(names, inner):
staged = self.staged.get(id(part))
if staged is None or staged[0] is not part:
return None
parts[name] = staged[1]
out = type(tensor).__tensor_unflatten__(parts, ctx, tensor.size(), tensor.stride())
if type(out) is not type(tensor) or not _same_device(out.device, self.device):
return None
return out
def __torch_function__(
self,
func,
types,
args = (),
kwargs = None,
):
import torch
kwargs = kwargs or {}
if func is torch.Tensor.to and args:
hit = self.staged.get(id(args[0]))
if hit is not None and hit[0] is args[0]:
try:
device, dtype, _non_blocking, memory_format = torch._C._nn._parse_to(
*args[1:], **kwargs
)
except Exception: # noqa: BLE001
device = None
if (
device is not None
and _same_device(device, self.device)
and (dtype is None or dtype == args[0].dtype)
and memory_format is None
and not kwargs.get("copy", False)
):
out = hit[1]
if args[0].requires_grad and torch.is_grad_enabled():
out = out.requires_grad_(True)
return out
elif self.wrapped and id(args[0]) in self.wrapped:
try:
device, dtype, _non_blocking, memory_format = torch._C._nn._parse_to(
*args[1:], **kwargs
)
out = None
if (
device is not None
and memory_format is None
and not kwargs.get("copy", False)
):
out = self._wrapped_copy(args[0], device, dtype)
except Exception: # noqa: BLE001 - the subclass's own .to below
out = None
if out is not None:
return out
return func(*args, **kwargs)
return _StagedToMode
def _StagedTo(
staged: dict[int, tuple[Any, Any]],
device: Any,
wrapped: Optional[dict[int, tuple[Any, list[str], Any, list[Any]]]] = None,
) -> Any:
return _staged_to_class()(staged, device, wrapped)