Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: prefetch an image load's weights and upload host tensors through a pinned ring (#12590)
Daniel Han124e89a64d68
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 1f1ff3445ec..476f1678b27 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -245,6 +245,7 @@ from .diffusion_precision import (torchao_quantize_importable,)from .diffusion_te_prequant import te_prequant_pipe_kwargs+from .diffusion_fast_load import start_load_prefetch, stop_prefetch, te_precast_componentsfrom .diffusion_flow_shift import apply_comfy_flow_shiftfrom .diffusion_text_length import (IDEOGRAM4_COMFY_GUIDANCE,@@ -2032,6 +2033,11 @@ class DiffusionBackend:# The INDEXED string, so _resolve_device_target can rebuild a selection an override would erase.return target.torch_device, target.dtype+ def _stop_load_prefetch(self) -> None:+ """End a load's page-cache prefetch (diffusion_fast_load.start_load_prefetch), if one is running."""+ stop_prefetch(getattr(self, "_load_prefetch", None))+ self._load_prefetch = None+def _raise_if_load_cancelled(self, token: int) -> None:# The epoch alone: unload() bumps _load_token before any teardown, so a load that started# after the eject carries the current token and was never cancelled.@@ -6328,6 +6334,20 @@ class DiffusionBackend:}if hf_token:pipe_kwargs["token"] = hf_token+ # Stopped once the pipeline is built.+ self._stop_load_prefetch()+ self._load_prefetch = start_load_prefetch(+ fam,+ _base_local_dir or fetch_base,+ prequant_scheme = pipeline_seed_scheme,+ prequant_path_override = transformer_prequant_path,+ prequant_base_repo = base,+ text_encoders_replaced = te_precast_components(+ fam, fetch_base, text_encoder_quant, target+ ),+ cache_dir = hub_cache_dir(),+ logger = logger,+ )if fam.name == HIDREAM_FAMILY_NAME:# The repo names a Llama text_encoder_4 it does not ship; supply it from the open# mirror@@ -6454,9 +6474,12 @@ class DiffusionBackend:)# The prefetched snapshot dir keeps from_pretrained off the hub (24 GB per FLUX.1# otherwise)- pipe = pipeline_cls.from_pretrained(- _base_local_dir or fetch_base, **pipe_kwargs- )+ try:+ pipe = pipeline_cls.from_pretrained(+ _base_local_dir or fetch_base, **pipe_kwargs+ )+ finally:+ self._stop_load_prefetch()if small_host is not None and small_host.engaged:plan = self._apply_small_host_route(pipe,@@ -7359,6 +7382,7 @@ class DiffusionBackend:_clear_exception_frames(exc)raisefinally:+ self._stop_load_prefetch()# Pre-commit failure: roll back the process-wide mutations (symmetric with _unload_locked).if not state_committed:# First: its gate sits in front of the CUDA-graph layer uninstalled by identity below.diff --git a/studio/backend/core/inference/diffusion_fast_load.py b/studio/backend/core/inference/diffusion_fast_load.pynew file mode 100644index 00000000000..f6cf8577e68--- /dev/null+++ b/studio/backend/core/inference/diffusion_fast_load.py@@ -0,0 +1,548 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Cold-start I/O for the image load path, byte-for-byte neutral: ``start_load_prefetch`` warms the page+cache with the files the load reads next (parallel slice reads beat mmap page faults); ``fast_upload``+moves plain host tensors through a pinned ring and hands them to the existing ``.to`` via a+``TorchFunctionMode``, so placement semantics stay the module's own. Kill switches+``UNSLOTH_DIFFUSION_PREFETCH=0`` / ``UNSLOTH_DIFFUSION_FAST_UPLOAD=0``.+"""++from __future__ import annotations++import contextlib+import json+import os+import re+import threading+from typing import Any, Iterable, Iterator, Optional, Sequence++PREFETCH_ENV = "UNSLOTH_DIFFUSION_PREFETCH"+FAST_UPLOAD_ENV = "UNSLOTH_DIFFUSION_FAST_UPLOAD"+_SWITCH_OFF = ("0", "off", "false", "no")++_PREFETCH_THREADS = 8+_PREFETCH_READ_BYTES = 64 << 20+_PREFETCH_MIN_BYTES = 256 << 20+_PREFETCH_THREAD_PREFIX = "unsloth-image-prefetch"++_UPLOAD_THREADS = 8+_UPLOAD_CHUNK_BYTES = 16 << 20+_UPLOAD_BUFFERS = 16+_UPLOAD_MIN_TENSOR_BYTES = 1 << 20+_UPLOAD_MIN_TOTAL_BYTES = 256 << 20++# variant=None: from_pretrained never opens *.fp16.safetensors-style twins.+_DEFAULT_WEIGHT_RE = re.compile(+ r"^(?:diffusion_pytorch_model|model)(?:-\d{5}-of-\d{5})?\.safetensors$"+)+_DENOISER_COMPONENTS = ("transformer", "unet")+++def _switch_on(env: str) -> bool:+ return (os.environ.get(env) or "").strip().lower() not in _SWITCH_OFF+++def prefetch_enabled() -> bool:+ return _switch_on(PREFETCH_ENV)+++def fast_upload_enabled() -> bool:+ return _switch_on(FAST_UPLOAD_ENV)+++class LoadPrefetch:+ """Handle on a running prefetch: ``stop`` ends the reads at the next request boundary."""++ def __init__(+ self, files: list[str], need: int, threads: list[threading.Thread], stop: threading.Event+ ):+ self.files = files+ self.need = need+ self.threads = threads+ self._stop = stop++ def stop(self) -> None:+ self._stop.set()++ def join(self, timeout: Optional[float] = None) -> bool:+ for thread in self.threads:+ thread.join(timeout)+ return not any(thread.is_alive() for thread in self.threads)+++def _uncached_bytes(path: str) -> int:+ from .video_ltx2 import _uncached_bytes as uncached+ return uncached(path)+++def _available_host_mib() -> Optional[int]:+ try:+ from .diffusion_memory import _available_system_memory_mib+ return _available_system_memory_mib()+ except Exception: # noqa: BLE001+ return None+++def _on_rotational_disk(path: str) -> bool:+ """True only when sysfs says the file's block device spins: parallel slice reads would seek-thrash it."""+ try:+ st = os.stat(path)+ base = f"/sys/dev/block/{os.major(st.st_dev)}:{os.minor(st.st_dev)}"+ for queue in (os.path.join(base, "queue"), os.path.join(base, "..", "queue")):+ flag = os.path.join(queue, "rotational")+ if os.path.isfile(flag):+ with open(flag, encoding = "utf-8") as handle:+ return handle.read().strip() == "1"+ except Exception: # noqa: BLE001 - unknown (non-Linux, overlay, network): not rotational+ pass+ return False+++def _slices(size: int, parts: int) -> list[tuple[int, int]]:+ """``parts`` contiguous [start, end) ranges covering ``size``, aligned to 1 MiB."""+ align = 1 << 20+ step = -(-size // parts)+ step = -(-step // align) * align+ return [(start, min(size, start + step)) for start in range(0, size, step)]+++def start_prefetch(+ paths: Sequence[str],+ *,+ logger: Any = None,+ threads: int = _PREFETCH_THREADS,+ min_bytes: int = _PREFETCH_MIN_BYTES,+) -> Optional[LoadPrefetch]:+ """Read the uncached parts of ``paths`` into the page cache on worker threads, in the given order.++ None when switched off, when nothing needs it, or when the uncached bytes exceed half the available+ host RAM (the reads would evict what the load itself needs). Never raises."""+ if not prefetch_enabled():+ return None+ try:+ seen: set[str] = set()+ files: list[str] = []+ for path in paths:+ if not path:+ continue+ real = os.path.realpath(path)+ if real in seen or not os.path.isfile(real):+ continue+ seen.add(real)+ files.append(real)+ pending = []+ need = 0+ for path in files:+ missing = _uncached_bytes(path)+ if missing > 0:+ pending.append(path)+ need += missing+ if not pending or need < min_bytes:+ return None+ if any(_on_rotational_disk(path) for path in pending):+ if logger is not None:+ logger.info("diffusion.prefetch: skipped, the weights are on a rotational disk")+ return None+ available = _available_host_mib()+ if available is None or need > (int(available) << 20) // 2:+ if logger is not None:+ logger.info(+ "diffusion.prefetch: skipped, %.1f GiB uncached against %s MiB available host RAM",+ need / 2**30,+ available,+ )+ return None+ threads = max(1, int(threads))+ # Thread i reads slice i of every file, files in load order, so the first file is warm first.+ plan: list[list[tuple[str, int, int]]] = [[] for _ in range(threads)]+ for path in pending:+ for index, (start, end) in enumerate(_slices(os.path.getsize(path), threads)):+ plan[index].append((path, start, end))+ except Exception as exc: # noqa: BLE001 - a prefetch is only a hint+ if logger is not None:+ logger.debug("diffusion.prefetch: skipped (%r)", exc)+ return None++ stop = threading.Event()++ def _worker(jobs: list[tuple[str, int, int]]) -> None:+ buffer = bytearray(_PREFETCH_READ_BYTES)+ view = memoryview(buffer)+ try:+ for path, start, end in jobs:+ if stop.is_set():+ return+ with open(path, "rb", buffering = 0) as handle:+ handle.seek(start)+ at = start+ while at < end and not stop.is_set():+ got = handle.readinto(view[: min(_PREFETCH_READ_BYTES, end - at)])+ if not got:+ break+ at += got+ except Exception: # noqa: BLE001 - a prefetch is only a hint+ pass++ workers = []+ for index, jobs in enumerate(plan):+ if not jobs:+ continue+ thread = threading.Thread(+ target = _worker, args = (jobs,), name = f"{_PREFETCH_THREAD_PREFIX}-{index}", daemon = True+ )+ thread.start()+ workers.append(thread)+ if logger is not None:+ logger.info(+ "diffusion.prefetch: warming %.1f GiB of %d file(s) on %d threads",+ need / 2**30,+ len(pending),+ len(workers),+ )+ return LoadPrefetch(pending, need, workers, stop)+++def stop_prefetch(handle: Optional[LoadPrefetch]) -> None:+ if handle is not None:+ try:+ handle.stop()+ except Exception: # noqa: BLE001+ pass+++def _snapshot_dir(+ base: Optional[str], base_local_dir: Optional[str], cache_dir: Optional[str]+) -> Optional[str]:+ if base_local_dir and os.path.isdir(base_local_dir):+ return base_local_dir+ if base and os.path.isdir(base):+ return base+ if not base:+ return None+ try:+ from huggingface_hub import try_to_load_from_cache+ for root in (cache_dir, None) if cache_dir else (None,):+ hit = try_to_load_from_cache(base, "model_index.json", cache_dir = root)+ if isinstance(hit, str):+ return os.path.dirname(hit)+ except Exception: # noqa: BLE001 - uncached (the load downloads it) or an unreadable cache+ return None+ return None+++def pipeline_component_files(+ snapshot: Optional[str],+ *,+ skip_denoiser: bool,+ skip_components: Iterable[str] = (),+) -> list[str]:+ """The weight files ``from_pretrained`` reads from a local diffusers snapshot: every component+ ``model_index.json`` names, text encoders first, then the VAE, then the denoiser. Cache only."""+ if not snapshot:+ return []+ try:+ with open(os.path.join(snapshot, "model_index.json"), "r", encoding = "utf-8") as handle:+ index = json.load(handle)+ except Exception: # noqa: BLE001+ return []+ components = [+ name+ for name, value in index.items()+ if not name.startswith("_")+ and isinstance(value, (list, tuple))+ and len(value) == 2+ and value[0]+ ]++ def _rank(name: str) -> int:+ if name.startswith("text_encoder"):+ return 0+ if name in _DENOISER_COMPONENTS:+ return 2+ return 1++ skip = set(skip_components or ())+ files: list[str] = []+ for name in sorted(components, key = lambda n: (_rank(n), n)):+ if skip_denoiser and name in _DENOISER_COMPONENTS:+ continue+ if name in skip:+ continue+ folder = os.path.join(snapshot, name)+ if not os.path.isdir(folder):+ continue+ try:+ entries = sorted(os.listdir(folder))+ except OSError:+ continue+ for entry in entries:+ if _DEFAULT_WEIGHT_RE.match(entry):+ path = os.path.join(folder, entry)+ if os.path.isfile(path):+ files.append(path)+ return files+++def te_precast_components(fam: Any, base: str, te_quant_mode: Any, target: Any) -> frozenset:+ """Encoders a hosted pre-cast checkpoint replaces; a runtime cast still reads the dense shards."""+ try:+ from .diffusion_te_prequant import te_prequant_sources_for_base+ return frozenset(+ te_prequant_sources_for_base(fam, base, te_quant_mode = te_quant_mode, target = target)+ )+ except Exception: # noqa: BLE001 - a prefetch is only a hint+ return frozenset()+++def start_load_prefetch(+ fam: Any,+ base: Optional[str],+ *,+ base_local_dir: Optional[str] = None,+ prequant_scheme: Optional[str] = None,+ prequant_path_override: Optional[str] = None,+ prequant_base_repo: Optional[str] = None,+ text_encoders_replaced: Iterable[str] = (),+ cache_dir: Optional[str] = None,+ logger: Any = None,+) -> Optional[LoadPrefetch]:+ """Prefetch what a diffusers pipeline load of ``base`` will read. Never raises."""+ if not prefetch_enabled():+ return None+ try:+ paths: list[str] = []+ seeded = False+ if prequant_scheme:+ from .diffusion_denoiser_prequant import denoiser_prequant_source+ source = denoiser_prequant_source(+ fam,+ prequant_scheme,+ base_repo = prequant_base_repo or base,+ path_override = prequant_path_override,+ )+ if source is not None:+ location = None+ if getattr(source, "kind", None) == "repo":+ from .diffusion_prequant import cached_checkpoint_path+ location = cached_checkpoint_path(source, cache_dir = cache_dir)+ else:+ location = getattr(source, "location", None)+ if location and os.path.isfile(str(location)):+ paths.append(str(location))+ seeded = True+ paths += pipeline_component_files(+ _snapshot_dir(base, base_local_dir, cache_dir),+ skip_denoiser = seeded,+ skip_components = text_encoders_replaced,+ )+ return start_prefetch(paths, logger = logger)+ except Exception as exc: # noqa: BLE001 - a prefetch is only a hint+ if logger is not None:+ logger.debug("diffusion.prefetch: skipped (%r)", exc)+ return None+++def _upload_candidates(modules: Sequence[Any], device: Any) -> list[Any]:+ """Plain, dense, contiguous CPU parameters and buffers worth a ring slot, each tensor once."""+ import torch++ out: list[Any] = []+ seen: set[int] = set()+ for module in modules:+ if module is None or not hasattr(module, "parameters"):+ continue+ tensors = list(module.parameters()) + list(module.buffers())+ for tensor in tensors:+ if id(tensor) in seen:+ continue+ seen.add(id(tensor))+ if type(tensor) not in (torch.Tensor, torch.nn.Parameter):+ continue # subclasses (torchao) keep their own .to+ if tensor.device.type != "cpu" or tensor.is_sparse or tensor.is_quantized:+ continue+ if not tensor.is_contiguous() or tensor.storage_offset() < 0:+ continue+ if tensor.numel() * tensor.element_size() < _UPLOAD_MIN_TENSOR_BYTES:+ continue+ out.append(tensor)+ return out+++def _ring_copy(sources: list[Any], device: Any) -> list[Any]:+ """Device copies of ``sources`` (byte-identical), read through a pinned ring on worker threads."""+ from concurrent.futures import ThreadPoolExecutor++ import torch++ targets = [torch.empty_like(src, device = device) for src in sources]+ jobs = []+ for src, dst in zip(sources, targets):+ nbytes = src.numel() * src.element_size()+ src8 = src.detach().reshape(-1).view(torch.uint8)+ dst8 = dst.view(-1).view(torch.uint8)+ for offset in range(0, nbytes, _UPLOAD_CHUNK_BYTES):+ jobs.append((src8, dst8, offset, min(_UPLOAD_CHUNK_BYTES, nbytes - offset)))+ if not jobs:+ return targets+ buffers = max(1, min(_UPLOAD_BUFFERS, len(jobs)))+ staging = [+ torch.empty(_UPLOAD_CHUNK_BYTES, dtype = torch.uint8, pin_memory = True) for _ in range(buffers)+ ]+ done: list[Any] = [None] * buffers+ stream = torch.cuda.Stream(device = device)++ def _fill(slot: int, src8: Any, offset: int, size: int) -> None:+ event = done[slot]+ if event is not None:+ event.synchronize()+ staging[slot][:size].copy_(src8[offset : offset + size])++ try:+ with ThreadPoolExecutor(+ max_workers = _UPLOAD_THREADS, thread_name_prefix = "unsloth-upload"+ ) as pool:+ pending = {}+ for index in range(buffers):+ src8, _dst8, offset, size = jobs[index]+ pending[index] = pool.submit(_fill, index, src8, offset, size)+ with torch.cuda.stream(stream):+ for index, (src8, dst8, offset, size) in enumerate(jobs):+ slot = index % buffers+ pending.pop(index).result()+ dst8[offset : offset + size].copy_(staging[slot][:size], non_blocking = True)+ event = torch.cuda.Event()+ event.record(stream)+ done[slot] = event+ following = index + buffers+ if following < len(jobs):+ nsrc8, _ndst8, noffset, nsize = jobs[following]+ pending[following] = pool.submit(_fill, slot, nsrc8, noffset, nsize)+ stream.synchronize()+ finally:+ try:+ stream.synchronize()+ except Exception: # noqa: BLE001+ pass+ del staging+ try:+ torch._C._host_emptyCache()+ except Exception: # noqa: BLE001 - older torch keeps the ring cached+ pass+ # The default stream must not read the targets before the side stream wrote them.+ torch.cuda.current_stream(device).wait_stream(stream)+ return targets+++def _same_device(a: Any, b: Any) -> bool:+ import torch++ a, b = torch.device(a), torch.device(b)+ if a.type != b.type:+ return False+ if a.type != "cuda":+ return a == b+ index_a = a.index if a.index is not None else torch.cuda.current_device()+ index_b = b.index if b.index is not None else torch.cuda.current_device()+ return index_a == index_b+++def fast_upload(+ modules: Sequence[Any],+ device: Any,+ *,+ logger: Any = None,+) -> Iterator[int]:+ """Inside the block, ``Tensor.to(device)`` on the modules' plain CPU tensors returns a ring-uploaded copy.++ Wrap the existing placement call (``module.to(device)`` / ``pipe.to(device)``): it still decides+ what moves and how parameters are re-wrapped; only the bytes arrive faster. Yields how many tensors+ were staged (0 = the block runs exactly as without this wrapper)."""+ staged: dict[int, tuple[Any, Any]] = {}+ mode = None+ try:+ if fast_upload_enabled():+ import torch+ target = torch.device(device)+ if target.type == "cuda" and torch.cuda.is_available():+ if target.index is None:+ target = torch.device("cuda", torch.cuda.current_device())+ sources = _upload_candidates(modules, target)+ total = sum(t.numel() * t.element_size() for t in sources)+ if sources and total >= _UPLOAD_MIN_TOTAL_BYTES:+ with torch.no_grad():+ copies = _ring_copy(sources, target)+ staged = {id(src): (src, dst) for src, dst in zip(sources, copies)}+ mode = _StagedTo(staged, target)+ if logger is not None:+ logger.info(+ "diffusion.fast_upload: staged %.1f GiB in %d tensor(s) onto %s",+ total / 2**30,+ len(sources),+ target,+ )+ except Exception as exc: # noqa: BLE001 - the stock .to() still runs inside the block+ if logger is not None:+ logger.warning("diffusion.fast_upload: falling back to the stock copy (%s)", exc)+ staged, mode = {}, None+ try:+ import torch+ torch.cuda.empty_cache()+ except Exception: # noqa: BLE001+ pass+ if mode is None:+ yield 0+ return+ with mode:+ yield len(staged)+ staged.clear()+++def _staged_to_class():+ from torch.overrides import TorchFunctionMode+ class _StagedToMode(TorchFunctionMode):+ """Answers ``Tensor.to`` for a staged source with its device copy; every other call runs as usual."""++ def __init__(self, staged: dict[int, tuple[Any, Any]], device: Any):+ super().__init__()+ self.staged = staged+ self.device = device++ def __torch_function__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ import torch++ kwargs = kwargs or {}+ if func is torch.Tensor.to and args:+ hit = self.staged.get(id(args[0]))+ if hit is not None and hit[0] is args[0]:+ try:+ device, dtype, _non_blocking, memory_format = torch._C._nn._parse_to(+ *args[1:], **kwargs+ )+ except Exception: # noqa: BLE001+ device = None+ if (+ device is not None+ and _same_device(device, self.device)+ and (dtype is None or dtype == args[0].dtype)+ and memory_format is None+ and not kwargs.get("copy", False)+ ):+ out = hit[1]+ if args[0].requires_grad and torch.is_grad_enabled():+ out = out.requires_grad_(True)+ return out+ return func(*args, **kwargs)++ return _StagedToMode+++def _StagedTo(staged: dict[int, tuple[Any, Any]], device: Any) -> Any:+ return _staged_to_class()(staged, device)diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex df2996ccb35..5b0498cbcbc 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -2873,7 +2873,12 @@ def apply_memory_plan(_fallback_to_model_offload()policy = OFFLOAD_MODELelse:- pipe.to(placement)+ from .diffusion_fast_load import fast_upload++ components = getattr(pipe, "components", None)+ modules = list(components.values()) if isinstance(components, dict) else []+ with fast_upload(modules, placement, logger = logger):+ pipe.to(placement)return policy, tiling_engageddiff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex ec44fb4a856..423ec583cd4 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -1265,7 +1265,10 @@ def load_prequantized_transformer(del state_dictdel ckpt- transformer = transformer.to(placement_device or device)+ from .diffusion_fast_load import fast_upload++ with fast_upload([transformer], placement_device or device, logger = logger):+ transformer = transformer.to(placement_device or device)if declares_rotation(metadata):try:import torchdiff --git a/studio/backend/hub/services/models/account_access.py b/studio/backend/hub/services/models/account_access.pyindex 98f3089b3f4..a537f55d1df 100644--- a/studio/backend/hub/services/models/account_access.py+++ b/studio/backend/hub/services/models/account_access.py@@ -180,9 +180,10 @@ def foreign_media_generations(account_id: str) -> int:total += sum(count for account, count in counts.items() if account != account_id)for holders in _generation_holders.values():total += sum(1 for holder in holders if holder != account_id)- # sys.modules, not an import: no video job is in flight before its module loads.+ # sys.modules, not an import: no job is in flight before (or while) its module loads.video = sys.modules.get("core.inference.video")- reserved = video.generation_account_in_flight() if video is not None else None+ in_flight = getattr(video, "generation_account_in_flight", None) if video is not None else None+ reserved = in_flight() if callable(in_flight) else Noneif reserved is not None and reserved != account_id:total += 1return totaldiff --git a/studio/backend/tests/test_diffusion_fast_load.py b/studio/backend/tests/test_diffusion_fast_load.pynew file mode 100644index 00000000000..70005007eff--- /dev/null+++ b/studio/backend/tests/test_diffusion_fast_load.py@@ -0,0 +1,367 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Image-load cold-start I/O: the page-cache prefetch and the pinned-ring upload (diffusion_fast_load)."""++import json+import threading++import pytest++from core.inference import diffusion_fast_load as fl++[email protected](autouse = True)+def _solid_state_disk(monkeypatch):+ # CI disks can report rotational=1; the rotational skip has its own test below.+ monkeypatch.setattr(fl, "_on_rotational_disk", lambda path: False)+++def _join_prefetch(handle, timeout = 30.0):+ assert handle is not None+ assert handle.join(timeout), "prefetch threads never finished"+ assert not [t for t in threading.enumerate() if t.name.startswith(fl._PREFETCH_THREAD_PREFIX)]+++def test_kill_switches(monkeypatch):+ for env, probe in (+ (fl.PREFETCH_ENV, fl.prefetch_enabled),+ (fl.FAST_UPLOAD_ENV, fl.fast_upload_enabled),+ ):+ monkeypatch.delenv(env, raising = False)+ assert probe()+ for off in ("0", "off", "false", "no", " OFF "):+ monkeypatch.setenv(env, off)+ assert not probe()+++def test_slices_cover_the_file_contiguously_on_mib_boundaries():+ for size, parts in ((1, 8), ((5 << 20) + 3, 8), (64 << 20, 8), ((100 << 20) + 1, 3)):+ got = fl._slices(size, parts)+ assert got[0][0] == 0 and got[-1][1] == size+ assert all(a[1] == b[0] for a, b in zip(got, got[1:]))+ assert all(start % (1 << 20) == 0 for start, _ in got)+ assert len(got) <= parts+++def test_prefetch_reads_every_uncached_byte_in_order(tmp_path, monkeypatch):+ files = []+ for name, size in (("dit.pt", (3 << 20) + 17), ("te.safetensors", 5 << 20)):+ path = tmp_path / name+ path.write_bytes(b"z" * size)+ files.append(str(path))+ read: list = []+ real_open = open++ class _Spy:+ def __init__(self, path, *args, **kwargs):+ self._handle = real_open(path, *args, **kwargs)+ self._path = path++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ self._handle.close()++ def seek(self, at):+ self._at = at+ return self._handle.seek(at)++ def readinto(self, view):+ got = self._handle.readinto(view)+ read.append((self._path, self._at, got))+ self._at += got+ return got++ monkeypatch.setattr(fl, "_uncached_bytes", lambda path: __import__("os").path.getsize(path))+ monkeypatch.setattr(fl, "_available_host_mib", lambda: 1 << 20)+ monkeypatch.setattr("builtins.open", _Spy)+ handle = fl.start_prefetch(+ files + [files[0], str(tmp_path / "missing")], min_bytes = 0, threads = 4+ )+ _join_prefetch(handle)+ assert handle.files == files # duplicates and missing paths dropped, order kept+ for path in files:+ size = __import__("os").path.getsize(path)+ covered = sorted((at, at + got) for p, at, got in read if p == path)+ assert covered[0][0] == 0 and covered[-1][1] == size+ assert all(a[1] == b[0] for a, b in zip(covered, covered[1:]))+++def test_prefetch_declines_cached_small_or_ram_tight_loads(tmp_path, monkeypatch):+ path = tmp_path / "w.safetensors"+ path.write_bytes(b"y" * (2 << 20))+ monkeypatch.setattr(fl, "_available_host_mib", lambda: 1 << 20)+ monkeypatch.setattr(fl, "_uncached_bytes", lambda p: 0)+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None # all cached+ monkeypatch.setattr(fl, "_uncached_bytes", lambda p: 2 << 20)+ assert fl.start_prefetch([str(path)]) is None # under the default floor+ monkeypatch.setattr(fl, "_available_host_mib", lambda: 3) # 2 MiB needed > half of 3 MiB+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None+ monkeypatch.setattr(fl, "_available_host_mib", lambda: None)+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None+ monkeypatch.setattr(fl, "_available_host_mib", lambda: 1 << 20)+ monkeypatch.setenv(fl.PREFETCH_ENV, "0")+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None+ monkeypatch.delenv(fl.PREFETCH_ENV)+ _join_prefetch(fl.start_prefetch([str(path)], min_bytes = 0))+ monkeypatch.setattr(+ fl, "_on_rotational_disk", lambda p: True+ ) # parallel slices would seek-thrash a disk+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None+++def _fake_snapshot(tmp_path):+ snap = tmp_path / "snap"+ index = {+ "_class_name": "FluxPipeline",+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],+ "text_encoder": ["transformers", "CLIPTextModel"],+ "text_encoder_2": ["transformers", "T5EncoderModel"],+ "tokenizer": ["transformers", "CLIPTokenizer"],+ "transformer": ["diffusers", "FluxTransformer2DModel"],+ "vae": ["diffusers", "AutoencoderKL"],+ "image_encoder": [None, None],+ }+ snap.mkdir()+ (snap / "model_index.json").write_text(json.dumps(index))+ layout = {+ "text_encoder": ["model.safetensors", "model.fp16.safetensors"],+ "text_encoder_2": [+ "model-00002-of-00002.safetensors",+ "model-00001-of-00002.safetensors",+ "index.json",+ ],+ "transformer": [+ "diffusion_pytorch_model.safetensors",+ "diffusion_pytorch_model.fp16-00001-of-00002.safetensors",+ ],+ "vae": ["diffusion_pytorch_model.safetensors", "diffusion_pytorch_model.bin"],+ "tokenizer": ["vocab.json"],+ }+ for folder, names in layout.items():+ (snap / folder).mkdir()+ for name in names:+ (snap / folder / name).write_bytes(b"w")+ return snap+++def test_component_files_are_the_weights_from_pretrained_reads_text_encoders_first(tmp_path):+ snap = _fake_snapshot(tmp_path)+ rel = lambda paths: [p[len(str(snap)) + 1 :] for p in paths] # noqa: E731+ assert rel(fl.pipeline_component_files(str(snap), skip_denoiser = False)) == [+ "text_encoder/model.safetensors",+ "text_encoder_2/model-00001-of-00002.safetensors",+ "text_encoder_2/model-00002-of-00002.safetensors",+ "vae/diffusion_pytorch_model.safetensors",+ "transformer/diffusion_pytorch_model.safetensors",+ ]+ assert rel(+ fl.pipeline_component_files(+ str(snap), skip_denoiser = True, skip_components = {"text_encoder_2"}+ )+ ) == [+ "text_encoder/model.safetensors",+ "vae/diffusion_pytorch_model.safetensors",+ ]+ assert fl.pipeline_component_files(None, skip_denoiser = False) == []+ assert fl.pipeline_component_files(str(tmp_path), skip_denoiser = False) == []+++def test_load_prefetch_puts_the_seeded_checkpoint_first_and_skips_the_dense_denoiser(+ tmp_path, monkeypatch+):+ snap = _fake_snapshot(tmp_path)+ ckpt = tmp_path / "FLUX.1-schnell-INT8.pt"+ ckpt.write_bytes(b"q")+ seen = {}++ class _Source:+ kind = "repo"++ import core.inference.diffusion_denoiser_prequant as dp+ import core.inference.diffusion_prequant as pq++ monkeypatch.setattr(+ dp,+ "denoiser_prequant_source",+ lambda fam, scheme, **kw: _Source() if scheme == "int8" else None,+ )+ monkeypatch.setattr(pq, "cached_checkpoint_path", lambda source, cache_dir = None: str(ckpt))+ monkeypatch.setattr(+ fl, "start_prefetch", lambda paths, logger = None: seen.setdefault("paths", list(paths))+ )++ fl.start_load_prefetch(object(), str(snap), prequant_scheme = "int8")+ assert seen.pop("paths")[0] == str(ckpt)+ fl.start_load_prefetch(object(), str(snap), prequant_scheme = "int8")+ assert not any("transformer/" in p for p in seen.pop("paths"))+ fl.start_load_prefetch(object(), str(snap), prequant_scheme = None)+ assert seen["paths"][-1].endswith("transformer/diffusion_pytorch_model.safetensors")+ monkeypatch.setattr(pq, "cached_checkpoint_path", lambda source, cache_dir = None: None)+ seen.clear()+ fl.start_load_prefetch(+ object(),+ str(snap),+ prequant_scheme = "int8",+ text_encoders_replaced = {"text_encoder", "text_encoder_2"},+ )+ assert [p.split("/snap/")[-1] for p in seen["paths"]] == [+ "vae/diffusion_pytorch_model.safetensors",+ "transformer/diffusion_pytorch_model.safetensors",+ ]+++def test_only_hosted_precast_encoders_leave_the_prefetch(monkeypatch):+ import types++ import torch+ import core.inference.diffusion_precision as prec+ from core.inference.diffusion_families import detect_family++ monkeypatch.setattr(prec, "te_quant_supported", lambda target, mode: True)+ fam = detect_family("black-forest-labs/FLUX.1-schnell")+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", dtype = torch.bfloat16)+ base = "black-forest-labs/FLUX.1-schnell"+ # A runtime cast loads the dense shards first, so they stay in the prefetch.+ for mode in (None, "off", "int8"):+ assert fl.te_precast_components(fam, base, mode, target) == frozenset()+ replaced = fl.te_precast_components(fam, base, "fp8", target)+ assert replaced and replaced <= {"text_encoder_2"}+ assert fl.te_precast_components(object(), base, "fp8", target) == frozenset()+++def test_load_prefetch_never_raises(monkeypatch):+ import core.inference.diffusion_denoiser_prequant as dp++ def _boom(*a, **k):+ raise RuntimeError("registry down")++ monkeypatch.setattr(dp, "denoiser_prequant_source", _boom)+ assert fl.start_load_prefetch(object(), "no/such-repo", prequant_scheme = "int8") is None+ fl.stop_prefetch(None)+++def test_fast_upload_is_inert_off_cuda():+ torch = pytest.importorskip("torch")+ module = torch.nn.Linear(4, 4)+ with fl.fast_upload([module], "cpu") as staged:+ module.to("cpu")+ assert staged == 0+++torch_cuda = pytest.mark.skipif(+ not (pytest.importorskip("torch").cuda.is_available()), reason = "needs a CUDA device"+)+++def _model(torch):+ torch.manual_seed(0)++ class _M(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.a = torch.nn.Linear(512, 1024).to(torch.bfloat16)+ self.b = torch.nn.Linear(1024, 512, bias = False).to(torch.bfloat16)+ self.c = torch.nn.Linear(512, 1024)+ self.c.weight = (+ self.a.weight+ ) # tied (a bf16 weight on an fp32 Linear is fine for .to())+ self.register_buffer("table", torch.randn(700, 1024))+ self.register_buffer("ids", torch.arange(300000, dtype = torch.int64))+ self.small = torch.nn.Parameter(torch.randn(3, 5), requires_grad = False)+ self.view_param = torch.nn.Parameter(+ torch.randn(1024, 600)[:, :512].clone().t(), requires_grad = False+ )++ return _M()+++@torch_cuda+def test_fast_upload_matches_a_plain_to_byte_for_byte(monkeypatch):+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)+ monkeypatch.setattr(fl, "_UPLOAD_CHUNK_BYTES", 1 << 20) # many chunks per tensor, ring wraps+ ref = _model(torch).to("cuda")+ got = _model(torch)+ names_before = {n: type(p) for n, p in got.named_parameters()}+ with fl.fast_upload([got, None, "tokenizer"], "cuda") as staged:+ out = got.to("cuda")+ assert out is got and staged > 0+ assert got.c.weight is got.a.weight # the tie survives+ ref_state, got_state = ref.state_dict(keep_vars = True), got.state_dict(keep_vars = True)+ assert ref_state.keys() == got_state.keys()+ for name, want in ref_state.items():+ have = got_state[name]+ assert (+ have.device.type == "cuda" and have.dtype == want.dtype and have.shape == want.shape+ ), name+ assert have.stride() == want.stride(), name+ assert have.requires_grad == want.requires_grad, name+ assert torch.equal(have, want), name+ assert {n: type(p) for n, p in got.named_parameters()} == names_before+++@torch_cuda+def test_fast_upload_leaves_other_to_calls_alone(monkeypatch):+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)+ module = _model(torch)+ weight = module.a.weight+ with fl.fast_upload([module], "cuda") as staged:+ assert staged > 0+ assert weight.to("cuda", dtype = torch.float32).dtype == torch.float32+ assert weight.to("cuda", copy = True).data_ptr() != weight.to("cuda").data_ptr()+ assert weight.to("cpu").device.type == "cpu"+ module.to("cuda")+ assert module.a.weight.device.type == "cuda"+++@torch_cuda+def test_fast_upload_kill_switch_and_failure_fall_back_to_the_stock_copy(monkeypatch):+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)+ monkeypatch.setenv(fl.FAST_UPLOAD_ENV, "0")+ module = _model(torch)+ with fl.fast_upload([module], "cuda") as staged:+ module.to("cuda")+ assert staged == 0 and module.a.weight.device.type == "cuda"+ monkeypatch.delenv(fl.FAST_UPLOAD_ENV)++ def _boom(*a, **k):+ raise RuntimeError("pinned memory exhausted")++ monkeypatch.setattr(fl, "_ring_copy", _boom)+ module = _model(torch)+ with fl.fast_upload([module], "cuda") as staged:+ module.to("cuda")+ assert staged == 0 and torch.equal(module.table, _model(torch).table.cuda())+++@torch_cuda+def test_fast_upload_passes_tensor_subclasses_through(monkeypatch):+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)++ class _Tagged(torch.Tensor):+ pass++ module = torch.nn.Linear(1024, 1024)+ module.weight = torch.nn.Parameter(+ module.weight.detach().as_subclass(_Tagged), requires_grad = False+ )+ assert fl._upload_candidates([module], "cuda") == [] or all(+ type(t) in (torch.Tensor, torch.nn.Parameter)+ for t in fl._upload_candidates([module], "cuda")+ )+ with fl.fast_upload([module], "cuda"):+ module.to("cuda")+ assert module.weight.device.type == "cuda"+++def test_rotational_lookup_never_raises(tmp_path):+ path = tmp_path / "w.safetensors"+ path.write_bytes(b"z")+ assert fl._on_rotational_disk(str(path)) in (True, False)+ assert fl._on_rotational_disk(str(tmp_path / "missing")) is Falsediff --git a/studio/backend/tests/test_media_generation_partial_video_import.py b/studio/backend/tests/test_media_generation_partial_video_import.pynew file mode 100644index 00000000000..f1ead5620c7--- /dev/null+++ b/studio/backend/tests/test_media_generation_partial_video_import.py@@ -0,0 +1,25 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A load posted while another thread is still importing core.inference.video must not 500 (AttributeError)."""++import sys+import types++from hub.services.models import account_access+++def test_a_video_module_still_importing_counts_as_no_video_job(monkeypatch):+ half_built = types.ModuleType("core.inference.video")+ monkeypatch.setitem(sys.modules, "core.inference.video", half_built)+ assert account_access.foreign_media_generations("someone") == 0+++def test_a_loaded_video_module_still_reports_its_in_flight_account(monkeypatch):+ loaded = types.ModuleType("core.inference.video")+ loaded.generation_account_in_flight = lambda: "other-account"+ monkeypatch.setitem(sys.modules, "core.inference.video", loaded)+ assert account_access.foreign_media_generations("someone") == 1+ assert account_access.foreign_media_generations("other-account") == 0+ loaded.generation_account_in_flight = lambda: None+ assert account_access.foreign_media_generations("someone") == 0