Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: prefetch an image load's weights and upload host tensors through a pinned ring (#12590)

Daniel Han124e89a64d68
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 1f1ff3445ec..476f1678b27 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -245,6 +245,7 @@ from .diffusion_precision import (
torchao_quantize_importable,
)
from .diffusion_te_prequant import te_prequant_pipe_kwargs
+from .diffusion_fast_load import start_load_prefetch, stop_prefetch, te_precast_components
from .diffusion_flow_shift import apply_comfy_flow_shift
from .diffusion_text_length import (
IDEOGRAM4_COMFY_GUIDANCE,
@@ -2032,6 +2033,11 @@ class DiffusionBackend:
# The INDEXED string, so _resolve_device_target can rebuild a selection an override would erase.
return target.torch_device, target.dtype
+ def _stop_load_prefetch(self) -> None:
+ """End a load's page-cache prefetch (diffusion_fast_load.start_load_prefetch), if one is running."""
+ stop_prefetch(getattr(self, "_load_prefetch", None))
+ self._load_prefetch = None
+
def _raise_if_load_cancelled(self, token: int) -> None:
# The epoch alone: unload() bumps _load_token before any teardown, so a load that started
# after the eject carries the current token and was never cancelled.
@@ -6328,6 +6334,20 @@ class DiffusionBackend:
}
if hf_token:
pipe_kwargs["token"] = hf_token
+ # Stopped once the pipeline is built.
+ self._stop_load_prefetch()
+ self._load_prefetch = start_load_prefetch(
+ fam,
+ _base_local_dir or fetch_base,
+ prequant_scheme = pipeline_seed_scheme,
+ prequant_path_override = transformer_prequant_path,
+ prequant_base_repo = base,
+ text_encoders_replaced = te_precast_components(
+ fam, fetch_base, text_encoder_quant, target
+ ),
+ cache_dir = hub_cache_dir(),
+ logger = logger,
+ )
if fam.name == HIDREAM_FAMILY_NAME:
# The repo names a Llama text_encoder_4 it does not ship; supply it from the open
# mirror
@@ -6454,9 +6474,12 @@ class DiffusionBackend:
)
# The prefetched snapshot dir keeps from_pretrained off the hub (24 GB per FLUX.1
# otherwise)
- pipe = pipeline_cls.from_pretrained(
- _base_local_dir or fetch_base, **pipe_kwargs
- )
+ try:
+ pipe = pipeline_cls.from_pretrained(
+ _base_local_dir or fetch_base, **pipe_kwargs
+ )
+ finally:
+ self._stop_load_prefetch()
if small_host is not None and small_host.engaged:
plan = self._apply_small_host_route(
pipe,
@@ -7359,6 +7382,7 @@ class DiffusionBackend:
_clear_exception_frames(exc)
raise
finally:
+ self._stop_load_prefetch()
# Pre-commit failure: roll back the process-wide mutations (symmetric with _unload_locked).
if not state_committed:
# First: its gate sits in front of the CUDA-graph layer uninstalled by identity below.
diff --git a/studio/backend/core/inference/diffusion_fast_load.py b/studio/backend/core/inference/diffusion_fast_load.py
new file mode 100644
index 00000000000..f6cf8577e68
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_fast_load.py
@@ -0,0 +1,548 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Cold-start I/O for the image load path, byte-for-byte neutral: ``start_load_prefetch`` warms the page
+cache with the files the load reads next (parallel slice reads beat mmap page faults); ``fast_upload``
+moves plain host tensors through a pinned ring and hands them to the existing ``.to`` via a
+``TorchFunctionMode``, so placement semantics stay the module's own. Kill switches
+``UNSLOTH_DIFFUSION_PREFETCH=0`` / ``UNSLOTH_DIFFUSION_FAST_UPLOAD=0``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import re
+import threading
+from typing import Any, Iterable, Iterator, Optional, Sequence
+
+PREFETCH_ENV = "UNSLOTH_DIFFUSION_PREFETCH"
+FAST_UPLOAD_ENV = "UNSLOTH_DIFFUSION_FAST_UPLOAD"
+_SWITCH_OFF = ("0", "off", "false", "no")
+
+_PREFETCH_THREADS = 8
+_PREFETCH_READ_BYTES = 64 << 20
+_PREFETCH_MIN_BYTES = 256 << 20
+_PREFETCH_THREAD_PREFIX = "unsloth-image-prefetch"
+
+_UPLOAD_THREADS = 8
+_UPLOAD_CHUNK_BYTES = 16 << 20
+_UPLOAD_BUFFERS = 16
+_UPLOAD_MIN_TENSOR_BYTES = 1 << 20
+_UPLOAD_MIN_TOTAL_BYTES = 256 << 20
+
+# variant=None: from_pretrained never opens *.fp16.safetensors-style twins.
+_DEFAULT_WEIGHT_RE = re.compile(
+ r"^(?:diffusion_pytorch_model|model)(?:-\d{5}-of-\d{5})?\.safetensors$"
+)
+_DENOISER_COMPONENTS = ("transformer", "unet")
+
+
+def _switch_on(env: str) -> bool:
+ return (os.environ.get(env) or "").strip().lower() not in _SWITCH_OFF
+
+
+def prefetch_enabled() -> bool:
+ return _switch_on(PREFETCH_ENV)
+
+
+def fast_upload_enabled() -> bool:
+ return _switch_on(FAST_UPLOAD_ENV)
+
+
+class LoadPrefetch:
+ """Handle on a running prefetch: ``stop`` ends the reads at the next request boundary."""
+
+ def __init__(
+ self, files: list[str], need: int, threads: list[threading.Thread], stop: threading.Event
+ ):
+ self.files = files
+ self.need = need
+ self.threads = threads
+ self._stop = stop
+
+ def stop(self) -> None:
+ self._stop.set()
+
+ def join(self, timeout: Optional[float] = None) -> bool:
+ for thread in self.threads:
+ thread.join(timeout)
+ return not any(thread.is_alive() for thread in self.threads)
+
+
+def _uncached_bytes(path: str) -> int:
+ from .video_ltx2 import _uncached_bytes as uncached
+ return uncached(path)
+
+
+def _available_host_mib() -> Optional[int]:
+ try:
+ from .diffusion_memory import _available_system_memory_mib
+ return _available_system_memory_mib()
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _on_rotational_disk(path: str) -> bool:
+ """True only when sysfs says the file's block device spins: parallel slice reads would seek-thrash it."""
+ try:
+ st = os.stat(path)
+ base = f"/sys/dev/block/{os.major(st.st_dev)}:{os.minor(st.st_dev)}"
+ for queue in (os.path.join(base, "queue"), os.path.join(base, "..", "queue")):
+ flag = os.path.join(queue, "rotational")
+ if os.path.isfile(flag):
+ with open(flag, encoding = "utf-8") as handle:
+ return handle.read().strip() == "1"
+ except Exception: # noqa: BLE001 - unknown (non-Linux, overlay, network): not rotational
+ pass
+ return False
+
+
+def _slices(size: int, parts: int) -> list[tuple[int, int]]:
+ """``parts`` contiguous [start, end) ranges covering ``size``, aligned to 1 MiB."""
+ align = 1 << 20
+ step = -(-size // parts)
+ step = -(-step // align) * align
+ return [(start, min(size, start + step)) for start in range(0, size, step)]
+
+
+def start_prefetch(
+ paths: Sequence[str],
+ *,
+ logger: Any = None,
+ threads: int = _PREFETCH_THREADS,
+ min_bytes: int = _PREFETCH_MIN_BYTES,
+) -> Optional[LoadPrefetch]:
+ """Read the uncached parts of ``paths`` into the page cache on worker threads, in the given order.
+
+ None when switched off, when nothing needs it, or when the uncached bytes exceed half the available
+ host RAM (the reads would evict what the load itself needs). Never raises."""
+ if not prefetch_enabled():
+ return None
+ try:
+ seen: set[str] = set()
+ files: list[str] = []
+ for path in paths:
+ if not path:
+ continue
+ real = os.path.realpath(path)
+ if real in seen or not os.path.isfile(real):
+ continue
+ seen.add(real)
+ files.append(real)
+ pending = []
+ need = 0
+ for path in files:
+ missing = _uncached_bytes(path)
+ if missing > 0:
+ pending.append(path)
+ need += missing
+ if not pending or need < min_bytes:
+ return None
+ if any(_on_rotational_disk(path) for path in pending):
+ if logger is not None:
+ logger.info("diffusion.prefetch: skipped, the weights are on a rotational disk")
+ return None
+ available = _available_host_mib()
+ if available is None or need > (int(available) << 20) // 2:
+ if logger is not None:
+ logger.info(
+ "diffusion.prefetch: skipped, %.1f GiB uncached against %s MiB available host RAM",
+ need / 2**30,
+ available,
+ )
+ return None
+ threads = max(1, int(threads))
+ # Thread i reads slice i of every file, files in load order, so the first file is warm first.
+ plan: list[list[tuple[str, int, int]]] = [[] for _ in range(threads)]
+ for path in pending:
+ for index, (start, end) in enumerate(_slices(os.path.getsize(path), threads)):
+ plan[index].append((path, start, end))
+ except Exception as exc: # noqa: BLE001 - a prefetch is only a hint
+ if logger is not None:
+ logger.debug("diffusion.prefetch: skipped (%r)", exc)
+ return None
+
+ stop = threading.Event()
+
+ def _worker(jobs: list[tuple[str, int, int]]) -> None:
+ buffer = bytearray(_PREFETCH_READ_BYTES)
+ view = memoryview(buffer)
+ try:
+ for path, start, end in jobs:
+ if stop.is_set():
+ return
+ with open(path, "rb", buffering = 0) as handle:
+ handle.seek(start)
+ at = start
+ while at < end and not stop.is_set():
+ got = handle.readinto(view[: min(_PREFETCH_READ_BYTES, end - at)])
+ if not got:
+ break
+ at += got
+ except Exception: # noqa: BLE001 - a prefetch is only a hint
+ pass
+
+ workers = []
+ for index, jobs in enumerate(plan):
+ if not jobs:
+ continue
+ thread = threading.Thread(
+ target = _worker, args = (jobs,), name = f"{_PREFETCH_THREAD_PREFIX}-{index}", daemon = True
+ )
+ thread.start()
+ workers.append(thread)
+ if logger is not None:
+ logger.info(
+ "diffusion.prefetch: warming %.1f GiB of %d file(s) on %d threads",
+ need / 2**30,
+ len(pending),
+ len(workers),
+ )
+ return LoadPrefetch(pending, need, workers, stop)
+
+
+def stop_prefetch(handle: Optional[LoadPrefetch]) -> None:
+ if handle is not None:
+ try:
+ handle.stop()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _snapshot_dir(
+ base: Optional[str], base_local_dir: Optional[str], cache_dir: Optional[str]
+) -> Optional[str]:
+ if base_local_dir and os.path.isdir(base_local_dir):
+ return base_local_dir
+ if base and os.path.isdir(base):
+ return base
+ if not base:
+ return None
+ try:
+ from huggingface_hub import try_to_load_from_cache
+ for root in (cache_dir, None) if cache_dir else (None,):
+ hit = try_to_load_from_cache(base, "model_index.json", cache_dir = root)
+ if isinstance(hit, str):
+ return os.path.dirname(hit)
+ except Exception: # noqa: BLE001 - uncached (the load downloads it) or an unreadable cache
+ return None
+ return None
+
+
+def pipeline_component_files(
+ snapshot: Optional[str],
+ *,
+ skip_denoiser: bool,
+ skip_components: Iterable[str] = (),
+) -> list[str]:
+ """The weight files ``from_pretrained`` reads from a local diffusers snapshot: every component
+ ``model_index.json`` names, text encoders first, then the VAE, then the denoiser. Cache only."""
+ if not snapshot:
+ return []
+ try:
+ with open(os.path.join(snapshot, "model_index.json"), "r", encoding = "utf-8") as handle:
+ index = json.load(handle)
+ except Exception: # noqa: BLE001
+ return []
+ components = [
+ name
+ for name, value in index.items()
+ if not name.startswith("_")
+ and isinstance(value, (list, tuple))
+ and len(value) == 2
+ and value[0]
+ ]
+
+ def _rank(name: str) -> int:
+ if name.startswith("text_encoder"):
+ return 0
+ if name in _DENOISER_COMPONENTS:
+ return 2
+ return 1
+
+ skip = set(skip_components or ())
+ files: list[str] = []
+ for name in sorted(components, key = lambda n: (_rank(n), n)):
+ if skip_denoiser and name in _DENOISER_COMPONENTS:
+ continue
+ if name in skip:
+ continue
+ folder = os.path.join(snapshot, name)
+ if not os.path.isdir(folder):
+ continue
+ try:
+ entries = sorted(os.listdir(folder))
+ except OSError:
+ continue
+ for entry in entries:
+ if _DEFAULT_WEIGHT_RE.match(entry):
+ path = os.path.join(folder, entry)
+ if os.path.isfile(path):
+ files.append(path)
+ return files
+
+
+def te_precast_components(fam: Any, base: str, te_quant_mode: Any, target: Any) -> frozenset:
+ """Encoders a hosted pre-cast checkpoint replaces; a runtime cast still reads the dense shards."""
+ try:
+ from .diffusion_te_prequant import te_prequant_sources_for_base
+ return frozenset(
+ te_prequant_sources_for_base(fam, base, te_quant_mode = te_quant_mode, target = target)
+ )
+ except Exception: # noqa: BLE001 - a prefetch is only a hint
+ return frozenset()
+
+
+def start_load_prefetch(
+ fam: Any,
+ base: Optional[str],
+ *,
+ base_local_dir: Optional[str] = None,
+ prequant_scheme: Optional[str] = None,
+ prequant_path_override: Optional[str] = None,
+ prequant_base_repo: Optional[str] = None,
+ text_encoders_replaced: Iterable[str] = (),
+ cache_dir: Optional[str] = None,
+ logger: Any = None,
+) -> Optional[LoadPrefetch]:
+ """Prefetch what a diffusers pipeline load of ``base`` will read. Never raises."""
+ if not prefetch_enabled():
+ return None
+ try:
+ paths: list[str] = []
+ seeded = False
+ if prequant_scheme:
+ from .diffusion_denoiser_prequant import denoiser_prequant_source
+ source = denoiser_prequant_source(
+ fam,
+ prequant_scheme,
+ base_repo = prequant_base_repo or base,
+ path_override = prequant_path_override,
+ )
+ if source is not None:
+ location = None
+ if getattr(source, "kind", None) == "repo":
+ from .diffusion_prequant import cached_checkpoint_path
+ location = cached_checkpoint_path(source, cache_dir = cache_dir)
+ else:
+ location = getattr(source, "location", None)
+ if location and os.path.isfile(str(location)):
+ paths.append(str(location))
+ seeded = True
+ paths += pipeline_component_files(
+ _snapshot_dir(base, base_local_dir, cache_dir),
+ skip_denoiser = seeded,
+ skip_components = text_encoders_replaced,
+ )
+ return start_prefetch(paths, logger = logger)
+ except Exception as exc: # noqa: BLE001 - a prefetch is only a hint
+ if logger is not None:
+ logger.debug("diffusion.prefetch: skipped (%r)", exc)
+ return None
+
+
+def _upload_candidates(modules: Sequence[Any], device: Any) -> list[Any]:
+ """Plain, dense, contiguous CPU parameters and buffers worth a ring slot, each tensor once."""
+ import torch
+
+ out: list[Any] = []
+ seen: set[int] = set()
+ for module in modules:
+ if module is None or not hasattr(module, "parameters"):
+ continue
+ tensors = list(module.parameters()) + list(module.buffers())
+ for tensor in tensors:
+ if id(tensor) in seen:
+ continue
+ seen.add(id(tensor))
+ if type(tensor) not in (torch.Tensor, torch.nn.Parameter):
+ continue # subclasses (torchao) keep their own .to
+ if tensor.device.type != "cpu" or tensor.is_sparse or tensor.is_quantized:
+ continue
+ if not tensor.is_contiguous() or tensor.storage_offset() < 0:
+ continue
+ if tensor.numel() * tensor.element_size() < _UPLOAD_MIN_TENSOR_BYTES:
+ continue
+ out.append(tensor)
+ return out
+
+
+def _ring_copy(sources: list[Any], device: Any) -> list[Any]:
+ """Device copies of ``sources`` (byte-identical), read through a pinned ring on worker threads."""
+ from concurrent.futures import ThreadPoolExecutor
+
+ import torch
+
+ targets = [torch.empty_like(src, device = device) for src in sources]
+ jobs = []
+ for src, dst in zip(sources, targets):
+ nbytes = src.numel() * src.element_size()
+ src8 = src.detach().reshape(-1).view(torch.uint8)
+ dst8 = dst.view(-1).view(torch.uint8)
+ for offset in range(0, nbytes, _UPLOAD_CHUNK_BYTES):
+ jobs.append((src8, dst8, offset, min(_UPLOAD_CHUNK_BYTES, nbytes - offset)))
+ if not jobs:
+ return targets
+ buffers = max(1, min(_UPLOAD_BUFFERS, len(jobs)))
+ staging = [
+ torch.empty(_UPLOAD_CHUNK_BYTES, dtype = torch.uint8, pin_memory = True) for _ in range(buffers)
+ ]
+ done: list[Any] = [None] * buffers
+ stream = torch.cuda.Stream(device = device)
+
+ def _fill(slot: int, src8: Any, offset: int, size: int) -> None:
+ event = done[slot]
+ if event is not None:
+ event.synchronize()
+ staging[slot][:size].copy_(src8[offset : offset + size])
+
+ try:
+ with ThreadPoolExecutor(
+ max_workers = _UPLOAD_THREADS, thread_name_prefix = "unsloth-upload"
+ ) as pool:
+ pending = {}
+ for index in range(buffers):
+ src8, _dst8, offset, size = jobs[index]
+ pending[index] = pool.submit(_fill, index, src8, offset, size)
+ with torch.cuda.stream(stream):
+ for index, (src8, dst8, offset, size) in enumerate(jobs):
+ slot = index % buffers
+ pending.pop(index).result()
+ dst8[offset : offset + size].copy_(staging[slot][:size], non_blocking = True)
+ event = torch.cuda.Event()
+ event.record(stream)
+ done[slot] = event
+ following = index + buffers
+ if following < len(jobs):
+ nsrc8, _ndst8, noffset, nsize = jobs[following]
+ pending[following] = pool.submit(_fill, slot, nsrc8, noffset, nsize)
+ stream.synchronize()
+ finally:
+ try:
+ stream.synchronize()
+ except Exception: # noqa: BLE001
+ pass
+ del staging
+ try:
+ torch._C._host_emptyCache()
+ except Exception: # noqa: BLE001 - older torch keeps the ring cached
+ pass
+ # The default stream must not read the targets before the side stream wrote them.
+ torch.cuda.current_stream(device).wait_stream(stream)
+ return targets
+
+
+def _same_device(a: Any, b: Any) -> bool:
+ import torch
+
+ a, b = torch.device(a), torch.device(b)
+ if a.type != b.type:
+ return False
+ if a.type != "cuda":
+ return a == b
+ index_a = a.index if a.index is not None else torch.cuda.current_device()
+ index_b = b.index if b.index is not None else torch.cuda.current_device()
+ return index_a == index_b
+
+
+def fast_upload(
+ modules: Sequence[Any],
+ device: Any,
+ *,
+ logger: Any = None,
+) -> Iterator[int]:
+ """Inside the block, ``Tensor.to(device)`` on the modules' plain CPU tensors returns a ring-uploaded copy.
+
+ Wrap the existing placement call (``module.to(device)`` / ``pipe.to(device)``): it still decides
+ what moves and how parameters are re-wrapped; only the bytes arrive faster. Yields how many tensors
+ were staged (0 = the block runs exactly as without this wrapper)."""
+ staged: dict[int, tuple[Any, Any]] = {}
+ mode = None
+ try:
+ if fast_upload_enabled():
+ import torch
+ target = torch.device(device)
+ if target.type == "cuda" and torch.cuda.is_available():
+ if target.index is None:
+ target = torch.device("cuda", torch.cuda.current_device())
+ sources = _upload_candidates(modules, target)
+ total = sum(t.numel() * t.element_size() for t in sources)
+ if sources and total >= _UPLOAD_MIN_TOTAL_BYTES:
+ with torch.no_grad():
+ copies = _ring_copy(sources, target)
+ staged = {id(src): (src, dst) for src, dst in zip(sources, copies)}
+ mode = _StagedTo(staged, target)
+ if logger is not None:
+ logger.info(
+ "diffusion.fast_upload: staged %.1f GiB in %d tensor(s) onto %s",
+ total / 2**30,
+ len(sources),
+ target,
+ )
+ except Exception as exc: # noqa: BLE001 - the stock .to() still runs inside the block
+ if logger is not None:
+ logger.warning("diffusion.fast_upload: falling back to the stock copy (%s)", exc)
+ staged, mode = {}, None
+ try:
+ import torch
+ torch.cuda.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+ if mode is None:
+ yield 0
+ return
+ with mode:
+ yield len(staged)
+ staged.clear()
+
+
+def _staged_to_class():
+ from torch.overrides import TorchFunctionMode
+ class _StagedToMode(TorchFunctionMode):
+ """Answers ``Tensor.to`` for a staged source with its device copy; every other call runs as usual."""
+
+ def __init__(self, staged: dict[int, tuple[Any, Any]], device: Any):
+ super().__init__()
+ self.staged = staged
+ self.device = device
+
+ def __torch_function__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ import torch
+
+ kwargs = kwargs or {}
+ if func is torch.Tensor.to and args:
+ hit = self.staged.get(id(args[0]))
+ if hit is not None and hit[0] is args[0]:
+ try:
+ device, dtype, _non_blocking, memory_format = torch._C._nn._parse_to(
+ *args[1:], **kwargs
+ )
+ except Exception: # noqa: BLE001
+ device = None
+ if (
+ device is not None
+ and _same_device(device, self.device)
+ and (dtype is None or dtype == args[0].dtype)
+ and memory_format is None
+ and not kwargs.get("copy", False)
+ ):
+ out = hit[1]
+ if args[0].requires_grad and torch.is_grad_enabled():
+ out = out.requires_grad_(True)
+ return out
+ return func(*args, **kwargs)
+
+ return _StagedToMode
+
+
+def _StagedTo(staged: dict[int, tuple[Any, Any]], device: Any) -> Any:
+ return _staged_to_class()(staged, device)
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index df2996ccb35..5b0498cbcbc 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -2873,7 +2873,12 @@ def apply_memory_plan(
_fallback_to_model_offload()
policy = OFFLOAD_MODEL
else:
- pipe.to(placement)
+ from .diffusion_fast_load import fast_upload
+
+ components = getattr(pipe, "components", None)
+ modules = list(components.values()) if isinstance(components, dict) else []
+ with fast_upload(modules, placement, logger = logger):
+ pipe.to(placement)
return policy, tiling_engaged
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index ec44fb4a856..423ec583cd4 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -1265,7 +1265,10 @@ def load_prequantized_transformer(
del state_dict
del ckpt
- transformer = transformer.to(placement_device or device)
+ from .diffusion_fast_load import fast_upload
+
+ with fast_upload([transformer], placement_device or device, logger = logger):
+ transformer = transformer.to(placement_device or device)
if declares_rotation(metadata):
try:
import torch
diff --git a/studio/backend/hub/services/models/account_access.py b/studio/backend/hub/services/models/account_access.py
index 98f3089b3f4..a537f55d1df 100644
--- a/studio/backend/hub/services/models/account_access.py
+++ b/studio/backend/hub/services/models/account_access.py
@@ -180,9 +180,10 @@ def foreign_media_generations(account_id: str) -> int:
total += sum(count for account, count in counts.items() if account != account_id)
for holders in _generation_holders.values():
total += sum(1 for holder in holders if holder != account_id)
- # sys.modules, not an import: no video job is in flight before its module loads.
+ # sys.modules, not an import: no job is in flight before (or while) its module loads.
video = sys.modules.get("core.inference.video")
- reserved = video.generation_account_in_flight() if video is not None else None
+ in_flight = getattr(video, "generation_account_in_flight", None) if video is not None else None
+ reserved = in_flight() if callable(in_flight) else None
if reserved is not None and reserved != account_id:
total += 1
return total
diff --git a/studio/backend/tests/test_diffusion_fast_load.py b/studio/backend/tests/test_diffusion_fast_load.py
new file mode 100644
index 00000000000..70005007eff
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_fast_load.py
@@ -0,0 +1,367 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Image-load cold-start I/O: the page-cache prefetch and the pinned-ring upload (diffusion_fast_load)."""
+
+import json
+import threading
+
+import pytest
+
+from core.inference import diffusion_fast_load as fl
+
+
[email protected](autouse = True)
+def _solid_state_disk(monkeypatch):
+ # CI disks can report rotational=1; the rotational skip has its own test below.
+ monkeypatch.setattr(fl, "_on_rotational_disk", lambda path: False)
+
+
+def _join_prefetch(handle, timeout = 30.0):
+ assert handle is not None
+ assert handle.join(timeout), "prefetch threads never finished"
+ assert not [t for t in threading.enumerate() if t.name.startswith(fl._PREFETCH_THREAD_PREFIX)]
+
+
+def test_kill_switches(monkeypatch):
+ for env, probe in (
+ (fl.PREFETCH_ENV, fl.prefetch_enabled),
+ (fl.FAST_UPLOAD_ENV, fl.fast_upload_enabled),
+ ):
+ monkeypatch.delenv(env, raising = False)
+ assert probe()
+ for off in ("0", "off", "false", "no", " OFF "):
+ monkeypatch.setenv(env, off)
+ assert not probe()
+
+
+def test_slices_cover_the_file_contiguously_on_mib_boundaries():
+ for size, parts in ((1, 8), ((5 << 20) + 3, 8), (64 << 20, 8), ((100 << 20) + 1, 3)):
+ got = fl._slices(size, parts)
+ assert got[0][0] == 0 and got[-1][1] == size
+ assert all(a[1] == b[0] for a, b in zip(got, got[1:]))
+ assert all(start % (1 << 20) == 0 for start, _ in got)
+ assert len(got) <= parts
+
+
+def test_prefetch_reads_every_uncached_byte_in_order(tmp_path, monkeypatch):
+ files = []
+ for name, size in (("dit.pt", (3 << 20) + 17), ("te.safetensors", 5 << 20)):
+ path = tmp_path / name
+ path.write_bytes(b"z" * size)
+ files.append(str(path))
+ read: list = []
+ real_open = open
+
+ class _Spy:
+ def __init__(self, path, *args, **kwargs):
+ self._handle = real_open(path, *args, **kwargs)
+ self._path = path
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ self._handle.close()
+
+ def seek(self, at):
+ self._at = at
+ return self._handle.seek(at)
+
+ def readinto(self, view):
+ got = self._handle.readinto(view)
+ read.append((self._path, self._at, got))
+ self._at += got
+ return got
+
+ monkeypatch.setattr(fl, "_uncached_bytes", lambda path: __import__("os").path.getsize(path))
+ monkeypatch.setattr(fl, "_available_host_mib", lambda: 1 << 20)
+ monkeypatch.setattr("builtins.open", _Spy)
+ handle = fl.start_prefetch(
+ files + [files[0], str(tmp_path / "missing")], min_bytes = 0, threads = 4
+ )
+ _join_prefetch(handle)
+ assert handle.files == files # duplicates and missing paths dropped, order kept
+ for path in files:
+ size = __import__("os").path.getsize(path)
+ covered = sorted((at, at + got) for p, at, got in read if p == path)
+ assert covered[0][0] == 0 and covered[-1][1] == size
+ assert all(a[1] == b[0] for a, b in zip(covered, covered[1:]))
+
+
+def test_prefetch_declines_cached_small_or_ram_tight_loads(tmp_path, monkeypatch):
+ path = tmp_path / "w.safetensors"
+ path.write_bytes(b"y" * (2 << 20))
+ monkeypatch.setattr(fl, "_available_host_mib", lambda: 1 << 20)
+ monkeypatch.setattr(fl, "_uncached_bytes", lambda p: 0)
+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None # all cached
+ monkeypatch.setattr(fl, "_uncached_bytes", lambda p: 2 << 20)
+ assert fl.start_prefetch([str(path)]) is None # under the default floor
+ monkeypatch.setattr(fl, "_available_host_mib", lambda: 3) # 2 MiB needed > half of 3 MiB
+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None
+ monkeypatch.setattr(fl, "_available_host_mib", lambda: None)
+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None
+ monkeypatch.setattr(fl, "_available_host_mib", lambda: 1 << 20)
+ monkeypatch.setenv(fl.PREFETCH_ENV, "0")
+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None
+ monkeypatch.delenv(fl.PREFETCH_ENV)
+ _join_prefetch(fl.start_prefetch([str(path)], min_bytes = 0))
+ monkeypatch.setattr(
+ fl, "_on_rotational_disk", lambda p: True
+ ) # parallel slices would seek-thrash a disk
+ assert fl.start_prefetch([str(path)], min_bytes = 0) is None
+
+
+def _fake_snapshot(tmp_path):
+ snap = tmp_path / "snap"
+ index = {
+ "_class_name": "FluxPipeline",
+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
+ "text_encoder": ["transformers", "CLIPTextModel"],
+ "text_encoder_2": ["transformers", "T5EncoderModel"],
+ "tokenizer": ["transformers", "CLIPTokenizer"],
+ "transformer": ["diffusers", "FluxTransformer2DModel"],
+ "vae": ["diffusers", "AutoencoderKL"],
+ "image_encoder": [None, None],
+ }
+ snap.mkdir()
+ (snap / "model_index.json").write_text(json.dumps(index))
+ layout = {
+ "text_encoder": ["model.safetensors", "model.fp16.safetensors"],
+ "text_encoder_2": [
+ "model-00002-of-00002.safetensors",
+ "model-00001-of-00002.safetensors",
+ "index.json",
+ ],
+ "transformer": [
+ "diffusion_pytorch_model.safetensors",
+ "diffusion_pytorch_model.fp16-00001-of-00002.safetensors",
+ ],
+ "vae": ["diffusion_pytorch_model.safetensors", "diffusion_pytorch_model.bin"],
+ "tokenizer": ["vocab.json"],
+ }
+ for folder, names in layout.items():
+ (snap / folder).mkdir()
+ for name in names:
+ (snap / folder / name).write_bytes(b"w")
+ return snap
+
+
+def test_component_files_are_the_weights_from_pretrained_reads_text_encoders_first(tmp_path):
+ snap = _fake_snapshot(tmp_path)
+ rel = lambda paths: [p[len(str(snap)) + 1 :] for p in paths] # noqa: E731
+ assert rel(fl.pipeline_component_files(str(snap), skip_denoiser = False)) == [
+ "text_encoder/model.safetensors",
+ "text_encoder_2/model-00001-of-00002.safetensors",
+ "text_encoder_2/model-00002-of-00002.safetensors",
+ "vae/diffusion_pytorch_model.safetensors",
+ "transformer/diffusion_pytorch_model.safetensors",
+ ]
+ assert rel(
+ fl.pipeline_component_files(
+ str(snap), skip_denoiser = True, skip_components = {"text_encoder_2"}
+ )
+ ) == [
+ "text_encoder/model.safetensors",
+ "vae/diffusion_pytorch_model.safetensors",
+ ]
+ assert fl.pipeline_component_files(None, skip_denoiser = False) == []
+ assert fl.pipeline_component_files(str(tmp_path), skip_denoiser = False) == []
+
+
+def test_load_prefetch_puts_the_seeded_checkpoint_first_and_skips_the_dense_denoiser(
+ tmp_path, monkeypatch
+):
+ snap = _fake_snapshot(tmp_path)
+ ckpt = tmp_path / "FLUX.1-schnell-INT8.pt"
+ ckpt.write_bytes(b"q")
+ seen = {}
+
+ class _Source:
+ kind = "repo"
+
+ import core.inference.diffusion_denoiser_prequant as dp
+ import core.inference.diffusion_prequant as pq
+
+ monkeypatch.setattr(
+ dp,
+ "denoiser_prequant_source",
+ lambda fam, scheme, **kw: _Source() if scheme == "int8" else None,
+ )
+ monkeypatch.setattr(pq, "cached_checkpoint_path", lambda source, cache_dir = None: str(ckpt))
+ monkeypatch.setattr(
+ fl, "start_prefetch", lambda paths, logger = None: seen.setdefault("paths", list(paths))
+ )
+
+ fl.start_load_prefetch(object(), str(snap), prequant_scheme = "int8")
+ assert seen.pop("paths")[0] == str(ckpt)
+ fl.start_load_prefetch(object(), str(snap), prequant_scheme = "int8")
+ assert not any("transformer/" in p for p in seen.pop("paths"))
+ fl.start_load_prefetch(object(), str(snap), prequant_scheme = None)
+ assert seen["paths"][-1].endswith("transformer/diffusion_pytorch_model.safetensors")
+ monkeypatch.setattr(pq, "cached_checkpoint_path", lambda source, cache_dir = None: None)
+ seen.clear()
+ fl.start_load_prefetch(
+ object(),
+ str(snap),
+ prequant_scheme = "int8",
+ text_encoders_replaced = {"text_encoder", "text_encoder_2"},
+ )
+ assert [p.split("/snap/")[-1] for p in seen["paths"]] == [
+ "vae/diffusion_pytorch_model.safetensors",
+ "transformer/diffusion_pytorch_model.safetensors",
+ ]
+
+
+def test_only_hosted_precast_encoders_leave_the_prefetch(monkeypatch):
+ import types
+
+ import torch
+ import core.inference.diffusion_precision as prec
+ from core.inference.diffusion_families import detect_family
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda target, mode: True)
+ fam = detect_family("black-forest-labs/FLUX.1-schnell")
+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", dtype = torch.bfloat16)
+ base = "black-forest-labs/FLUX.1-schnell"
+ # A runtime cast loads the dense shards first, so they stay in the prefetch.
+ for mode in (None, "off", "int8"):
+ assert fl.te_precast_components(fam, base, mode, target) == frozenset()
+ replaced = fl.te_precast_components(fam, base, "fp8", target)
+ assert replaced and replaced <= {"text_encoder_2"}
+ assert fl.te_precast_components(object(), base, "fp8", target) == frozenset()
+
+
+def test_load_prefetch_never_raises(monkeypatch):
+ import core.inference.diffusion_denoiser_prequant as dp
+
+ def _boom(*a, **k):
+ raise RuntimeError("registry down")
+
+ monkeypatch.setattr(dp, "denoiser_prequant_source", _boom)
+ assert fl.start_load_prefetch(object(), "no/such-repo", prequant_scheme = "int8") is None
+ fl.stop_prefetch(None)
+
+
+def test_fast_upload_is_inert_off_cuda():
+ torch = pytest.importorskip("torch")
+ module = torch.nn.Linear(4, 4)
+ with fl.fast_upload([module], "cpu") as staged:
+ module.to("cpu")
+ assert staged == 0
+
+
+torch_cuda = pytest.mark.skipif(
+ not (pytest.importorskip("torch").cuda.is_available()), reason = "needs a CUDA device"
+)
+
+
+def _model(torch):
+ torch.manual_seed(0)
+
+ class _M(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.a = torch.nn.Linear(512, 1024).to(torch.bfloat16)
+ self.b = torch.nn.Linear(1024, 512, bias = False).to(torch.bfloat16)
+ self.c = torch.nn.Linear(512, 1024)
+ self.c.weight = (
+ self.a.weight
+ ) # tied (a bf16 weight on an fp32 Linear is fine for .to())
+ self.register_buffer("table", torch.randn(700, 1024))
+ self.register_buffer("ids", torch.arange(300000, dtype = torch.int64))
+ self.small = torch.nn.Parameter(torch.randn(3, 5), requires_grad = False)
+ self.view_param = torch.nn.Parameter(
+ torch.randn(1024, 600)[:, :512].clone().t(), requires_grad = False
+ )
+
+ return _M()
+
+
+@torch_cuda
+def test_fast_upload_matches_a_plain_to_byte_for_byte(monkeypatch):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)
+ monkeypatch.setattr(fl, "_UPLOAD_CHUNK_BYTES", 1 << 20) # many chunks per tensor, ring wraps
+ ref = _model(torch).to("cuda")
+ got = _model(torch)
+ names_before = {n: type(p) for n, p in got.named_parameters()}
+ with fl.fast_upload([got, None, "tokenizer"], "cuda") as staged:
+ out = got.to("cuda")
+ assert out is got and staged > 0
+ assert got.c.weight is got.a.weight # the tie survives
+ ref_state, got_state = ref.state_dict(keep_vars = True), got.state_dict(keep_vars = True)
+ assert ref_state.keys() == got_state.keys()
+ for name, want in ref_state.items():
+ have = got_state[name]
+ assert (
+ have.device.type == "cuda" and have.dtype == want.dtype and have.shape == want.shape
+ ), name
+ assert have.stride() == want.stride(), name
+ assert have.requires_grad == want.requires_grad, name
+ assert torch.equal(have, want), name
+ assert {n: type(p) for n, p in got.named_parameters()} == names_before
+
+
+@torch_cuda
+def test_fast_upload_leaves_other_to_calls_alone(monkeypatch):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)
+ module = _model(torch)
+ weight = module.a.weight
+ with fl.fast_upload([module], "cuda") as staged:
+ assert staged > 0
+ assert weight.to("cuda", dtype = torch.float32).dtype == torch.float32
+ assert weight.to("cuda", copy = True).data_ptr() != weight.to("cuda").data_ptr()
+ assert weight.to("cpu").device.type == "cpu"
+ module.to("cuda")
+ assert module.a.weight.device.type == "cuda"
+
+
+@torch_cuda
+def test_fast_upload_kill_switch_and_failure_fall_back_to_the_stock_copy(monkeypatch):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)
+ monkeypatch.setenv(fl.FAST_UPLOAD_ENV, "0")
+ module = _model(torch)
+ with fl.fast_upload([module], "cuda") as staged:
+ module.to("cuda")
+ assert staged == 0 and module.a.weight.device.type == "cuda"
+ monkeypatch.delenv(fl.FAST_UPLOAD_ENV)
+
+ def _boom(*a, **k):
+ raise RuntimeError("pinned memory exhausted")
+
+ monkeypatch.setattr(fl, "_ring_copy", _boom)
+ module = _model(torch)
+ with fl.fast_upload([module], "cuda") as staged:
+ module.to("cuda")
+ assert staged == 0 and torch.equal(module.table, _model(torch).table.cuda())
+
+
+@torch_cuda
+def test_fast_upload_passes_tensor_subclasses_through(monkeypatch):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)
+
+ class _Tagged(torch.Tensor):
+ pass
+
+ module = torch.nn.Linear(1024, 1024)
+ module.weight = torch.nn.Parameter(
+ module.weight.detach().as_subclass(_Tagged), requires_grad = False
+ )
+ assert fl._upload_candidates([module], "cuda") == [] or all(
+ type(t) in (torch.Tensor, torch.nn.Parameter)
+ for t in fl._upload_candidates([module], "cuda")
+ )
+ with fl.fast_upload([module], "cuda"):
+ module.to("cuda")
+ assert module.weight.device.type == "cuda"
+
+
+def test_rotational_lookup_never_raises(tmp_path):
+ path = tmp_path / "w.safetensors"
+ path.write_bytes(b"z")
+ assert fl._on_rotational_disk(str(path)) in (True, False)
+ assert fl._on_rotational_disk(str(tmp_path / "missing")) is False
diff --git a/studio/backend/tests/test_media_generation_partial_video_import.py b/studio/backend/tests/test_media_generation_partial_video_import.py
new file mode 100644
index 00000000000..f1ead5620c7
--- /dev/null
+++ b/studio/backend/tests/test_media_generation_partial_video_import.py
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A load posted while another thread is still importing core.inference.video must not 500 (AttributeError)."""
+
+import sys
+import types
+
+from hub.services.models import account_access
+
+
+def test_a_video_module_still_importing_counts_as_no_video_job(monkeypatch):
+ half_built = types.ModuleType("core.inference.video")
+ monkeypatch.setitem(sys.modules, "core.inference.video", half_built)
+ assert account_access.foreign_media_generations("someone") == 0
+
+
+def test_a_loaded_video_module_still_reports_its_in_flight_account(monkeypatch):
+ loaded = types.ModuleType("core.inference.video")
+ loaded.generation_account_in_flight = lambda: "other-account"
+ monkeypatch.setitem(sys.modules, "core.inference.video", loaded)
+ assert account_access.foreign_media_generations("someone") == 1
+ assert account_access.foreign_media_generations("other-account") == 0
+ loaded.generation_account_in_flight = lambda: None
+ assert account_access.foreign_media_generations("someone") == 0