Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep host memory unpinned on integrated GPUs (#15077)

Daniel Hancfc593f0bdfd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index fbb9f926775..9a40a9d868f 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -14861,6 +14861,8 @@ def _offload_controlnet_module(cn_model: Any, device: str, logger: Any) -> bool:
import torch
from diffusers.hooks import apply_group_offloading
+ from .host_pinning import host_pinning_allowed
+
onload = torch.device(device)
apply_group_offloading(
cn_model,
@@ -14868,7 +14870,7 @@ def _offload_controlnet_module(cn_model: Any, device: str, logger: Any) -> bool:
offload_device = torch.device("cpu"),
offload_type = "block_level",
num_blocks_per_group = 1,
- use_stream = onload.type == "cuda",
+ use_stream = onload.type == "cuda" and host_pinning_allowed(onload),
)
return True
except Exception as exc: # noqa: BLE001 - offload is best-effort; resident is the fallback
diff --git a/studio/backend/core/inference/diffusion_fast_load.py b/studio/backend/core/inference/diffusion_fast_load.py
index aa3d477fbb9..2cd121f39ae 100644
--- a/studio/backend/core/inference/diffusion_fast_load.py
+++ b/studio/backend/core/inference/diffusion_fast_load.py
@@ -525,8 +525,11 @@ def fast_upload(
try:
if fast_upload_enabled():
import torch
+ from .host_pinning import host_pinning_allowed
+
target = torch.device(device)
- if target.type == "cuda" and torch.cuda.is_available():
+ # The ring is pinned memory: an integrated GPU takes the stock ``.to`` instead.
+ if target.type == "cuda" and torch.cuda.is_available() and host_pinning_allowed(target):
if target.index is None:
target = torch.device("cuda", torch.cuda.current_device())
sources = _upload_candidates(modules, target)
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index e5ee65dca0c..0f105cb97d3 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -25,6 +25,7 @@ import sys
import threading
from dataclasses import dataclass, field, replace
from typing import Any, Callable, Iterator, Optional
+from .host_pinning import host_pinning_allowed
from .rocm_bf16 import is_rocm_torch
@@ -317,7 +318,7 @@ def _pin_host_weights(
import torch
mode = _pin_mode()
- if mode == "off" or not torch.cuda.is_available():
+ if mode == "off" or not torch.cuda.is_available() or not host_pinning_allowed():
return 0
# (name, owner tensor, inner attribute or None, the tensor to pin)
params: list = []
@@ -5360,7 +5361,7 @@ def _streamed_pin_plan(
) -> tuple[bool, bool]:
"""(pin transformer, pin encoders), transformer first."""
forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
- if forced in ("0", "off", "false", "no"):
+ if forced in ("0", "off", "false", "no") or not host_pinning_allowed():
return False, False
if forced in ("1", "on", "true", "yes"):
return True, True
@@ -5434,7 +5435,7 @@ def _background_pin_enabled() -> bool:
"off",
"false",
"no",
- )
+ ) and host_pinning_allowed()
_DENOISER_NAMES = ("transformer", "transformer_2", "unconditional_transformer", "unet")
@@ -5514,7 +5515,7 @@ def _fast_pin_supported() -> bool:
try:
import torch
- if is_rocm_torch(torch) or not torch.cuda.is_available():
+ if is_rocm_torch(torch) or not torch.cuda.is_available() or not host_pinning_allowed():
return False
cudart = torch.cuda.cudart()
return hasattr(cudart, "cudaHostRegister") and hasattr(cudart, "cudaHostUnregister")
@@ -5967,6 +5968,8 @@ def _pin_top_level_group(
``.data``) and when the copy does not fit the pinnable host RAM."""
if (os.environ.get(PIN_TOP_GROUP_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
return False
+ if not host_pinning_allowed():
+ return False
try:
import torch
from diffusers.hooks import group_offloading as go
@@ -6216,7 +6219,8 @@ def _apply_group_offload(
streamed_encoders[name] = comp
onload = torch.device(device)
- use_stream = onload.type == "cuda" # overlap H2D copies with compute
+ # overlap H2D copies with compute; a stream makes diffusers pin, so not on an integrated GPU
+ use_stream = onload.type == "cuda" and host_pinning_allowed(onload)
gkwargs: dict[str, Any] = {
"onload_device": onload,
"offload_device": torch.device("cpu"),
@@ -6415,7 +6419,7 @@ def stream_text_encoders_onto(
install_group_offload_buffer_restore()
install_group_offload_hooks_eager()
onload = torch.device(device)
- use_stream = onload.type == "cuda"
+ use_stream = onload.type == "cuda" and host_pinning_allowed(onload)
kwargs: dict[str, Any] = {
"onload_device": onload,
"offload_device": torch.device("cpu"),
@@ -6990,7 +6994,9 @@ def _apply_streaming_offload(
onload = torch.device(device)
offload = torch.device("cpu")
params = inspect.signature(apply_group_offloading).parameters
- use_stream = onload.type == "cuda" and "low_cpu_mem_usage" in params
+ use_stream = (
+ onload.type == "cuda" and "low_cpu_mem_usage" in params and host_pinning_allowed(onload)
+ )
# Keep small companions such as the tiled VAE resident. Every transformer and text encoder remains on CPU behind
# a granular hook.
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index 3f2c408575c..107ad19f63e 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -780,6 +780,10 @@ def _adopt_top_group(
return None
if str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower() in off:
return None
+ from .host_pinning import host_pinning_allowed
+
+ if not host_pinning_allowed():
+ return None
go = _go()
registry = getattr(module, "_diffusers_hook", None)
get_hook = getattr(registry, "get_hook", None)
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 41bfefb07ed..441e9715d8d 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -3449,8 +3449,11 @@ def stream_prequantized_module(
module.requires_grad_(False)
install_group_offload_buffer_restore()
install_group_offload_hooks_eager()
- use_stream = onload.type == "cuda" and _weights_pinnable(module)
- if onload.type == "cuda" and not use_stream:
+ from .host_pinning import host_pinning_allowed
+
+ pinning = host_pinning_allowed(onload)
+ use_stream = onload.type == "cuda" and pinning and _weights_pinnable(module)
+ if onload.type == "cuda" and pinning and not use_stream:
# Sync copies measured 16.4 s/step vs 0.8 s resident (B200): rebuild v1 int8 as pinnable Int8Tensor.
from .prequant_legacy_int8 import convert_legacy_int8_weights
converted = convert_legacy_int8_weights(module)
diff --git a/studio/backend/core/inference/diffusion_preview.py b/studio/backend/core/inference/diffusion_preview.py
index 682c6c064b4..70f052bd1af 100644
--- a/studio/backend/core/inference/diffusion_preview.py
+++ b/studio/backend/core/inference/diffusion_preview.py
@@ -258,6 +258,7 @@ class LatentPreviewer:
# One slot per planned snapshot: with the host far ahead every one can be in flight at once.
self.stride = max(1, math.ceil(total / max(1, int(max_snapshots))))
n_slots = total // self.stride + 2
+ # Pinned even on an integrated GPU: a few MB, and an unpinned copy would make the denoise thread wait each snapshot.
pinned = torch.empty(n_slots * cap, dtype = torch.uint8, pin_memory = True)
self._slots = [
{
diff --git a/studio/backend/core/inference/diffusion_segment.py b/studio/backend/core/inference/diffusion_segment.py
index f88a6e53632..9b1ee40fb15 100644
--- a/studio/backend/core/inference/diffusion_segment.py
+++ b/studio/backend/core/inference/diffusion_segment.py
@@ -1391,7 +1391,11 @@ class MaskSegmenter:
self._timer = None
started = time.perf_counter()
model = self._model
- host = self._host or _HostWeights(pin = (os.environ.get(ENV_PARK_PIN) or "").strip() == "1")
+ from .host_pinning import host_pinning_allowed
+
+ host = self._host or _HostWeights(
+ pin = (os.environ.get(ENV_PARK_PIN) or "").strip() == "1" and host_pinning_allowed()
+ )
try:
host.park(model)
copied = time.perf_counter() - started
diff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.py
index 5040a820af4..bcd16521278 100644
--- a/studio/backend/core/inference/diffusion_small_host.py
+++ b/studio/backend/core/inference/diffusion_small_host.py
@@ -967,6 +967,10 @@ def install_encoder_prefetch(
if _env(GROUP_OFFLOAD_PIN_ENV) in ("0", "off", "false", "no"):
return 0 # the ring is pinned memory: the user's "pin nothing" override wins
+ from .host_pinning import host_pinning_allowed
+
+ if not host_pinning_allowed(device):
+ return 0 # integrated GPU: pinned memory only takes RAM from the GPU
groups = [
g
diff --git a/studio/backend/core/inference/host_pinning.py b/studio/backend/core/inference/host_pinning.py
new file mode 100644
index 00000000000..64618a39a02
--- /dev/null
+++ b/studio/backend/core/inference/host_pinning.py
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Integrated GPUs skip pinned host memory: their VRAM is system RAM, so a pinned copy only takes RAM from the GPU
+(ComfyUI does the same). An unclassifiable device answers discrete (pinning allowed)."""
+
+from __future__ import annotations
+
+import threading
+from typing import Any, Optional
+
+_LOCK = threading.Lock()
+_CACHE: dict[int, bool] = {}
+
+
+def _device_index(torch: Any, device: Any) -> Optional[int]:
+ if device is None:
+ return int(torch.cuda.current_device())
+ if isinstance(device, int):
+ return device
+ dev = torch.device(device)
+ if dev.type != "cuda":
+ return None
+ return int(dev.index) if dev.index is not None else int(torch.cuda.current_device())
+
+
+def _classify(torch: Any, index: int) -> bool:
+ props = torch.cuda.get_device_properties(index)
+ from .rocm_bf16 import is_rocm_torch
+
+ if is_rocm_torch(torch):
+ from utils.hardware.hardware import _rocm_props_unified_status
+ return _rocm_props_unified_status(props) is True
+ from utils.hardware.hardware import _cuda_props_are_integrated
+
+ return bool(_cuda_props_are_integrated(props))
+
+
+def device_is_integrated(device: Any = None) -> bool:
+ """Never raises: no GPU, a non-CUDA device or an unreadable property answers False (discrete)."""
+ try:
+ import torch
+
+ if not torch.cuda.is_available():
+ return False
+ index = _device_index(torch, device)
+ if index is None:
+ return False
+ with _LOCK:
+ hit = _CACHE.get(index)
+ if hit is not None:
+ return hit
+ answer = bool(_classify(torch, index))
+ with _LOCK:
+ _CACHE[index] = answer
+ return answer
+ except Exception: # noqa: BLE001 - unclassifiable: keep the discrete path
+ return False
+
+
+def host_pinning_allowed(device: Any = None) -> bool:
+ return not device_is_integrated(device)
+
+
+def reset_cache() -> None:
+ with _LOCK:
+ _CACHE.clear()
diff --git a/studio/backend/core/inference/video_frames.py b/studio/backend/core/inference/video_frames.py
index 3ff4a2be740..93fc1284ca4 100644
--- a/studio/backend/core/inference/video_frames.py
+++ b/studio/backend/core/inference/video_frames.py
@@ -22,7 +22,9 @@ def device_uint8(frames: Any) -> Optional[Any]:
count, channels, height, width = frames.shape
shape = (count, height, width, channels)
host = None
- if device.type == "cuda":
+ from .host_pinning import host_pinning_allowed
+
+ if device.type == "cuda" and host_pinning_allowed(device):
try:
host = torch.empty(shape, dtype = torch.uint8, device = "cpu", pin_memory = True)
except Exception: # noqa: BLE001 - pinned allocation refused (host limits); a pageable copy is still exact
diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.py
index 1223194bbdb..66057f8e6fb 100644
--- a/studio/backend/core/inference/video_ltx2.py
+++ b/studio/backend/core/inference/video_ltx2.py
@@ -499,7 +499,11 @@ def _copy_file_chunks(
got += read
return
buffers = max(1, min(buffers, len(jobs)))
- staging = [torch.empty(chunk_bytes, dtype = torch.uint8, pin_memory = True) for _ in range(buffers)]
+ from .host_pinning import host_pinning_allowed
+
+ # Pageable on an integrated GPU: the copy then completes before copy_ returns, so slot reuse stays safe.
+ pin = host_pinning_allowed(device)
+ staging = [torch.empty(chunk_bytes, dtype = torch.uint8, pin_memory = pin) for _ in range(buffers)]
done_events: list[Any] = [None] * buffers
handles: "queue.SimpleQueue[Any]" = queue.SimpleQueue()
opened: list[Any] = []
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index 9a02cedeb65..bc0a3730ccc 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -516,8 +516,10 @@ def h3_te_pin_allowed(payload_bytes: int = 0) -> bool:
from .diffusion_memory import GROUP_OFFLOAD_PIN_ENV, _pin_budget_mib, _pinned_memory_capped
+ from .host_pinning import host_pinning_allowed
+
forced = str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower()
- if forced in ("0", "off", "false", "no"):
+ if forced in ("0", "off", "false", "no") or not host_pinning_allowed():
return False
if forced in ("1", "on", "true", "yes"):
return True
@@ -619,7 +621,8 @@ def stream_h3_text_encoder(
"""Stream MiniMax-H3's conditioner leaf by leaf via group offloading, outside the ComponentsManager rotation.
Hooks go on ``text_encoder.model``, which the H3 encode step calls directly. ``pin=None`` decides from
- ``h3_te_pin_allowed``. Returns ``"stream"`` (pinned) / ``"stream_lazy"``, or None with nothing changed."""
+ ``h3_te_pin_allowed``. Returns ``"stream"`` (pinned) / ``"stream_lazy"`` / ``"sync"`` (integrated GPU: no stream,
+ nothing pinned), or None with nothing changed."""
import torch
onload = torch.device(device)
@@ -640,8 +643,14 @@ def stream_h3_text_encoder(
)
from .diffusion_prequant import _evict_rotation_hook, _unhook_from_manager
+ from .host_pinning import host_pinning_allowed
+
params = inspect.signature(apply_group_offloading).parameters
- if pin is None:
+ # A stream makes diffusers pin each leaf; an integrated GPU streams synchronously from pageable memory instead.
+ stream = host_pinning_allowed(onload)
+ if not stream:
+ pin = False
+ elif pin is None:
pin = h3_te_pin_allowed(_module_payload_bytes(text_encoder))
if pin:
try:
@@ -654,17 +663,18 @@ def stream_h3_text_encoder(
"onload_device": onload,
"offload_device": torch.device("cpu"),
"offload_type": "leaf_level",
- "use_stream": True,
+ "use_stream": stream,
}
- if "non_blocking" in params:
- kwargs["non_blocking"] = True
- if "record_stream" in params:
- kwargs["record_stream"] = True
- if "low_cpu_mem_usage" in params:
- # Already pinned in place, so diffusers' pin_memory() returns the same tensor: no second host copy.
- kwargs["low_cpu_mem_usage"] = not pin
- elif not pin:
- return None
+ if stream:
+ if "non_blocking" in params:
+ kwargs["non_blocking"] = True
+ if "record_stream" in params:
+ kwargs["record_stream"] = True
+ if "low_cpu_mem_usage" in params:
+ # Already pinned in place, so diffusers' pin_memory() returns the same tensor: no second host copy.
+ kwargs["low_cpu_mem_usage"] = not pin
+ elif not pin:
+ return None
try:
text_encoder.requires_grad_(False)
# The int8 weights are buffers, which stock diffusers leaves on the device after offload.
@@ -686,7 +696,7 @@ def stream_h3_text_encoder(
return None
# The manager only evicts inside a managed pre_forward; else the last decode's VAE sits beside the encode.
target.register_forward_pre_hook(_evict_rotation_hook(manager, onload))
- mode = "stream" if pin else "stream_lazy"
+ mode = ("stream" if pin else "stream_lazy") if stream else "sync"
if logger is not None:
logger.info(
"video.h3_te_stream: conditioner streamed leaf by leaf on %s (%s host copy)",
diff --git a/studio/backend/core/inference/video_stream_encode.py b/studio/backend/core/inference/video_stream_encode.py
index 4cf2f6c0491..8352e580527 100644
--- a/studio/backend/core/inference/video_stream_encode.py
+++ b/studio/backend/core/inference/video_stream_encode.py
@@ -19,6 +19,8 @@ import threading
from pathlib import Path
from typing import Any, Iterator, Optional
+from .host_pinning import host_pinning_allowed
+
STREAM_ENCODE_ENV = "UNSLOTH_STUDIO_VIDEO_STREAM_ENCODE"
QUEUE_DEPTH = 8
_STOP = object()
@@ -132,7 +134,7 @@ class StreamEncode:
self._side = torch.cuda.Stream(device = device)
ready = torch.cuda.Event()
ready.record(torch.cuda.current_stream(device))
- host = torch.empty(x.shape, dtype = torch.uint8, pin_memory = True)
+ host = torch.empty(x.shape, dtype = torch.uint8, pin_memory = host_pinning_allowed(device))
with torch.cuda.stream(self._side):
self._side.wait_event(ready)
host.copy_(x, non_blocking = True)
@@ -396,7 +398,7 @@ class H3StreamEncode(StreamEncode):
pair = []
for values in self._pixel:
host = torch.tensor(values, dtype = torch.float32)
- if device.type == "cuda":
+ if device.type == "cuda" and host_pinning_allowed(device):
host = host.pin_memory()
pair.append(host.to(device, non_blocking = True).view(1, -1, 1, 1, 1))
held = (device, tuple(pair))
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 6538f5dddd8..73753897d5b 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -2121,13 +2121,17 @@ def _build_latent_cache(
len(image_paths), cfg.cache_variants, cfg.center_crop, cfg.random_flip, cfg.seed
)
+ from core.inference.host_pinning import host_pinning_allowed
+
+ pin = device == "cuda" and host_pinning_allowed()
+
def _hold(t):
if t is None:
return None
import torch
t = t.to(torch.float32).cpu()
- if device == "cuda":
+ if pin:
try:
t = t.pin_memory()
except RuntimeError:
@@ -2237,13 +2241,17 @@ def _load_warm_conditioning(
return None, None
embeds.append(entry)
+ from core.inference.host_pinning import host_pinning_allowed
+
+ pin = device == "cuda" and host_pinning_allowed()
+
def _hold(t):
if t is None:
return None
import torch # noqa: F401 -- pin_memory needs torch initialised
t = t.contiguous()
- if device == "cuda":
+ if pin:
try:
t = t.pin_memory()
except RuntimeError:
diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.py
index 1217088a88f..1e846812c62 100644
--- a/studio/backend/core/training/diffusion_lora_trainer.py
+++ b/studio/backend/core/training/diffusion_lora_trainer.py
@@ -230,9 +230,13 @@ def _build_sdxl_latent_cache(
len(image_paths), cfg.cache_variants, cfg.center_crop, cfg.random_flip, cfg.seed
)
+ from core.inference.host_pinning import host_pinning_allowed
+
+ pin = device == "cuda" and host_pinning_allowed()
+
def _hold(t):
t = t.to(torch.float32).cpu()
- if device == "cuda":
+ if pin:
try:
t = t.pin_memory()
except RuntimeError:
diff --git a/studio/backend/tests/test_integrated_gpu_no_host_pin.py b/studio/backend/tests/test_integrated_gpu_no_host_pin.py
new file mode 100644
index 00000000000..30f4c8b59ad
--- /dev/null
+++ b/studio/backend/tests/test_integrated_gpu_no_host_pin.py
@@ -0,0 +1,533 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""No page-locked host memory on an integrated GPU (Strix Halo, Jetson, DGX Spark), as ComfyUI does.
+
+The device is stubbed through ``torch.cuda.get_device_properties(...).is_integrated``: 1 must reach no pinning call
+at any diffusion / video site, 0 (a discrete card) must reach the same calls as before. GPU-free throughout."""
+
+from __future__ import annotations
+
+import ast
+import inspect
+import os
+import types
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_memory as dm # noqa: E402
+
+# Imported before any test stubs the device properties: importing diffusers under the stub reads fields it lacks.
+pytest.importorskip("diffusers.hooks.group_offloading")
+
+BACKEND = Path(__file__).resolve().parents[1]
+
+
+class _Stop(Exception):
+ pass
+
+
+def _props(integrated: bool, arch: str = "") -> types.SimpleNamespace:
+ flag = 1 if integrated else 0
+ return types.SimpleNamespace(
+ is_integrated = flag,
+ integrated = flag,
+ name = "Fake GPU",
+ gcnArchName = arch,
+ total_memory = 16 << 30,
+ major = 9,
+ minor = 0,
+ )
+
+
+def _forget_answers() -> None:
+ try:
+ from core.inference import host_pinning
+ except ImportError:
+ return
+ host_pinning.reset_cache()
+
+
+def gpu(monkeypatch):
+ """``gpu(True)`` stubs an integrated device, ``gpu(False)`` a discrete one."""
+
+ def make(integrated: bool, arch: str = "") -> None:
+ props = _props(integrated, arch)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)
+ _forget_answers()
+
+ yield make
+ _forget_answers()
+
+
+def pins(monkeypatch):
+ """Every pinning request (``torch.empty(pin_memory=True)``, ``Tensor.pin_memory()``), served as plain host memory."""
+ calls: list = []
+ real_empty = torch.empty
+
+ def empty(*args, **kwargs):
+ if kwargs.pop("pin_memory", False):
+ calls.append("empty")
+ return real_empty(*args, **kwargs)
+
+ def pin_memory(self, *args, **kwargs):
+ calls.append("pin_memory")
+ return self.clone()
+
+ monkeypatch.setattr(torch, "empty", empty)
+ monkeypatch.setattr(torch.Tensor, "pin_memory", pin_memory)
+ return calls
+
+
[email protected]("integrated", [True, False])
+def test_the_predicate_reads_the_driver_flag(gpu, integrated):
+ from core.inference import host_pinning
+
+ gpu(integrated)
+ assert host_pinning.device_is_integrated() is integrated
+ assert host_pinning.host_pinning_allowed() is (not integrated)
+ assert host_pinning.host_pinning_allowed("cuda:0") is (not integrated)
+ assert host_pinning.host_pinning_allowed(torch.device("cuda")) is (not integrated)
+
+
+def test_the_predicate_keeps_the_discrete_answer_without_a_gpu_or_on_another_device(
+ monkeypatch, gpu
+):
+ from core.inference import host_pinning
+
+ gpu(True)
+ assert host_pinning.host_pinning_allowed("cpu") is True
+ assert host_pinning.host_pinning_allowed("mps") is True
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ host_pinning.reset_cache()
+ assert host_pinning.host_pinning_allowed() is True
+
+ def boom(*a, **k):
+ raise RuntimeError("no driver")
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "get_device_properties", boom)
+ host_pinning.reset_cache()
+ assert host_pinning.host_pinning_allowed() is True
+
+
+ "arch, flag, integrated",
+ [("gfx1151", 1, True), ("gfx1151", 0, True), ("gfx1100", 0, False), ("gfx942", 0, False)],
+)
+def test_rocm_uses_studios_apu_classifier(monkeypatch, gpu, arch, flag, integrated):
+ from core.inference import host_pinning
+
+ gpu(bool(flag), arch)
+ monkeypatch.setattr(torch.version, "hip", "6.4.0", raising = False)
+ host_pinning.reset_cache()
+ assert host_pinning.device_is_integrated() is integrated
+
+
+def test_the_answer_is_cached_per_device(monkeypatch, gpu):
+ from core.inference import host_pinning
+
+ gpu(True)
+ asked: list = []
+ props = _props(True)
+ monkeypatch.setattr(
+ torch.cuda, "get_device_properties", lambda i, *a, **k: (asked.append(i), props)[1]
+ )
+ for _ in range(3):
+ host_pinning.host_pinning_allowed("cuda:0")
+ assert asked == [0]
+
+
+def _host_of(module):
+ return {name: p.data for name, p in module.named_parameters()}
+
+
[email protected]("integrated", [True, False])
+def test_keep_cpu_weights_are_not_pinned_on_an_integrated_gpu(monkeypatch, gpu, pins, integrated):
+ gpu(integrated)
+ monkeypatch.delenv(dm.OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 64 << 10)
+ monkeypatch.setattr(dm, "_host_ram_capacity_mib", lambda: 128 << 10)
+ lin = torch.nn.Linear(8, 8)
+ pinned = dm._pin_host_weights(lin, _host_of(lin))
+ if integrated:
+ assert pinned == 0 and pins == []
+ else:
+ assert pinned > 0 and pins
+
+
[email protected]("integrated", [True, False])
+def test_the_streamed_pin_plan_pins_nothing_on_an_integrated_gpu(monkeypatch, gpu, integrated):
+ gpu(integrated)
+ monkeypatch.delenv(dm.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: False)
+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 1 << 20)
+ want = (False, False) if integrated else (True, True)
+ assert dm._streamed_pin_plan(1024, 1024) == want
+
+
[email protected]("integrated", [True, False])
+def test_background_and_registered_pins_are_off_on_an_integrated_gpu(monkeypatch, gpu, integrated):
+ gpu(integrated)
+ monkeypatch.delenv(dm.BACKGROUND_PIN_ENV, raising = False)
+ monkeypatch.delenv(dm.FAST_PIN_ENV, raising = False)
+ monkeypatch.setattr(dm.sys, "platform", "linux")
+ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: False)
+ registered: list = []
+ cudart = types.SimpleNamespace(
+ cudaHostRegister = lambda *a: registered.append(a) or 0,
+ cudaHostUnregister = lambda *a: 0,
+ )
+ monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
+ assert dm._background_pin_enabled() is (not integrated)
+ assert dm._fast_pin_supported() is (not integrated)
+ assert registered == []
+
+
[email protected]("integrated", [True, False])
+def test_the_top_level_group_is_not_pinned_on_an_integrated_gpu(monkeypatch, gpu, integrated):
+ gpu(integrated)
+ monkeypatch.delenv(dm.PIN_TOP_GROUP_ENV, raising = False)
+ looked: list = []
+ module = types.SimpleNamespace(
+ _diffusers_hook = types.SimpleNamespace(get_hook = lambda name: looked.append(name))
+ )
+ assert dm._pin_top_level_group(module) is False
+ assert bool(looked) is (not integrated)
+
+
+def _record_group_offload(monkeypatch):
+ from diffusers import hooks as dh
+
+ real = dh.apply_group_offloading
+ seen: list = []
+
+ def apply_group_offloading(module, **kwargs):
+ seen.append(kwargs)
+
+ apply_group_offloading.__signature__ = inspect.signature(real)
+ monkeypatch.setattr(dh, "apply_group_offloading", apply_group_offloading)
+ monkeypatch.setattr(dm, "install_group_offload_buffer_restore", lambda: None)
+ monkeypatch.setattr(dm, "install_group_offload_hooks_eager", lambda: None)
+ return seen
+
+
[email protected]("integrated", [True, False])
+def test_streamed_text_encoders_do_not_use_a_pinning_stream_on_an_integrated_gpu(
+ monkeypatch, gpu, integrated
+):
+ pytest.importorskip("diffusers")
+ gpu(integrated)
+ seen = _record_group_offload(monkeypatch)
+ monkeypatch.setattr(dm, "_pin_vision_embedding_device", lambda module: 0)
+ pipe = types.SimpleNamespace(components = {"text_encoder": torch.nn.Linear(4, 4)})
+ assert dm.stream_text_encoders_onto(pipe, "cuda") == ["text_encoder"]
+ (kwargs,) = seen
+ assert kwargs["use_stream"] is (not integrated)
+ if integrated:
+ assert "record_stream" not in kwargs and "non_blocking" not in kwargs
+
+
[email protected]("integrated", [True, False])
+def test_the_controlnet_offload_streams_without_pinning_on_an_integrated_gpu(
+ monkeypatch, gpu, integrated
+):
+ pytest.importorskip("diffusers")
+ from core.inference import diffusion
+
+ gpu(integrated)
+ seen = _record_group_offload(monkeypatch)
+ assert diffusion._offload_controlnet_module(torch.nn.Linear(4, 4), "cuda", None) is True
+ assert seen[0]["use_stream"] is (not integrated)
+
+
[email protected]("integrated", [True, False])
+def test_the_fast_upload_ring_is_skipped_on_an_integrated_gpu(monkeypatch, gpu, integrated):
+ from core.inference import diffusion_fast_load as fl
+
+ gpu(integrated)
+ monkeypatch.delenv(fl.FAST_UPLOAD_ENV, raising = False)
+ asked: list = []
+ monkeypatch.setattr(
+ fl, "_upload_candidates", lambda modules, device: asked.append(device) or []
+ )
+ with fl.fast_upload([torch.nn.Linear(2, 2)], "cuda") as staged:
+ assert staged == 0
+ assert bool(asked) is (not integrated)
+
+
[email protected]("integrated", [True, False])
+def test_the_encoder_prefetch_ring_is_skipped_on_an_integrated_gpu(monkeypatch, gpu, integrated):
+ from core.inference import diffusion_small_host as sh
+
+ gpu(integrated)
+ monkeypatch.delenv(dm.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(sh, "encoder_prefetch_disabled", lambda: False)
+ asked: list = []
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: asked.append(module) or [])
+ assert sh.install_encoder_prefetch(torch.nn.Linear(2, 2), "cuda") == 0
+ assert bool(asked) is (not integrated)
+
+
[email protected]("integrated", [True, False])
+def test_the_torchao_top_group_is_not_adopted_on_an_integrated_gpu(monkeypatch, gpu, integrated):
+ from core.inference import diffusion_offload_prefetch as op
+
+ gpu(integrated)
+ monkeypatch.delenv(dm.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.delenv(dm.PIN_TOP_GROUP_ENV, raising = False)
+ asked: list = []
+ monkeypatch.setattr(op, "_go", lambda: asked.append(1) or types.SimpleNamespace())
+ assert op._adopt_top_group(types.SimpleNamespace(), object()) is None
+ assert bool(asked) is (not integrated)
+
+
[email protected]("integrated", [True, False])
+def test_video_frame_readback_is_pageable_on_an_integrated_gpu(monkeypatch, gpu, integrated):
+ from core.inference import video_frames
+
+ gpu(integrated)
+ asked: list = []
+ real_empty = torch.empty
+
+ def empty(*args, **kwargs):
+ asked.append(bool(kwargs.pop("pin_memory", False)))
+ if len(asked) > 1 or not asked[0]:
+ raise _Stop
+ return real_empty(*args, **kwargs)
+
+ monkeypatch.setattr(torch, "empty", empty)
+ frames = types.SimpleNamespace(
+ device = torch.device("cuda", 0), numel = lambda: 6, shape = (1, 3, 1, 2)
+ )
+ if integrated:
+ with pytest.raises(_Stop):
+ video_frames.device_uint8(frames)
+ assert asked == [False]
+ else:
+ with pytest.raises(Exception): # noqa: B017 - the fake frames cannot be sliced
+ video_frames.device_uint8(frames)
+ assert asked[0] is True
+
+
[email protected]("integrated", [True, False])
+def test_the_ltx2_direct_load_staging_is_pageable_on_an_integrated_gpu(
+ monkeypatch, gpu, integrated
+):
+ from core.inference import video_ltx2
+
+ gpu(integrated)
+ asked: list = []
+
+ def empty(*args, **kwargs):
+ asked.append(bool(kwargs.get("pin_memory", False)))
+ raise _Stop
+
+ monkeypatch.setattr(torch, "empty", empty)
+ with pytest.raises(_Stop):
+ video_ltx2._copy_file_chunks("unused", [(0, 4, None, 0)], "cuda", 4, 1, 1)
+ assert asked == [not integrated]
+
+
+class _FakeHost:
+ def __init__(self, calls):
+ self.calls = calls
+
+ def pin_memory(self):
+ self.calls.append("pin_memory")
+ return self
+
+ def to(self, *a, **k):
+ return self
+
+ def view(self, *a):
+ return self
+
+ def copy_(self, *a, **k):
+ return self
+
+
+def _fake_torch(calls):
+ cuda = types.SimpleNamespace(
+ Stream = lambda device = None: types.SimpleNamespace(wait_event = lambda e: None),
+ Event = lambda: types.SimpleNamespace(record = lambda *a: None),
+ current_stream = lambda device = None: object(),
+ stream = lambda s: __import__("contextlib").nullcontext(),
+ )
+
+ def empty(*a, **k):
+ if k.get("pin_memory"):
+ calls.append("empty")
+ return _FakeHost(calls)
+
+ return types.SimpleNamespace(
+ tensor = lambda *a, **k: _FakeHost(calls), float32 = None, uint8 = None, empty = empty, cuda = cuda
+ )
+
+
[email protected]("integrated", [True, False])
+def test_stream_encode_host_buffers_are_pageable_on_an_integrated_gpu(monkeypatch, gpu, integrated):
+ from core.inference import video_stream_encode as vse
+
+ gpu(integrated)
+ calls: list = []
+ fake = types.SimpleNamespace(_torch = _fake_torch(calls), _pixel = ((0.5,), (0.5,)))
+ vse.H3StreamEncode._pixel_tensors(fake, torch.device("cuda", 0))
+ assert bool(calls) is (not integrated)
+
+ calls.clear()
+
+ class _X:
+ device = torch.device("cuda", 0)
+ shape = (1, 1, 1, 3)
+
+ def to(self, *a, **k):
+ return self
+
+ def __mul__(self, other):
+ return self
+
+ def round_(self):
+ return self
+
+ def permute(self, *a):
+ return self
+
+ def contiguous(self):
+ return self
+
+ def record_stream(self, s):
+ pass
+
+ import queue
+
+ sess = types.SimpleNamespace(
+ _torch = fake._torch, _side = None, chunks = [], _thread = object(), _queue = queue.SimpleQueue()
+ )
+ vse.StreamEncode._push_uint8(sess, _X())
+ assert calls == ([] if integrated else ["empty"])
+
+
[email protected]("integrated", [True, False])
+def test_the_h3_pin_gate_refuses_an_integrated_gpu_even_when_forced(monkeypatch, gpu, integrated):
+ from core.inference import video_minimax_h3_te as te
+
+ gpu(integrated)
+ monkeypatch.setenv(dm.GROUP_OFFLOAD_PIN_ENV, "1")
+ assert te.h3_te_pin_allowed(1 << 20) is (not integrated)
+
+
+# Sites with no gate of their own, each reached only through one that asks host_pinning_allowed.
+_REACHED_THROUGH_A_GATE = {
+ ("core/inference/diffusion_fast_load.py", "_ring_copy"): "fast_upload",
+ (
+ "core/inference/diffusion_memory.py",
+ "install_torchao_v1_int8_pin_ops._pinned",
+ ): "torchao's pin op: runs when a gated site pins",
+ ("core/inference/diffusion_memory.py", "_RegisteredHostBuffer.register"): "_fast_pin_supported",
+ (
+ "core/inference/diffusion_memory.py",
+ "_GroupPinner._fast_result",
+ ): "_background_pin_enabled / use_stream",
+ (
+ "core/inference/diffusion_memory.py",
+ "_GroupPinner._run",
+ ): "_background_pin_enabled / use_stream",
+ (
+ "core/inference/diffusion_offload_prefetch.py",
+ "GroupPrefetcher._issue",
+ ): "streamed groups only (use_stream)",
+ (
+ "core/inference/diffusion_pinned_arena.py",
+ "_alloc_slab",
+ ): "group offload with use_stream only",
+ (
+ "core/inference/diffusion_pinned_arena.py",
+ "PinnedArena.pin_plain",
+ ): "group offload with use_stream only",
+ (
+ "core/inference/diffusion_segment.py",
+ "_HostWeights._repack",
+ ): "MaskSegmenter park passes pin=",
+ (
+ "core/inference/diffusion_small_host.py",
+ "_EncoderPrefetcher._ensure_stage",
+ ): "install_encoder_prefetch",
+ (
+ "core/inference/video_minimax_h3_te.py",
+ "pin_module_in_place",
+ ): "h3_te_pin_allowed / stream_h3_text_encoder",
+ ("core/inference/diffusion_preview.py", "_device_map"): "exempt: tiny, async preview",
+ (
+ "core/inference/diffusion_preview.py",
+ "LatentPreviewer.__init__",
+ ): "exempt: tiny, async preview",
+}
+
+
+def _pin_sites():
+ for path in sorted((BACKEND / "core").rglob("*.py")):
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ rel = path.relative_to(BACKEND).as_posix()
+
+ def visit(node, chain):
+ for child in ast.iter_child_nodes(node):
+ here = chain
+ if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ here = chain + [child]
+ if isinstance(child, ast.Call):
+ func = child.func
+ hit = isinstance(func, ast.Attribute) and func.attr in (
+ "pin_memory",
+ "cudaHostRegister",
+ )
+ for kw in child.keywords:
+ if kw.arg == "pin_memory" and not (
+ isinstance(kw.value, ast.Constant) and kw.value.value is False
+ ):
+ hit = True
+ if hit:
+ yield rel, ".".join(n.name for n in here), here, child.lineno
+ yield from visit(child, here)
+
+ yield from visit(tree, [])
+
+
+def test_every_pinning_site_asks_the_integrated_gpu_predicate_or_is_reached_through_one():
+ ungated = []
+ for rel, qualname, chain, line in _pin_sites():
+ funcs = [n for n in chain if not isinstance(n, ast.ClassDef)]
+ if any("host_pinning_allowed" in ast.unparse(n) for n in funcs):
+ continue
+ if (rel, qualname) in _REACHED_THROUGH_A_GATE:
+ continue
+ ungated.append(f"{rel}:{line} {qualname}")
+ assert not ungated, "pinning without asking host_pinning_allowed: " + ", ".join(ungated)
+
+
+def test_every_cuda_group_offload_stream_asks_the_predicate():
+ # diffusers pins every streamed group's host copy, so use_stream is itself a pinning decision.
+ loose = []
+ for path in sorted((BACKEND / "core" / "inference").glob("*.py")):
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ value = None
+ if isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id == "use_stream" for t in node.targets
+ ):
+ value = node.value
+ elif isinstance(node, ast.keyword) and node.arg == "use_stream":
+ value = node.value
+ if value is None:
+ continue
+ text = ast.unparse(value)
+ if "'cuda'" in text and "host_pinning_allowed" not in text and "pinning" not in text:
+ loose.append(f"{path.name}:{getattr(value, 'lineno', '?')} {text}")
+ assert not loose, loose