Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio Images: pin the parked SAM 3 copy with registered host memory by default (unpark 0.36 -> 0.14 s) (#16791)
Daniel Han0ab3f0773de4
diff --git a/studio/backend/core/inference/diffusion_segment.py b/studio/backend/core/inference/diffusion_segment.pyindex 3ed07a7e873..82106d477ae 100644--- a/studio/backend/core/inference/diffusion_segment.py+++ b/studio/backend/core/inference/diffusion_segment.py@@ -127,7 +127,7 @@ ENV_KEEP_ALIVE = "UNSLOTH_SAM3_KEEP_ALIVE"# Seconds a host-RAM parked model is kept; 0 never parks.ENV_PARK_SECONDS = "UNSLOTH_SAM3_PARK_SECONDS"PARK_SECONDS = 600.0-# "1" pins the parked copy (0.03 s unpark vs ~0.6 s); opt-in since pinning 1.7 GB took minutes on a loaded host.+# Unset: registered host memory (torch's pinned allocator took minutes for 1.7 GB on a loaded host); "1" torch's, "0" pageable.ENV_PARK_PIN = "UNSLOTH_SAM3_PARK_PIN"# ── ComfyUI SAM3_Detect defaults ────────────────────────────────────────────────────────────────────────────────────@@ -1058,8 +1058,13 @@ class _HostWeights:_ALIGN = 256- def __init__(self, pin: bool) -> None:- self.pin = pin+ def __init__(+ self,+ pin: bool,+ fast: bool = False,+ ) -> None:+ self.pin = pin or fast+ self.fast = fastself.buffer: Any = Noneself.storage: Any = Noneself.total = 0@@ -1112,7 +1117,14 @@ class _HostWeights:slots[key] = (storage, offset)offset += -(-storage.nbytes() // self._ALIGN) * self._ALIGNself.total = offset- self.buffer = torch.empty(self.total, dtype = torch.uint8, pin_memory = self.pin)+ self.buffer = None+ if self.fast:+ from .diffusion_memory import registered_pinned_empty+ self.buffer = registered_pinned_empty(self.total)+ # No registered memory here: pageable, never torch's slow pinned allocator.+ self.pin = self.buffer is not None+ if self.buffer is None:+ self.buffer = torch.empty(self.total, dtype = torch.uint8, pin_memory = self.pin)storages = [st for st, _off in slots.values()]if storages:torch._foreach_copy_(@@ -1415,8 +1427,11 @@ class MaskSegmenter:model = self._modelfrom .host_pinning import host_pinning_allowed+ raw = (os.environ.get(ENV_PARK_PIN) or "").strip().lower()+ allowed = host_pinning_allowed(device)host = self._host or _HostWeights(- pin = (os.environ.get(ENV_PARK_PIN) or "").strip() == "1" and host_pinning_allowed(device)+ pin = raw == "1" and allowed,+ fast = raw not in ("0", "off", "false", "no", "1") and allowed,)try:host.park(model)diff --git a/studio/backend/tests/test_diffusion_segment.py b/studio/backend/tests/test_diffusion_segment.pyindex af39b63406f..666bdbfca45 100644--- a/studio/backend/tests/test_diffusion_segment.py+++ b/studio/backend/tests/test_diffusion_segment.py@@ -464,6 +464,62 @@ def _weights(model):return {k: v.detach().clone() for k, v in model.state_dict().items()}+ "env,pin,fast",+ [+ (None, False, True),+ ("", False, True),+ ("0", False, False),+ ("off", False, False),+ ("1", True, False),+ ],+)+def test_park_pins_with_registered_memory_by_default(monkeypatch, env, pin, fast):+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ _gpu_loaded(monkeypatch, segmenter, _small_model())+ if env is None:+ monkeypatch.delenv(seg.ENV_PARK_PIN, raising = False)+ else:+ monkeypatch.setenv(seg.ENV_PARK_PIN, env)+ monkeypatch.setattr(+ "core.inference.host_pinning.host_pinning_allowed", lambda device = None: True+ )+ made = []++ class _Recorder(seg._HostWeights):+ def __init__(+ self,+ pin,+ fast = False,+ ):+ made.append((pin, fast))+ super().__init__(False)++ monkeypatch.setattr(seg, "_HostWeights", _Recorder)+ monkeypatch.setattr(segmenter, "_load_weights", lambda *a, **k: None)+ assert segmenter.release("image generation", wait = True, park = True) is True+ assert made == [(pin, fast)]+++def test_fast_park_pin_falls_back_to_pageable_not_the_torch_allocator(monkeypatch):+ import torch++ from core.inference import diffusion_memory as dm++ monkeypatch.setattr(dm, "registered_pinned_empty", lambda nbytes: None)+ allocs = []+ real_empty = torch.empty++ def _empty(*a, **k):+ allocs.append(k.get("pin_memory", False))+ return real_empty(*a, **{key: v for key, v in k.items() if key != "pin_memory"})++ monkeypatch.setattr(torch, "empty", _empty)+ host = seg._HostWeights(False, fast = True)+ host.park(_small_model())+ assert host.pin is False and allocs[0] is False++def test_park_keeps_the_model_in_host_ram_and_the_next_load_reuses_it(monkeypatch):"""The render (or the Detailer) parks SAM 3 instead of dropping it: its weight storages are emptied, and the nextcall on the same card gives the same module its exact bytes back with no second load."""