Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep the LoRA fuse host backup off integrated GPU pins and inside the host RAM reserve (#17086)
Daniel Hanffeb8f7f2037
diff --git a/studio/backend/core/inference/diffusion_lora_fuse.py b/studio/backend/core/inference/diffusion_lora_fuse.pyindex 2fe8a42f7de..4ae7ace166d 100644--- a/studio/backend/core/inference/diffusion_lora_fuse.py+++ b/studio/backend/core/inference/diffusion_lora_fuse.py@@ -6,7 +6,8 @@PEFT runs every adapted layer as ``base(x) + B(A(x)) * scale``: two extra GEMMs per layer on every step. Once thesame set renders twice, the second render folds ``B @ A * scale`` into the base weights (fp32 add, one rounding to theweight dtype) and marks each layer merged, so PEFT's forward runs the base layer alone. The original weights are kept-on the card when it has ample room (a restore is then a device copy), else in pinned host chunks, and copied back+on the card when it has ample room (a restore is then a device copy), else in host chunks (pinned on a discrete card,+only while the host keeps its RAM reserve), and copied backbefore any change, so the weights-only fast path (``set_adapters``) runs on exactly the weights it would have seenunfused. Opt out with UNSLOTH_DIFFUSION_LORA_FUSE=0."""@@ -17,6 +18,7 @@ from typing import Any, OptionalLORA_FUSE_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE"BACKUP_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_BACKUP"+HOST_GUARD_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_HOST_GUARD"_DEVICE_BACKUP_MIN_FREE = 8 << 30_FUSED_ATTR = "_unsloth_lora_fused"_ALIGN = 64@@ -128,13 +130,19 @@ def fuse(used += nbytestotal = sum(sizes)on_device = _backup_on_device(device, total)+ if not on_device and not _host_backup_fits(total):+ if logger is not None:+ logger.info(+ "diffusion.lora_fuse: a %d MiB host backup would cut into the host RAM reserve, staying unfused",+ total >> 20,+ )+ return Falsetry:if on_device:chunks = [torch.empty(n, dtype = torch.uint8, device = device) for n in sizes]else:- chunks = [- torch.empty(n, dtype = torch.uint8, pin_memory = device.type == "cuda") for n in sizes- ]+ pin = _host_backup_pinned(device)+ chunks = [torch.empty(n, dtype = torch.uint8, pin_memory = pin) for n in sizes]except Exception: # noqa: BLE001 - no room for the backup: stay unfusedreturn Falseentries = []@@ -195,6 +203,35 @@ def _backup_on_device(device: Any, nbytes: int) -> bool:return int(free) - int(nbytes) >= max(_DEVICE_BACKUP_MIN_FREE, int(total) // 4)+def _host_guard_enabled() -> bool:+ return (os.environ.get(HOST_GUARD_ENV) or "").strip().lower() not in ("0", "off", "false", "no")+++def _host_backup_pinned(device: Any) -> bool:+ """Pinned only on a discrete card: an integrated GPU's memory is this RAM (host_pinning)."""+ if device.type != "cuda":+ return False+ if not _host_guard_enabled():+ return True+ from .host_pinning import host_pinning_allowed++ return host_pinning_allowed(device)+++def _host_backup_fits(nbytes: int) -> bool:+ """Leaves the small-host reserve of RAM free; an unreadable host keeps the allocation."""+ if not _host_guard_enabled():+ return True+ try:+ from .diffusion_small_host import host_ram_mib, host_reserve_mib+ total_mib, available_mib = host_ram_mib()+ except Exception: # noqa: BLE001 - unreadable: keep the allocation+ return True+ if available_mib is None:+ return True+ return int(nbytes) + (host_reserve_mib(total_mib) << 20) <= int(available_mib) << 20++def _sync(device: Any) -> None:if device.type == "cuda":import torchdiff --git a/studio/backend/tests/test_diffusion_lora_fuse.py b/studio/backend/tests/test_diffusion_lora_fuse.pyindex 55ab2c9051e..98485bac04c 100644--- a/studio/backend/tests/test_diffusion_lora_fuse.py+++ b/studio/backend/tests/test_diffusion_lora_fuse.py@@ -16,6 +16,7 @@ pytest.importorskip("peft")from core.inference import diffusion as dmod # noqa: E402FUSE_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE"+HOST_GUARD_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_HOST_GUARD"def _unet_with_lora():@@ -73,6 +74,13 @@ class _Pipe(types.SimpleNamespace):@pytest.fixturedef engine(monkeypatch):monkeypatch.delenv(FUSE_ENV, raising = False)+ from core.inference import diffusion_memory+ from core.inference.diffusion_small_host import HOST_RAM_LIMIT_ENV++ monkeypatch.delenv(HOST_GUARD_ENV, raising = False)+ monkeypatch.delenv(HOST_RAM_LIMIT_ENV, raising = False)+ # CI runners can sit below the reserve; the RAM tests cap it explicitly.+ monkeypatch.setattr(diffusion_memory, "_available_system_memory_mib", lambda: 1 << 20)monkeypatch.setattr(dmod, "_refuse_lora_without_loader", lambda *a, **k: None)monkeypatch.setattr(dmod, "_lora_load_block_reason", lambda pipe: None)monkeypatch.setattr(dmod, "invalidate_cfg_replica", lambda *a, **k: None)@@ -238,3 +246,57 @@ def test_fold_and_restore_on_the_gpu_in_bf16(engine, monkeypatch, backup):assert _merged_layers(engine.unet) == 0after = _base_weights(engine.unet)assert all(torch.equal(before[k], after[k]) for k in before)+++def test_a_host_backup_that_cuts_into_the_ram_reserve_stays_unfused(engine, monkeypatch):+ from core.inference import diffusion_lora_fuse as lf+ from core.inference.diffusion_small_host import HOST_RAM_LIMIT_ENV++ monkeypatch.setenv(HOST_RAM_LIMIT_ENV, "2048") # below the 3 GiB reserve alone+ before = _base_weights(engine.unet)+ engine.apply(1.0)+ assert _merged_layers(engine.unet) == 0+ assert not lf.fused(engine.pipe)+ assert all(torch.equal(before[k], v) for k, v in _base_weights(engine.unet).items())+ monkeypatch.setenv(HOST_GUARD_ENV, "0")+ engine.apply(1.0)+ assert _merged_layers(engine.unet) > 0+++ "available_mib, nbytes_mib, fits",+ [+ (1 << 20, 4 << 10, True),+ (4 << 10, 512, True),+ (4 << 10, 2 << 10, False),+ (None, 8 << 10, True), # unreadable host: the allocation decides+ ],+)+def test_the_host_backup_keeps_the_small_host_reserve(monkeypatch, available_mib, nbytes_mib, fits):+ from core.inference import diffusion_lora_fuse as lf+ from core.inference import diffusion_small_host++ monkeypatch.delenv(lf.HOST_GUARD_ENV, raising = False)+ monkeypatch.setattr(diffusion_small_host, "host_ram_mib", lambda: (16 << 10, available_mib))+ assert lf._host_backup_fits(nbytes_mib << 20) is fits+ monkeypatch.setenv(lf.HOST_GUARD_ENV, "0")+ assert lf._host_backup_fits(nbytes_mib << 20) is True++[email protected]("integrated", [True, False])+def test_the_host_backup_is_pageable_on_an_integrated_gpu(monkeypatch, integrated):+ from core.inference import diffusion_lora_fuse as lf+ from core.inference import host_pinning++ monkeypatch.delenv(lf.HOST_GUARD_ENV, raising = False)+ asked = []+ monkeypatch.setattr(+ host_pinning, "device_is_integrated", lambda device = None: asked.append(device) or integrated+ )+ assert lf.HOST_GUARD_ENV == HOST_GUARD_ENV+ cuda = torch.device("cuda", 1)+ assert lf._host_backup_pinned(cuda) is (not integrated)+ assert asked == [cuda] # the weights' card, not the current device+ assert lf._host_backup_pinned(torch.device("cpu")) is False+ monkeypatch.setenv(lf.HOST_GUARD_ENV, "0")+ assert lf._host_backup_pinned(cuda) is Truediff --git a/studio/backend/tests/test_integrated_gpu_no_host_pin.py b/studio/backend/tests/test_integrated_gpu_no_host_pin.pyindex c31ade49232..e104cb9ed2b 100644--- a/studio/backend/tests/test_integrated_gpu_no_host_pin.py+++ b/studio/backend/tests/test_integrated_gpu_no_host_pin.py@@ -473,6 +473,7 @@ _REACHED_THROUGH_A_GATE = {"_pin_host_tensor",): "_H3BlockPinner._run only, built by defer_streamed_block_pins behind _background_pin_enabled / ""_fast_pin_supported / h3_te_pin_allowed (the admit too), all on the target device",+ ("core/inference/diffusion_lora_fuse.py", "fuse"): "_host_backup_pinned",("core/inference/diffusion_preview.py", "_device_map"): "exempt: tiny, async preview",("core/inference/diffusion_preview.py",