Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run image and video denoises on one render thread so cuDNN caches are reused (#11843)
Daniel Han6e046f9af476
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 23dc2201131..6cdd9e09911 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -161,6 +161,7 @@ from . import diffusion_cond_cache as cond_cachefrom . import diffusion_prompt_cache as prompt_cachefrom . import diffusion_gguf_compile as gguf_compilefrom . import diffusion_cuda_graph as cuda_graph+from . import diffusion_render_thread as render_threadfrom .diffusion_batched import (chunk_jobs,is_oom_error,@@ -8614,7 +8615,9 @@ class DiffusionBackend:try:# inference_mode is faster than no_grad and numerically identical here.with torch.inference_mode(), protect_ctx:- out = pipe(**chunk_kwargs).images+ out = render_thread.run(+ "diffusion", lambda: pipe(**chunk_kwargs).images+ )except Exception as exc: # noqa: BLE001 - reraised unless a splittable OOMoom = is_oom_error(exc)if oom:diff --git a/studio/backend/core/inference/diffusion_render_thread.py b/studio/backend/core/inference/diffusion_render_thread.pynew file mode 100644index 00000000000..6de37c30890--- /dev/null+++ b/studio/backend/core/inference/diffusion_render_thread.py@@ -0,0 +1,65 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""One persistent denoise thread per backend per card: cuDNN caches conv/SDPA plans thread_local (ATen Conv_v8.cpp, MHA.cpp)."""++from __future__ import annotations++import contextvars+import os+import threading+from concurrent.futures import ThreadPoolExecutor+from typing import Any, Callable++_ENV = "UNSLOTH_DIFFUSION_RENDER_THREAD"+_EXECUTORS: dict[str, ThreadPoolExecutor] = {}+_RENDER_THREAD_IDS: set[int] = set()+_LOCK = threading.Lock()+++def enabled() -> bool:+ if os.environ.get(_ENV, "").strip().lower() in ("0", "false", "no", "off"):+ return False+ try:+ import torch # noqa: PLC0415+ return bool(torch.cuda.is_available()) and getattr(torch.version, "hip", None) is None+ except Exception: # noqa: BLE001+ return False+++def _mark_render_thread() -> None:+ _RENDER_THREAD_IDS.add(threading.get_ident())+++def _executor(name: str) -> ThreadPoolExecutor:+ with _LOCK:+ ex = _EXECUTORS.get(name)+ if ex is None:+ ex = ThreadPoolExecutor(+ max_workers = 1,+ thread_name_prefix = f"unsloth-{name}-render",+ initializer = _mark_render_thread,+ )+ _EXECUTORS[name] = ex+ return ex+++def run(name: str, fn: Callable[[], Any]) -> Any:+ if threading.get_ident() in _RENDER_THREAD_IDS or not enabled():+ return fn()+ import torch # noqa: PLC0415++ device = torch.cuda.current_device()+ inference = torch.is_inference_mode_enabled()+ grad = torch.is_grad_enabled()+ ctx = contextvars.copy_context()++ def call() -> Any:+ torch.cuda.set_device(device)+ if inference:+ with torch.inference_mode():+ return fn()+ with torch.set_grad_enabled(grad):+ return fn()++ return _executor(f"{name}-cuda{device}").submit(ctx.run, call).result()diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 4fbecb9a35e..129353a7a4c 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -100,6 +100,7 @@ from .diffusion_memory import (settled_snapshot_device_memory,)from .diffusion_torchao_patches import install_torchao_int_mm_patch+from . import diffusion_render_thread as render_threadfrom .diffusion_speed import (SPEED_DEFAULT,SPEED_EAGER,@@ -7119,7 +7120,7 @@ class VideoBackend:self._reset_step_cache(pipe)try:with torch.inference_mode(), protect_ctx, progress_ctx(), sigma_ctx:- output = pipe(**kwargs)+ output = render_thread.run("video", lambda: pipe(**kwargs))except _VideoGenerationCancelled:# Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules would# stay on the GPU.diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 68015365ad1..32a831d953e 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -12496,6 +12496,21 @@ def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypaassert "prequant artifact missing" in str(failures[0])+def test_generate_runs_the_pipeline_through_the_render_thread(fake_runtime, tmp_path, monkeypatch):+ from core.inference import diffusion as diff_mod++ names = []++ def run(name, fn):+ names.append(name)+ return fn()++ monkeypatch.setattr(diff_mod.render_thread, "run", run)+ backend = _loaded_backend(tmp_path)+ assert len(backend.generate(prompt = "a sloth", steps = 2)["images"]) == 1+ assert names == ["diffusion"]++class _StopAfterInstallGate(Exception):"""Raised just past the FlashInfer pre-install hop."""diff --git a/studio/backend/tests/test_diffusion_render_thread.py b/studio/backend/tests/test_diffusion_render_thread.pynew file mode 100644index 00000000000..906f0245031--- /dev/null+++ b/studio/backend/tests/test_diffusion_render_thread.py@@ -0,0 +1,77 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import contextvars+import threading++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_render_thread as rt # noqa: E402+++def armed(monkeypatch):+ monkeypatch.setattr(rt, "enabled", lambda: True)+ devices = {"current": 0, "set": []}+ monkeypatch.setattr(torch.cuda, "current_device", lambda: devices["current"])+ monkeypatch.setattr(torch.cuda, "set_device", lambda d: devices["set"].append(d))+ return devices+++def test_every_call_lands_on_one_thread_whatever_the_caller(armed):+ idents = []+ for _ in range(3):+ t = threading.Thread(target = lambda: idents.append(rt.run("t1", threading.get_ident)))+ t.start()+ t.join()+ idents.append(rt.run("t1", threading.get_ident))+ assert len(set(idents)) == 1+ assert idents[0] != threading.get_ident()+++def test_caller_state_is_carried(armed):+ var = contextvars.ContextVar("account", default = None)+ var.set("alice")+ armed["current"] = 3+ with torch.inference_mode():+ seen = rt.run("t2", lambda: (var.get(), torch.is_inference_mode_enabled()))+ assert seen == ("alice", True)+ assert armed["set"][-1] == 3+ assert rt.run("t2", torch.is_inference_mode_enabled) is False+++def test_exceptions_reach_the_caller(armed):+ class Boom(RuntimeError):+ pass++ def fail():+ raise Boom("x")++ with pytest.raises(Boom):+ rt.run("t3", fail)+ assert rt.run("t3", lambda: 5) == 5+++def test_nested_call_runs_inline(armed):+ outer = rt.run("t4", lambda: (threading.get_ident(), rt.run("t4", threading.get_ident)))+ assert outer[0] == outer[1]+++def test_disabled_runs_inline(monkeypatch):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_RENDER_THREAD", "0")+ assert rt.enabled() is False+ assert rt.run("t5", threading.get_ident) == threading.get_ident()+++def test_one_thread_per_card(armed):+ armed["current"] = 0+ a = rt.run("t6", threading.get_ident)+ armed["current"] = 1+ b = rt.run("t6", threading.get_ident)+ armed["current"] = 0+ assert rt.run("t6", threading.get_ident) == a+ assert a != bdiff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex e076f904e3b..b5b375cfca0 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -10541,6 +10541,24 @@ def test_the_boundary_marker_waits_out_a_busy_capture_lock(fake_runtime, monkeypassert at_decode.get("phase") == "decode"+def test_generate_runs_the_video_pipeline_through_the_render_thread(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import video as video_mod++ names = []++ def run(name, fn):+ names.append(name)+ return fn()++ monkeypatch.setattr(video_mod.render_thread, "run", run)+ backend = _load_ltx23_from_dir(tmp_path)+ backend.generate(prompt = "a sloth")+ assert names == ["video"]+ assert backend._state.pipe.last_kwargs["num_inference_steps"] == 8++@pytest.mark.parametrize("resident", [True, False])def test_a_failed_replacement_keeps_the_resident_models_nvfp4_state(monkeypatch, resident):"""A failed replacement keeps the old model, whose CUDA graph still uses the NVFP4 tensors."""