Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run image and video denoises on one render thread so cuDNN caches are reused (#11843)

Daniel Han6e046f9af476
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 23dc2201131..6cdd9e09911 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -161,6 +161,7 @@ from . import diffusion_cond_cache as cond_cache
from . import diffusion_prompt_cache as prompt_cache
from . import diffusion_gguf_compile as gguf_compile
from . import diffusion_cuda_graph as cuda_graph
+from . import diffusion_render_thread as render_thread
from .diffusion_batched import (
chunk_jobs,
is_oom_error,
@@ -8614,7 +8615,9 @@ class DiffusionBackend:
try:
# inference_mode is faster than no_grad and numerically identical here.
with torch.inference_mode(), protect_ctx:
- out = pipe(**chunk_kwargs).images
+ out = render_thread.run(
+ "diffusion", lambda: pipe(**chunk_kwargs).images
+ )
except Exception as exc: # noqa: BLE001 - reraised unless a splittable OOM
oom = is_oom_error(exc)
if oom:
diff --git a/studio/backend/core/inference/diffusion_render_thread.py b/studio/backend/core/inference/diffusion_render_thread.py
new file mode 100644
index 00000000000..6de37c30890
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_render_thread.py
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""One persistent denoise thread per backend per card: cuDNN caches conv/SDPA plans thread_local (ATen Conv_v8.cpp, MHA.cpp)."""
+
+from __future__ import annotations
+
+import contextvars
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Callable
+
+_ENV = "UNSLOTH_DIFFUSION_RENDER_THREAD"
+_EXECUTORS: dict[str, ThreadPoolExecutor] = {}
+_RENDER_THREAD_IDS: set[int] = set()
+_LOCK = threading.Lock()
+
+
+def enabled() -> bool:
+ if os.environ.get(_ENV, "").strip().lower() in ("0", "false", "no", "off"):
+ return False
+ try:
+ import torch # noqa: PLC0415
+ return bool(torch.cuda.is_available()) and getattr(torch.version, "hip", None) is None
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _mark_render_thread() -> None:
+ _RENDER_THREAD_IDS.add(threading.get_ident())
+
+
+def _executor(name: str) -> ThreadPoolExecutor:
+ with _LOCK:
+ ex = _EXECUTORS.get(name)
+ if ex is None:
+ ex = ThreadPoolExecutor(
+ max_workers = 1,
+ thread_name_prefix = f"unsloth-{name}-render",
+ initializer = _mark_render_thread,
+ )
+ _EXECUTORS[name] = ex
+ return ex
+
+
+def run(name: str, fn: Callable[[], Any]) -> Any:
+ if threading.get_ident() in _RENDER_THREAD_IDS or not enabled():
+ return fn()
+ import torch # noqa: PLC0415
+
+ device = torch.cuda.current_device()
+ inference = torch.is_inference_mode_enabled()
+ grad = torch.is_grad_enabled()
+ ctx = contextvars.copy_context()
+
+ def call() -> Any:
+ torch.cuda.set_device(device)
+ if inference:
+ with torch.inference_mode():
+ return fn()
+ with torch.set_grad_enabled(grad):
+ return fn()
+
+ return _executor(f"{name}-cuda{device}").submit(ctx.run, call).result()
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 4fbecb9a35e..129353a7a4c 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -100,6 +100,7 @@ from .diffusion_memory import (
settled_snapshot_device_memory,
)
from .diffusion_torchao_patches import install_torchao_int_mm_patch
+from . import diffusion_render_thread as render_thread
from .diffusion_speed import (
SPEED_DEFAULT,
SPEED_EAGER,
@@ -7119,7 +7120,7 @@ class VideoBackend:
self._reset_step_cache(pipe)
try:
with torch.inference_mode(), protect_ctx, progress_ctx(), sigma_ctx:
- output = pipe(**kwargs)
+ output = render_thread.run("video", lambda: pipe(**kwargs))
except _VideoGenerationCancelled:
# Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules would
# stay on the GPU.
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 68015365ad1..32a831d953e 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -12496,6 +12496,21 @@ def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypa
assert "prequant artifact missing" in str(failures[0])
+def test_generate_runs_the_pipeline_through_the_render_thread(fake_runtime, tmp_path, monkeypatch):
+ from core.inference import diffusion as diff_mod
+
+ names = []
+
+ def run(name, fn):
+ names.append(name)
+ return fn()
+
+ monkeypatch.setattr(diff_mod.render_thread, "run", run)
+ backend = _loaded_backend(tmp_path)
+ assert len(backend.generate(prompt = "a sloth", steps = 2)["images"]) == 1
+ assert names == ["diffusion"]
+
+
class _StopAfterInstallGate(Exception):
"""Raised just past the FlashInfer pre-install hop."""
diff --git a/studio/backend/tests/test_diffusion_render_thread.py b/studio/backend/tests/test_diffusion_render_thread.py
new file mode 100644
index 00000000000..906f0245031
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_render_thread.py
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import contextvars
+import threading
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_render_thread as rt # noqa: E402
+
+
+def armed(monkeypatch):
+ monkeypatch.setattr(rt, "enabled", lambda: True)
+ devices = {"current": 0, "set": []}
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: devices["current"])
+ monkeypatch.setattr(torch.cuda, "set_device", lambda d: devices["set"].append(d))
+ return devices
+
+
+def test_every_call_lands_on_one_thread_whatever_the_caller(armed):
+ idents = []
+ for _ in range(3):
+ t = threading.Thread(target = lambda: idents.append(rt.run("t1", threading.get_ident)))
+ t.start()
+ t.join()
+ idents.append(rt.run("t1", threading.get_ident))
+ assert len(set(idents)) == 1
+ assert idents[0] != threading.get_ident()
+
+
+def test_caller_state_is_carried(armed):
+ var = contextvars.ContextVar("account", default = None)
+ var.set("alice")
+ armed["current"] = 3
+ with torch.inference_mode():
+ seen = rt.run("t2", lambda: (var.get(), torch.is_inference_mode_enabled()))
+ assert seen == ("alice", True)
+ assert armed["set"][-1] == 3
+ assert rt.run("t2", torch.is_inference_mode_enabled) is False
+
+
+def test_exceptions_reach_the_caller(armed):
+ class Boom(RuntimeError):
+ pass
+
+ def fail():
+ raise Boom("x")
+
+ with pytest.raises(Boom):
+ rt.run("t3", fail)
+ assert rt.run("t3", lambda: 5) == 5
+
+
+def test_nested_call_runs_inline(armed):
+ outer = rt.run("t4", lambda: (threading.get_ident(), rt.run("t4", threading.get_ident)))
+ assert outer[0] == outer[1]
+
+
+def test_disabled_runs_inline(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_RENDER_THREAD", "0")
+ assert rt.enabled() is False
+ assert rt.run("t5", threading.get_ident) == threading.get_ident()
+
+
+def test_one_thread_per_card(armed):
+ armed["current"] = 0
+ a = rt.run("t6", threading.get_ident)
+ armed["current"] = 1
+ b = rt.run("t6", threading.get_ident)
+ armed["current"] = 0
+ assert rt.run("t6", threading.get_ident) == a
+ assert a != b
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index e076f904e3b..b5b375cfca0 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -10541,6 +10541,24 @@ def test_the_boundary_marker_waits_out_a_busy_capture_lock(fake_runtime, monkeyp
assert at_decode.get("phase") == "decode"
+def test_generate_runs_the_video_pipeline_through_the_render_thread(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import video as video_mod
+
+ names = []
+
+ def run(name, fn):
+ names.append(name)
+ return fn()
+
+ monkeypatch.setattr(video_mod.render_thread, "run", run)
+ backend = _load_ltx23_from_dir(tmp_path)
+ backend.generate(prompt = "a sloth")
+ assert names == ["video"]
+ assert backend._state.pipe.last_kwargs["num_inference_steps"] == 8
+
+
@pytest.mark.parametrize("resident", [True, False])
def test_a_failed_replacement_keeps_the_resident_models_nvfp4_state(monkeypatch, resident):
"""A failed replacement keeps the old model, whose CUDA graph still uses the NVFP4 tensors."""