Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

2.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""One persistent denoise thread per backend per card: cuDNN caches conv/SDPA plans thread_local (ATen Conv_v8.cpp, MHA.cpp)."""

from __future__ import annotations

import contextvars
import os
import threading
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Callable
from .rocm_bf16 import is_rocm_torch

_ENV = "UNSLOTH_DIFFUSION_RENDER_THREAD"
_EXECUTORS: dict[str, ThreadPoolExecutor] = {}
_RENDER_THREAD_IDS: set[int] = set()
_LOCK = threading.Lock()


def enabled() -> bool:
    if os.environ.get(_ENV, "").strip().lower() in ("0", "false", "no", "off"):
        return False
    try:
        import torch  # noqa: PLC0415
        return bool(torch.cuda.is_available()) and not is_rocm_torch(torch)
    except Exception:  # noqa: BLE001
        return False


def _mark_render_thread() -> None:
    _RENDER_THREAD_IDS.add(threading.get_ident())


def _executor(name: str) -> ThreadPoolExecutor:
    with _LOCK:
        ex = _EXECUTORS.get(name)
        if ex is None:
            ex = ThreadPoolExecutor(
                max_workers = 1,
                thread_name_prefix = f"unsloth-{name}-render",
                initializer = _mark_render_thread,
            )
            _EXECUTORS[name] = ex
        return ex


def _apply_compile_config() -> None:
    try:
        from . import diffusion_compile_config  # noqa: PLC0415
        diffusion_compile_config.apply()
    except Exception:  # noqa: BLE001 - never fail a render over a config write
        pass


def run(name: str, fn: Callable[[], Any]) -> Any:
    if threading.get_ident() in _RENDER_THREAD_IDS or not enabled():
        _apply_compile_config()
        return fn()
    import torch  # noqa: PLC0415

    device = torch.cuda.current_device()
    inference = torch.is_inference_mode_enabled()
    grad = torch.is_grad_enabled()
    ctx = contextvars.copy_context()

    def call() -> Any:
        torch.cuda.set_device(device)
        _apply_compile_config()
        if inference:
            with torch.inference_mode():
                return fn()
        with torch.set_grad_enabled(grad):
            return fn()

    return _executor(f"{name}-cuda{device}").submit(ctx.run, call).result()