Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: make CPU RAG embedding fast on Windows ROCm torch and honour Windows CPU affinity (#14354)
Daniel Hanb2ccaa01a0e6
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex b8d7e509d36..ce08397b54a 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -28,6 +28,7 @@ from contextvars import ContextVarfrom functools import lru_cachefrom typing import Any, Callable+from utils.cpu_threads import fit_threads_to_affinity, runtime_capped_openblas_loadedfrom utils.hardware.hardware import DeviceType, get_devicefrom utils.transformers_dtype import dtype_kwargsfrom utils.utils import hf_env_offline@@ -558,6 +559,52 @@ def _gate_st_custom_modules() -> None:setattr(module, "import_module_class", import_module_class)+def _route_linears_through_onednn(model) -> int:+ """Run a CPU model's fp32 Linear layers in oneDNN when torch's BLAS is Windows ROCm's OpenBLAS.++ That DLL does ~4 GFLOPS per core against MKL's ~130, so bge-small indexed ~7x slower on all cores+ (#14313). oneDNN ships in the same wheel; the vectors match to ~2e-7, so the index stays valid.+ Probed on the model's own first layer; any doubt keeps the stock path. Returns layers rerouted.+ """+ if not runtime_capped_openblas_loaded():+ return 0+ try:+ import torch++ if not getattr(torch.version, "hip", None) or not torch.backends.mkldnn.is_available():+ return 0+ linears = [+ m+ for m in model.modules()+ if type(m) is torch.nn.Linear+ and m.weight.device.type == "cpu"+ and m.weight.dtype == torch.float32+ ]+ if not linears:+ return 0++ class OneDnnLinear(torch.nn.Linear):+ def forward(self, x):+ return torch.ops.mkldnn._linear_pointwise(x, self.weight, self.bias, "none", [], "")++ probe = linears[0]+ with torch.inference_mode():+ x = torch.randn(2, 5, probe.in_features)+ expected = torch.nn.functional.linear(x, probe.weight, probe.bias)+ got = OneDnnLinear.forward(probe, x)+ if got.shape != expected.shape or not torch.allclose(got, expected, rtol = 1e-4, atol = 1e-5):+ return 0+ except Exception: # noqa: BLE001 -- the stock path is always correct+ return 0+ for module in linears:+ module.__class__ = OneDnnLinear+ logger.info(+ "embedding model: %d Linear layers run in oneDNN (ROCm OpenBLAS is slow on CPU)",+ len(linears),+ )+ return len(linears)++def _get(model_name: str | None = None):"""Cached SentenceTransformer, (re)loading on a name change. Loaded in fp16 on anaccelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU; see ``_load_dtype``@@ -652,6 +699,7 @@ def _get(model_name: str | None = None):_model = SentenceTransformer(load_target, **st_kwargs)finally:_emit_load_reports(report)+ _route_linears_through_onednn(_model)_name = nameif download_pending:# Retire only once the model is constructed: retiring earlier let a failed construction fall through@@ -692,6 +740,8 @@ def _st_encode(# Admission and model lookup are one lease: lookup first would let unload clear the globals while# this call still held a strong reference.model = _get(model_name)+ # Affinity narrowed in Task Manager mid-index must reach this thread's pool (#14313).+ fit_threads_to_affinity()os.environ["TOKENIZERS_PARALLELISM"] = "true"try:with _inference_ctx():diff --git a/studio/backend/tests/test_cpu_threads_windows_affinity.py b/studio/backend/tests/test_cpu_threads_windows_affinity.pynew file mode 100644index 00000000000..d238380edab--- /dev/null+++ b/studio/backend/tests/test_cpu_threads_windows_affinity.py@@ -0,0 +1,233 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Windows CPU affinity reaches the thread pools (#14313); sys.platform and kernel32 are faked."""++import ctypes+import os+import sys++import pytest++import utils.cpu_threads as cpu_threads+++def _kernel32(+ process_mask,+ system_mask,+ ok = True,+):+ class _GetProcessAffinityMask:+ argtypes = restype = None++ def __call__(self, handle, process, system):+ process._obj.value = process_mask+ system._obj.value = system_mask+ return ok++ class _GetCurrentProcess:+ restype = None++ def __call__(self):+ return -1++ class _Kernel32:+ GetProcessAffinityMask = _GetProcessAffinityMask()+ GetCurrentProcess = _GetCurrentProcess()++ return _Kernel32()+++def windows_mask(monkeypatch):+ """Set the faked (process, system) affinity masks: a 32-CPU host, unrestricted until told otherwise."""+ masks = {"process": (1 << 32) - 1, "system": (1 << 32) - 1, "ok": True}++ def fake_windll(name, use_last_error = False):+ assert name == "kernel32"+ return _kernel32(masks["process"], masks["system"], masks["ok"])++ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setattr(ctypes, "WinDLL", fake_windll, raising = False)+ # Python on Windows: process_cpu_count is cpu_count and there is no sched_getaffinity.+ monkeypatch.setattr(os, "cpu_count", lambda: 32)+ monkeypatch.setattr(os, "process_cpu_count", lambda: 32, raising = False)+ monkeypatch.delattr(os, "sched_getaffinity", raising = False)+ monkeypatch.setattr(cpu_threads, "_openblas_memory_headroom", lambda: None)+ monkeypatch.setattr(cpu_threads, "_torch_threads_unfitted", None, raising = False)+ monkeypatch.setattr(cpu_threads, "_fitted_affinity", None, raising = False)+ return masks+++ "process, system, ok, expected",+ [+ (0b1, (1 << 32) - 1, True, 1),+ (0b1100, (1 << 32) - 1, True, 2),+ ((1 << 32) - 1, (1 << 32) - 1, True, None),+ # threads in several processor groups+ (0, 0, True, None),+ (0b1, (1 << 32) - 1, False, None),+ ],+)+def test_windows_affinity_reader(windows_mask, process, system, ok, expected):+ windows_mask.update(process = process, system = system, ok = ok)++ assert cpu_threads._windows_affinity_cpus() == expected+++# The defect: one allowed CPU of 32 still sized every pool for 32.+def test_usable_cpus_follows_a_windows_affinity_mask(windows_mask):+ windows_mask["process"] = 0b1++ assert cpu_threads._usable_cpus() == 1+ assert cpu_threads.default_openblas_threads() == 1+++def test_unrestricted_windows_keeps_the_core_count(windows_mask):+ assert cpu_threads._usable_cpus() == 32+ assert cpu_threads.default_openblas_threads() == 8+++def test_an_unreadable_mask_keeps_the_core_count(windows_mask, monkeypatch):+ def broken(name, use_last_error = False):+ raise OSError("no kernel32")++ monkeypatch.setattr(ctypes, "WinDLL", broken, raising = False)++ assert cpu_threads._usable_cpus() == 32+++class _Torch:+ def __init__(self, threads):+ self.threads = threads+ self.set_calls = []++ def get_num_threads(self):+ return self.threads++ def set_num_threads(self, n):+ self.set_calls.append(n)+ self.threads = n+++def dll_calls(windows_mask, monkeypatch):+ calls = []+ monkeypatch.setattr(+ cpu_threads,+ "apply_openblas_runtime_cap",+ lambda: calls.append(cpu_threads._openblas_thread_target()),+ )+ saved = {k: os.environ.get(k) for k in ("OPENBLAS_NUM_THREADS", "UNSLOTH_OPENBLAS_DEFAULTED")}+ os.environ["OPENBLAS_NUM_THREADS"] = os.environ["UNSLOTH_OPENBLAS_DEFAULTED"] = "8"+ yield calls+ for key, value in saved.items():+ if value is None:+ os.environ.pop(key, None)+ else:+ os.environ[key] = value+++# Task Manager narrows a running backend, then widens it again.+def test_fit_follows_affinity_narrowed_and_widened_at_runtime(windows_mask, dll_calls, monkeypatch):+ torch = _Torch(16)+ monkeypatch.setitem(sys.modules, "torch", torch)++ cpu_threads.fit_threads_to_affinity()+ assert torch.set_calls == [] and dll_calls == []++ windows_mask["process"] = 0b1+ cpu_threads.fit_threads_to_affinity()+ cpu_threads.fit_threads_to_affinity()+ assert torch.set_calls == [1]+ assert dll_calls == [1]++ windows_mask["process"] = 0b1111+ cpu_threads.fit_threads_to_affinity()+ assert torch.set_calls == [1, 4]+ assert dll_calls == [1, 4]++ windows_mask["process"] = (1 << 32) - 1+ cpu_threads.fit_threads_to_affinity()+ assert torch.set_calls == [1, 4, 16]+ assert dll_calls == [1, 4, 16]+++def test_fit_never_raises_a_smaller_user_count(windows_mask, dll_calls, monkeypatch):+ torch = _Torch(2)+ monkeypatch.setitem(sys.modules, "torch", torch)+ windows_mask["process"] = 0b1111++ cpu_threads.fit_threads_to_affinity()++ assert torch.set_calls == []+ assert dll_calls == [2]+++def test_a_user_openblas_value_is_clamped_to_the_allowed_cpus(windows_mask, dll_calls, monkeypatch):+ monkeypatch.setitem(sys.modules, "torch", _Torch(16))+ os.environ["OPENBLAS_NUM_THREADS"] = "6"+ windows_mask["process"] = 0b11++ cpu_threads.fit_threads_to_affinity()++ assert dll_calls == [2]+++def test_fit_is_a_noop_before_torch_or_off_windows(windows_mask, dll_calls, monkeypatch):+ windows_mask["process"] = 0b1+ monkeypatch.delitem(sys.modules, "torch", raising = False)+ cpu_threads.fit_threads_to_affinity()++ torch = _Torch(16)+ monkeypatch.setitem(sys.modules, "torch", torch)+ monkeypatch.setattr(sys, "platform", "linux")+ cpu_threads.fit_threads_to_affinity()++ assert torch.set_calls == [] and dll_calls == []+++def test_a_failing_torch_never_breaks_the_batch(windows_mask, dll_calls, monkeypatch):+ class _Broken:+ def get_num_threads(self):+ raise RuntimeError("torch went away")++ monkeypatch.setitem(sys.modules, "torch", _Broken())+ windows_mask["process"] = 0b1++ cpu_threads.fit_threads_to_affinity()++ assert dll_calls == []+++# A process launched already narrowed gets fitted pools right after torch imports.+def test_torch_import_fits_an_inherited_affinity(windows_mask, dll_calls, monkeypatch):+ torch = _Torch(16)+ monkeypatch.setitem(sys.modules, "torch", torch)+ windows_mask["process"] = 0b1++ assert cpu_threads.install_openblas_runtime_cap()++ assert torch.set_calls == [1]+ assert dll_calls == [1, 1]+++def test_every_cpu_embedding_batch_refits(monkeypatch):+ import ast+ from pathlib import Path++ source = (Path(__file__).resolve().parent.parent / "core/rag/embeddings.py").read_text(+ encoding = "utf-8"+ )+ encode = next(+ node+ for node in ast.walk(ast.parse(source))+ if isinstance(node, ast.FunctionDef) and node.name == "_st_encode"+ )+ called = [+ node.func.id+ for node in ast.walk(encode)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)+ ]+ assert "fit_threads_to_affinity" in calleddiff --git a/studio/backend/tests/test_embedding_onednn_linear.py b/studio/backend/tests/test_embedding_onednn_linear.pynew file mode 100644index 00000000000..f84c658c8a2--- /dev/null+++ b/studio/backend/tests/test_embedding_onednn_linear.py@@ -0,0 +1,110 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Windows ROCm: the CPU embedder's Linear layers run in oneDNN instead of rocm-openblas.dll (#14313).++The DLL lookup and torch.version.hip are faked; the oneDNN op itself is real, so this needs a torch with mkldnn.+"""++import ast+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+if not torch.backends.mkldnn.is_available():+ pytest.skip("torch built without oneDNN", allow_module_level = True)++import core.rag.embeddings as embeddings # noqa: E402+++def rocm_windows(monkeypatch):+ monkeypatch.setattr(embeddings, "runtime_capped_openblas_loaded", lambda: True)+ monkeypatch.setattr(torch.version, "hip", "7.14.0", raising = False)+++def _model():+ torch.manual_seed(0)+ return torch.nn.Sequential(+ torch.nn.Linear(384, 1536), torch.nn.GELU(), torch.nn.Linear(1536, 384, bias = False)+ )+++def test_linears_run_in_onednn_with_the_same_vectors(rocm_windows):+ model = _model()+ x = torch.randn(4, 37, 384)+ with torch.inference_mode():+ expected = model(x)++ assert embeddings._route_linears_through_onednn(model) == 2++ assert all(type(m).__name__ == "OneDnnLinear" for m in model if isinstance(m, torch.nn.Linear))+ with torch.inference_mode():+ got = model(x)+ assert torch.allclose(got, expected, rtol = 1e-4, atol = 1e-5)+++def test_nothing_changes_without_rocm_openblas(monkeypatch):+ monkeypatch.setattr(embeddings, "runtime_capped_openblas_loaded", lambda: False)+ model = _model()++ assert embeddings._route_linears_through_onednn(model) == 0+ assert type(model[0]) is torch.nn.Linear+++def test_a_cuda_wheel_on_windows_keeps_its_blas(monkeypatch):+ monkeypatch.setattr(embeddings, "runtime_capped_openblas_loaded", lambda: True)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ model = _model()++ assert embeddings._route_linears_through_onednn(model) == 0+ assert type(model[0]) is torch.nn.Linear+++def test_non_fp32_layers_are_left_alone(rocm_windows):+ model = _model().to(torch.bfloat16)++ assert embeddings._route_linears_through_onednn(model) == 0+ assert type(model[0]) is torch.nn.Linear+++def test_a_probe_mismatch_keeps_the_stock_path(rocm_windows, monkeypatch):+ monkeypatch.setattr(torch, "allclose", lambda *a, **k: False)+ model = _model()++ assert embeddings._route_linears_through_onednn(model) == 0+ assert type(model[0]) is torch.nn.Linear+++def test_a_missing_onednn_op_keeps_the_stock_path(rocm_windows, monkeypatch):+ class _NoMkldnn:+ def __getattr__(self, name):+ raise AttributeError(name)++ monkeypatch.setattr(torch.ops, "mkldnn", _NoMkldnn(), raising = False)+ model = _model()++ assert embeddings._route_linears_through_onednn(model) == 0+ assert type(model[0]) is torch.nn.Linear+++def test_a_model_without_modules_is_ignored(rocm_windows):+ assert embeddings._route_linears_through_onednn(object()) == 0+++def test_every_load_routes_the_model():+ source = (Path(__file__).resolve().parent.parent / "core/rag/embeddings.py").read_text(+ encoding = "utf-8"+ )+ get = next(+ node+ for node in ast.walk(ast.parse(source))+ if isinstance(node, ast.FunctionDef) and node.name == "_get"+ )+ called = [+ node.func.id+ for node in ast.walk(get)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)+ ]+ assert "_route_linears_through_onednn" in calleddiff --git a/studio/backend/utils/cpu_threads.py b/studio/backend/utils/cpu_threads.pyindex 22b348fd542..0a8c4b28f1f 100644--- a/studio/backend/utils/cpu_threads.py+++ b/studio/backend/utils/cpu_threads.py@@ -32,8 +32,44 @@ def _threads_that_fit(threads: int, cost: int) -> int:return max(1, min(threads, int(headroom * _OPENBLAS_MEMORY_SHARE) // cost))+def _windows_affinity_cpus() -> Optional[int]:+ """CPUs in this process's Windows affinity mask when narrower than the system's, else None.++ Python reads no affinity on Windows (process_cpu_count is cpu_count there). Zero masks mean the+ process spans processor groups, and a full group mask says nothing about the others.+ """+ try:+ import ctypes+ from ctypes import wintypes++ kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)+ kernel32.GetCurrentProcess.restype = wintypes.HANDLE+ get_mask = kernel32.GetProcessAffinityMask+ get_mask.argtypes = [+ wintypes.HANDLE,+ ctypes.POINTER(ctypes.c_size_t),+ ctypes.POINTER(ctypes.c_size_t),+ ]+ get_mask.restype = wintypes.BOOL+ process_mask, system_mask = ctypes.c_size_t(), ctypes.c_size_t()+ if not get_mask(+ kernel32.GetCurrentProcess(), ctypes.byref(process_mask), ctypes.byref(system_mask)+ ):+ return None+ allowed = bin(process_mask.value).count("1")+ if not allowed or allowed >= bin(system_mask.value).count("1"):+ return None+ return allowed+ except Exception: # noqa: BLE001 -- unknown affinity means the core count+ return None++def _usable_cpus() -> int:"""CPUs this process may run on: an affinity mask (taskset, Slurm, cpusets) can be far below the host's."""+ if sys.platform == "win32":+ allowed = _windows_affinity_cpus()+ if allowed is not None:+ return allowedcount = getattr(os, "process_cpu_count", None) # 3.13+if count is not None:return count() or 1@@ -229,7 +265,22 @@ def _openblas_thread_target(environ: Optional[MutableMapping[str, str]] = None)if target < 1:return None# OpenBLAS clamps an env value to the core count; its runtime setter only to MAX_THREADS.- return min(target, os.cpu_count() or target)+ return min(target, os.cpu_count() or target, _usable_cpus())+++def runtime_capped_openblas_loaded() -> bool:+ """Whether torch's CPU BLAS is AMD's Windows ROCm OpenBLAS: about 30x slower per core than MKL (#14313)."""+ if sys.platform != "win32":+ return False+ try:+ import ctypes++ get_module = ctypes.WinDLL("kernel32", use_last_error = True).GetModuleHandleW+ get_module.argtypes = [ctypes.c_wchar_p]+ get_module.restype = ctypes.c_void_p+ return any(get_module(name) for name in _RUNTIME_CAPPED_OPENBLAS)+ except Exception: # noqa: BLE001+ return Falsedef apply_openblas_runtime_cap() -> list:@@ -263,6 +314,45 @@ def apply_openblas_runtime_cap() -> list:return capped+_affinity_lock = threading.Lock()+_torch_threads_unfitted: Optional[int] = None+_fitted_affinity: Optional[int] = None+++def fit_threads_to_affinity() -> None:+ """Fit torch's pool on this thread, and the ROCm OpenBLAS one, to the CPUs Windows lets this process use.++ Affinity is often narrowed after launch (Task Manager), and neither pool reads it: 16 threads on one+ allowed CPU spin against each other (#14313). Widening it again restores the launch counts. Cheap+ enough to call before every CPU batch; a no-op off Windows, before torch loads, or while unrestricted.+ """+ if sys.platform != "win32" or "torch" not in sys.modules:+ return+ global _torch_threads_unfitted, _fitted_affinity+ with _affinity_lock:+ allowed = _windows_affinity_cpus()+ if allowed is None and _torch_threads_unfitted is None:+ return+ try:+ torch = sys.modules["torch"]+ current = int(torch.get_num_threads())+ if _torch_threads_unfitted is None:+ _torch_threads_unfitted = current+ target = (+ _torch_threads_unfitted+ if allowed is None+ else min(_torch_threads_unfitted, allowed)+ )+ # Per thread under OpenMP: the setter reaches only the calling thread's pool.+ if current != target:+ torch.set_num_threads(target)+ except Exception: # noqa: BLE001 -- a thread cap must never break a CPU batch+ return+ if allowed != _fitted_affinity:+ _fitted_affinity = allowed+ apply_openblas_runtime_cap()++class _TorchLoader(importlib.abc.Loader):__slots__ = ("_loader",)@@ -284,6 +374,7 @@ class _TorchLoader(importlib.abc.Loader):if getattr(module, "__loader__", None) is self:module.__loader__ = self._loaderapply_openblas_runtime_cap()+ fit_threads_to_affinity()def __getattr__(self, attribute):if attribute == "_loader":@@ -328,6 +419,7 @@ def install_openblas_runtime_cap() -> bool:return Falseif "torch" in sys.modules:apply_openblas_runtime_cap()+ fit_threads_to_affinity()return Trueif not any(getattr(finder, _OPENBLAS_CAP_SENTINEL, False) for finder in sys.meta_path):sys.meta_path.insert(0, _TorchImportFinder())