Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: make CPU RAG embedding fast on Windows ROCm torch and honour Windows CPU affinity (#14354)

Daniel Hanb2ccaa01a0e6
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index b8d7e509d36..ce08397b54a 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -28,6 +28,7 @@ from contextvars import ContextVar
from functools import lru_cache
from typing import Any, Callable
+from utils.cpu_threads import fit_threads_to_affinity, runtime_capped_openblas_loaded
from utils.hardware.hardware import DeviceType, get_device
from utils.transformers_dtype import dtype_kwargs
from utils.utils import hf_env_offline
@@ -558,6 +559,52 @@ def _gate_st_custom_modules() -> None:
setattr(module, "import_module_class", import_module_class)
+def _route_linears_through_onednn(model) -> int:
+ """Run a CPU model's fp32 Linear layers in oneDNN when torch's BLAS is Windows ROCm's OpenBLAS.
+
+ That DLL does ~4 GFLOPS per core against MKL's ~130, so bge-small indexed ~7x slower on all cores
+ (#14313). oneDNN ships in the same wheel; the vectors match to ~2e-7, so the index stays valid.
+ Probed on the model's own first layer; any doubt keeps the stock path. Returns layers rerouted.
+ """
+ if not runtime_capped_openblas_loaded():
+ return 0
+ try:
+ import torch
+
+ if not getattr(torch.version, "hip", None) or not torch.backends.mkldnn.is_available():
+ return 0
+ linears = [
+ m
+ for m in model.modules()
+ if type(m) is torch.nn.Linear
+ and m.weight.device.type == "cpu"
+ and m.weight.dtype == torch.float32
+ ]
+ if not linears:
+ return 0
+
+ class OneDnnLinear(torch.nn.Linear):
+ def forward(self, x):
+ return torch.ops.mkldnn._linear_pointwise(x, self.weight, self.bias, "none", [], "")
+
+ probe = linears[0]
+ with torch.inference_mode():
+ x = torch.randn(2, 5, probe.in_features)
+ expected = torch.nn.functional.linear(x, probe.weight, probe.bias)
+ got = OneDnnLinear.forward(probe, x)
+ if got.shape != expected.shape or not torch.allclose(got, expected, rtol = 1e-4, atol = 1e-5):
+ return 0
+ except Exception: # noqa: BLE001 -- the stock path is always correct
+ return 0
+ for module in linears:
+ module.__class__ = OneDnnLinear
+ logger.info(
+ "embedding model: %d Linear layers run in oneDNN (ROCm OpenBLAS is slow on CPU)",
+ len(linears),
+ )
+ return len(linears)
+
+
def _get(model_name: str | None = None):
"""Cached SentenceTransformer, (re)loading on a name change. Loaded in fp16 on an
accelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU; see ``_load_dtype``
@@ -652,6 +699,7 @@ def _get(model_name: str | None = None):
_model = SentenceTransformer(load_target, **st_kwargs)
finally:
_emit_load_reports(report)
+ _route_linears_through_onednn(_model)
_name = name
if download_pending:
# Retire only once the model is constructed: retiring earlier let a failed construction fall through
@@ -692,6 +740,8 @@ def _st_encode(
# Admission and model lookup are one lease: lookup first would let unload clear the globals while
# this call still held a strong reference.
model = _get(model_name)
+ # Affinity narrowed in Task Manager mid-index must reach this thread's pool (#14313).
+ fit_threads_to_affinity()
os.environ["TOKENIZERS_PARALLELISM"] = "true"
try:
with _inference_ctx():
diff --git a/studio/backend/tests/test_cpu_threads_windows_affinity.py b/studio/backend/tests/test_cpu_threads_windows_affinity.py
new file mode 100644
index 00000000000..d238380edab
--- /dev/null
+++ b/studio/backend/tests/test_cpu_threads_windows_affinity.py
@@ -0,0 +1,233 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Windows CPU affinity reaches the thread pools (#14313); sys.platform and kernel32 are faked."""
+
+import ctypes
+import os
+import sys
+
+import pytest
+
+import utils.cpu_threads as cpu_threads
+
+
+def _kernel32(
+ process_mask,
+ system_mask,
+ ok = True,
+):
+ class _GetProcessAffinityMask:
+ argtypes = restype = None
+
+ def __call__(self, handle, process, system):
+ process._obj.value = process_mask
+ system._obj.value = system_mask
+ return ok
+
+ class _GetCurrentProcess:
+ restype = None
+
+ def __call__(self):
+ return -1
+
+ class _Kernel32:
+ GetProcessAffinityMask = _GetProcessAffinityMask()
+ GetCurrentProcess = _GetCurrentProcess()
+
+ return _Kernel32()
+
+
+def windows_mask(monkeypatch):
+ """Set the faked (process, system) affinity masks: a 32-CPU host, unrestricted until told otherwise."""
+ masks = {"process": (1 << 32) - 1, "system": (1 << 32) - 1, "ok": True}
+
+ def fake_windll(name, use_last_error = False):
+ assert name == "kernel32"
+ return _kernel32(masks["process"], masks["system"], masks["ok"])
+
+ monkeypatch.setattr(sys, "platform", "win32")
+ monkeypatch.setattr(ctypes, "WinDLL", fake_windll, raising = False)
+ # Python on Windows: process_cpu_count is cpu_count and there is no sched_getaffinity.
+ monkeypatch.setattr(os, "cpu_count", lambda: 32)
+ monkeypatch.setattr(os, "process_cpu_count", lambda: 32, raising = False)
+ monkeypatch.delattr(os, "sched_getaffinity", raising = False)
+ monkeypatch.setattr(cpu_threads, "_openblas_memory_headroom", lambda: None)
+ monkeypatch.setattr(cpu_threads, "_torch_threads_unfitted", None, raising = False)
+ monkeypatch.setattr(cpu_threads, "_fitted_affinity", None, raising = False)
+ return masks
+
+
+ "process, system, ok, expected",
+ [
+ (0b1, (1 << 32) - 1, True, 1),
+ (0b1100, (1 << 32) - 1, True, 2),
+ ((1 << 32) - 1, (1 << 32) - 1, True, None),
+ # threads in several processor groups
+ (0, 0, True, None),
+ (0b1, (1 << 32) - 1, False, None),
+ ],
+)
+def test_windows_affinity_reader(windows_mask, process, system, ok, expected):
+ windows_mask.update(process = process, system = system, ok = ok)
+
+ assert cpu_threads._windows_affinity_cpus() == expected
+
+
+# The defect: one allowed CPU of 32 still sized every pool for 32.
+def test_usable_cpus_follows_a_windows_affinity_mask(windows_mask):
+ windows_mask["process"] = 0b1
+
+ assert cpu_threads._usable_cpus() == 1
+ assert cpu_threads.default_openblas_threads() == 1
+
+
+def test_unrestricted_windows_keeps_the_core_count(windows_mask):
+ assert cpu_threads._usable_cpus() == 32
+ assert cpu_threads.default_openblas_threads() == 8
+
+
+def test_an_unreadable_mask_keeps_the_core_count(windows_mask, monkeypatch):
+ def broken(name, use_last_error = False):
+ raise OSError("no kernel32")
+
+ monkeypatch.setattr(ctypes, "WinDLL", broken, raising = False)
+
+ assert cpu_threads._usable_cpus() == 32
+
+
+class _Torch:
+ def __init__(self, threads):
+ self.threads = threads
+ self.set_calls = []
+
+ def get_num_threads(self):
+ return self.threads
+
+ def set_num_threads(self, n):
+ self.set_calls.append(n)
+ self.threads = n
+
+
+def dll_calls(windows_mask, monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ cpu_threads,
+ "apply_openblas_runtime_cap",
+ lambda: calls.append(cpu_threads._openblas_thread_target()),
+ )
+ saved = {k: os.environ.get(k) for k in ("OPENBLAS_NUM_THREADS", "UNSLOTH_OPENBLAS_DEFAULTED")}
+ os.environ["OPENBLAS_NUM_THREADS"] = os.environ["UNSLOTH_OPENBLAS_DEFAULTED"] = "8"
+ yield calls
+ for key, value in saved.items():
+ if value is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = value
+
+
+# Task Manager narrows a running backend, then widens it again.
+def test_fit_follows_affinity_narrowed_and_widened_at_runtime(windows_mask, dll_calls, monkeypatch):
+ torch = _Torch(16)
+ monkeypatch.setitem(sys.modules, "torch", torch)
+
+ cpu_threads.fit_threads_to_affinity()
+ assert torch.set_calls == [] and dll_calls == []
+
+ windows_mask["process"] = 0b1
+ cpu_threads.fit_threads_to_affinity()
+ cpu_threads.fit_threads_to_affinity()
+ assert torch.set_calls == [1]
+ assert dll_calls == [1]
+
+ windows_mask["process"] = 0b1111
+ cpu_threads.fit_threads_to_affinity()
+ assert torch.set_calls == [1, 4]
+ assert dll_calls == [1, 4]
+
+ windows_mask["process"] = (1 << 32) - 1
+ cpu_threads.fit_threads_to_affinity()
+ assert torch.set_calls == [1, 4, 16]
+ assert dll_calls == [1, 4, 16]
+
+
+def test_fit_never_raises_a_smaller_user_count(windows_mask, dll_calls, monkeypatch):
+ torch = _Torch(2)
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ windows_mask["process"] = 0b1111
+
+ cpu_threads.fit_threads_to_affinity()
+
+ assert torch.set_calls == []
+ assert dll_calls == [2]
+
+
+def test_a_user_openblas_value_is_clamped_to_the_allowed_cpus(windows_mask, dll_calls, monkeypatch):
+ monkeypatch.setitem(sys.modules, "torch", _Torch(16))
+ os.environ["OPENBLAS_NUM_THREADS"] = "6"
+ windows_mask["process"] = 0b11
+
+ cpu_threads.fit_threads_to_affinity()
+
+ assert dll_calls == [2]
+
+
+def test_fit_is_a_noop_before_torch_or_off_windows(windows_mask, dll_calls, monkeypatch):
+ windows_mask["process"] = 0b1
+ monkeypatch.delitem(sys.modules, "torch", raising = False)
+ cpu_threads.fit_threads_to_affinity()
+
+ torch = _Torch(16)
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ monkeypatch.setattr(sys, "platform", "linux")
+ cpu_threads.fit_threads_to_affinity()
+
+ assert torch.set_calls == [] and dll_calls == []
+
+
+def test_a_failing_torch_never_breaks_the_batch(windows_mask, dll_calls, monkeypatch):
+ class _Broken:
+ def get_num_threads(self):
+ raise RuntimeError("torch went away")
+
+ monkeypatch.setitem(sys.modules, "torch", _Broken())
+ windows_mask["process"] = 0b1
+
+ cpu_threads.fit_threads_to_affinity()
+
+ assert dll_calls == []
+
+
+# A process launched already narrowed gets fitted pools right after torch imports.
+def test_torch_import_fits_an_inherited_affinity(windows_mask, dll_calls, monkeypatch):
+ torch = _Torch(16)
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ windows_mask["process"] = 0b1
+
+ assert cpu_threads.install_openblas_runtime_cap()
+
+ assert torch.set_calls == [1]
+ assert dll_calls == [1, 1]
+
+
+def test_every_cpu_embedding_batch_refits(monkeypatch):
+ import ast
+ from pathlib import Path
+
+ source = (Path(__file__).resolve().parent.parent / "core/rag/embeddings.py").read_text(
+ encoding = "utf-8"
+ )
+ encode = next(
+ node
+ for node in ast.walk(ast.parse(source))
+ if isinstance(node, ast.FunctionDef) and node.name == "_st_encode"
+ )
+ called = [
+ node.func.id
+ for node in ast.walk(encode)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
+ ]
+ assert "fit_threads_to_affinity" in called
diff --git a/studio/backend/tests/test_embedding_onednn_linear.py b/studio/backend/tests/test_embedding_onednn_linear.py
new file mode 100644
index 00000000000..f84c658c8a2
--- /dev/null
+++ b/studio/backend/tests/test_embedding_onednn_linear.py
@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Windows ROCm: the CPU embedder's Linear layers run in oneDNN instead of rocm-openblas.dll (#14313).
+
+The DLL lookup and torch.version.hip are faked; the oneDNN op itself is real, so this needs a torch with mkldnn.
+"""
+
+import ast
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+if not torch.backends.mkldnn.is_available():
+ pytest.skip("torch built without oneDNN", allow_module_level = True)
+
+import core.rag.embeddings as embeddings # noqa: E402
+
+
+def rocm_windows(monkeypatch):
+ monkeypatch.setattr(embeddings, "runtime_capped_openblas_loaded", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", "7.14.0", raising = False)
+
+
+def _model():
+ torch.manual_seed(0)
+ return torch.nn.Sequential(
+ torch.nn.Linear(384, 1536), torch.nn.GELU(), torch.nn.Linear(1536, 384, bias = False)
+ )
+
+
+def test_linears_run_in_onednn_with_the_same_vectors(rocm_windows):
+ model = _model()
+ x = torch.randn(4, 37, 384)
+ with torch.inference_mode():
+ expected = model(x)
+
+ assert embeddings._route_linears_through_onednn(model) == 2
+
+ assert all(type(m).__name__ == "OneDnnLinear" for m in model if isinstance(m, torch.nn.Linear))
+ with torch.inference_mode():
+ got = model(x)
+ assert torch.allclose(got, expected, rtol = 1e-4, atol = 1e-5)
+
+
+def test_nothing_changes_without_rocm_openblas(monkeypatch):
+ monkeypatch.setattr(embeddings, "runtime_capped_openblas_loaded", lambda: False)
+ model = _model()
+
+ assert embeddings._route_linears_through_onednn(model) == 0
+ assert type(model[0]) is torch.nn.Linear
+
+
+def test_a_cuda_wheel_on_windows_keeps_its_blas(monkeypatch):
+ monkeypatch.setattr(embeddings, "runtime_capped_openblas_loaded", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ model = _model()
+
+ assert embeddings._route_linears_through_onednn(model) == 0
+ assert type(model[0]) is torch.nn.Linear
+
+
+def test_non_fp32_layers_are_left_alone(rocm_windows):
+ model = _model().to(torch.bfloat16)
+
+ assert embeddings._route_linears_through_onednn(model) == 0
+ assert type(model[0]) is torch.nn.Linear
+
+
+def test_a_probe_mismatch_keeps_the_stock_path(rocm_windows, monkeypatch):
+ monkeypatch.setattr(torch, "allclose", lambda *a, **k: False)
+ model = _model()
+
+ assert embeddings._route_linears_through_onednn(model) == 0
+ assert type(model[0]) is torch.nn.Linear
+
+
+def test_a_missing_onednn_op_keeps_the_stock_path(rocm_windows, monkeypatch):
+ class _NoMkldnn:
+ def __getattr__(self, name):
+ raise AttributeError(name)
+
+ monkeypatch.setattr(torch.ops, "mkldnn", _NoMkldnn(), raising = False)
+ model = _model()
+
+ assert embeddings._route_linears_through_onednn(model) == 0
+ assert type(model[0]) is torch.nn.Linear
+
+
+def test_a_model_without_modules_is_ignored(rocm_windows):
+ assert embeddings._route_linears_through_onednn(object()) == 0
+
+
+def test_every_load_routes_the_model():
+ source = (Path(__file__).resolve().parent.parent / "core/rag/embeddings.py").read_text(
+ encoding = "utf-8"
+ )
+ get = next(
+ node
+ for node in ast.walk(ast.parse(source))
+ if isinstance(node, ast.FunctionDef) and node.name == "_get"
+ )
+ called = [
+ node.func.id
+ for node in ast.walk(get)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
+ ]
+ assert "_route_linears_through_onednn" in called
diff --git a/studio/backend/utils/cpu_threads.py b/studio/backend/utils/cpu_threads.py
index 22b348fd542..0a8c4b28f1f 100644
--- a/studio/backend/utils/cpu_threads.py
+++ b/studio/backend/utils/cpu_threads.py
@@ -32,8 +32,44 @@ def _threads_that_fit(threads: int, cost: int) -> int:
return max(1, min(threads, int(headroom * _OPENBLAS_MEMORY_SHARE) // cost))
+def _windows_affinity_cpus() -> Optional[int]:
+ """CPUs in this process's Windows affinity mask when narrower than the system's, else None.
+
+ Python reads no affinity on Windows (process_cpu_count is cpu_count there). Zero masks mean the
+ process spans processor groups, and a full group mask says nothing about the others.
+ """
+ try:
+ import ctypes
+ from ctypes import wintypes
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
+ kernel32.GetCurrentProcess.restype = wintypes.HANDLE
+ get_mask = kernel32.GetProcessAffinityMask
+ get_mask.argtypes = [
+ wintypes.HANDLE,
+ ctypes.POINTER(ctypes.c_size_t),
+ ctypes.POINTER(ctypes.c_size_t),
+ ]
+ get_mask.restype = wintypes.BOOL
+ process_mask, system_mask = ctypes.c_size_t(), ctypes.c_size_t()
+ if not get_mask(
+ kernel32.GetCurrentProcess(), ctypes.byref(process_mask), ctypes.byref(system_mask)
+ ):
+ return None
+ allowed = bin(process_mask.value).count("1")
+ if not allowed or allowed >= bin(system_mask.value).count("1"):
+ return None
+ return allowed
+ except Exception: # noqa: BLE001 -- unknown affinity means the core count
+ return None
+
+
def _usable_cpus() -> int:
"""CPUs this process may run on: an affinity mask (taskset, Slurm, cpusets) can be far below the host's."""
+ if sys.platform == "win32":
+ allowed = _windows_affinity_cpus()
+ if allowed is not None:
+ return allowed
count = getattr(os, "process_cpu_count", None) # 3.13+
if count is not None:
return count() or 1
@@ -229,7 +265,22 @@ def _openblas_thread_target(environ: Optional[MutableMapping[str, str]] = None)
if target < 1:
return None
# OpenBLAS clamps an env value to the core count; its runtime setter only to MAX_THREADS.
- return min(target, os.cpu_count() or target)
+ return min(target, os.cpu_count() or target, _usable_cpus())
+
+
+def runtime_capped_openblas_loaded() -> bool:
+ """Whether torch's CPU BLAS is AMD's Windows ROCm OpenBLAS: about 30x slower per core than MKL (#14313)."""
+ if sys.platform != "win32":
+ return False
+ try:
+ import ctypes
+
+ get_module = ctypes.WinDLL("kernel32", use_last_error = True).GetModuleHandleW
+ get_module.argtypes = [ctypes.c_wchar_p]
+ get_module.restype = ctypes.c_void_p
+ return any(get_module(name) for name in _RUNTIME_CAPPED_OPENBLAS)
+ except Exception: # noqa: BLE001
+ return False
def apply_openblas_runtime_cap() -> list:
@@ -263,6 +314,45 @@ def apply_openblas_runtime_cap() -> list:
return capped
+_affinity_lock = threading.Lock()
+_torch_threads_unfitted: Optional[int] = None
+_fitted_affinity: Optional[int] = None
+
+
+def fit_threads_to_affinity() -> None:
+ """Fit torch's pool on this thread, and the ROCm OpenBLAS one, to the CPUs Windows lets this process use.
+
+ Affinity is often narrowed after launch (Task Manager), and neither pool reads it: 16 threads on one
+ allowed CPU spin against each other (#14313). Widening it again restores the launch counts. Cheap
+ enough to call before every CPU batch; a no-op off Windows, before torch loads, or while unrestricted.
+ """
+ if sys.platform != "win32" or "torch" not in sys.modules:
+ return
+ global _torch_threads_unfitted, _fitted_affinity
+ with _affinity_lock:
+ allowed = _windows_affinity_cpus()
+ if allowed is None and _torch_threads_unfitted is None:
+ return
+ try:
+ torch = sys.modules["torch"]
+ current = int(torch.get_num_threads())
+ if _torch_threads_unfitted is None:
+ _torch_threads_unfitted = current
+ target = (
+ _torch_threads_unfitted
+ if allowed is None
+ else min(_torch_threads_unfitted, allowed)
+ )
+ # Per thread under OpenMP: the setter reaches only the calling thread's pool.
+ if current != target:
+ torch.set_num_threads(target)
+ except Exception: # noqa: BLE001 -- a thread cap must never break a CPU batch
+ return
+ if allowed != _fitted_affinity:
+ _fitted_affinity = allowed
+ apply_openblas_runtime_cap()
+
+
class _TorchLoader(importlib.abc.Loader):
__slots__ = ("_loader",)
@@ -284,6 +374,7 @@ class _TorchLoader(importlib.abc.Loader):
if getattr(module, "__loader__", None) is self:
module.__loader__ = self._loader
apply_openblas_runtime_cap()
+ fit_threads_to_affinity()
def __getattr__(self, attribute):
if attribute == "_loader":
@@ -328,6 +419,7 @@ def install_openblas_runtime_cap() -> bool:
return False
if "torch" in sys.modules:
apply_openblas_runtime_cap()
+ fit_threads_to_affinity()
return True
if not any(getattr(finder, _OPENBLAS_CAP_SENTINEL, False) for finder in sys.meta_path):
sys.meta_path.insert(0, _TorchImportFinder())