Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: share one kernel install core with unsloth install-kernels (#12828)
Daniel Han2622a5d07dda
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 5c1fce92634..466c013a045 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -64,6 +64,20 @@ from core.training.dataset_bounds import ()from core.training.resume import _checkpoint_state, session_eta_secondsfrom utils.training_runs import build_default_output_dir_name+from utils.kernel_install import (+ CAUSAL_CONV1D,+ MAMBA_SSM,+ PinnedKernel,+ hipcc_gcc_install_dir,+ install_prebuilt,+ source_build_command,+ source_build_run_kwargs,+ uninstall_command,+)+from utils.ssm_runtime import (+ CAUSAL_CONV1D_MODEL_SUBSTRINGS as _CAUSAL_CONV1D_MODEL_SUBSTRINGS,+ SSM_MODEL_SUBSTRINGS as _SSM_MODEL_SUBSTRINGS,+)from utils.wheel_utils import (direct_wheel_url,flash_attn_wheel_url,@@ -1045,30 +1059,6 @@ def _model_load_security_error(config: dict, load_target: str, hf_token: str | N}-_CAUSAL_CONV1D_RELEASE_TAG = "v1.6.1.post4"-_CAUSAL_CONV1D_PACKAGE_VERSION = "1.6.1"-_CAUSAL_CONV1D_MODEL_SUBSTRINGS = (- "qwen3.5",- "qwen3_5",- "qwen3.6",- "qwen3_6",- "qwen3-next",- "qwen3_next",- "nemotron_h",- "nemotron-h",- "nemotron-3-nano",- "falcon_h1",- "falcon-h1",- "granite-4.0-h",- "granitemoehybrid",- "lfm2",- "mamba",- "jamba",- "zamba",- "bamba",-)-_MAMBA_SSM_RELEASE_TAG = "v2.3.1"-_MAMBA_SSM_PACKAGE_VERSION = "2.3.1"_FLASH_ATTN_RUNTIME_MIN_SEQ_LEN = 32768_FLASH_ATTN_SKIP_ENV = "UNSLOTH_STUDIO_SKIP_FLASHATTN_INSTALL"_FAST_PATH_HOOKS_SKIP_ENV = "UNSLOTH_STUDIO_SKIP_FAST_PATH_HOOKS"@@ -1195,24 +1185,7 @@ def _model_wants_causal_conv1d(model_name: str) -> bool:return any(key in name for key in _CAUSAL_CONV1D_MODEL_SUBSTRINGS)-def _hipcc_gcc_install_dir() -> str | None:- """Highest-numbered ``/usr/lib/gcc/x86_64-linux-gnu/<N>`` that has BOTH the gcc runtime dir AND- ``/usr/include/c++/<N>`` headers, or None. Ubuntu 24.04 ships gcc-14 runtime but not- ``/usr/include/c++/14``; ROCm clang-20 picks the highest runtime dir, finds no ``<cstdlib>``,- and the HIP build fails. The returned path is passed to clang via ``--gcc-install-dir``.- Mirrors bbf004c in studio/setup.sh (PR #5301)."""- if not sys.platform.startswith("linux"):- return None- import platform as _platform-- if _platform.machine().lower() != "x86_64":- return None- for _ver in (14, 13, 12, 11):- _runtime = f"/usr/lib/gcc/x86_64-linux-gnu/{_ver}/include"- _headers = f"/usr/include/c++/{_ver}"- if os.path.isdir(_runtime) and os.path.isdir(_headers):- return f"/usr/lib/gcc/x86_64-linux-gnu/{_ver}"- return None+_hipcc_gcc_install_dir = hipcc_gcc_install_dirdef _is_importable(import_name: str) -> bool:@@ -1259,12 +1232,8 @@ def _is_importable_isolated(import_name: str) -> bool:def _uninstall_package(pypi_name: str, display_name: str) -> bool:"""Remove a distribution. True iff it is gone afterwards."""- if shutil.which("uv"):- cmd = ["uv", "pip", "uninstall", "--python", sys.executable, pypi_name]- else:- cmd = [sys.executable, "-m", "pip", "uninstall", "-y", pypi_name]result = _sp.run(- cmd,+ uninstall_command(pypi_name, use_uv = bool(shutil.which("uv"))),stdout = _sp.PIPE,stderr = _sp.STDOUT,text = True,@@ -1377,30 +1346,29 @@ def _attempt_package_install(logger.info("No compatible %s wheel candidate", display_name)elif wheel_available:_send_status(event_queue, f"Installing {display_name} for faster training...")- for installer, result in install_wheel(+ outcome = install_prebuilt(wheel_url,- python_executable = sys.executable,- use_uv = bool(shutil.which("uv")),- run = _sp.run,- ):- if result.returncode == 0:- # A wheel can install yet fail to import (CUDA/ABI or arch mismatch), so verify rather than trust the- # exit code, and do it out of process: a bad one can take the worker down with it.- if _is_importable_isolated(import_name):- logger.info("Installed prebuilt %s wheel successfully", display_name)- return True- logger.warning(- "%s wheel installed but is not importable; falling back to PyPI",- display_name,- )- wheel_rejected = True- break- logger.warning(+ install = install_wheel,+ # Out of process: a bad wheel can take the worker down with it.+ verify = lambda: _is_importable_isolated(import_name),+ on_failed = lambda installer, result: logger.warning("%s failed to install %s wheel:\n%s",installer,display_name,result.stdout,+ ),+ use_uv = bool(shutil.which("uv")),+ run = _sp.run,+ )+ if outcome == "installed":+ logger.info("Installed prebuilt %s wheel successfully", display_name)+ return True+ if outcome == "rejected":+ logger.warning(+ "%s wheel installed but is not importable; falling back to PyPI",+ display_name,)+ wheel_rejected = Trueelif wheel_available is None:_send_status(event_queue,@@ -1456,63 +1424,19 @@ def _attempt_package_install(else:pypi_cmd = [sys.executable, "-m", "pip", "install", pypi_spec]else:- if shutil.which("uv"):- pypi_cmd = [- "uv",- "pip",- "install",- "--python",- sys.executable,- "--no-build-isolation",- "--no-deps",- ]- # Avoid stale cache artifacts from partial HIP source builds- if is_hip:- pypi_cmd.append("--no-cache")- pypi_cmd.append(pypi_spec)- else:- pypi_cmd = [- sys.executable,- "-m",- "pip",- "install",- "--no-build-isolation",- "--no-deps",- "--no-cache-dir",- pypi_spec,- ]+ pypi_cmd = source_build_command(+ pypi_spec, use_uv = bool(shutil.which("uv")), is_hip = bool(is_hip), reinstall = False+ )- # ROCm source compilation can take 10-30 min; use a generous timeout. Non-HIP installs keep the pre-existing "no- # timeout" behaviour so unrelated slow builds (causal-conv1d on aarch64, unsupported torch/CUDA combos) aren't- # aborted at 5 minutes.- _run_kwargs: dict[str, Any] = {- "stdout": _sp.PIPE,- "stderr": _sp.STDOUT,- "text": True,- "encoding": "utf-8",- "errors": "replace",- # Make the Python child emit the UTF-8 we decode above.- "env": utf8_child_env(),- }- if is_hip:- _run_kwargs["timeout"] = 1800- # On Ubuntu 24.04 + ROCm clang-20 the HIP source build dies on a missing <cstdlib> (gcc-14 runtime dir lacks- # C++ headers). Inject --gcc-install-dir for a gcc whose headers exist, respecting any pre-existing one.- # Mirrors bbf004c in setup.sh (PR #5301).- _existing_flags = os.environ.get("HIPCC_COMPILE_FLAGS_APPEND", "")- if "--gcc-install-dir" not in _existing_flags:- _gcc_dir = _hipcc_gcc_install_dir()- if _gcc_dir is not None:- _appended = (f"{_existing_flags} --gcc-install-dir={_gcc_dir}").strip()- _env = _run_kwargs.get("env", os.environ).copy()- _env["HIPCC_COMPILE_FLAGS_APPEND"] = _appended- _run_kwargs["env"] = _env- logger.info(- "HIP source build for %s: appended "- "--gcc-install-dir=%s to HIPCC_COMPILE_FLAGS_APPEND",- display_name,- _gcc_dir,- )+ _run_kwargs, _gcc_dir = source_build_run_kwargs(+ is_hip = bool(is_hip), gcc_install_dir = _hipcc_gcc_install_dir+ )+ if _gcc_dir is not None:+ logger.info(+ "HIP source build for %s: appended --gcc-install-dir=%s to HIPCC_COMPILE_FLAGS_APPEND",+ display_name,+ _gcc_dir,+ )try:result = _sp.run(pypi_cmd, **_run_kwargs)@@ -1573,6 +1497,19 @@ def _attempt_package_install(return True+def _pinned_kernel_kwargs(kernel: PinnedKernel) -> dict[str, str]:+ """_install_package_wheel_first arguments for a pinned kernel release."""+ return {+ "import_name": kernel.import_name,+ "display_name": kernel.display_name,+ "pypi_name": kernel.pypi_name,+ "pypi_version": kernel.package_version,+ "filename_prefix": kernel.import_name,+ "release_tag": kernel.release_tag,+ "release_base_url": kernel.release_base_url,+ }++def _ensure_causal_conv1d_fast_path(event_queue: Any,model_name: str,@@ -1587,16 +1524,7 @@ def _ensure_causal_conv1d_fast_path(logger.info("causal-conv1d: no prebuilt wheel for Windows; skipping")return- _install_package_wheel_first(- event_queue = event_queue,- import_name = "causal_conv1d",- display_name = "causal-conv1d",- pypi_name = "causal-conv1d",- pypi_version = _CAUSAL_CONV1D_PACKAGE_VERSION,- filename_prefix = "causal_conv1d",- release_tag = _CAUSAL_CONV1D_RELEASE_TAG,- release_base_url = "https://github.com/Dao-AILab/causal-conv1d/releases/download",- )+ _install_package_wheel_first(event_queue = event_queue, **_pinned_kernel_kwargs(CAUSAL_CONV1D))def _flash_linear_attention_importable() -> bool:@@ -1613,32 +1541,12 @@ def _flash_linear_attention_importable() -> bool:return False-_SSM_MODEL_SUBSTRINGS = (- "nemotron_h",- "nemotron-h",- "nemotron-3-nano",- "falcon_h1",- "falcon-h1",- "granite-4.0-h",- "granitemoehybrid",-)--def _ensure_mamba_ssm(event_queue: Any, model_name: str) -> None:if not any(sub in model_name.lower() for sub in _SSM_MODEL_SUBSTRINGS):returnlogger.info("SSM model detected; setting up mamba-ssm after causal-conv1d")- _install_package_wheel_first(- event_queue = event_queue,- import_name = "mamba_ssm",- display_name = "mamba-ssm",- pypi_name = "mamba-ssm",- pypi_version = _MAMBA_SSM_PACKAGE_VERSION,- filename_prefix = "mamba_ssm",- release_tag = _MAMBA_SSM_RELEASE_TAG,- release_base_url = "https://github.com/state-spaces/mamba/releases/download",- )+ _install_package_wheel_first(event_queue = event_queue, **_pinned_kernel_kwargs(MAMBA_SSM))def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:@@ -1954,16 +1862,7 @@ def _install_fast_path_hooks(if sys.platform == "win32":logger.info("causal-conv1d: no prebuilt wheel for Windows; skipping")return False- ok = _install_package_wheel_first(- event_queue = eq,- import_name = "causal_conv1d",- display_name = "causal-conv1d",- pypi_name = "causal-conv1d",- pypi_version = _CAUSAL_CONV1D_PACKAGE_VERSION,- filename_prefix = "causal_conv1d",- release_tag = _CAUSAL_CONV1D_RELEASE_TAG,- release_base_url = ("https://github.com/Dao-AILab/causal-conv1d/releases/download"),- )+ ok = _install_package_wheel_first(event_queue = eq, **_pinned_kernel_kwargs(CAUSAL_CONV1D))return bool(ok)hooks: list[tuple[str, Callable[[Any], bool]]] = []diff --git a/studio/backend/tests/test_kernel_install.py b/studio/backend/tests/test_kernel_install.pyindex 4e01ddaf572..1e45f1e4fdb 100644--- a/studio/backend/tests/test_kernel_install.py+++ b/studio/backend/tests/test_kernel_install.py@@ -396,3 +396,166 @@ def test_capability_probe_takes_the_best_visible_gpu():assert kernel_install._gpu_capability(run) == (9, 0)assert "max(torch.cuda.get_device_capability(i)" in seen["check"] and seen["timeout"]+++def test_pinned_kernels_match_the_wheel_utils_pins():+ from utils import wheel_utils++ cc1d, mamba = kernel_install.CAUSAL_CONV1D, kernel_install.MAMBA_SSM+ assert (cc1d.package_version, cc1d.release_tag, cc1d.release_base_url) == (+ wheel_utils.CAUSAL_CONV1D_PACKAGE_VERSION,+ wheel_utils.CAUSAL_CONV1D_RELEASE_TAG,+ wheel_utils.CAUSAL_CONV1D_RELEASE_BASE_URL,+ )+ assert (mamba.package_version, mamba.release_tag, mamba.release_base_url) == (+ wheel_utils.MAMBA_SSM_PACKAGE_VERSION,+ wheel_utils.MAMBA_SSM_RELEASE_TAG,+ wheel_utils.MAMBA_SSM_RELEASE_BASE_URL,+ )+ assert (cc1d.import_name, cc1d.pypi_name, mamba.import_name, mamba.pypi_name) == (+ "causal_conv1d",+ "causal-conv1d",+ "mamba_ssm",+ "mamba-ssm",+ )+ url = cc1d.wheel_url(_env("2.10.0+cu128", "12.8"))+ assert url.startswith(f"{_CC1D}/causal_conv1d-1.6.1+cu12torch2.10")+ assert kernel_install.resolve_wheel_url("causal_conv1d", _env("2.10.0+cu128", "12.8")) == url+++def _ok(code):+ return SimpleNamespace(returncode = code, stdout = f"out{code}")+++ "attempts, verified, outcome, failed",+ [+ ([("uv", 0)], True, "installed", []),+ ([("uv", 1), ("pip", 0)], True, "installed", ["uv"]),+ ([("uv", 0)], False, "rejected", []),+ ([("uv", 1), ("pip", 1)], True, "failed", ["uv", "pip"]),+ ],+)+def test_install_prebuilt_outcomes(attempts, verified, outcome, failed):+ seen, failures, verifies = {}, [], []++ def install(url, **kwargs):+ seen["url"], seen["kwargs"] = url, kwargs+ return [(installer, _ok(code)) for installer, code in attempts]++ result = kernel_install.install_prebuilt(+ "https://example.invalid/k.whl",+ install = install,+ verify = lambda: verifies.append(1) or verified,+ on_failed = lambda installer, result: failures.append(installer),+ use_uv = True,+ )+ assert result == outcome+ assert failures == failed+ assert len(verifies) == (outcome != "failed")+ # Only what the caller passed is forwarded, so each caller's installer flags stay its own.+ assert seen == {+ "url": "https://example.invalid/k.whl",+ "kwargs": {"python_executable": sys.executable, "use_uv": True},+ }+++ "use_uv, is_hip, reinstall, expected",+ [+ (+ True,+ False,+ False,+ ["uv", "pip", "install", "--python", "PY", "--no-build-isolation", "--no-deps", "k==1"],+ ),+ (+ True,+ True,+ True,+ [+ "uv",+ "pip",+ "install",+ "--python",+ "PY",+ "--no-build-isolation",+ "--no-deps",+ "--reinstall",+ "--no-cache",+ "k==1",+ ],+ ),+ (+ False,+ False,+ False,+ [+ "PY",+ "-m",+ "pip",+ "install",+ "--no-build-isolation",+ "--no-deps",+ "--no-cache-dir",+ "k==1",+ ],+ ),+ (+ False,+ True,+ True,+ [+ "PY",+ "-m",+ "pip",+ "install",+ "--no-build-isolation",+ "--no-deps",+ "--no-cache-dir",+ "--force-reinstall",+ "k==1",+ ],+ ),+ ],+)+def test_source_build_command(use_uv, is_hip, reinstall, expected):+ cmd = kernel_install.source_build_command(+ "k==1", use_uv = use_uv, is_hip = is_hip, reinstall = reinstall+ )+ assert cmd == [sys.executable if part == "PY" else part for part in expected]+++def test_source_build_run_kwargs(monkeypatch):+ monkeypatch.delenv("HIPCC_COMPILE_FLAGS_APPEND", raising = False)+ kwargs, gcc = kernel_install.source_build_run_kwargs(+ is_hip = False, gcc_install_dir = lambda: pytest.fail("non-HIP never looks for gcc")+ )+ assert gcc is None and "timeout" not in kwargs+ assert kwargs["encoding"] == "utf-8" and kwargs["stderr"] == subprocess.STDOUT++ kwargs, gcc = kernel_install.source_build_run_kwargs(+ is_hip = True, gcc_install_dir = lambda: "/usr/lib/gcc/x86_64-linux-gnu/13"+ )+ assert gcc == "/usr/lib/gcc/x86_64-linux-gnu/13" and kwargs["timeout"] == 1800+ assert kwargs["env"]["HIPCC_COMPILE_FLAGS_APPEND"] == f"--gcc-install-dir={gcc}"+ assert kwargs["env"]["PYTHONIOENCODING"] == "utf-8"++ monkeypatch.setenv("HIPCC_COMPILE_FLAGS_APPEND", "--gcc-install-dir=/mine")+ kwargs, gcc = kernel_install.source_build_run_kwargs(+ is_hip = True, gcc_install_dir = lambda: pytest.fail("an explicit dir is respected")+ )+ assert gcc is None and kwargs["env"]["HIPCC_COMPILE_FLAGS_APPEND"] == "--gcc-install-dir=/mine"+++ "use_uv, system, expected",+ [+ (True, False, ["uv", "pip", "uninstall", "--python", "PY", "k"]),+ (True, True, ["uv", "pip", "uninstall", "--system", "--python", "PY", "k"]),+ (False, True, ["PY", "-m", "pip", "uninstall", "-y", "k"]),+ ],+)+def test_uninstall_command(use_uv, system, expected):+ cmd = kernel_install.uninstall_command("k", use_uv = use_uv, uv_needs_system = system)+ assert cmd == [sys.executable if part == "PY" else part for part in expected]diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.pyindex 55964cfae00..bb6214d3f57 100644--- a/studio/backend/tests/test_mlx_training_worker_config.py+++ b/studio/backend/tests/test_mlx_training_worker_config.py@@ -26,6 +26,9 @@ def _load_worker_module():"utils.training_runs","utils.wheel_utils","utils.training_runs",+ # Imported for real against the stubs above, so dropped again afterwards.+ "utils.kernel_install",+ "utils.ssm_runtime",)previous_modules = {name: sys.modules.get(name) for name in stub_names}@@ -82,6 +85,21 @@ def _load_worker_module():"url_exists",):setattr(wheel_utils, name, lambda *_args, **_kwargs: None)+ # Read at import by utils.kernel_install, which the worker imports.+ for name in (+ "redact_url_credentials",+ "xformers_wheel_url",+ ):+ setattr(wheel_utils, name, lambda *_args, **_kwargs: None)+ for name in (+ "CAUSAL_CONV1D_PACKAGE_VERSION",+ "CAUSAL_CONV1D_RELEASE_BASE_URL",+ "CAUSAL_CONV1D_RELEASE_TAG",+ "MAMBA_SSM_PACKAGE_VERSION",+ "MAMBA_SSM_RELEASE_BASE_URL",+ "MAMBA_SSM_RELEASE_TAG",+ ):+ setattr(wheel_utils, name, "")sys.modules["utils.wheel_utils"] = wheel_utilsspec = importlib.util.spec_from_file_location("mlx_training_worker_under_test", worker_path)diff --git a/studio/backend/tests/test_ssm_runtime.py b/studio/backend/tests/test_ssm_runtime.pyindex 14b705e2bfe..8d09c7ef418 100644--- a/studio/backend/tests/test_ssm_runtime.py+++ b/studio/backend/tests/test_ssm_runtime.py@@ -4,8 +4,7 @@"""Tests for utils.ssm_runtime: the inference-side auto-install of SSM/Mamba kernels.Covers detection, wheel-first install, idempotency, the failure path, the inference-worker wiring, and a drift guard so the constants/detection stay in lockstep with the-training worker (the original source of this behaviour).+worker wiring, and that the training worker shares the same tables and pinned kernels."""import json@@ -692,23 +691,21 @@ def test_security_gates_run_before_ssm_install():assert min(gates) < min(ssm), f"{fn} must gate before installing SSM kernels"-# ── drift guard vs the training worker (single source of truth) ───────────────+# ── training worker shares these tables and pins ──────────────────────────────-def test_constants_match_training_worker():+def test_constants_shared_with_training_worker():try:from core.training import worker as twexcept Exception as exc: # pragma: no cover - only when training deps absentpytest.skip(f"training worker not importable here: {exc}")+ from utils import kernel_install- assert set(ssm_runtime.SSM_MODEL_SUBSTRINGS) == set(tw._SSM_MODEL_SUBSTRINGS)- assert set(ssm_runtime.CAUSAL_CONV1D_MODEL_SUBSTRINGS) == set(- tw._CAUSAL_CONV1D_MODEL_SUBSTRINGS- )- assert ssm_runtime.MAMBA_SSM_PACKAGE_VERSION == tw._MAMBA_SSM_PACKAGE_VERSION- assert ssm_runtime.MAMBA_SSM_RELEASE_TAG == tw._MAMBA_SSM_RELEASE_TAG- assert ssm_runtime.CAUSAL_CONV1D_PACKAGE_VERSION == tw._CAUSAL_CONV1D_PACKAGE_VERSION- assert ssm_runtime.CAUSAL_CONV1D_RELEASE_TAG == tw._CAUSAL_CONV1D_RELEASE_TAG+ assert tw._SSM_MODEL_SUBSTRINGS is ssm_runtime.SSM_MODEL_SUBSTRINGS+ assert tw._CAUSAL_CONV1D_MODEL_SUBSTRINGS is ssm_runtime.CAUSAL_CONV1D_MODEL_SUBSTRINGS+ assert tw.CAUSAL_CONV1D is ssm_runtime.CAUSAL_CONV1D is kernel_install.CAUSAL_CONV1D+ assert tw.MAMBA_SSM is ssm_runtime.MAMBA_SSM is kernel_install.MAMBA_SSM+ assert tw._hipcc_gcc_install_dir is ssm_runtime._hipcc_gcc_install_dir# detection must agree with the training worker across SSM + non-SSM namesfor name in (diff --git a/studio/backend/tests/test_training_worker_flash_attn.py b/studio/backend/tests/test_training_worker_flash_attn.pyindex cf22c7a1424..9f34dbe32bf 100644--- a/studio/backend/tests/test_training_worker_flash_attn.py+++ b/studio/backend/tests/test_training_worker_flash_attn.py@@ -588,9 +588,9 @@ def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch)import_name = "causal_conv1d",display_name = "causal-conv1d",pypi_name = "causal-conv1d",- pypi_version = worker._CAUSAL_CONV1D_PACKAGE_VERSION,+ pypi_version = worker.CAUSAL_CONV1D.package_version,filename_prefix = "causal_conv1d",- release_tag = worker._CAUSAL_CONV1D_RELEASE_TAG,+ release_tag = worker.CAUSAL_CONV1D.release_tag,release_base_url = "https://github.com/Dao-AILab/causal-conv1d/releases/download",)@@ -626,9 +626,9 @@ def test_mamba_ssm_path_preserves_wheel_first_install_args(monkeypatch):import_name = "mamba_ssm",display_name = "mamba-ssm",pypi_name = "mamba-ssm",- pypi_version = worker._MAMBA_SSM_PACKAGE_VERSION,+ pypi_version = worker.MAMBA_SSM.package_version,filename_prefix = "mamba_ssm",- release_tag = worker._MAMBA_SSM_RELEASE_TAG,+ release_tag = worker.MAMBA_SSM.release_tag,release_base_url = "https://github.com/state-spaces/mamba/releases/download",)diff --git a/studio/backend/utils/kernel_install.py b/studio/backend/utils/kernel_install.pyindex a713326816c..d401e2ad731 100644--- a/studio/backend/utils/kernel_install.py+++ b/studio/backend/utils/kernel_install.py@@ -1,20 +1,25 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""`unsloth install-kernels [names]`: prebuilt kernel wheels matched to the resident torch, all by default.+"""Kernel install pieces shared by Studio setup, the training worker, the SSM runtime and+`unsloth install-kernels [names]`.-Wheel-only by design: no wheel for this torch / CUDA / Python means nothing is installed and the-model keeps its torch fallback, never a source build. Installs use --no-deps so torch is never touched.+Imports only stdlib, wheel_utils and child_stdio so setup and torch-less hosts can import it.+The CLI is wheel-only (never a source build) and uses --no-deps so torch is never touched."""from __future__ import annotationsimport argparse+import os+import platformimport shutilimport subprocessimport sys-from typing import Callable+from dataclasses import dataclass+from typing import Any, Callable+from utils.child_stdio import utf8_child_envfrom utils.wheel_utils import (CAUSAL_CONV1D_PACKAGE_VERSION,CAUSAL_CONV1D_RELEASE_BASE_URL,@@ -31,6 +36,158 @@ from utils.wheel_utils import (xformers_wheel_url,)++@dataclass(frozen = True)+class PinnedKernel:+ """A kernel installed from a pinned release: prebuilt wheel first, else `pypi_name==version`."""++ import_name: str+ display_name: str+ pypi_name: str+ package_version: str+ release_tag: str+ release_base_url: str++ def wheel_url(self, env: dict[str, str] | None) -> str | None:+ return direct_wheel_url(+ filename_prefix = self.import_name,+ package_version = self.package_version,+ release_tag = self.release_tag,+ release_base_url = self.release_base_url,+ env = env,+ )+++CAUSAL_CONV1D = PinnedKernel(+ import_name = "causal_conv1d",+ display_name = "causal-conv1d",+ pypi_name = "causal-conv1d",+ package_version = CAUSAL_CONV1D_PACKAGE_VERSION,+ release_tag = CAUSAL_CONV1D_RELEASE_TAG,+ release_base_url = CAUSAL_CONV1D_RELEASE_BASE_URL,+)+MAMBA_SSM = PinnedKernel(+ import_name = "mamba_ssm",+ display_name = "mamba-ssm",+ pypi_name = "mamba-ssm",+ package_version = MAMBA_SSM_PACKAGE_VERSION,+ release_tag = MAMBA_SSM_RELEASE_TAG,+ release_base_url = MAMBA_SSM_RELEASE_BASE_URL,+)+++def install_prebuilt(+ wheel_url: str,+ *,+ install: Callable[..., list[tuple[str, Any]]],+ verify: Callable[[], bool],+ on_failed: Callable[[str, Any], None],+ **install_kwargs: Any,+) -> str:+ """Install and verify a prebuilt wheel: "installed", "rejected" (installed, `verify()` failed)+ or "failed". `install_kwargs` are forwarded as given so each caller keeps its own installer flags."""+ for installer, result in install(wheel_url, python_executable = sys.executable, **install_kwargs):+ if getattr(result, "returncode", 1) == 0:+ # A wheel can install yet not load (CUDA/ABI or arch mismatch), so the exit code is no proof.+ return "installed" if verify() else "rejected"+ on_failed(installer, result)+ return "failed"+++def hipcc_gcc_install_dir() -> str | None:+ """Highest-numbered ``/usr/lib/gcc/x86_64-linux-gnu/<N>`` that has BOTH the gcc runtime dir AND+ ``/usr/include/c++/<N>`` headers, or None. Ubuntu 24.04 ships gcc-14 runtime but not+ ``/usr/include/c++/14``; ROCm clang-20 picks the highest runtime dir, finds no ``<cstdlib>``,+ and the HIP build fails, hence ``--gcc-install-dir``. Mirrors studio/setup.sh (PR #5301)."""+ if not sys.platform.startswith("linux") or platform.machine().lower() != "x86_64":+ return None+ for ver in (14, 13, 12, 11):+ if os.path.isdir(f"/usr/lib/gcc/x86_64-linux-gnu/{ver}/include") and os.path.isdir(+ f"/usr/include/c++/{ver}"+ ):+ return f"/usr/lib/gcc/x86_64-linux-gnu/{ver}"+ return None+++def source_build_command(spec: str, *, use_uv: bool, is_hip: bool, reinstall: bool) -> list[str]:+ """`--no-build-isolation --no-deps` install of *spec* against the resident torch; the cache is+ skipped on HIP (uv) or always (pip) so stale partial build artifacts are never reused."""+ if use_uv:+ cmd = [+ "uv",+ "pip",+ "install",+ "--python",+ sys.executable,+ "--no-build-isolation",+ "--no-deps",+ ]+ if reinstall:+ cmd.append("--reinstall")+ if is_hip:+ cmd.append("--no-cache")+ else:+ cmd = [+ sys.executable,+ "-m",+ "pip",+ "install",+ "--no-build-isolation",+ "--no-deps",+ "--no-cache-dir",+ ]+ if reinstall:+ cmd.append("--force-reinstall")+ cmd.append(spec)+ return cmd+++def source_build_run_kwargs(+ *, is_hip: bool, gcc_install_dir: Callable[[], str | None]+) -> tuple[dict[str, Any], str | None]:+ """subprocess.run kwargs for a source build, and the gcc dir added to HIPCC_COMPILE_FLAGS_APPEND+ (None when nothing was added). Non-HIP builds have no timeout so slow builds (causal-conv1d on+ aarch64) are not aborted; ROCm builds take 10-30 min, bounded at 1800 s."""+ kwargs: dict[str, Any] = {+ "stdout": subprocess.PIPE,+ "stderr": subprocess.STDOUT,+ "text": True,+ # pip and the compilers it drives write UTF-8 down this pipe; the Windows ANSI codepage would mojibake or raise over a fine install.+ "encoding": "utf-8",+ "errors": "replace",+ "env": utf8_child_env(),+ }+ if not is_hip:+ return kwargs, None+ kwargs["timeout"] = 1800+ existing = os.environ.get("HIPCC_COMPILE_FLAGS_APPEND", "")+ if "--gcc-install-dir" in existing:+ return kwargs, None+ gcc_dir = gcc_install_dir()+ if not gcc_dir:+ return kwargs, None+ env = dict(kwargs["env"])+ env["HIPCC_COMPILE_FLAGS_APPEND"] = f"{existing} --gcc-install-dir={gcc_dir}".strip()+ kwargs["env"] = env+ return kwargs, gcc_dir+++def uninstall_command(+ distribution: str,+ *,+ use_uv: bool,+ uv_needs_system: bool = False,+) -> list[str]:+ """Uninstall *distribution* from ``sys.executable``. uv gets --python as well as --system:+ --system alone would remove from the system Python and leave the venv copy behind."""+ if use_uv:+ system = ["--system"] if uv_needs_system else []+ return ["uv", "pip", "uninstall", *system, "--python", sys.executable, distribution]+ return [sys.executable, "-m", "pip", "uninstall", "-y", distribution]+++# `unsloth install-kernels` from here on.+# name -> (pip distribution, check that loads the compiled extension). The package imports are no# proof: `import causal_conv1d` never loads it, and `import mamba_ssm` fails on einops under --no-deps.# Install order: mamba_ssm's fast path imports causal_conv1d.@@ -44,14 +201,7 @@ KERNELS = {"mamba_ssm": ("mamba-ssm", "import torch, selective_scan_cuda"),}-_RELEASES = {- "causal_conv1d": (- CAUSAL_CONV1D_PACKAGE_VERSION,- CAUSAL_CONV1D_RELEASE_TAG,- CAUSAL_CONV1D_RELEASE_BASE_URL,- ),- "mamba_ssm": (MAMBA_SSM_PACKAGE_VERSION, MAMBA_SSM_RELEASE_TAG, MAMBA_SSM_RELEASE_BASE_URL),-}+_PINNED = {"causal_conv1d": CAUSAL_CONV1D, "mamba_ssm": MAMBA_SSM}def resolve_wheel_url(name: str, env: dict[str, str] | None) -> str | None:@@ -64,14 +214,7 @@ def resolve_wheel_url(name: str, env: dict[str, str] | None) -> str | None:return Noneif name == "flash_attn":return flash_attn_wheel_url(env)- package_version, release_tag, release_base_url = _RELEASES[name]- return direct_wheel_url(- filename_prefix = name,- package_version = package_version,- release_tag = release_tag,- release_base_url = release_base_url,- env = env,- )+ return _PINNED[name].wheel_url(env)def _gpu_capability(run: Callable[..., subprocess.CompletedProcess]) -> tuple[int, int] | None:@@ -112,11 +255,9 @@ def _outside_venv() -> bool:def _uninstall(distribution: str, run: Callable[..., subprocess.CompletedProcess]) -> bool:- if shutil.which("uv"):- system = ["--system"] if _outside_venv() else []- cmd = ["uv", "pip", "uninstall", *system, "--python", sys.executable, distribution]- else:- cmd = [sys.executable, "-m", "pip", "uninstall", "-y", distribution]+ cmd = uninstall_command(+ distribution, use_uv = bool(shutil.which("uv")), uv_needs_system = _outside_venv()+ )return run(cmd, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL).returncode == 0@@ -154,22 +295,24 @@ def install_kernel(if _loads(check, run):print(f"Unsloth: {name} already installed and loads.")return 0- attempts = install_wheel(+ failures: list[subprocess.CompletedProcess] = []+ outcome = install_prebuilt(url,- python_executable = sys.executable,+ install = install_wheel,+ verify = lambda: _loads(check, run),+ on_failed = lambda _installer, result: failures.append(result),use_uv = True,uv_needs_system = _outside_venv(),reinstall = True,run = run,)- result = attempts[-1][1]- if result.returncode != 0:+ if outcome == "failed":print(f"Unsloth: installing {name} from {shown} failed; using the torch fallback.\n"- + result.stdout[-2000:].replace(url, shown)+ + failures[-1].stdout[-2000:].replace(url, shown))return 1- if not _loads(check, run):+ if outcome == "rejected":# A wheel that imports but whose extension cannot load would be picked up and crash later.removed = ("removed it" if _uninstall(distribution, run) else f"run `pip uninstall {distribution}`"diff --git a/studio/backend/utils/ssm_runtime.py b/studio/backend/utils/ssm_runtime.pyindex 9c3afd57b9e..c60f1e618b0 100644--- a/studio/backend/utils/ssm_runtime.py+++ b/studio/backend/utils/ssm_runtime.py@@ -1,13 +1,14 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Auto-install the SSM/Mamba kernels a hybrid model needs before it loads. Mamba/SSM hybrids (Nemotron-H/Nano, Falcon-H1, Granite-4.0-H, GraniteMoEHybrid, ...) lazy-``import mamba_ssm`` / ``causal_conv1d`` in their ``modeling_*.py`` during ``from_pretrained``; absent, the load dies with "mamba-ssm is required ... cannot be imported". The training worker installs them wheel-first before a fine-tune; this is the shared, callback-based version the inference load path calls so chat behaves the same. Detection and versions mirror the training worker (``tests/test_ssm_runtime.py`` guards drift)."""+"""Auto-install the SSM/Mamba kernels a hybrid model lazy-imports during ``from_pretrained``;+the inference-path counterpart of the training worker's install.+"""from __future__ import annotationsimport importlibimport os-import platformimport shutilimport subprocessimport sys@@ -17,14 +18,16 @@ from pathlib import Pathfrom typing import Any, Callable, Iterator, Optionalfrom loggers import get_logger-from utils.child_stdio import utf8_child_env+from utils.kernel_install import (+ CAUSAL_CONV1D,+ MAMBA_SSM,+ PinnedKernel,+ hipcc_gcc_install_dir,+ install_prebuilt,+ source_build_command,+ source_build_run_kwargs,+)from utils.wheel_utils import (- CAUSAL_CONV1D_PACKAGE_VERSION,- CAUSAL_CONV1D_RELEASE_BASE_URL,- CAUSAL_CONV1D_RELEASE_TAG,- MAMBA_SSM_PACKAGE_VERSION,- MAMBA_SSM_RELEASE_BASE_URL,- MAMBA_SSM_RELEASE_TAG,direct_wheel_url,install_wheel,probe_torch_wheel_env,@@ -35,7 +38,7 @@ logger = get_logger(__name__)StatusCb = Optional[Callable[[str], None]]-# Lowercased-id substring matches, mirroring the training worker. mamba-ssm models are a subset of the causal-conv1d set.+# Lowercased-id substring matches, shared with the training worker. mamba-ssm models are a subset of the causal-conv1d set.SSM_MODEL_SUBSTRINGS = ("nemotron_h","nemotron-h",@@ -214,16 +217,7 @@ def _emit(status_cb: StatusCb, message: str) -> None:logger.debug("ssm_runtime status callback raised", exc_info = True)-def _hipcc_gcc_install_dir() -> Optional[str]:- """Highest gcc dir with both runtime and C++ headers, for ROCm clang's ``--gcc-install-dir`` (Ubuntu 24.04 ships gcc-14 runtime without its headers)."""- if not sys.platform.startswith("linux") or platform.machine().lower() != "x86_64":- return None- for ver in (14, 13, 12, 11):- if os.path.isdir(f"/usr/lib/gcc/x86_64-linux-gnu/{ver}/include") and os.path.isdir(- f"/usr/include/c++/{ver}"- ):- return f"/usr/lib/gcc/x86_64-linux-gnu/{ver}"- return None+_hipcc_gcc_install_dir = hipcc_gcc_install_dir# Keep quiet downloads and builds inside the orchestrator's inactivity deadline.@@ -297,27 +291,26 @@ def _install_kernel(f"Still installing {display_name} (prebuilt kernel)...",):# A cold first import can also stay quiet for tens of seconds.- for installer, result in install_wheel(+ outcome = install_prebuilt(wheel_url,- python_executable = sys.executable,- use_uv = bool(shutil.which("uv")),- run = run,- ):- if getattr(result, "returncode", 1) == 0:- # A wheel can install yet fail to import (CUDA/ABI mismatch); verify before trusting it, else source-build to match the local ABI.- if _is_importable(import_name):- logger.info("Installed prebuilt %s wheel", display_name)- return True- logger.warning(- "%s wheel installed but not importable; building from source",- display_name,- )- break- logger.warning(+ install = install_wheel,+ verify = lambda: _is_importable(import_name),+ on_failed = lambda installer, result: logger.warning("%s could not install %s wheel:\n%s",installer,display_name,getattr(result, "stdout", ""),+ ),+ use_uv = bool(shutil.which("uv")),+ run = run,+ )+ if outcome == "installed":+ logger.info("Installed prebuilt %s wheel", display_name)+ return True+ if outcome == "rejected":+ logger.warning(+ "%s wheel installed but not importable; building from source",+ display_name,)elif wheel_available is None:_emit(@@ -341,56 +334,9 @@ def _install_kernel(status_cb,f"Building {display_name} from source for this model (this can take several minutes)...",)- # Reinstall so the source build replaces a broken wheel instead of no-opping as "already satisfied"; --no-cache avoids stale partial HIP build artifacts.- if shutil.which("uv"):- cmd = [- "uv",- "pip",- "install",- "--python",- sys.executable,- "--no-build-isolation",- "--no-deps",- "--reinstall",- ]- if is_hip:- cmd.append("--no-cache")- cmd.append(spec)- else:- cmd = [- sys.executable,- "-m",- "pip",- "install",- "--no-build-isolation",- "--no-deps",- "--no-cache-dir",- "--force-reinstall",- spec,- ]-- run_kwargs: dict[str, Any] = {- "stdout": subprocess.PIPE,- "stderr": subprocess.STDOUT,- "text": True,- # pip and the compilers it drives write UTF-8 down this pipe; the Windows ANSI codepage would mojibake or raise over a fine install.- "encoding": "utf-8",- "errors": "replace",- # Make the Python child emit the UTF-8 we decode above.- "env": utf8_child_env(),- }- if is_hip:- run_kwargs["timeout"] = 1800- existing = os.environ.get("HIPCC_COMPILE_FLAGS_APPEND", "")- if "--gcc-install-dir" not in existing:- gcc_dir = _hipcc_gcc_install_dir()- if gcc_dir:- # Extends the UTF-8 env above rather than replacing it.- _env = dict(run_kwargs["env"])- _env["HIPCC_COMPILE_FLAGS_APPEND"] = (- f"{existing} --gcc-install-dir={gcc_dir}".strip()- )- run_kwargs["env"] = _env+ # Reinstall so the source build replaces a broken wheel instead of no-opping as "already satisfied".+ cmd = source_build_command(spec, use_uv = bool(shutil.which("uv")), is_hip = is_hip, reinstall = True)+ run_kwargs, _ = source_build_run_kwargs(is_hip = is_hip, gcc_install_dir = _hipcc_gcc_install_dir)try:result = _run_with_heartbeat(run, cmd, status_cb, display_name, **run_kwargs)except subprocess.TimeoutExpired:@@ -402,6 +348,18 @@ def _install_kernel(return _is_importable(import_name)+def _pinned_kwargs(kernel: PinnedKernel) -> dict[str, str]:+ """_install_kernel arguments for a pinned kernel release."""+ return {+ "import_name": kernel.import_name,+ "display_name": kernel.display_name,+ "pypi_name": kernel.pypi_name,+ "package_version": kernel.package_version,+ "release_tag": kernel.release_tag,+ "release_base_url": kernel.release_base_url,+ }++def ensure_ssm_runtime(model_name: str,*,@@ -423,25 +381,9 @@ def ensure_ssm_runtime(# causal-conv1d first: SSM modeling files lazy-import it, and mamba-ssm's fast path uses it.if wants_causal_conv1d and not _install_kernel(- import_name = "causal_conv1d",- display_name = "causal-conv1d",- pypi_name = "causal-conv1d",- package_version = CAUSAL_CONV1D_PACKAGE_VERSION,- release_tag = CAUSAL_CONV1D_RELEASE_TAG,- release_base_url = CAUSAL_CONV1D_RELEASE_BASE_URL,- status_cb = status_cb,- run = run,+ **_pinned_kwargs(CAUSAL_CONV1D), status_cb = status_cb, run = run):logger.warning("causal-conv1d unavailable; continuing on the model's torch fallback")- if is_ssm and not _install_kernel(- import_name = "mamba_ssm",- display_name = "mamba-ssm",- pypi_name = "mamba-ssm",- package_version = MAMBA_SSM_PACKAGE_VERSION,- release_tag = MAMBA_SSM_RELEASE_TAG,- release_base_url = MAMBA_SSM_RELEASE_BASE_URL,- status_cb = status_cb,- run = run,- ):+ if is_ssm and not _install_kernel(**_pinned_kwargs(MAMBA_SSM), status_cb = status_cb, run = run):raise RuntimeError("Could not install mamba-ssm, required by this Mamba model.")diff --git a/studio/backend/utils/wheel_utils.py b/studio/backend/utils/wheel_utils.pyindex 3c693a8ba80..327461789c8 100644--- a/studio/backend/utils/wheel_utils.py+++ b/studio/backend/utils/wheel_utils.py@@ -23,7 +23,7 @@ from utils.subprocess_compat import windows_hidden_subprocess_kwargs_logger = logging.getLogger(__name__)FLASH_ATTN_RELEASE_BASE_URL = "https://github.com/Dao-AILab/flash-attention/releases/download"-# Pinned wheels, kept in lockstep with core/training/worker.py by tests/test_ssm_runtime.py.+# Pinned wheels; utils.kernel_install wraps them for the training worker, SSM runtime and CLI.CAUSAL_CONV1D_PACKAGE_VERSION = "1.6.1"CAUSAL_CONV1D_RELEASE_TAG = "v1.6.1.post4"CAUSAL_CONV1D_RELEASE_BASE_URL = "https://github.com/Dao-AILab/causal-conv1d/releases/download"diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex 80db869e25f..afc7f97e088 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -51,6 +51,7 @@ def _nvidia_library_inventory():return _nvidia_probe.probe() if _nvidia_probe is not None else None+from backend.utils.kernel_install import install_prebuilt, uninstall_commandfrom backend.utils.wheel_utils import (flash_attn_package_version,flash_attn_wheel_url,@@ -5445,18 +5446,10 @@ def _warn_still_cpu(expected: str) -> bool:def _uninstall_distribution(name: str) -> bool:"""Remove one distribution from the venv this script targets. True iff it is gone.-- Same shape as the flash-attn removal below: --python sys.executable so a uv that- also needs --system cannot remove from the system Python instead, and a pip- fallback for the same interpreter. Output is swallowed; the caller reports.- """- if USE_UV and shutil.which("uv"):- cmd = ["uv", "pip", "uninstall"]- if UV_NEEDS_SYSTEM:- cmd.append("--system")- cmd.extend(["--python", sys.executable, name])- else:- cmd = [sys.executable, "-m", "pip", "uninstall", "-y", name]+ Output is swallowed; the caller reports."""+ cmd = uninstall_command(+ name, use_uv = USE_UV and bool(shutil.which("uv")), uv_needs_system = UV_NEEDS_SYSTEM+ )_count_install_action()removed = subprocess.run(cmd, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL)return removed.returncode == 0@@ -8149,21 +8142,10 @@ def _flash_attn_importable() -> bool:def _remove_rejected_flash_attn() -> bool:"""Uninstall a flash-attn that installed but will not import. True iff it is gone.- Targets the interpreter install_wheel installed into, always ``sys.executable``: its uv- command passes --python as well as --system, and its pip fallback runs that interpreter.- --system ALONE would remove from the system Python, leaving the rejected wheel in the- venv while setup reported it gone.+ uv gets --python as well as --system: --system ALONE would remove from the system+ Python, leaving the rejected wheel in the venv while setup reported it gone."""- if USE_UV and shutil.which("uv"):- cmd = ["uv", "pip", "uninstall"]- if UV_NEEDS_SYSTEM:- cmd.append("--system")- cmd.extend(["--python", sys.executable, "flash-attn"])- else:- cmd = [sys.executable, "-m", "pip", "uninstall", "-y", "flash-attn"]- _count_install_action()- removed = subprocess.run(cmd, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL)- return removed.returncode == 0+ return _uninstall_distribution("flash-attn")def _ensure_flash_attn() -> None:@@ -8182,39 +8164,37 @@ def _ensure_flash_attn() -> None:if wheel_available:# Counted: it lands a distribution, so the caches keyed on the counter must be rebuilt._count_install_action()- for installer, wheel_result in install_wheel(+ outcome = install_prebuilt(wheel_url,- python_executable = sys.executable,- use_uv = USE_UV,- uv_needs_system = UV_NEEDS_SYSTEM,- ):- if wheel_result.returncode == 0:- # Verify rather than trust the exit code, so setup reports what happened.- if _flash_attn_importable():- return- # Remove it before giving up. Left installed, unsloth/models/_utils.py finds- # it by metadata (_package_available) and then imports the native module- # in process, so a wheel that killed the probe would kill training too.- if _remove_rejected_flash_attn():- _step(- "warning",- "flash-attn wheel installed but is not importable on this GPU; removed it",- _cyan,- )- else:- # Say so plainly: it is still importable in process, so this is not the- # same state as never having installed it.- _step(- "warning",- "flash-attn wheel is not importable on this GPU and could not be "- "removed; uninstall flash-attn manually before training",- _cyan,- )- break- _print_optional_install_failure(+ install = install_wheel,+ verify = _flash_attn_importable,+ on_failed = lambda installer, wheel_result: _print_optional_install_failure(f"Installing flash-attn prebuilt wheel with {installer}",wheel_result,- )+ ),+ use_uv = USE_UV,+ uv_needs_system = UV_NEEDS_SYSTEM,+ )+ if outcome == "installed":+ return+ if outcome == "rejected":+ # Remove it before giving up. Left installed, unsloth/models/_utils.py finds+ # it by metadata (_package_available) and then imports the native module+ # in process, so a wheel that killed the probe would kill training too.+ if _remove_rejected_flash_attn():+ _step(+ "warning",+ "flash-attn wheel installed but is not importable on this GPU; removed it",+ _cyan,+ )+ else:+ # Still importable in process, unlike never having installed it.+ _step(+ "warning",+ "flash-attn wheel is not importable on this GPU and could not be "+ "removed; uninstall flash-attn manually before training",+ _cyan,+ )_step("warning", "Continuing without flash-attn", _cyan)return