Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the on-demand kernels install importable next to transformers (#17576)

Daniel Han9c4e41c39082
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 07648c24d6d..808b6f27e15 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -3180,6 +3180,9 @@ def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:
# A hidden package stays unusable until restart; skip without recording an attempt.
if module in sys.modules and sys.modules[module] is None:
reason = f"{module} is disabled in this process because it requires a different torch"
+ if module == "kernels":
+ from utils.kernels_compat import hidden_reason
+ reason = hidden_reason() or reason
if logger is not None:
logger.warning(
"diffusion.attention: not installing %s for backend=%s: %s; restart Unsloth "
@@ -3242,6 +3245,15 @@ def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:
_INSTALL_ATTEMPTED.add(package)
import subprocess
+ requirement = package
+ previous_kernels = None
+ if package == "kernels":
+ # Unpinned, pip takes the newest kernels, which an older transformers cannot import next to.
+ from utils.kernels_compat import current_kernels_install_spec
+
+ requirement = current_kernels_install_spec()
+ previous_kernels = _dist_version("kernels")
+ display = requirement
if logger is not None:
logger.info(
"diffusion.attention: installing %s for backend=%s (wheel-only)", display, backend
@@ -3257,7 +3269,7 @@ def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:
"--only-binary",
":all:",
"--no-deps",
- package,
+ requirement,
],
capture_output = True,
timeout = 600,
@@ -3267,7 +3279,11 @@ def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:
# the new wheel.
importlib.invalidate_caches()
if package == "kernels":
- _refresh_diffusers_kernels_version(logger)
+ from utils.kernels_compat import check_kernels_install
+
+ # A kernels transformers cannot import breaks every later process: roll it back instead.
+ if check_kernels_install(previous_kernels, logger):
+ _refresh_diffusers_kernels_version(logger)
except Exception as exc: # noqa: BLE001 - no wheel / no network -> native fallback
if logger is not None:
# CalledProcessError.str() shows only the exit code; surface stderr so the fallback is diagnosable.
diff --git a/studio/backend/tests/test_diffusion_attention.py b/studio/backend/tests/test_diffusion_attention.py
index 6ece3ba3c32..e34bb1ac092 100644
--- a/studio/backend/tests/test_diffusion_attention.py
+++ b/studio/backend/tests/test_diffusion_attention.py
@@ -247,6 +247,11 @@ class _Recorder:
return types.SimpleNamespace(returncode = 0)
+def _pip_calls(run):
+ # A kernels install is followed by a `python -c "import transformers.activations"` check.
+ return [c for c in run.calls if "pip" in c]
+
+
def _stub_subprocess(monkeypatch, run):
import subprocess
monkeypatch.setattr(subprocess, "run", run)
@@ -283,7 +288,7 @@ def test_install_runs_wheel_only_for_missing_kernel(monkeypatch):
run = _Recorder()
_stub_subprocess(monkeypatch, run)
att._ensure_attention_backend_installed("sage")
- assert len(run.calls) == 1
+ assert len(_pip_calls(run)) == 1
cmd = run.calls[0]
assert "--only-binary" in cmd and ":all:" in cmd
@@ -299,7 +304,7 @@ def test_sage_install_never_asks_pypi_for_sageattention(monkeypatch):
_stub_subprocess(monkeypatch, run)
att._ensure_attention_backend_installed("sage")
assert run.calls and not any("sageattention" in part for cmd in run.calls for part in cmd)
- assert run.calls[0][-1] == "kernels"
+ assert run.calls[0][-1] in ("kernels==0.12.3", "kernels>=0.12.3")
_XFORMERS_WHEEL = "https://download.pytorch.org/whl/cu130/xformers-0.0.34-cp39-abi3-win_amd64.whl"
@@ -524,8 +529,8 @@ def test_matched_wheel_path_does_not_touch_other_backends(monkeypatch):
monkeypatch.setattr(att, "_pip_sage2_installed", lambda: False)
monkeypatch.setattr(att, "_kernels_hub_compatible", lambda: True)
att._ensure_attention_backend_installed("sage")
- assert len(run.calls) == 1
- assert run.calls[0][-1] == "kernels"
+ assert len(_pip_calls(run)) == 1
+ assert run.calls[0][-1] in ("kernels==0.12.3", "kernels>=0.12.3")
def test_failed_install_not_retried_in_same_process(monkeypatch):
@@ -658,7 +663,8 @@ def test_kernels_install_allowed_on_supported_hub(monkeypatch):
run = _Recorder()
_stub_subprocess(monkeypatch, run)
att._ensure_attention_backend_installed("flash_4_hub")
- assert len(run.calls) == 1 and "kernels" in run.calls[0]
+ pip = _pip_calls(run)
+ assert len(pip) == 1 and pip[0][-1] in ("kernels==0.12.3", "kernels>=0.12.3")
def test_kernels_gate_also_covers_sage(monkeypatch):
diff --git a/studio/backend/tests/test_diffusion_attention_install.py b/studio/backend/tests/test_diffusion_attention_install.py
index f01c2d4673c..64438257ca0 100644
--- a/studio/backend/tests/test_diffusion_attention_install.py
+++ b/studio/backend/tests/test_diffusion_attention_install.py
@@ -250,7 +250,7 @@ def test_sage_install_never_asks_pypi_for_sageattention(real_install, monkeypatc
monkeypatch.setattr(importlib.util, "find_spec", lambda name: None)
monkeypatch.setattr(att, "_pip_sage2_installed", lambda: False)
att._ensure_attention_backend_installed("sage")
- assert run.calls and run.calls[0][-1] == "kernels"
+ assert run.calls and run.calls[0][-1] in ("kernels==0.12.3", "kernels>=0.12.3")
assert not any("sageattention" in part for cmd in run.calls for part in cmd)
diff --git a/studio/backend/tests/test_kernels_compat.py b/studio/backend/tests/test_kernels_compat.py
new file mode 100644
index 00000000000..62a0aa38b6b
--- /dev/null
+++ b/studio/backend/tests/test_kernels_compat.py
@@ -0,0 +1,322 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""kernels >= 0.15 next to transformers < 5.13 breaks `import transformers.activations` at import time."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import textwrap
+import types
+
+import pytest
+
+import utils.kernels_compat as kc
+
+_MARKER_ERR = b"ValueError: Either a revision or a version must be specified.\n"
+
+
+class _Logger:
+ def __init__(self):
+ self.warnings: list = []
+ self.infos: list = []
+
+ def warning(self, msg, *args):
+ self.warnings.append(msg % args if args else msg)
+
+ def info(self, msg, *args):
+ self.infos.append(msg % args if args else msg)
+
+ def debug(self, msg, *args):
+ pass
+
+
+class _Run:
+ """Recorded subprocess.run: pip calls succeed; the import check answers from ``import_ok``."""
+
+ def __init__(self, import_ok):
+ self.calls: list = []
+ self.import_ok = list(import_ok)
+
+ def __call__(self, cmd, **kwargs):
+ cmd = list(cmd)
+ self.calls.append(cmd)
+ if cmd[1:3] == ["-c", kc._VERIFY_CODE]:
+ ok = self.import_ok.pop(0) if len(self.import_ok) > 1 else self.import_ok[0]
+ return types.SimpleNamespace(
+ returncode = 0 if ok else 1, stderr = b"" if ok else _MARKER_ERR
+ )
+ return types.SimpleNamespace(returncode = 0, stderr = b"")
+
+ def pip(self):
+ return [c for c in self.calls if "pip" in c]
+
+
[email protected](autouse = True)
+def _isolated(monkeypatch):
+ monkeypatch.setattr(kc, "_healed", False)
+ monkeypatch.setattr(kc, "_hidden_reason", None)
+ monkeypatch.setattr(kc.importlib, "invalidate_caches", lambda: None)
+ monkeypatch.delitem(sys.modules, "kernels", raising = False)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ATTENTION_INSTALL", "auto")
+ yield
+ if sys.modules.get("kernels", object()) is None:
+ del sys.modules["kernels"]
+
+
+def _dists(monkeypatch, **versions):
+ table = {k.replace("_", "-") if k != "huggingface_hub" else k: v for k, v in versions.items()}
+ monkeypatch.setattr(kc, "_dist_version", lambda name: table.get(name))
+ return table
+
+
+ "tf, hub, spec",
+ [
+ ("5.5.0", "1.33.0", "kernels==0.12.3"),
+ ("5.8.0", None, "kernels==0.12.3"),
+ ("5.12.1", "1.33.0", "kernels==0.12.3"),
+ ("5.13.0rc0", "1.33.0", "kernels==0.12.3"),
+ ("5.13.0.dev0", "1.33.0", "kernels==0.12.3"),
+ ("5.13.0", "1.33.0", "kernels>=0.12.3"),
+ ("5.17.0", "1.33.0", "kernels>=0.12.3"),
+ ("6.0.0", None, "kernels>=0.12.3"),
+ (None, None, "kernels==0.12.3"),
+ ("garbage", None, "kernels==0.12.3"),
+ ("5.17.0", "1.5.0", "kernels==0.12.3"),
+ ],
+)
+def test_install_spec_follows_transformers(tf, hub, spec):
+ assert kc.kernels_install_spec(tf, hub) == spec
+
+
+ "kernels, tf, conflict",
+ [
+ ("0.18.2", "5.5.0", True),
+ ("0.15.1", "5.12.1", True),
+ ("0.14.1", "5.5.0", False),
+ ("0.12.3", "5.5.0", False),
+ ("0.18.2", "5.13.0", False),
+ ("0.18.2", "5.17.0", False),
+ (None, "5.5.0", False),
+ ("0.18.2", None, False),
+ ],
+)
+def test_conflict_prediction_matches_the_measured_matrix(kernels, tf, conflict):
+ assert kc.predicted_conflict(kernels, tf) is conflict
+
+
+def test_on_demand_install_pins_kernels_for_old_transformers(monkeypatch):
+ import core.inference.diffusion_attention as att
+ import importlib.util
+
+ _dists(monkeypatch, transformers = "5.5.0", huggingface_hub = "1.33.0")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ATTENTION_INSTALL", "auto")
+ monkeypatch.setattr(att, "_INSTALL_ATTEMPTED", set())
+ monkeypatch.setattr(att, "_kernels_hub_compatible", lambda: True)
+ monkeypatch.setattr(importlib.util, "find_spec", lambda name: None)
+ refreshed = []
+ monkeypatch.setattr(
+ att, "_refresh_diffusers_kernels_version", lambda log = None: refreshed.append(1)
+ )
+ run = _Run([True])
+ monkeypatch.setattr(subprocess, "run", run)
+ assert att._ensure_backend_package("sage", _Logger()) is None
+ assert run.pip()[0][-1] == "kernels==0.12.3"
+ assert run.calls[1][1:] == ["-c", "import transformers.activations"]
+ assert refreshed == [1]
+ assert "kernels" not in sys.modules
+
+
+def test_failed_import_check_rolls_the_install_back_and_hides_kernels(monkeypatch):
+ import core.inference.diffusion_attention as att
+ import importlib.util
+
+ table = _dists(monkeypatch, transformers = "5.5.0")
+ monkeypatch.setattr(att, "_dist_version", lambda name: table.get(name))
+ monkeypatch.setattr(att, "_INSTALL_ATTEMPTED", set())
+ monkeypatch.setattr(att, "_kernels_hub_compatible", lambda: True)
+ monkeypatch.setattr(importlib.util, "find_spec", lambda name: None)
+ refreshed = []
+ monkeypatch.setattr(
+ att, "_refresh_diffusers_kernels_version", lambda log = None: refreshed.append(1)
+ )
+
+ def run(cmd, **kw):
+ out = _Run([False])(cmd, **kw)
+ calls.append(list(cmd))
+ if "install" in cmd:
+ table["kernels"] = "0.18.2"
+ return out
+
+ calls: list = []
+ monkeypatch.setattr(subprocess, "run", run)
+ log = _Logger()
+ assert att._ensure_backend_package("sage", log) is None
+ assert calls[-1][-3:] == ["uninstall", "-y", "kernels"]
+ assert sys.modules["kernels"] is None
+ assert refreshed == []
+ assert any("removed it again" in w and "default attention backend" in w for w in log.warnings)
+ # The hidden package is reported with the real reason on the next request.
+ assert "breaks `import transformers.activations`" in att._ensure_backend_package("sage", log)
+ with pytest.raises(ImportError):
+ import kernels # noqa: F401
+
+
+def test_rollback_reinstalls_the_previous_version(monkeypatch):
+ run = _Run([False])
+ monkeypatch.setattr(subprocess, "run", run)
+ _dists(monkeypatch, transformers = "5.5.0", kernels = "0.18.2")
+ assert kc.check_kernels_install("0.11.7", _Logger()) is False
+ assert run.calls[-1][-1] == "kernels==0.11.7"
+
+
+def test_unconfirmed_import_check_keeps_the_install(monkeypatch):
+ def run(cmd, **kw):
+ raise subprocess.TimeoutExpired(cmd, 1)
+
+ monkeypatch.setattr(subprocess, "run", run)
+ _dists(monkeypatch, transformers = "5.5.0", kernels = "0.14.1")
+ log = _Logger()
+ assert kc.check_kernels_install(None, log) is True
+ assert "kernels" not in sys.modules and log.warnings
+
+
+def test_heal_is_metadata_only_without_a_conflict(monkeypatch):
+ run = _Run([True])
+ monkeypatch.setattr(subprocess, "run", run)
+ _dists(monkeypatch, transformers = "5.5.0", kernels = "0.12.3")
+ assert kc.heal_kernels_conflict(_Logger()) is None
+ assert run.calls == []
+
+
+def test_heal_repairs_a_broken_install(monkeypatch):
+ run = _Run([False, True])
+ monkeypatch.setattr(subprocess, "run", run)
+ _dists(monkeypatch, transformers = "5.5.0", kernels = "0.18.2", huggingface_hub = "1.33.0")
+ log = _Logger()
+ assert kc.heal_kernels_conflict(log) == "repaired: kernels==0.12.3"
+ assert run.pip()[0][-1] == "kernels==0.12.3" and "--no-deps" in run.pip()[0]
+ assert "kernels" not in sys.modules
+ assert any("reinstalled" in w for w in log.warnings)
+ # Once per process.
+ assert kc.heal_kernels_conflict(log) is None and len(run.calls) == 3
+
+
+def test_heal_hides_kernels_when_installs_are_off(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ATTENTION_INSTALL", "0")
+ run = _Run([False])
+ monkeypatch.setattr(subprocess, "run", run)
+ _dists(monkeypatch, transformers = "5.5.0", kernels = "0.18.2")
+ log = _Logger()
+ assert kc.heal_kernels_conflict(log).startswith("hidden:")
+ assert run.pip() == []
+ assert sys.modules["kernels"] is None
+ assert any(
+ "hub kernels are disabled" in w and "pip install --no-deps" in w for w in log.warnings
+ )
+
+
+def test_heal_hides_kernels_already_imported_even_after_a_repair(monkeypatch):
+ monkeypatch.setitem(sys.modules, "kernels", types.ModuleType("kernels"))
+ run = _Run([False, True])
+ monkeypatch.setattr(subprocess, "run", run)
+ _dists(monkeypatch, transformers = "5.5.0", kernels = "0.18.2")
+ log = _Logger()
+ assert kc.heal_kernels_conflict(log).startswith("hidden:")
+ assert run.pip() and sys.modules["kernels"] is None
+ assert any("after Unsloth restarts" in w for w in log.warnings)
+
+
+def test_heal_leaves_a_working_pair_alone(monkeypatch):
+ run = _Run([True])
+ monkeypatch.setattr(subprocess, "run", run)
+ _dists(monkeypatch, transformers = "5.5.0", kernels = "0.18.2")
+ assert kc.heal_kernels_conflict(_Logger()) is None
+ assert run.pip() == [] and "kernels" not in sys.modules
+
+
+def test_heal_runs_before_every_diffusers_import_window(monkeypatch):
+ import utils.torch_warmup as tw
+
+ seen = []
+ monkeypatch.setattr(kc, "heal_kernels_conflict", lambda log = None: seen.append(log))
+ with tw.background_media_import():
+ pass
+ monkeypatch.setattr(tw, "ensure_dynamo_imported", lambda **kw: True)
+ monkeypatch.setattr(tw, "_prime_cuda_driver", lambda: None)
+ monkeypatch.setattr(tw, "claim_media_import_window", lambda log = None: True)
+ tw.close_dynamo_import_window(_Logger())
+ assert len(seen) == 2
+
+
+def test_hidden_kernels_lets_real_transformers_import(tmp_path):
+ """A kernels whose LayerRepository raises like 0.15+ does breaks the real transformers import; hidden, it imports."""
+ pytest.importorskip("transformers")
+ fake = tmp_path / "kernels"
+ fake.mkdir()
+ (fake / "__init__.py").write_text(
+ textwrap.dedent(
+ """
+ class _Any:
+ def __init__(self, *a, **k):
+ raise ValueError("Either a revision or a version must be specified.")
+ def __getattr__(name):
+ return _Any
+ """
+ )
+ )
+ from transformers.utils import import_utils
+
+ # A version this transformers treats as available, so hub_kernels really imports it.
+ ver = getattr(import_utils, "KERNELS_MIN_VERSION", "0.18.2")
+ meta = tmp_path / f"kernels-{ver}.dist-info"
+ meta.mkdir()
+ (meta / "METADATA").write_text(f"Metadata-Version: 2.1\nName: kernels\nVersion: {ver}\n")
+ backend = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ env = dict(os.environ, PYTHONPATH = os.pathsep.join([str(tmp_path), backend]))
+ code = "import transformers.activations"
+ broken = subprocess.run([sys.executable, "-c", code], env = env, capture_output = True, timeout = 300)
+ assert broken.returncode != 0 and kc.KERNELS_CONFLICT_MARKER.encode() in broken.stderr
+ hidden = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ "import utils.kernels_compat as kc; kc.hide_kernels('test'); " + code,
+ ],
+ env = env,
+ capture_output = True,
+ timeout = 300,
+ )
+ assert hidden.returncode == 0, hidden.stderr.decode()[-800:]
+
+
+def test_a_concurrent_caller_waits_for_the_repair_in_flight(monkeypatch):
+ import threading
+
+ monkeypatch.setattr(kc, "_healed", False)
+ started, release = threading.Event(), threading.Event()
+ calls = []
+
+ def slow_heal(logger):
+ calls.append(1)
+ started.set()
+ release.wait(5)
+ return "repaired"
+
+ monkeypatch.setattr(kc, "_heal", slow_heal)
+ first = threading.Thread(target = kc.heal_kernels_conflict)
+ first.start()
+ assert started.wait(5)
+ second_done = threading.Event()
+ second = threading.Thread(target = lambda: (kc.heal_kernels_conflict(), second_done.set()))
+ second.start()
+ # The second caller must not return (and import diffusers) while the repair is still running.
+ assert not second_done.wait(0.3)
+ release.set()
+ first.join(5)
+ second.join(5)
+ assert second_done.is_set() and calls == [1]
diff --git a/studio/backend/utils/kernels_compat.py b/studio/backend/utils/kernels_compat.py
new file mode 100644
index 00000000000..00c98416074
--- /dev/null
+++ b/studio/backend/utils/kernels_compat.py
@@ -0,0 +1,274 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Keeps ``kernels`` at a version the installed transformers imports: transformers < 5.13 builds unversioned
+``LayerRepository`` objects at import, which kernels >= 0.15 rejects. Stdlib only."""
+
+from __future__ import annotations
+
+import importlib
+import importlib.metadata
+import os
+import re
+import subprocess
+import sys
+import threading
+from typing import Any, Optional
+
+# flash_4_hub's floor and the installer's pin (requirements/extras-no-deps.txt).
+KERNELS_FLOOR = "0.12.3"
+# First kernels release that rejects a LayerRepository without a revision or version.
+KERNELS_STRICT = (0, 15)
+# First transformers release whose hub_kernels mapping passes version= to every LayerRepository.
+TRANSFORMERS_KERNELS_STRICT_OK = (5, 13)
+# kernels >= 0.14.1 requires huggingface-hub >= 1.10.0.
+_KERNELS_HUB_FLOOR = (1, 10)
+KERNELS_CONFLICT_MARKER = "Either a revision or a version must be specified"
+
+_ATTENTION_INSTALL_ENV = "UNSLOTH_DIFFUSION_ATTENTION_INSTALL"
+_VERIFY_TIMEOUT_S = 180
+_VERIFY_CODE = "import transformers.activations"
+
+_heal_lock = threading.Lock()
+_healed = False
+_hidden_reason: Optional[str] = None
+
+
+def _dist_version(name: str) -> Optional[str]:
+ try:
+ return importlib.metadata.version(name)
+ except Exception: # noqa: BLE001 - missing or broken metadata reads as not installed
+ return None
+
+
+def _release(text: Optional[str]) -> Optional[tuple[int, ...]]:
+ if not text:
+ return None
+ m = re.match(r"\s*v?(\d+(?:\.\d+)*)", text)
+ if m is None:
+ return None
+ return tuple(int(p) for p in m.group(1).split("."))
+
+
+def _is_prerelease(text: str) -> bool:
+ return re.search(r"(dev|a|alpha|b|beta|rc|c|pre|preview)\d*$", text.strip().lower()) is not None
+
+
+def transformers_accepts_strict_kernels(transformers_version: Optional[str]) -> bool:
+ """Known to import next to kernels >= 0.15 (not a 5.13.0 pre-release or an unknown version)."""
+ have = _release(transformers_version)
+ if have is None:
+ return False
+ if _is_prerelease(transformers_version or "") and tuple(
+ have[:3]
+ ) == TRANSFORMERS_KERNELS_STRICT_OK + (0,):
+ return False
+ return have[:2] >= TRANSFORMERS_KERNELS_STRICT_OK
+
+
+def kernels_install_spec(
+ transformers_version: Optional[str] = None, hub_version: Optional[str] = None
+) -> str:
+ """Pip requirement for the ``--no-deps`` install. Exact below 5.13: 0.14.x needs ``kernels-data``, which
+ ``--no-deps`` leaves out."""
+ hub = _release(hub_version)
+ if not transformers_accepts_strict_kernels(transformers_version) or (
+ hub is not None and hub[:2] < _KERNELS_HUB_FLOOR
+ ):
+ return f"kernels=={KERNELS_FLOOR}"
+ return f"kernels>={KERNELS_FLOOR}"
+
+
+def current_kernels_install_spec() -> str:
+ return kernels_install_spec(_dist_version("transformers"), _dist_version("huggingface_hub"))
+
+
+def predicted_conflict(kernels_version: Optional[str], transformers_version: Optional[str]) -> bool:
+ kv = _release(kernels_version)
+ if kv is None or transformers_version is None or _release(transformers_version) is None:
+ return False
+ return kv[:2] >= KERNELS_STRICT and not transformers_accepts_strict_kernels(
+ transformers_version
+ )
+
+
+def verify_transformers_import(timeout: float = _VERIFY_TIMEOUT_S) -> tuple[Optional[bool], str]:
+ """Fresh-interpreter import check: True, False (with stderr tail) or None (could not tell)."""
+ try:
+ proc = subprocess.run(
+ [sys.executable, "-c", _VERIFY_CODE],
+ capture_output = True,
+ timeout = timeout,
+ env = dict(os.environ),
+ )
+ except subprocess.TimeoutExpired:
+ return None, f"the import check timed out after {timeout:.0f}s"
+ except Exception as exc: # noqa: BLE001
+ return None, f"the import check could not run ({type(exc).__name__}: {exc})"
+ if proc.returncode == 0:
+ return True, ""
+ err = proc.stderr
+ if isinstance(err, bytes):
+ err = err.decode("utf-8", errors = "replace")
+ lines = [ln for ln in (err or "").strip().splitlines() if ln.strip()]
+ return False, (lines[-1] if lines else f"exit code {proc.returncode}")[-500:]
+
+
+def hide_kernels(reason: str) -> None:
+ """Make ``kernels`` unimportable here; transformers' hub_kernels tolerates the ImportError."""
+ global _hidden_reason
+ for name in [m for m in sys.modules if m == "kernels" or m.startswith("kernels.")]:
+ sys.modules.pop(name, None)
+ sys.modules["kernels"] = None
+ _hidden_reason = reason
+ _reset_import_caches()
+
+
+def hidden_reason() -> Optional[str]:
+ return _hidden_reason if sys.modules.get("kernels", object()) is None else None
+
+
+def _reset_import_caches() -> None:
+ importlib.invalidate_caches()
+ for module_name in ("transformers.utils.import_utils", "diffusers.utils.import_utils"):
+ mod = sys.modules.get(module_name)
+ fn = getattr(mod, "is_kernels_available", None) if mod is not None else None
+ cache_clear = getattr(fn, "cache_clear", None)
+ if callable(cache_clear):
+ try:
+ cache_clear()
+ except Exception: # noqa: BLE001
+ pass
+ mod = sys.modules.get("diffusers.utils.import_utils")
+ if mod is not None and sys.modules.get("kernels", object()) is None:
+ for attr, value in (("_kernels_available", False), ("_kernels_version", "N/A")):
+ if hasattr(mod, attr):
+ setattr(mod, attr, value)
+
+
+def _pip(args: list[str], timeout: float = 600) -> subprocess.CompletedProcess:
+ return subprocess.run(
+ [sys.executable, "-m", "pip", "--disable-pip-version-check", *args],
+ capture_output = True,
+ timeout = timeout,
+ check = True,
+ )
+
+
+def restore_kernels(previous_version: Optional[str]) -> bool:
+ try:
+ if previous_version is None:
+ _pip(["uninstall", "-y", "kernels"])
+ else:
+ _pip(["install", "--only-binary", ":all:", "--no-deps", f"kernels=={previous_version}"])
+ importlib.invalidate_caches()
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+def check_kernels_install(previous_version: Optional[str], logger: Any = None) -> bool:
+ """True when transformers still imports after the install; else roll back and hide kernels."""
+ if _dist_version("transformers") is None:
+ return True
+ ok, detail = verify_transformers_import()
+ if ok is not False:
+ if ok is None and logger is not None:
+ logger.warning(
+ "diffusion.attention: could not confirm transformers still imports after installing kernels %s: %s",
+ _dist_version("kernels"),
+ detail,
+ )
+ return True
+ installed = _dist_version("kernels")
+ restored = restore_kernels(previous_version)
+ reason = (
+ f"kernels {installed} breaks `import transformers.activations` with transformers "
+ f"{_dist_version('transformers')} ({detail})"
+ )
+ hide_kernels(reason)
+ if logger is not None:
+ logger.warning(
+ "diffusion.attention: %s; %s, using the default attention backend",
+ reason,
+ (
+ (
+ "removed it again"
+ if previous_version is None
+ else f"restored kernels {previous_version}"
+ )
+ if restored
+ else "could not roll the install back"
+ ),
+ )
+ return False
+
+
+def heal_kernels_conflict(logger: Any = None) -> Optional[str]:
+ """Once per process before diffusers imports: repair or hide a kernels transformers cannot import."""
+ global _healed
+ if _healed:
+ return None
+ with _heal_lock:
+ if _healed:
+ return None
+ try:
+ return _heal(logger)
+ except Exception as exc: # noqa: BLE001 - must never break an import
+ if logger is not None:
+ logger.debug("kernels compatibility check failed: %r", exc)
+ return None
+ finally:
+ # Set only once the repair is done, so a concurrent caller waits on the lock instead of importing.
+ _healed = True
+
+
+def _heal(logger: Any) -> Optional[str]:
+ kernels_version = _dist_version("kernels")
+ transformers_version = _dist_version("transformers")
+ if not predicted_conflict(kernels_version, transformers_version):
+ return None
+ if sys.modules.get("kernels", object()) is None:
+ return None
+ ok, detail = verify_transformers_import()
+ if ok is not False or KERNELS_CONFLICT_MARKER not in detail:
+ return None
+ spec = kernels_install_spec(transformers_version, _dist_version("huggingface_hub"))
+ gate = os.environ.get(_ATTENTION_INSTALL_ENV, "auto").strip().lower()
+ already_imported = "kernels" in sys.modules
+ if gate not in ("0", "false", "no", "off"):
+ try:
+ _pip(["install", "--only-binary", ":all:", "--no-deps", spec])
+ importlib.invalidate_caches()
+ ok, detail = verify_transformers_import()
+ except Exception as exc: # noqa: BLE001
+ stderr = getattr(exc, "stderr", None)
+ if isinstance(stderr, bytes):
+ stderr = stderr.decode("utf-8", errors = "replace")
+ ok, detail = False, ((stderr or "").strip()[-300:] or str(exc))
+ if ok:
+ if not already_imported:
+ if logger is not None:
+ logger.warning(
+ "kernels %s cannot be imported by transformers %s; reinstalled %s (now %s)",
+ kernels_version,
+ transformers_version,
+ spec,
+ _dist_version("kernels"),
+ )
+ _reset_import_caches()
+ return f"repaired: {spec}"
+ reason = (
+ f"kernels {kernels_version} cannot be imported by transformers {transformers_version} "
+ f"({KERNELS_CONFLICT_MARKER})"
+ )
+ hide_kernels(reason)
+ if logger is not None:
+ logger.warning(
+ "%s; Hugging Face hub kernels are disabled in this process%s. To repair, run: "
+ "python -m pip install --no-deps '%s'",
+ reason,
+ " (the repaired version takes effect after Unsloth restarts)" if ok else "",
+ spec,
+ )
+ return f"hidden: {reason}"
diff --git a/studio/backend/utils/torch_warmup.py b/studio/backend/utils/torch_warmup.py
index 18d7e9f2af7..febdc41c678 100644
--- a/studio/backend/utils/torch_warmup.py
+++ b/studio/backend/utils/torch_warmup.py
@@ -330,8 +330,18 @@ def ensure_dynamo_imported(
_dynamo_lock.release()
+def _heal_kernels_conflict(log) -> None:
+ """A kernels install transformers cannot import fails every diffusers import: repair or hide it first."""
+ try:
+ from utils.kernels_compat import heal_kernels_conflict
+ except Exception: # noqa: BLE001
+ return
+ heal_kernels_conflict(log or logger)
+
+
def close_dynamo_import_window(log) -> bool:
"""Call before importing diffusers, itself a dynamo importer that cannot recover a lost race; wrap the import of this module too."""
+ _heal_kernels_conflict(log)
imported = ensure_dynamo_imported(log = log, reason = "diffusers import")
# `import diffusers` probes CUDA at module level (diffusers.utils.torch_utils) with the GIL held.
_prime_cuda_driver()
@@ -356,6 +366,7 @@ _media_import_owner: Optional[int] = None
def background_media_import():
"""Hold the window for background work; yields False once a load has claimed it (skip then)."""
global _media_import_owner
+ _heal_kernels_conflict(None)
with _media_import_lock:
_media_import_owner = threading.get_ident()
try: