Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: restore Qwen causal-conv1d on demand, narrow fatal mamba-ssm, provision SSM kernels in export (#13971)
Daniel Han7729c37fd09e
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.jsonindex d31bf604ced..8cd6e9a1e6d 100644--- a/scripts/risky_loaders_baseline.json+++ b/scripts/risky_loaders_baseline.json@@ -38,7 +38,7 @@"file": "studio/backend/core/export/worker.py","rule": "trust-remote-code","sink": "keyword",- "digest": "58d5d0f2c979e2ae",+ "digest": "35a68c33b5a8b4ad","count": 1,"reason": "Argument to the consent evaluator inside `if trust_remote_code:`; it scans the code and blocks unless approved."},diff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.pyindex 53754e39189..6f54406d5cd 100644--- a/studio/backend/core/export/worker.py+++ b/studio/backend/core/export/worker.py@@ -192,7 +192,10 @@ def _offline_window_if_unreachable(step = "loading"):offline constant and cached sessions, so env alone is too late: defer to the loader's_force_hf_offline (env + in-process flags + session reset). Pre-import (activation),huggingface_hub is not loaded yet, so setting the env vars suffices for its urllib probes."""+ from utils.wheel_utils import HF_OFFLINE_AUTO_ENV+saved: dict[str, str | None] = {}+ saved_marker: list = [] # [prior value] once this window marked HF offline as auto-setforce_ctx = Nonetry:from utils.transformers_version import _env_offline, hf_endpoint_unreachable@@ -204,6 +207,10 @@ def _offline_window_if_unreachable(step = "loading"):)if not _env_offline() and probe_enabled and hf_endpoint_unreachable():logger.warning("Hugging Face endpoint unreachable; %s offline", step)+ # Only the Hub is known to be down: mark this offline as ours so a required kernel+ # install (GitHub / PyPI) is still allowed; an operator's HF offline never enters here.+ saved_marker.append(os.environ.get(HF_OFFLINE_AUTO_ENV))+ os.environ[HF_OFFLINE_AUTO_ENV] = "1"if "huggingface_hub" in sys.modules:try:from unsloth.models.loader_utils import _force_hf_offline@@ -230,6 +237,11 @@ def _offline_window_if_unreachable(step = "loading"):os.environ.pop(k, None)else:os.environ[k] = v+ if saved_marker:+ if saved_marker[0] is None:+ os.environ.pop(HF_OFFLINE_AUTO_ENV, None)+ else:+ os.environ[HF_OFFLINE_AUTO_ENV] = saved_marker[0]def _send_response(resp_queue: Any, response: dict) -> None:@@ -239,70 +251,49 @@ def _send_response(resp_queue: Any, response: dict) -> None:logger.error("Failed to send response: %s", exc)-def _handle_load(backend, cmd: dict, resp_queue: Any) -> None:- from hub.utils.hf_tokens import hf_token_arg-- checkpoint_path = cmd["checkpoint_path"]- # The preflight helpers read the policy off the token, so rebuild it once here.- hf_token = hf_token_arg(cmd.get("hf_token"), allow_ambient_token = cmd.get("allow_ambient", True))- max_seq_length = cmd.get("max_seq_length", 2048)- load_in_4bit = cmd.get("load_in_4bit", True)- # Latest-sidecar checkpoints load 16-bit here too: bnb 4-bit feeds quantized- # expert weights into unvalidated paths (same flip as the chat worker).- if load_in_4bit:- from utils.transformers_version import latest_tier_active_for-- # Plain token, like the activation below: the sentinel reads no cache offline,- # misses the sidecar and leaves 4-bit on.- if latest_tier_active_for(checkpoint_path, hf_token or None):- load_in_4bit = False- logger.info(- "Latest-transformers sidecar active for %s - forcing a 16-bit "- "export load (4-bit is disabled for brand-new architectures)",- checkpoint_path,- )- trust_remote_code = cmd.get("trust_remote_code", False)- base_model = cmd.get("base_model") or None-- # Auto-enable trust_remote_code for NemotronH/Nano models.- if not trust_remote_code:- from utils.security.trusted_org import is_trusted_org_repo-- _NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano")- _cp_lower = checkpoint_path.lower()- if (- any(sub in _cp_lower for sub in _NEMOTRON_TRUST_SUBSTRINGS)- and (_cp_lower.startswith("unsloth/") or _cp_lower.startswith("nvidia/"))- # Genuine first-party Hub repo only (not a local/spoof name starting- # with "unsloth/"); authenticated so private repos resolve.- and is_trusted_org_repo(checkpoint_path, hf_token = hf_token)- ):- trust_remote_code = True- logger.info(- "Auto-enabled trust_remote_code for Nemotron model: %s",- checkpoint_path,- )-- # Malware gate: a poisoned pickle deserializes on load even with- # trust_remote_code False, so check HF's security scan (metadata-only) every- # load. Local checkpoints have no Hub scan and are skipped in the helper; a- # LoRA merges its base weights, so gate that repo too.- from utils.security import evaluate_file_security, load_scan_target, security_load_subdirs-- requested_security_targets = [checkpoint_path]- try:- from utils.models.model_config import get_base_model_from_lora_identifier+def _nemotron_auto_trust(checkpoint_path: str, hf_token) -> bool:+ """Whether a NemotronH/Nano checkpoint gets trust_remote_code without the user asking: a+ genuine first-party Hub repo only (not a local/spoof name starting with "unsloth/"),+ authenticated so private repos resolve."""+ _NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano")+ _cp_lower = str(checkpoint_path).lower()+ if not (+ any(sub in _cp_lower for sub in _NEMOTRON_TRUST_SUBSTRINGS)+ and (_cp_lower.startswith("unsloth/") or _cp_lower.startswith("nvidia/"))+ ):+ return False+ from utils.security.trusted_org import is_trusted_org_repo++ return is_trusted_org_repo(checkpoint_path, hf_token = hf_token)+++def _run_export_security_gates(+ targets: list,+ *,+ trust_remote_code: bool,+ hf_token,+ approved_fingerprint,+ resp_queue: Any,+ compute_subdirs: bool = True,+ subject = None,+) -> bool:+ """Malware + (when *trust_remote_code*) remote-code consent gates over *targets* (checkpoint ++ LoRA base). Sends the matching 'loaded' failure and returns False if blocked, True when every+ target is clear.++ ``compute_subdirs=False`` keeps the gate transformers-free (``security_load_subdirs`` imports+ ``model_config`` -> ``transformers``, which would snapshot optional-backend availability before+ the SSM kernels are installed): used for the preflight before kernel provisioning, where+ ``_handle_load`` re-runs the authoritative gate with full subdir scoping."""+ from utils.security import evaluate_file_security, load_scan_target++ if compute_subdirs:+ from utils.security import security_load_subdirs- # Resolve a LOCAL or REMOTE adapter's base so a remote LoRA base is gated too.- _base = base_model or get_base_model_from_lora_identifier(checkpoint_path, hf_token)- if _base:- requested_security_targets.append(_base)- except Exception as exc:- logger.debug("Could not resolve LoRA base for malware scan: %s", exc)security_targets: list[str] = []consent_load_subdirs: dict[str, tuple] = {}- for requested_target in dict.fromkeys(requested_security_targets):- load_subdirs = security_load_subdirs(requested_target, hf_token)+ for requested_target in dict.fromkeys(t for t in targets if t):+ load_subdirs = security_load_subdirs(requested_target, hf_token) if compute_subdirs else ()target, load_subdirs = load_scan_target(requested_target, load_subdirs)if target not in consent_load_subdirs:security_targets.append(target)@@ -329,20 +320,19 @@ def _handle_load(backend, cmd: dict, resp_queue: Any) -> None:"ts": time.time(),},)- return+ return False- # Consent gate: scan auto_map code before it runs; block CRITICAL/HIGH unless- # pinned-approved. A LoRA merges its base model, whose code runs, so gate it too.+ # Consent gate: scan auto_map code before it runs; block CRITICAL/HIGH unless pinned-approved.+ # A LoRA merges its base model, whose code runs, so gate it too. Adapter + base are scanned as+ # one combined unit, pinned by a single fingerprint.if trust_remote_code:from utils.security import evaluate_remote_code_consent_for_targets-- # Scan adapter + base as one combined unit, pinned by a single fingerprint._rc = evaluate_remote_code_consent_for_targets(security_targets,hf_token = hf_token,trust_remote_code = True,- approved_fingerprint = cmd.get("approved_remote_code_fingerprint"),- subject = cmd.get("subject"),+ approved_fingerprint = approved_fingerprint,+ subject = subject,load_subdirs_by_target = consent_load_subdirs,)if _rc.blocked:@@ -361,7 +351,175 @@ def _handle_load(backend, cmd: dict, resp_queue: Any) -> None:"ts": time.time(),},)- return+ return False++ return True+++def _local_adapter_base(checkpoint_path: str):+ """``base_model_name_or_path`` of a local adapter checkpoint, read straight from its+ adapter_config.json (no ML import), else None."""+ try:+ import json+ cfg = Path(checkpoint_path) / "adapter_config.json"+ if cfg.is_file():+ return (+ json.loads(cfg.read_text(encoding = "utf-8-sig")).get("base_model_name_or_path")+ or None+ )+ except Exception:+ pass+ return None+++def _preflight_security_gates(cmd: dict, resp_queue: Any) -> bool:+ """The security gates, transformers-free, BEFORE any SSM kernel is provisioned: a blocked+ checkpoint (or LoRA base) must not trigger a native build or a wheel install on its behalf.+ Gates the checkpoint plus its LoRA base (the explicit one, a local adapter_config.json, or the+ Hub adapter config), under the trust the load will use. ``_handle_load`` re-runs the+ authoritative gates afterwards. Returns False after sending the 'loaded' failure."""+ checkpoint_path = cmd["checkpoint_path"]+ hf_token = cmd.get("hf_token") or None+ base = cmd.get("base_model") or _local_adapter_base(checkpoint_path)+ if not base:+ try:+ from utils.transformers_version import _remote_lora_base+ base = _remote_lora_base(checkpoint_path, hf_token)+ except Exception as exc:+ logger.debug("Could not resolve the adapter base for the preflight gates: %s", exc)+ trust_remote_code = bool(cmd.get("trust_remote_code")) or _nemotron_auto_trust(+ checkpoint_path, hf_token+ )+ return _run_export_security_gates(+ [checkpoint_path, base],+ trust_remote_code = trust_remote_code,+ hf_token = hf_token,+ approved_fingerprint = cmd.get("approved_remote_code_fingerprint"),+ resp_queue = resp_queue,+ compute_subdirs = False, # stay transformers-free until the SSM kernels are installed+ subject = cmd.get("subject"),+ )+++def _export_uses_cuda_kernels(device_backend) -> bool:+ """causal-conv1d / mamba-ssm are CUDA / ROCm extensions, so only a CUDA export (ROCm reports as+ "cuda" too) provisions them: MLX, CPU, XPU, an unspecified backend and any macOS export never+ load them. ssm_runtime then checks the torch build itself."""+ return sys.platform != "darwin" and str(device_backend or "") in ("cuda", "rocm")+++def _provision_ssm_kernels(+ checkpoint_path: str,+ base_model: str | None,+ hf_token,+ resp_queue: Any,+ device_backend = None,+ trust_remote_code = None,+) -> bool:+ """Install the causal-conv1d / mamba-ssm a checkpoint needs before Unsloth and its modeling+ import, decided from its config (a LoRA's base config), as a training or chat load does.+ Returns False after reporting a failure only for a model that cannot load without mamba-ssm."""+ if not _export_uses_cuda_kernels(device_backend):+ return True+ try:+ from utils.ssm_runtime import provision_ssm_kernels, ssm_probe_identifier+ from utils.transformers_version import _remote_lora_base+ except Exception as exc:+ logger.debug("ssm_runtime unavailable (%s); skipping SSM kernel install", exc)+ return True+ base = base_model+ if not base:+ try:+ # Raw HTTP, no ML import; a local adapter's base is read by the provisioner itself.+ base = _remote_lora_base(checkpoint_path, hf_token or None)+ except Exception as exc:+ logger.debug("Could not resolve the adapter base for SSM kernels: %s", exc)+ status = lambda message: _send_response(+ resp_queue, {"type": "status", "message": message, "ts": time.time()}+ )+ try:+ # The trust the load will use decides whether an auto_map model runs its remote code.+ trusted = bool(trust_remote_code) or _nemotron_auto_trust(checkpoint_path, hf_token or None)+ provision_ssm_kernels(+ ssm_probe_identifier(checkpoint_path, base),+ checkpoint_path,+ hf_token or None,+ adapter_base = base,+ trust_remote_code = trusted,+ status_cb = status,+ )+ return True+ except Exception as exc:+ _send_response(+ resp_queue,+ {+ "type": "error",+ "error": (+ "This model needs SSM kernel libraries (causal-conv1d / mamba-ssm) "+ f"that could not be installed: {exc}"+ ),+ "ts": time.time(),+ },+ )+ return False+++def _handle_load(backend, cmd: dict, resp_queue: Any) -> None:+ from hub.utils.hf_tokens import hf_token_arg++ checkpoint_path = cmd["checkpoint_path"]+ # The preflight helpers read the policy off the token, so rebuild it once here.+ hf_token = hf_token_arg(cmd.get("hf_token"), allow_ambient_token = cmd.get("allow_ambient", True))+ max_seq_length = cmd.get("max_seq_length", 2048)+ load_in_4bit = cmd.get("load_in_4bit", True)+ # Latest-sidecar checkpoints load 16-bit here too: bnb 4-bit feeds quantized+ # expert weights into unvalidated paths (same flip as the chat worker).+ if load_in_4bit:+ from utils.transformers_version import latest_tier_active_for++ # Plain token, like the activation below: the sentinel reads no cache offline,+ # misses the sidecar and leaves 4-bit on.+ if latest_tier_active_for(checkpoint_path, hf_token or None):+ load_in_4bit = False+ logger.info(+ "Latest-transformers sidecar active for %s - forcing a 16-bit "+ "export load (4-bit is disabled for brand-new architectures)",+ checkpoint_path,+ )+ trust_remote_code = cmd.get("trust_remote_code", False)+ base_model = cmd.get("base_model") or None++ # Auto-enable trust_remote_code for NemotronH/Nano models.+ if not trust_remote_code and _nemotron_auto_trust(checkpoint_path, hf_token):+ trust_remote_code = True+ logger.info(+ "Auto-enabled trust_remote_code for Nemotron model: %s",+ checkpoint_path,+ )++ # Malware gate: a poisoned pickle deserializes on load even with trust_remote_code False, so+ # check HF's security scan (metadata-only) every load. Local checkpoints have no Hub scan and+ # are skipped in the helper; a LoRA merges its base weights, so gate that repo too. This is+ # the authoritative check (full subdir scoping); the pre-provisioning preflight is a subset.+ requested_security_targets = [checkpoint_path]+ try:+ from utils.models.model_config import get_base_model_from_lora_identifier++ # Resolve a LOCAL or REMOTE adapter's base so a remote LoRA base is gated too.+ _base = base_model or get_base_model_from_lora_identifier(checkpoint_path, hf_token)+ if _base:+ requested_security_targets.append(_base)+ except Exception as exc:+ logger.debug("Could not resolve LoRA base for malware scan: %s", exc)+ if not _run_export_security_gates(+ requested_security_targets,+ trust_remote_code = trust_remote_code,+ hf_token = hf_token,+ approved_fingerprint = cmd.get("approved_remote_code_fingerprint"),+ resp_queue = resp_queue,+ subject = cmd.get("subject"),+ ):+ returntry:_send_response(@@ -598,6 +756,23 @@ def run_export_process(*, cmd_queue: Any, resp_queue: Any, config: dict) -> None)return+ # Before Unsloth and the modeling import: transformers captures its kernel gates at import,+ # and some remote modeling code (Nemotron-H) cannot import at all without mamba-ssm. Inside+ # the activation window, so it adds no Hub reachability probe of its own; an MLX / CPU /+ # macOS export installs nothing. The security gates run first, so a blocked checkpoint+ # never triggers a kernel build on its behalf.+ if not _preflight_security_gates(config, resp_queue):+ return+ if not _provision_ssm_kernels(+ checkpoint_path,+ config.get("base_model") or None,+ config.get("hf_token"),+ resp_queue,+ config.get("device_backend"),+ trust_remote_code = config.get("trust_remote_code"),+ ):+ return+# Importable Triton isn't enough on AMD: its clang-cl JIT also needs the MSVC CRT headers (#7595).if sys.platform == "win32":from core._msvc_env import gate_torch_compile_on_windows@@ -686,8 +861,20 @@ def run_export_process(*, cmd_queue: Any, resp_queue: Any, config: dict) -> Noneif cmd_type == "load":# Load a new checkpoint, reusing this subprocess.backend.cleanup_memory()- # Offline window also covers this load's Hub preflights (re-probed per load).+ # Offline window covers this load's Hub preflights (re-probed per load). The+ # security gates run before any SSM kernel is provisioned for the checkpoint.with _offline_window_if_unreachable():+ if not _preflight_security_gates(cmd, resp_queue):+ continue+ if not _provision_ssm_kernels(+ cmd["checkpoint_path"],+ cmd.get("base_model") or None,+ cmd.get("hf_token"),+ resp_queue,+ config.get("device_backend"),+ trust_remote_code = cmd.get("trust_remote_code"),+ ):+ continue_handle_load(backend, cmd, resp_queue)elif cmd_type == "export":diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 799eca015a8..cf9600e9459 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -873,6 +873,16 @@ class InferenceOrchestrator:with self._dispatcher_lifecycle_lock:self._worker_reserved_for = None+ def _shutdown_before_respawn(self) -> None:+ """Tear the current worker down before a retry spawns its replacement. A worker that+ survives terminate/kill keeps its handle (see the model-switch guard in load_model), and a+ second worker must not be spawned over it."""+ if self._shutdown_subprocess(timeout = 5) is False:+ raise RuntimeError(+ "The inference worker did not exit and still holds GPU memory; "+ "not starting a new one over it. Retry shortly."+ )+def _shutdown_subprocess(self, timeout: float = 10.0) -> bool:with self._subprocess_shutdown_lock:managed = getattr(self, "_managed_engine", None)@@ -2251,7 +2261,11 @@ class InferenceOrchestrator:os.environ.get("HF_HUB_DISABLE_XET") == "1")- for attempt in range(2):+ # One spawn, plus at most one Xet-off retry after a stalled download and one respawn+ # to bind SSM kernels the worker just installed: each is used once, in either order.+ xet_retry_used = False+ ssm_restart_used = False+ for attempt in range(3):if model_name not in self.loading_models or (load_cancel_event is not None and load_cancel_event.is_set()):@@ -2265,7 +2279,7 @@ class InferenceOrchestrator:return Falselogger.info("Spawning fresh inference subprocess for '%s' "- "(transformers %s.x, attempt %d/2%s)",+ "(transformers %s.x, attempt %d/3%s)",model_name,needed_major,attempt + 1,@@ -2312,18 +2326,23 @@ class InferenceOrchestrator:return Falseexcept DownloadStallError:# First stall with Xet on -> retry with Xet disabled- if attempt == 0 and not disable_xet:+ if not xet_retry_used and not disable_xet:+ xet_retry_used = Truelogger.warning("Download stalled for '%s' -- retrying with HF_HUB_DISABLE_XET=1",model_name,)- self._shutdown_subprocess(timeout = 5)+ self._shutdown_before_respawn()disable_xet = Truecontinueself._shutdown_subprocess(timeout = 5)+ if disable_xet:+ raise RuntimeError(+ f"Download stalled for '{model_name}' even with "+ f"HF_HUB_DISABLE_XET=1 -- check your network connection"+ )raise RuntimeError(- f"Download stalled for '{model_name}' even with "- f"HF_HUB_DISABLE_XET=1 -- check your network connection"+ f"Download stalled for '{model_name}' -- check your network connection")if resp.get("success"):@@ -2395,6 +2414,19 @@ class InferenceOrchestrator:logger.info("Model '%s' loaded successfully in subprocess", model_name)return Trueelse:+ if (+ resp.get("error_kind") == "ssm_runtime_restart_required"+ and not ssm_restart_used+ ):+ # The worker installed an SSM kernel after its transformers had already+ # imported the model's modeling module; a fresh worker binds it.+ ssm_restart_used = True+ logger.info(+ "Respawning the inference worker for '%s' to pick up new SSM kernels",+ model_name,+ )+ self._shutdown_before_respawn()+ continue# Worker reports failures (consent gate included) under "message".error = resp.get("message") or resp.get("error") or "Failed to load model"self.active_model_name = Nonediff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 06e165a8cd1..a663a746b73 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -386,25 +386,28 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool:return load_in_4bit+SSM_RESTART_REQUIRED = "ssm_runtime_restart_required"++def _ensure_ssm_kernels(targets: list,resp_queue: Any,hf_token: str | None = None,+ trust_remote_code: bool | None = None,) -> bool:"""Install the SSM kernels the given model(s) lazy-import in from_pretrained; no-op fornon-SSM models, idempotent. *targets* are ``(probe_identifier, config_target)`` pairs: the- kernels are decided from the config at ``config_target`` (the resolved base), the name only- when there is none. Returns True on success; on a fatal mamba-ssm failure sends a 'loaded'- failure response and returns False. Call BEFORE importing transformers, which snapshots its- optional-backend gates at import; an in-process reload clears them after installing.+ kernels are decided from the config at ``config_target`` (the resolved base, or a local+ adapter's recorded base), the name only when there is none. Call BEFORE importing+ transformers, which snapshots its optional-backend gates at import.++ Returns True on success. Returns False after sending a 'loaded' failure: on a fatal mamba-ssm+ failure, or when a kernel was newly installed after this process already imported the model's+ modeling module, whose captured kernel functions only a fresh worker picks up (the+ orchestrator respawns once on ``SSM_RESTART_REQUIRED``)."""try:- from utils.ssm_runtime import (- ensure_ssm_runtime,- refresh_transformers_kernel_gates,- resolved_model_is_ssm,- resolved_model_wants_causal_conv1d,- )+ from utils.ssm_runtime import modeling_modules_needing_restart, provision_ssm_kernelsexcept Exception as exc:logger.debug("ssm_runtime unavailable (%s); skipping SSM kernel pre-install", exc)return True@@ -412,17 +415,39 @@ def _ensure_ssm_kernels(_ssm_status = lambda m: _send_response(resp_queue, {"type": "status", "message": m})try:for probe, config_target in dict.fromkeys(t for t in targets if t and t[0]):- config_target = config_target or probe- ensure_ssm_runtime(+ # trust_remote_code decides whether an auto_map model loads its remote code, and so+ # whether a missing mamba-ssm refuses the load (see ssm_runtime.mamba_ssm_is_fatal).+ kernels, installed = provision_ssm_kernels(probe,+ config_target or probe,+ hf_token,+ trust_remote_code = trust_remote_code,status_cb = _ssm_status,- # transformers-free: this runs before the worker's transformers import.- wants_causal_conv1d = resolved_model_wants_causal_conv1d(- probe, config_target, hf_token, scan_modeling_source = False- ),- is_ssm = resolved_model_is_ssm(probe, config_target, hf_token),)- refresh_transformers_kernel_gates()+ stale = (+ modeling_modules_needing_restart(kernels.model_types, kernels.remote_code)+ if installed+ else []+ )+ if stale:+ logger.info(+ "Installed %s after %s was imported; restarting the worker to pick it up",+ ", ".join(installed),+ ", ".join(stale),+ )+ _send_response(+ resp_queue,+ {+ "type": "loaded",+ "success": False,+ "message": (+ f"Installed {', '.join(installed)} for this model; "+ "restarting the model worker to use it."+ ),+ "error_kind": SSM_RESTART_REQUIRED,+ },+ )+ return Falsereturn Trueexcept Exception as exc:_send_response(@@ -717,7 +742,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:_ssm_base or config["model_name"],)]- if not _ensure_ssm_kernels(ssm_targets, resp_queue, hf_token):+ if not _ensure_ssm_kernels(ssm_targets, resp_queue, hf_token, trust_remote_code):return# Heartbeat keeps the orchestrator's inactivity deadline alive during slow@@ -2490,7 +2515,7 @@ def run_inference_process(from utils.ssm_runtime import ssm_probe_identifier_ssm_targets = [(ssm_probe_identifier(model_name, _base), _base or model_name)]- if not _ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token):+ if not _ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token, _trust_remote_code):returntry:diff --git a/studio/backend/core/systemone/clef_runtime.py b/studio/backend/core/systemone/clef_runtime.pyindex 4a6cb10596f..547a6414e10 100644--- a/studio/backend/core/systemone/clef_runtime.py+++ b/studio/backend/core/systemone/clef_runtime.py@@ -46,19 +46,10 @@ class _Events:def _fast_paths(folder: str) -> None:- # Clef is a Qwen3.5 backbone: unsloth_zoo ships its gated delta kernels and a Triton causal conv,- # so nothing is installed. Where that conv cannot run (switched off, ROCm, no Triton) it gets the- # prebuilt causal-conv1d a training run would; a failed install leaves transformers' torch path.+ # The same prebuilt causal-conv1d a Qwen3.5 training run gets; the gated delta kernels ship with+ # unsloth_zoo. A failed install leaves transformers' torch path, which is slower but right.try:- from utils.ssm_runtime import zoo_triton_causal_conv1d_usable-- if zoo_triton_causal_conv1d_usable():- # The hooks below also steer FLA off TileLang where it faults; keep that part.- from core.training.worker import _guard_fla_tilelang- _guard_fla_tilelang()- returnfrom core.training.worker import _ensure_causal_conv1d_fast_path, _install_fast_path_hooks-_ensure_causal_conv1d_fast_path(_Events(), folder, required = True)_install_fast_path_hooks(_Events(), folder, install_causal_conv1d = True)except Exception as exc:diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex 0204ab91d61..8e95901766a 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -105,6 +105,38 @@ _pyplot = None_pyplot_failed = False+def _terminate_descendants(pid: int) -> list:+ """SIGTERM every process *pid* started (recursively) and return them for _reap_descendants.+ Collected before the worker dies: once it is gone they are re-parented and lost."""+ try:+ import psutil+ descendants = psutil.Process(pid).children(recursive = True)+ except Exception:+ return []+ for child in descendants:+ try:+ child.terminate()+ except Exception:+ pass+ return descendants+++def _reap_descendants(descendants: list, timeout: float = 5.0) -> None:+ """Wait for terminated descendants and SIGKILL any still running."""+ if not descendants:+ return+ try:+ import psutil+ _, alive = psutil.wait_procs(descendants, timeout = timeout)+ except Exception:+ alive = descendants+ for child in alive:+ try:+ child.kill()+ except Exception:+ pass++@dataclass(frozen = True)class TrainingStartRequestRecord:start_request_id: str@@ -2473,8 +2505,12 @@ class TrainingBackend:proc = self._procif target_proc is not None and proc is not target_proc:return+ descendants = []if proc is not None and proc.is_alive():logger.info("Force-terminating training subprocess (pid=%s)", proc.pid)+ # First the processes it started: a kernel source build runs in its own session+ # (so its compilers can be stopped together) and would outlive the worker.+ descendants = _terminate_descendants(proc.pid)proc.terminate()cancelled = self._cancel_requestedoutput_dir = self._cancel_cleanup_output_dir or self._output_dir@@ -2484,6 +2520,7 @@ class TrainingBackend:if proc.is_alive():proc.kill()proc.join(timeout = 2.0)+ _reap_descendants(descendants)# Wait for pump thread to finish DB finalization (8s covers SQLite's 5s lock timeout).if self._pump_thread is not None and self._pump_thread.is_alive():diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 5bf50dbcb82..d7955ec596c 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -85,6 +85,7 @@ from utils.wheel_utils import (flash_attn_wheel_url,install_wheel,is_windows_wheel_env,+ pinned_source_build_supported,probe_torch_wheel_env,url_exists,)@@ -306,6 +307,22 @@ def _is_model_cache_artifact_error(error: BaseException | None) -> bool:return False+def _package_installs_offline() -> bool:+ # UV_OFFLINE too: uv would refuse the index and install_wheel's pip fallback would not.+ try:+ from utils.utils import package_installs_offline+ return package_installs_offline()+ except Exception:+ return os.environ.get("UV_OFFLINE", "").strip().lower() in {+ "1",+ "true",+ "t",+ "yes",+ "y",+ "on",+ }++def _model_offline_mode_enabled() -> bool:try:from utils.utils import hf_env_offline@@ -1421,7 +1438,13 @@ def _reject_install(event_queue: Any, pypi_name: str, display_name: str, reason:def _install_package_wheel_first(- *, event_queue: Any, import_name: str, display_name: str, pypi_name: str, **kwargs: Any+ *,+ event_queue: Any,+ import_name: str,+ display_name: str,+ pypi_name: str,+ required: bool = False,+ **kwargs: Any,) -> bool:"""Install a fast-path package, wheel first, and never leave an unusable one behind. The two"touch nothing" guards run here, outside the cleanup: an already-working package returns@@ -1433,23 +1456,59 @@ def _install_package_wheel_first(logger.info("%s already installed", display_name)return True- if _model_offline_mode_enabled():+ if _model_offline_mode_enabled() or _package_installs_offline():logger.info("Skipping %s installation while offline", display_name)return False- installed = False- try:- installed = _attempt_package_install(- event_queue = event_queue,- import_name = import_name,- display_name = display_name,- pypi_name = pypi_name,- **kwargs,- )- return installed- finally:- if not installed:- _reject_install(event_queue, pypi_name, display_name, "will not import")+ # The runtime's failed-attempt memo (same key): the eager install and the fast-path hook, or a+ # chat load in the same process, never repeat a failed (possibly half-hour) build.+ from utils.ssm_runtime import _FAILED_INSTALLS, _failure_key++ failure_key = _failure_key(import_name)+ if failure_key in _FAILED_INSTALLS:+ logger.info("%s already failed to install in this process; not retrying", display_name)+ return False++ from utils.kernel_install import (+ KERNEL_LOCK_WAIT,+ OPTIONAL_KERNEL_LOCK_WAIT,+ kernel_install_lock,+ )++ # One install per package and environment at a time, across Studio workers: precheck, install,+ # verify and cleanup are one transaction, or a failed peer could remove what another verified.+ # An optional acceleration does not hold a run for a peer's half-hour build.+ with kernel_install_lock(+ import_name,+ on_wait = lambda: _send_status(+ event_queue, f"Waiting for another process installing {display_name}..."+ ),+ wait_seconds = KERNEL_LOCK_WAIT if required else OPTIONAL_KERNEL_LOCK_WAIT,+ ) as held:+ if not held:+ _send_status(+ event_queue,+ f"Could not take the {display_name} install lock (another process is installing "+ "it, or the lock is unavailable); continuing without it",+ )+ return False+ if _is_importable(import_name):+ logger.info("%s installed by another process", display_name)+ return True+ installed = False+ try:+ installed = _attempt_package_install(+ event_queue = event_queue,+ import_name = import_name,+ display_name = display_name,+ pypi_name = pypi_name,+ **kwargs,+ )+ return installed+ finally:+ if not installed:+ _FAILED_INSTALLS.add(failure_key)+ _reject_install(event_queue, pypi_name, display_name, "will not import")def _attempt_package_install(@@ -1465,12 +1524,19 @@ def _attempt_package_install(wheel_url_builder: Callable[[dict[str, str] | None], str | None] | None = None,pypi_spec: str | None = None,pypi_status_message: str | None = None,+ allow_source_build: bool = True,) -> bool:"""The install itself. Call it through _install_package_wheel_first, never directly."""# Set when a wheel installed but would not import; see the uninstall before the fallback.wheel_rejected = Falseenv = probe_torch_wheel_env(timeout = 30, include_windows = True)+ if not env or not (env.get("cuda_version") or env.get("cuda_major") or env.get("hip_version")):+ # These are CUDA / ROCm extensions: CPU-only, XPU, MPS or an unreadable torch builds none.+ _send_status(+ event_queue, f"{display_name} needs a CUDA or ROCm build of torch; skipping it"+ )+ return False# Windows is wheel-only: a PyPI fallback there is a source build needing MSVC + the CUDA toolkit.wheel_only = sys.platform == "win32" or is_windows_wheel_env(env)if wheel_url_builder is not None:@@ -1487,7 +1553,14 @@ def _attempt_package_install(wheel_available = url_exists(wheel_url) if wheel_url else Falseif wheel_url is None:logger.info("No compatible %s wheel candidate", display_name)- elif wheel_available:+ elif wheel_available is not False:+ # Only a 404 proves nothing is published: an unanswered HEAD (a proxy refusing HEAD but+ # serving the download) still tries the wheel, and falls back only if that fails.+ if wheel_available is None:+ _send_status(+ event_queue,+ f"Could not check the {display_name} prebuilt wheel; trying to download it.",+ )_send_status(event_queue, f"Installing {display_name} for faster training...")outcome = install_prebuilt(wheel_url,@@ -1513,18 +1586,24 @@ def _attempt_package_install("removing it" if wheel_only else "falling back to PyPI",)wheel_rejected = True- elif wheel_available is None:- _send_status(- event_queue,- f"Could not check the {display_name} prebuilt wheel; "- + ("skipping it." if wheel_only else "installing from PyPI."),- )else:logger.info("No published %s wheel found: %s", display_name, wheel_url)if wheel_only:logger.info("%s: no usable prebuilt wheel for Windows; skipping", display_name)return False+ if not allow_source_build:+ _send_status(event_queue, f"No usable prebuilt {display_name} for this torch; skipping it")+ return False+ if pypi_version is not None and not pinned_source_build_supported(env):+ # The pinned causal-conv1d / mamba-ssm sources build at C++17 and torch 2.13+ needs+ # C++20: a certain failure after minutes of compiling.+ _send_status(+ event_queue,+ f"No prebuilt {display_name} for torch {env.get('torch_mm')}, and it cannot be built "+ "from source for it; skipping it",+ )+ return Falseis_hip = env and env.get("hip_version")if is_hip and not shutil.which("hipcc"):@@ -1587,8 +1666,11 @@ def _attempt_package_install(_gcc_dir,)+ from utils.kernel_install import run_process_tree+try:- result = _sp.run(pypi_cmd, **_run_kwargs)+ # Bounded, and a timeout stops the compilers pip / uv started too.+ result = run_process_tree(pypi_cmd, run = _sp.run, **_run_kwargs)except _sp.TimeoutExpired:logger.error("%s installation timed out after %ds",@@ -1683,20 +1765,43 @@ def _ensure_mamba_ssm(model_name: str,*,required: bool | None = None,+ essential: bool | None = None,) -> None:# `required` is the config-resolved answer (resolved_model_is_ssm); the name is the fallback.+ # `essential`: Nemotron-H, Falcon-H1, Granite-4.0-H, which may build it from source. Training+ # stays best effort as at 9f5b13325: their native transformers path can run without it, so a+ # missing mamba-ssm is left to the model load. The others have a torch path and get mamba-ssm+ # only as a prebuilt wheel where unsloth_zoo keeps its fast path on, never as a source build.if required is None:required = _model_is_ssm(model_name)if not required:return+ if essential is None:+ # Unspecified (the worker passes the config's answer): only the families named for it+ # (Nemotron-H, Falcon-H1, Granite-4.0-H) may build from source, never Mamba / Mamba2 /+ # Jamba / Zamba / Bamba, which a caller leaving it out must not turn into a source build.+ from utils.ssm_runtime import model_requires_mamba_ssm+ essential = model_requires_mamba_ssm(model_name)if sys.platform == "win32" and importlib.util.find_spec("triton") is None:# mamba_ssm imports triton at load and the wheel goes in --no-deps._send_status(event_queue, "mamba-ssm needs Triton on Windows (triton-windows); skipping it")return+ if not essential and not _is_importable(MAMBA_SSM.import_name):+ from utils.ssm_runtime import _mamba_ssm_floor_reason+ reason = _mamba_ssm_floor_reason()+ if reason is not None:+ _send_status(event_queue, f"Skipping mamba-ssm, which {reason}")+ return+logger.info("SSM model detected; setting up mamba-ssm after causal-conv1d")- _install_package_wheel_first(event_queue = event_queue, **_pinned_kernel_kwargs(MAMBA_SSM))+ _install_package_wheel_first(+ event_queue = event_queue,+ **_pinned_kernel_kwargs(MAMBA_SSM),+ allow_source_build = essential,+ required = essential,+ )def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:@@ -2144,9 +2249,40 @@ def _should_try_runtime_flash_attn_install(max_seq_length: int) -> bool:return sys.platform.startswith("linux") or sys.platform == "win32"+def _flash_attn_gpu_below_floor() -> bool:+ """Whether the best visible GPU is below sm80, where Unsloth leaves FlashAttention 2 off. Asks+ the torch this worker already imported when there is one (no child process); ROCm never takes+ the NVIDIA floor; no visible GPU is not below it."""+ torch = sys.modules.get("torch")+ if torch is not None:+ try:+ if getattr(torch.version, "hip", None):+ return False+ if not torch.cuda.is_available():+ return False+ count = torch.cuda.device_count()+ return count > 0 and max(+ tuple(torch.cuda.get_device_capability(i)) for i in range(count)+ ) < (8, 0)+ except Exception as exc:+ logger.debug("In-process GPU capability check failed (%s); probing a child", exc)+ from utils.kernel_install import gpu_below_flash_attn_floor++ return gpu_below_flash_attn_floor()++def _ensure_flash_attn_for_long_context(event_queue: Any, max_seq_length: int) -> None:if not _should_try_runtime_flash_attn_install(max_seq_length):return+ if _is_importable("flash_attn"):+ # Already usable: nothing to download, so no capability probe either.+ logger.info("flash-attn already installed")+ return++ if _flash_attn_gpu_below_floor():+ # Unsloth only turns FlashAttention 2 on from sm80 (a T4 is sm75): nothing to download.+ _send_status(event_queue, "flash-attn needs an sm80 or newer GPU; continuing without it")+ returninstalled = _install_package_wheel_first(event_queue = event_queue,@@ -4028,25 +4164,25 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -># 2) mamba-ssm + flash-attn keep their substring / size gates.# 3) FLA gated-delta kernels: vendored by unsloth_zoo, nothing to install.# Laya decision models are ModernBERT encoders: none of these apply.- # Clef decision models are Qwen3.5 backbones: zoo's gated-delta and Triton conv kernels cover them.+ # Clef decision models are Qwen3.5 backbones and need the same gated-delta / conv kernels.if (not config.get("is_decision")or config.get("decision_layout") in ("clef", "llm")or _decision_has_llm_backbone(model_load_target, _worker_hf_token(config))):try:- from utils.ssm_runtime import (- resolved_model_is_ssm,- resolved_model_wants_causal_conv1d,- )+ from utils.ssm_runtime import resolve_ssm_kernels- is_ssm = resolved_model_is_ssm(model_name, model_load_target, _worker_hf_token(config))- # Every mamba-ssm model also runs causal-conv1d.- wants_causal_conv1d = is_ssm or resolved_model_wants_causal_conv1d(+ # A continued local adapter loads from its folder, but its base decides the kernels:+ # the authorized pin when the run was checked, else the adapter's recorded base.+ kernels = resolve_ssm_kernels(model_name,model_load_target,_worker_hf_token(config),+ adapter_base = config.get("adapter_base_pin") or None,)+ is_ssm = kernels.mamba_ssm+ wants_causal_conv1d = kernels.causal_conv1d_ensure_causal_conv1d_fast_path(event_queue,model_name,@@ -4057,7 +4193,12 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->model_name,install_causal_conv1d = wants_causal_conv1d,)- _ensure_mamba_ssm(event_queue, model_name, required = is_ssm)+ _ensure_mamba_ssm(+ event_queue,+ model_name,+ required = is_ssm,+ essential = kernels.mamba_ssm_required,+ )_ensure_flash_attn_for_long_context(event_queue,int(config.get("max_seq_length", 2048)),diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 649906732a1..43d214cdb5a 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -1465,3 +1465,14 @@ def traced_offload_hooks(monkeypatch):orig = getattr(fn, "_unsloth_orig", None)if orig is not None:monkeypatch.setattr(cls, name, orig)++[email protected](scope = "session")+def _kernel_lock_dir(tmp_path_factory):+ return str(tmp_path_factory.mktemp("kernel-locks"))++[email protected](autouse = True)+def _isolated_kernel_install_locks(monkeypatch, _kernel_lock_dir):+ # Tests never take (or create) the environment's real kernel install locks.+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", _kernel_lock_dir)diff --git a/studio/backend/tests/test_clef_fast_paths.py b/studio/backend/tests/test_clef_fast_paths.pyindex 3105cb5dcc5..26838a370f1 100644--- a/studio/backend/tests/test_clef_fast_paths.py+++ b/studio/backend/tests/test_clef_fast_paths.py@@ -1,8 +1,8 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Clef's fast-path setup: nothing is installed where zoo's Triton conv runs, but the FLA TileLang-guard still applies; elsewhere the prebuilt causal-conv1d is installed as before."""+"""Clef (a Qwen3.5 backbone) installs the prebuilt causal-conv1d and its availability hook, whose+setup also applies the FLA TileLang guard."""import sysfrom pathlib import Path@@ -13,33 +13,27 @@ if str(_BACKEND) not in sys.path:from core.systemone import clef_runtime # noqa: E402from core.training import worker # noqa: E402-from utils import ssm_runtime # noqa: E402-def _record(monkeypatch, usable: bool) -> list[str]:- calls: list[str] = []- monkeypatch.setattr(ssm_runtime, "zoo_triton_causal_conv1d_usable", lambda: usable)- monkeypatch.setattr(worker, "_guard_fla_tilelang", lambda: calls.append("guard"))+def test_clef_installs_causal_conv1d_and_its_hooks(monkeypatch):+ calls: list = []monkeypatch.setattr(worker,"_ensure_causal_conv1d_fast_path",- lambda *args, **kwargs: calls.append("install"),+ lambda *args, **kwargs: calls.append(("install", kwargs.get("required"))),)monkeypatch.setattr(worker,"_install_fast_path_hooks",- lambda *args, **kwargs: calls.append("hooks"),+ lambda *args, **kwargs: calls.append(("hooks", kwargs.get("install_causal_conv1d"))),)- return calls---def test_clef_installs_nothing_but_keeps_the_tilelang_guard(monkeypatch):- calls = _record(monkeypatch, usable = True)clef_runtime._fast_paths("/models/clef")- assert calls == ["guard"]+ assert calls == [("install", True), ("hooks", True)]-def test_clef_installs_causal_conv1d_where_zoo_conv_cannot_run(monkeypatch):- calls = _record(monkeypatch, usable = False)- clef_runtime._fast_paths("/models/clef")- assert calls == ["install", "hooks"]+def test_the_fast_path_hooks_apply_the_tilelang_guard(monkeypatch):+ guarded = []+ monkeypatch.setattr(worker, "_guard_fla_tilelang", lambda: guarded.append(True))+ monkeypatch.setenv(worker._FAST_PATH_HOOKS_SKIP_ENV, "1")+ worker._install_fast_path_hooks(event_queue = [], model_name = "/models/clef")+ assert guarded == [True]diff --git a/studio/backend/tests/test_export_offline_window.py b/studio/backend/tests/test_export_offline_window.pyindex b5bde893fea..875660fe3f6 100644--- a/studio/backend/tests/test_export_offline_window.py+++ b/studio/backend/tests/test_export_offline_window.py@@ -100,3 +100,132 @@ def test_push_export_is_not_pinned_offline(monkeypatch):export = {"type": "export", "export_type": "merged", "push_to_hub": True}calls, _ = _run_worker(monkeypatch, [export], lambda: True)assert calls == [(False, True)]+++# ── An auto-set (Hub unreachable) offline does not block kernel installs ──────++import contextlib # noqa: E402++import pytest # noqa: E402+++def _isolated_offline_env(monkeypatch, **preset):+ for name in (+ "HF_HUB_OFFLINE",+ "TRANSFORMERS_OFFLINE",+ "UV_OFFLINE",+ "_UNSLOTH_HF_OFFLINE_AUTO",+ ):+ monkeypatch.delenv(name, raising = False)+ for name, value in preset.items():+ monkeypatch.setenv(name, value)+ monkeypatch.setenv("UNSLOTH_OFFLINE_PROBE", "1")++[email protected](params = ["env", "force"])+def window_branch(request, monkeypatch):+ """Both ways the window goes offline: env vars only (huggingface_hub not yet usable) or the+ loader's _force_hf_offline (post-import)."""+ from utils.utils import force_hf_offline++ if request.param == "env":+ monkeypatch.setitem(sys.modules, "unsloth.models.loader_utils", None) # import fails+ else:+ fake_loader = types.ModuleType("unsloth.models.loader_utils")+ fake_loader._force_hf_offline = force_hf_offline+ monkeypatch.setitem(sys.modules, "unsloth.models.loader_utils", fake_loader)+ return request.param+++def _install_kernel_reaches_the_lock(monkeypatch):+ """Whether ssm_runtime._install_kernel gets past its offline guard (to the kernel lock, which+ is stubbed busy so nothing installs)."""+ from utils import kernel_install, ssm_runtime++ reached = []++ @contextlib.contextmanager+ def _busy(name, **kwargs):+ reached.append(name)+ yield False++ monkeypatch.setattr(kernel_install, "kernel_install_lock", _busy)+ monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: False)+ monkeypatch.setattr(ssm_runtime, "_FAILED_INSTALLS", set())+ monkeypatch.setattr(+ ssm_runtime, "install_wheel", lambda *a, **k: pytest.fail("no install in this test")+ )+ ssm_runtime._install_kernel(+ import_name = "causal_conv1d",+ display_name = "causal-conv1d",+ pypi_name = "causal-conv1d",+ package_version = "1.6.1",+ release_tag = "v1.6.1.post4",+ release_base_url = "x",+ status_cb = None,+ run = lambda *a, **k: pytest.fail("no subprocess in this test"),+ )+ return bool(reached)+++def test_an_unreachable_hub_still_allows_kernel_installs_inside_the_window(+ monkeypatch, window_branch+):+ from core.export import worker+ from utils import transformers_version+ from utils.utils import hf_env_offline, package_installs_offline+ from utils.wheel_utils import package_installs_offline_env++ _isolated_offline_env(monkeypatch)+ monkeypatch.setattr(transformers_version, "hf_endpoint_unreachable", lambda: True)+ with worker._offline_window_if_unreachable(step = "activating transformers"):+ assert hf_env_offline() # the Hub side is offline+ assert os.environ.get("HF_HUB_OFFLINE") == "1"+ assert package_installs_offline() is False+ assert package_installs_offline_env() is False+ assert _install_kernel_reaches_the_lock(monkeypatch)+ # Marker and env restored on exit.+ assert "_UNSLOTH_HF_OFFLINE_AUTO" not in os.environ+ assert "HF_HUB_OFFLINE" not in os.environ+++def test_uv_offline_still_blocks_installs_inside_the_window(monkeypatch, window_branch):+ from core.export import worker+ from utils import transformers_version+ from utils.utils import package_installs_offline+ from utils.wheel_utils import package_installs_offline_env++ _isolated_offline_env(monkeypatch, UV_OFFLINE = "1")+ monkeypatch.setattr(transformers_version, "hf_endpoint_unreachable", lambda: True)+ with worker._offline_window_if_unreachable():+ assert package_installs_offline() is True+ assert package_installs_offline_env() is True+ assert not _install_kernel_reaches_the_lock(monkeypatch)++[email protected]("variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"])+def test_an_operator_set_hf_offline_still_blocks_installs(monkeypatch, variable):+ from core.export import worker+ from utils import transformers_version+ from utils.utils import package_installs_offline+ from utils.wheel_utils import package_installs_offline_env++ _isolated_offline_env(monkeypatch, **{variable: "1"})+ monkeypatch.setattr(transformers_version, "hf_endpoint_unreachable", lambda: True)+ with worker._offline_window_if_unreachable():+ assert "_UNSLOTH_HF_OFFLINE_AUTO" not in os.environ+ assert package_installs_offline() is True+ assert package_installs_offline_env() is True+ assert not _install_kernel_reaches_the_lock(monkeypatch)+ assert package_installs_offline() is True+++def test_a_reachable_hub_sets_no_marker(monkeypatch):+ from core.export import worker+ from utils import transformers_version++ _isolated_offline_env(monkeypatch)+ monkeypatch.setattr(transformers_version, "hf_endpoint_unreachable", lambda: False)+ with worker._offline_window_if_unreachable():+ assert "_UNSLOTH_HF_OFFLINE_AUTO" not in os.environ+ assert "HF_HUB_OFFLINE" not in os.environdiff --git a/studio/backend/tests/test_export_ssm_kernels.py b/studio/backend/tests/test_export_ssm_kernels.pynew file mode 100644index 00000000000..3685fe45fd3--- /dev/null+++ b/studio/backend/tests/test_export_ssm_kernels.py@@ -0,0 +1,417 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Export installs the SSM kernels a checkpoint needs before Unsloth and its modeling import, as a+training or chat load does: setup no longer installs them, and Nemotron-H's remote modeling code+cannot even import without mamba-ssm."""++import json+import sys+import types+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++from core.export import worker as export_worker # noqa: E402+from utils import ssm_runtime # noqa: E402+++class _Queue(list):+ def put(self, item):+ self.append(item)+++class _ClosedQueue:+ def get(self, timeout = None):+ raise EOFError+++def test_a_nemotron_h_export_provisions_kernels_before_the_backend_import(monkeypatch, tmp_path):+ order = [+ step+ for step in _run_export(monkeypatch, tmp_path, {"device_backend": "cuda"})+ if step[0] != "window"+ ]+ assert order[0] == ("provision", True, True)+ assert order.index(("provision", True, True)) < order.index(("import",))+ assert ("load",) in order++[email protected]("device_backend", ["cpu", "mlx"])+def test_a_cpu_or_mlx_export_makes_no_extra_connectivity_probe(+ monkeypatch, tmp_path, device_backend+):+ # 9f5b13325 entered no "installing SSM kernels" window; an export that installs nothing must not.+ order = _run_export(monkeypatch, tmp_path, {"device_backend": device_backend})+ assert ("window", "installing SSM kernels") not in order+ assert not any(step[0] == "provision" for step in order)+ assert ("load",) in order++[email protected]("device_backend", ["cuda", "cpu", "mlx"])+def test_provisioning_adds_no_connectivity_window(monkeypatch, tmp_path, device_backend):+ # Same window count as 9f5b13325: provisioning rides the activation window.+ order = _run_export(monkeypatch, tmp_path, {"device_backend": device_backend})+ windows = [step for step in order if step[0] == "window"]+ assert windows == [("window", "activating transformers"), ("window", None)]+++def test_a_cuda_export_installs_inside_the_activation_window(monkeypatch, tmp_path):+ order = _run_export(monkeypatch, tmp_path, {"device_backend": "cuda"})+ activation = order.index(("window", "activating transformers"))+ provision = order.index(("provision", True, True))+ assert activation < provision < order.index(("import",))+ assert not any(step[0] == "window" for step in order[activation + 1 : provision])+++def _run_export(monkeypatch, tmp_path, extra_config):+ import contextlib++ monkeypatch.setattr(export_worker.sys, "platform", "linux")+ checkpoint = tmp_path / "nemotron-export"+ checkpoint.mkdir()+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "nemotron_h"}))+ order: list = []++ @contextlib.contextmanager+ def _window(step = None):+ order.append(("window", step))+ yield++ monkeypatch.setattr(export_worker, "_offline_window_if_unreachable", _window)++ def _ensure(probe, **kwargs):+ order.append(("provision", kwargs["is_ssm"], kwargs["mamba_ssm_required"]))+ return []++ monkeypatch.setattr(ssm_runtime, "ensure_ssm_runtime", _ensure)+ monkeypatch.setattr(export_worker, "_setup_log_capture", lambda queue: None)+ monkeypatch.setattr(export_worker, "_activate_transformers_version", lambda *a, **k: None)+ monkeypatch.setattr(export_worker, "apply_gpu_ids", lambda *a, **k: None)+ monkeypatch.setattr(+ "core._torchao_stub.install_torchao_windows_rocm_real_or_stub", lambda: None+ )+ monkeypatch.setattr("core.import_guards.ensure_real_packages", lambda *names: None)++ class _Backend:+ def __init__(self):+ order.append(("backend",))++ fake_export = types.ModuleType("core.export.export")++ def _getattr(name):+ if name == "ExportBackend":+ order.append(("import",))+ return _Backend+ raise AttributeError(name)++ fake_export.__getattr__ = _getattr+ monkeypatch.setitem(sys.modules, "core.export.export", fake_export)+ monkeypatch.setattr(export_worker, "_handle_load", lambda *a, **k: order.append(("load",)))+ # run_export_process configures its own process; keep that from leaking into this one.+ for key in ("TQDM_MININTERVAL", "TOKENIZERS_PARALLELISM", "PYTHONWARNINGS", "PYTHONUNBUFFERED"):+ monkeypatch.setenv(key, "")+ monkeypatch.delenv(key)+ monkeypatch.setenv("ENVIRONMENT_TYPE", "development")+ monkeypatch.setattr("loggers.config.LogConfig.setup_logging", lambda *a, **k: None)+ monkeypatch.setattr("loggers.config.allow_progress_bars", lambda: None)++ queue = _Queue()+ export_worker.run_export_process(+ cmd_queue = _ClosedQueue(),+ resp_queue = queue,+ config = {"checkpoint_path": str(checkpoint), "allow_ambient": True, **extra_config},+ )+ return order+++def test_a_failed_required_install_stops_the_export_before_the_import(monkeypatch, tmp_path):+ checkpoint = tmp_path / "falcon"+ checkpoint.mkdir()+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "falcon_h1"}))+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ monkeypatch.setattr(ssm_runtime, "_torch_env", lambda: {"cuda_version": "12.8"})+ # A transformers without the native Falcon-H1 model cannot load it without mamba-ssm.+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda model_type: False)+ queue = _Queue()+ assert export_worker._provision_ssm_kernels(str(checkpoint), None, None, queue, "cuda") is False+ assert queue[-1]["type"] == "error"+ assert "mamba-ssm" in queue[-1]["error"]+++def test_a_native_required_hybrid_exports_without_mamba_ssm(monkeypatch, tmp_path):+ checkpoint = tmp_path / "falcon"+ checkpoint.mkdir()+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "falcon_h1"}))+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ monkeypatch.setattr(ssm_runtime, "_torch_env", lambda: {"cuda_version": "12.8"})+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda model_type: True)+ queue = _Queue()+ assert export_worker._provision_ssm_kernels(str(checkpoint), None, None, queue, "cuda") is True+ assert not any(item.get("type") == "error" for item in queue)+ assert any("Warning: could not install mamba-ssm" in item.get("message", "") for item in queue)+++def test_an_optional_ssm_family_exports_on_the_torch_path(monkeypatch, tmp_path):+ checkpoint = tmp_path / "jamba"+ checkpoint.mkdir()+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "jamba"}))+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ queue = _Queue()+ assert export_worker._provision_ssm_kernels(str(checkpoint), None, None, queue, "cuda") is True+ assert not any(item.get("type") == "error" for item in queue)++[email protected]("pinned", [True, False])+def test_a_lora_export_resolves_its_base(monkeypatch, tmp_path, pinned):+ base = tmp_path / "base"+ base.mkdir()+ (base / "config.json").write_text(json.dumps({"model_type": "falcon_h1"}))+ adapter = tmp_path / "my-adapter"+ adapter.mkdir()+ (adapter / "adapter_config.json").write_text(json.dumps({"base_model_name_or_path": str(base)}))+ seen = []+ monkeypatch.setattr(+ ssm_runtime, "ensure_ssm_runtime", lambda probe, **k: seen.append(k["is_ssm"]) or []+ )+ monkeypatch.setattr("utils.transformers_version._remote_lora_base", lambda *a, **k: None)+ assert export_worker._provision_ssm_kernels(+ str(adapter), str(base) if pinned else None, None, _Queue(), "cuda"+ )+ assert seen == [True]+++ "platform, device_backend",+ [("darwin", None), ("darwin", "mlx"), ("linux", "mlx"), ("linux", "cpu")],+)+def test_mlx_cpu_and_macos_exports_install_no_cuda_kernels(+ monkeypatch, tmp_path, platform, device_backend+):+ # causal-conv1d / mamba-ssm are CUDA / ROCm extensions: an MLX or CPU export never loads them,+ # and a source build there would only delay (or fail) a valid export.+ checkpoint = tmp_path / "qwen"+ checkpoint.mkdir()+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "qwen3_5"}))+ monkeypatch.setattr(export_worker.sys, "platform", platform)+ monkeypatch.setattr(+ ssm_runtime,+ "provision_ssm_kernels",+ lambda *a, **k: pytest.fail("no SSM kernel provisioning on MLX / CPU / macOS"),+ )+ queue = _Queue()+ assert (+ export_worker._provision_ssm_kernels(str(checkpoint), None, None, queue, device_backend)+ is True+ )+ assert queue == []++[email protected]("device_backend", ["cuda", "rocm"])+def test_cuda_exports_still_provision(monkeypatch, tmp_path, device_backend):+ checkpoint = tmp_path / "qwen"+ checkpoint.mkdir()+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "qwen3_5"}))+ monkeypatch.setattr(export_worker.sys, "platform", "linux")+ seen = []+ monkeypatch.setattr(+ ssm_runtime, "provision_ssm_kernels", lambda *a, **k: seen.append(a) or (None, [])+ )+ assert export_worker._provision_ssm_kernels(+ str(checkpoint), None, None, _Queue(), device_backend+ )+ assert seen+++def test_both_export_provisioning_sites_pass_the_device_backend():+ import ast+ import inspect++ tree = ast.parse(inspect.getsource(export_worker.run_export_process))+ calls = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "_provision_ssm_kernels"+ ]+ assert len(calls) == 2+ for call in calls:+ assert ast.unparse(call.args[4]) == "config.get('device_backend')"++[email protected]("device_backend", ["xpu", None, ""])+def test_xpu_or_unspecified_exports_install_no_cuda_kernels(monkeypatch, tmp_path, device_backend):+ # Only a CUDA / ROCm export provisions: XPU builds none, and an unknown backend is not CUDA.+ monkeypatch.setattr(export_worker.sys, "platform", "linux")+ monkeypatch.setattr(+ ssm_runtime,+ "provision_ssm_kernels",+ lambda *a, **k: pytest.fail("no SSM kernel provisioning off CUDA / ROCm"),+ )+ assert export_worker._provision_ssm_kernels("acme/hybrid", None, None, _Queue(), device_backend)+++# ── Security gates run before any SSM kernel is provisioned ──────────────────+++class _CmdQueue:+ def __init__(self, cmds):+ self._cmds = list(cmds)++ def get(self, timeout = None):+ if self._cmds:+ return self._cmds.pop(0)+ raise EOFError+++class _Verdict:+ def __init__(self, blocked):+ self.blocked = blocked+ self.reason = "flagged pickle"+ self.model_name = "acme/evil"+ self.max_severity = "CRITICAL"++ def response_payload(self):+ return {"blocked": self.blocked}+++def _gated_export(+ monkeypatch,+ tmp_path,+ *,+ malware,+ consent,+ cmds = (),+ trust = True,+):+ """Run the export worker with the Hub security verdicts stubbed: *malware* / *consent* say+ whether each gate blocks, *cmds* are fed to the command loop after startup."""+ import contextlib++ monkeypatch.setattr(export_worker.sys, "platform", "linux")+ order: list = []+ monkeypatch.setattr(+ export_worker, "_offline_window_if_unreachable", lambda step = None: contextlib.nullcontext()+ )+ monkeypatch.setattr(+ export_worker,+ "_provision_ssm_kernels",+ lambda *a, **k: order.append(("provision", a[0])) or True,+ )+ monkeypatch.setattr("utils.transformers_version._remote_lora_base", lambda *a, **k: None)+ monkeypatch.setattr(export_worker, "_nemotron_auto_trust", lambda *a, **k: False)++ def _file_security(target, **kwargs):+ order.append(("malware_scan", target, kwargs["load_subdirs"]))+ return _Verdict(target in malware if isinstance(malware, set) else malware)++ def _consent(targets, **kwargs):+ order.append(("consent_scan", tuple(targets)))+ return _Verdict(consent)++ monkeypatch.setattr("utils.security.evaluate_file_security", _file_security)+ monkeypatch.setattr("utils.security.evaluate_remote_code_consent_for_targets", _consent)+ # The preflight must stay transformers-free: no load-subdir computation.+ monkeypatch.setattr(+ "utils.security.security_load_subdirs",+ lambda *a, **k: pytest.fail("preflight computed load subdirs (imports transformers)"),+ )+ monkeypatch.setattr(export_worker, "_setup_log_capture", lambda queue: None)+ monkeypatch.setattr(export_worker, "_activate_transformers_version", lambda *a, **k: None)+ monkeypatch.setattr(export_worker, "apply_gpu_ids", lambda *a, **k: None)+ monkeypatch.setattr(+ "core._torchao_stub.install_torchao_windows_rocm_real_or_stub", lambda: None+ )+ monkeypatch.setattr("core.import_guards.ensure_real_packages", lambda *names: None)++ class _Backend:+ def cleanup_memory(self):+ pass++ fake_export = types.ModuleType("core.export.export")+ fake_export.ExportBackend = _Backend+ monkeypatch.setitem(sys.modules, "core.export.export", fake_export)+ monkeypatch.setattr(+ export_worker,+ "_handle_load",+ lambda b, cmd, q: order.append(("load", cmd["checkpoint_path"])),+ )+ for key in ("TQDM_MININTERVAL", "TOKENIZERS_PARALLELISM", "PYTHONWARNINGS", "PYTHONUNBUFFERED"):+ monkeypatch.setenv(key, "")+ monkeypatch.delenv(key)+ monkeypatch.setenv("ENVIRONMENT_TYPE", "development")+ monkeypatch.setattr("loggers.config.LogConfig.setup_logging", lambda *a, **k: None)+ monkeypatch.setattr("loggers.config.allow_progress_bars", lambda: None)++ queue = _Queue()+ export_worker.run_export_process(+ cmd_queue = _CmdQueue(cmds),+ resp_queue = queue,+ config = {+ "checkpoint_path": "acme/evil",+ "base_model": "acme/evil-base",+ "trust_remote_code": trust,+ "allow_ambient": True,+ "device_backend": "cuda",+ },+ )+ return order, queue+++def test_a_malware_blocked_export_provisions_no_kernels(monkeypatch, tmp_path):+ order, queue = _gated_export(monkeypatch, tmp_path, malware = True, consent = False)+ assert not any(step[0] == "provision" for step in order)+ assert not any(step[0] == "load" for step in order)+ loaded = [r for r in queue if r.get("type") == "loaded"]+ assert loaded and loaded[0]["error_kind"] == "malware_blocked"+ assert loaded[0]["success"] is False+ # Transformers-free preflight: no subdir scoping.+ assert order[0] == ("malware_scan", "acme/evil", ())+++def test_a_consent_blocked_trusted_export_provisions_no_kernels(monkeypatch, tmp_path):+ order, queue = _gated_export(monkeypatch, tmp_path, malware = False, consent = True)+ assert ("consent_scan", ("acme/evil", "acme/evil-base")) in order+ assert not any(step[0] == "provision" for step in order)+ loaded = [r for r in queue if r.get("type") == "loaded"]+ assert loaded and loaded[0]["error_kind"] == "remote_code_blocked"+ assert "remote_code" in loaded[0]+++def test_an_untrusted_export_skips_the_consent_scan_and_provisions(monkeypatch, tmp_path):+ order, queue = _gated_export(monkeypatch, tmp_path, malware = False, consent = True, trust = False)+ assert not any(step[0] == "consent_scan" for step in order)+ assert ("provision", "acme/evil") in order+ assert order.index(("malware_scan", "acme/evil", ())) < order.index(("provision", "acme/evil"))+++def test_a_blocked_load_command_provisions_no_kernels(monkeypatch, tmp_path):+ # Startup is clear; the later "load" command names a flagged checkpoint.+ order, queue = _gated_export(+ monkeypatch,+ tmp_path,+ malware = {"acme/evil2"},+ consent = False,+ cmds = [{"type": "load", "checkpoint_path": "acme/evil2", "trust_remote_code": True}],+ )+ assert ("provision", "acme/evil") in order # startup was clear+ assert ("load", "acme/evil") in order+ assert ("provision", "acme/evil2") not in order+ assert ("load", "acme/evil2") not in order+ loaded = [r for r in queue if r.get("type") == "loaded"]+ assert [r["error_kind"] for r in loaded] == ["malware_blocked"]+++def test_a_clear_load_command_gates_then_provisions_then_loads(monkeypatch, tmp_path):+ order, queue = _gated_export(+ monkeypatch,+ tmp_path,+ malware = False,+ consent = False,+ cmds = [{"type": "load", "checkpoint_path": "acme/ok2"}],+ )+ scan = order.index(("malware_scan", "acme/ok2", ()))+ assert scan < order.index(("provision", "acme/ok2")) < order.index(("load", "acme/ok2"))diff --git a/studio/backend/tests/test_kernel_install.py b/studio/backend/tests/test_kernel_install.pyindex 9f6b7c02135..6848275995a 100644--- a/studio/backend/tests/test_kernel_install.py+++ b/studio/backend/tests/test_kernel_install.py@@ -7,6 +7,7 @@ from __future__ import annotationsimport osimport subprocess+import timeimport sysfrom pathlib import Pathfrom types import SimpleNamespace@@ -19,6 +20,8 @@ if str(_BACKEND) not in sys.path:from utils import kernel_install # noqa: E402+_REAL_LOCK_PATH = kernel_install._kernel_lock_path+_REPO = _BACKEND.parent.parent_PT = "https://download.pytorch.org/whl"_CC1D = "https://github.com/Dao-AILab/causal-conv1d/releases/download/v1.6.1.post4"@@ -139,6 +142,20 @@ _COLAB = _env("2.11.0+cu130", "13.0")_COLAB_XFORMERS = f"{_PT}/cu130/xformers-0.0.35-py39-none-manylinux_2_28_x86_64.whl"[email protected](autouse = True)+def _nothing_installed_and_online(monkeypatch, tmp_path):+ monkeypatch.setattr(+ kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / f"{name}.lock")+ )+ # Every copy of the module (and any child process) uses the test's folder too.+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", str(tmp_path / "locks"))+ # Tests that need an installed distribution say so; the host's own packages and offline+ # switches must not decide them.+ monkeypatch.setattr(kernel_install, "_installed_version", lambda distribution: None)+ for name in ("UV_OFFLINE", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"):+ monkeypatch.delenv(name, raising = False)++@pytest.fixturedef uv(monkeypatch):def use(available, outside_venv = False):@@ -513,12 +530,150 @@ def test_main_installs_all_by_default_in_dependency_order(monkeypatch, argv, exporder = []monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: _COLAB)monkeypatch.setattr(- kernel_install, "install_kernel", lambda name, env, dry_run = False: order.append(name) or 0+ kernel_install, "install_kernel", lambda name, env, **kwargs: order.append(name) or 0)assert kernel_install.main(argv) == 0assert order == expected+# A GPU-less image build (Docker, CI) provisions for the GPU it will run on only when asked to.++[email protected]("opt_in", ["flag", "env"])+def test_install_kernels_without_gpu_installs_mamba_ssm_headless(uv, monkeypatch, opt_in):+ uv(False)+ monkeypatch.setattr(kernel_install.sys, "platform", "linux")+ monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: _COLAB)+ monkeypatch.setattr(kernel_install, "url_exists", lambda url: True)+ run = _Runner([False, True], capability = "None")+ real = kernel_install.install_kernel+ monkeypatch.setattr(+ kernel_install,+ "install_kernel",+ lambda name, env, **kwargs: real(name, env, run = run, exists = lambda url: True, **kwargs),+ )+ argv = ["mamba_ssm"]+ monkeypatch.delenv("UNSLOTH_INSTALL_KERNELS_WITHOUT_GPU", raising = False)+ if opt_in == "flag":+ argv.append("--without-gpu")+ else:+ monkeypatch.setenv("UNSLOTH_INSTALL_KERNELS_WITHOUT_GPU", "1")+ assert kernel_install.main(argv) == 0+ assert run.installer_calls[0][-1].endswith(+ "mamba_ssm-2.3.1+cu13torch2.10cxx11abiTRUE-cp313-cp313-linux_x86_64.whl"+ )++[email protected]("strict, code", [(False, 0), (True, 2)])+def test_a_requested_kernel_skipped_without_a_gpu_is_listed_and_fails_only_with_strict(+ monkeypatch, capsys, strict, code+):+ # 9f5b13325 and 4a6429a65 exit 0 here, and `set -e` install scripts rely on it.+ monkeypatch.delenv("UNSLOTH_INSTALL_KERNELS_WITHOUT_GPU", raising = False)+ monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: _COLAB)+ run = _Runner([], capability = "None")+ real = kernel_install.install_kernel+ monkeypatch.setattr(+ kernel_install,+ "install_kernel",+ lambda name, env, **kwargs: real(name, env, run = run, exists = lambda url: True, **kwargs),+ )+ assert kernel_install.main(["mamba_ssm"] + (["--strict"] if strict else [])) == code+ assert run.installer_calls == []+ out = capsys.readouterr().out+ assert "--without-gpu" in out+ assert "not installed on this machine: mamba_ssm" in out+++ "name, env, capability",+ [+ # A ROCm source build: no CUDA wheel resolves.+ ("causal_conv1d", None, "9 0"),+ # A GPU-hidden builder with an earlier install.+ ("mamba_ssm", _COLAB, "None"),+ ("flash_attn", _COLAB, "None"),+ ],+)+def test_an_already_working_kernel_is_success_before_any_skip(+ monkeypatch, capsys, name, env, capability+):+ monkeypatch.setattr(+ kernel_install,+ "_installed_version",+ lambda distribution: "1.0" if distribution == kernel_install.KERNELS[name][0] else None,+ )+ monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: env)+ run = _Runner([True], capability = capability)+ real = kernel_install.install_kernel+ monkeypatch.setattr(+ kernel_install,+ "install_kernel",+ lambda n, e, **kwargs: real(n, e, run = run, exists = lambda url: True, **kwargs),+ )+ assert kernel_install.main([name, "--strict"]) == 0+ assert run.installer_calls == []+ assert not any("get_device_capability" in " ".join(c) for c in run.calls)+ out = capsys.readouterr().out+ assert f"{name} already installed and loads" in out+ assert "not installed on this machine" not in out+++def test_an_installed_kernel_that_does_not_load_still_installs(uv, monkeypatch):+ uv(False)+ run = _Runner([False, False, True], capability = "9 0")+ monkeypatch.setattr(kernel_install, "_installed_version", lambda distribution: "1.0")+ assert (+ kernel_install.install_kernel("causal_conv1d", _COLAB, run = run, exists = lambda url: True)+ == 0+ )+ assert run.installer_calls++[email protected]("variable", ["UV_OFFLINE", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"])+def test_the_cli_installs_nothing_offline(monkeypatch, capsys, variable):+ monkeypatch.setenv(variable, "1")+ run = _Runner([], capability = "9 0")+ url_checks = []+ assert (+ kernel_install.install_kernel(+ "causal_conv1d", _COLAB, run = run, exists = lambda url: url_checks.append(url)+ )+ == 0+ )+ assert run.calls == [] and url_checks == []+ assert "offline" in capsys.readouterr().out+++def test_offline_an_already_working_kernel_still_succeeds(monkeypatch, capsys):+ monkeypatch.setenv("UV_OFFLINE", "1")+ monkeypatch.setattr(kernel_install, "_installed_version", lambda distribution: "1.0")+ run = _Runner([True], capability = "9 0")+ assert (+ kernel_install.install_kernel("causal_conv1d", _COLAB, run = run, exists = lambda url: True)+ == 0+ )+ assert "already installed and loads" in capsys.readouterr().out+++def test_the_default_run_lists_skipped_kernels_and_keeps_its_exit_code(monkeypatch, capsys):+ monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: _COLAB)++ def _install(+ name,+ env,+ skipped = None,+ **kwargs,+ ):+ if name == "mamba_ssm":+ skipped.append(name)+ return 0++ monkeypatch.setattr(kernel_install, "install_kernel", _install)+ assert kernel_install.main([]) == 0+ assert "not installed on this machine: mamba_ssm" in capsys.readouterr().out++def test_flash_attn_without_a_wheel_never_probes_the_gpu():run = _Runner([])assert kernel_install.install_kernel("flash_attn", None, run = run, exists = lambda url: True) == 0@@ -676,7 +831,8 @@ def test_source_build_run_kwargs(monkeypatch):kwargs, gcc = kernel_install.source_build_run_kwargs(is_hip = False, gcc_install_dir = lambda: pytest.fail("non-HIP never looks for gcc"))- assert gcc is None and "timeout" not in kwargs+ # CUDA builds get their own, far looser deadline (a slow aarch64 build must not be aborted).+ assert gcc is None and kwargs["timeout"] == kernel_install.SOURCE_BUILD_TIMEOUT_CUDAassert kwargs["encoding"] == "utf-8" and kwargs["stderr"] == subprocess.STDOUTkwargs, gcc = kernel_install.source_build_run_kwargs(@@ -693,6 +849,73 @@ def test_source_build_run_kwargs(monkeypatch):assert gcc is None and kwargs["env"]["HIPCC_COMPILE_FLAGS_APPEND"] == "--gcc-install-dir=/mine"+ "raw, expected",+ [+ (None, 7200),+ ("", 7200),+ ("10800", 10800),+ ("3600.5", 3600),+ ("0", 7200),+ ("-5", 7200),+ ("x", 7200),+ ],+)+def test_the_cuda_source_build_timeout_is_7200_unless_overridden(monkeypatch, raw, expected):+ if raw is None:+ monkeypatch.delenv("UNSLOTH_SOURCE_BUILD_TIMEOUT", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_SOURCE_BUILD_TIMEOUT", raw)+ assert kernel_install._cuda_source_build_timeout() == expected+++def test_hip_keeps_1800_and_the_lock_wait_outlasts_both_deadlines():+ assert kernel_install.SOURCE_BUILD_TIMEOUT == 1800+ assert kernel_install.SOURCE_BUILD_TIMEOUT_CUDA >= 1800+ assert kernel_install.KERNEL_LOCK_WAIT > kernel_install.SOURCE_BUILD_TIMEOUT+ assert kernel_install.KERNEL_LOCK_WAIT > kernel_install.SOURCE_BUILD_TIMEOUT_CUDA++[email protected]("raw, cuda", [(None, 7200), ("14400", 14400), ("600", 600)])+def test_the_env_override_is_read_at_import_and_the_lock_wait_follows(raw, cuda):+ import json as _json++ env = {k: v for k, v in os.environ.items() if k != "UNSLOTH_SOURCE_BUILD_TIMEOUT"}+ if raw is not None:+ env["UNSLOTH_SOURCE_BUILD_TIMEOUT"] = raw+ code = (+ "import json, sys; sys.path.insert(0, sys.argv[1]); from utils import kernel_install as k; "+ "print(json.dumps([k.SOURCE_BUILD_TIMEOUT, k.SOURCE_BUILD_TIMEOUT_CUDA, k.KERNEL_LOCK_WAIT]))"+ )+ out = subprocess.run(+ [sys.executable, "-c", code, str(_BACKEND)],+ env = env,+ capture_output = True,+ text = True,+ timeout = 120,+ check = True,+ ).stdout+ hip, cuda_seen, lock_wait = _json.loads(out.strip().splitlines()[-1])+ assert (hip, cuda_seen) == (1800, cuda)+ assert lock_wait == max(1800, cuda) + 600+ assert lock_wait > hip and lock_wait > cuda_seen+++def test_a_dry_run_prints_the_url_even_offline(monkeypatch, capsys):+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")+ run = _Runner([])+ assert (+ kernel_install.install_kernel(+ "xformers", _COLAB, dry_run = True, run = run, exists = lambda url: True+ )+ == 0+ )+ assert run.calls == []+ out = capsys.readouterr().out+ assert "offline" not in out+ assert out.strip().endswith("xformers-0.0.35-py39-none-manylinux_2_28_x86_64.whl")++@pytest.mark.parametrize("use_uv, system, expected",[@@ -737,3 +960,195 @@ def test_windows_mamba_ssm_installs_with_triton(uv, monkeypatch):assert run.installer_calls[0][-1].endswith("mamba_ssm-2.3.2.post1+cu13torch2.13cxx11abiTRUE-cp313-cp313-win_amd64.whl")+++ "capability, below",+ [((7, 5), True), ((7, 0), True), ((8, 0), False), ((12, 0), False), (None, False)],+)+def test_flash_attn_floor_is_sm80_and_no_visible_gpu_is_not_below(monkeypatch, capability, below):+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: capability)+ monkeypatch.setattr(kernel_install, "_torch_is_hip", lambda run: False)+ assert kernel_install.gpu_below_flash_attn_floor() is below+++def test_rocm_never_takes_the_sm80_floor(monkeypatch):+ # ROCm reports a gfx version as its capability; the sm80 floor is an NVIDIA rule.+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: (7, 5))+ monkeypatch.setattr(kernel_install, "_torch_is_hip", lambda run: True)+ assert kernel_install.gpu_below_flash_attn_floor() is False+++def test_the_hip_probe_reads_torch_version_hip():+ def run(cmd, **kwargs):+ assert "torch.version.hip" in cmd[-1]+ return SimpleNamespace(returncode = 0, stdout = "True\n")++ assert kernel_install._torch_is_hip(run) is True+++def test_install_wheel_never_falls_back_to_pip_under_uv_offline(monkeypatch):+ # uv refuses the index under UV_OFFLINE; pip has no such switch and would reach it.+ from utils import wheel_utils++ monkeypatch.setattr(wheel_utils.shutil, "which", lambda name: "/usr/bin/uv")+ calls = []++ def run(cmd, **kwargs):+ calls.append(cmd)+ return subprocess.CompletedProcess(cmd, 1, stdout = "offline")++ monkeypatch.setenv("UV_OFFLINE", "1")+ attempts = wheel_utils.install_wheel(+ "https://example.invalid/k.whl", python_executable = sys.executable, use_uv = True, run = run+ )+ assert [installer for installer, _ in attempts] == ["uv"]+ assert all(cmd[0] == "uv" for cmd in calls)++ monkeypatch.delenv("UV_OFFLINE")+ calls.clear()+ attempts = wheel_utils.install_wheel(+ "https://example.invalid/k.whl", python_executable = sys.executable, use_uv = True, run = run+ )+ assert [installer for installer, _ in attempts] == ["uv", "pip"]++[email protected](os.name == "nt", reason = "POSIX process groups")+def test_a_timed_out_build_stops_its_whole_process_tree(tmp_path):+ # pip / uv start nvcc / hipcc / ninja; a timeout must stop those too, not just the leader.+ pid_file = tmp_path / "grandchild.pid"+ script = (+ "import subprocess, sys, time; "+ "p = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)']); "+ f"open({str(pid_file)!r}, 'w').write(str(p.pid)); "+ "time.sleep(60)"+ )+ # A generous timeout: on a loaded host the child can take seconds to start its grandchild.+ with pytest.raises(subprocess.TimeoutExpired):+ kernel_install.run_process_tree(+ [sys.executable, "-c", script], timeout = 15, stdout = subprocess.PIPE+ )+ assert pid_file.exists(), "the build never started its grandchild within the timeout"+ grandchild = int(pid_file.read_text())+ for _ in range(100):+ try:+ os.kill(grandchild, 0)+ except ProcessLookupError:+ break+ # Reaped by init once killed; a zombie still answers kill(0) briefly.+ try:+ with open(f"/proc/{grandchild}/stat") as stat:+ if stat.read().split()[2] == "Z":+ break+ except FileNotFoundError:+ break+ time.sleep(0.1)+ else:+ os.kill(grandchild, 9)+ pytest.fail("the build's grandchild outlived the timeout")+++def test_a_test_double_runner_is_called_as_given():+ calls = []++ def run(cmd, **kwargs):+ calls.append(kwargs)+ return subprocess.CompletedProcess(cmd, 0)++ kernel_install.run_process_tree(["x"], run = run, timeout = 5, text = True)+ assert calls == [{"timeout": 5, "text": True}]+++def test_the_kernel_lock_is_exclusive_bounded_and_beats_while_waiting(tmp_path, monkeypatch):+ monkeypatch.setattr(kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / name))+ beats = []+ with kernel_install.kernel_install_lock("causal_conv1d") as first:+ assert first is True+ with kernel_install.kernel_install_lock(+ "causal_conv1d", on_wait = lambda: beats.append(1), wait_seconds = 1.2, beat_seconds = 0.4+ ) as second:+ assert second is False # held elsewhere past the bound: install nothing+ with kernel_install.kernel_install_lock("mamba_ssm", wait_seconds = 0) as other:+ assert other is True # one lock per kernel+ assert len(beats) >= 2+ with kernel_install.kernel_install_lock("causal_conv1d", wait_seconds = 0) as again:+ assert again is True # released+++def test_the_kernel_lock_is_per_environment(monkeypatch, tmp_path):+ monkeypatch.delenv("UNSLOTH_KERNEL_LOCK_DIR", raising = False)+ first_env, other_env = tmp_path / "a", tmp_path / "b"+ first_env.mkdir()+ other_env.mkdir()+ monkeypatch.setattr(kernel_install.sys, "prefix", str(first_env))+ first = _REAL_LOCK_PATH("causal_conv1d")+ monkeypatch.setattr(kernel_install.sys, "prefix", str(other_env))+ assert _REAL_LOCK_PATH("causal_conv1d") != first+++def test_the_lock_lives_beside_the_environment_when_writable(monkeypatch, tmp_path):+ monkeypatch.delenv("UNSLOTH_KERNEL_LOCK_DIR", raising = False)+ env = tmp_path / "venv"+ env.mkdir()+ monkeypatch.setattr(kernel_install.sys, "prefix", str(env))+ assert _REAL_LOCK_PATH("mamba_ssm") == str(env / ".unsloth-kernel-locks" / "mamba_ssm.lock")++[email protected](os.name == "nt" or os.geteuid() == 0, reason = "POSIX permissions, non-root")+def test_a_read_only_environment_falls_back_to_one_temp_lock(monkeypatch, tmp_path):+ monkeypatch.delenv("UNSLOTH_KERNEL_LOCK_DIR", raising = False)+ env = tmp_path / "venv"+ env.mkdir()+ env.chmod(0o555)+ try:+ monkeypatch.setattr(kernel_install.sys, "prefix", str(env))+ monkeypatch.setattr("tempfile.tempdir", str(tmp_path / "tmp"))+ (tmp_path / "tmp").mkdir()+ first = _REAL_LOCK_PATH("mamba_ssm")+ assert first.startswith(str(tmp_path / "tmp")) and first == _REAL_LOCK_PATH("mamba_ssm")+ finally:+ env.chmod(0o755)++[email protected](os.name == "nt" or os.geteuid() == 0, reason = "POSIX permissions, non-root")+def test_a_lock_that_cannot_be_opened_installs_nothing(monkeypatch, tmp_path):+ lock = tmp_path / "ro.lock"+ lock.write_text("")+ lock.chmod(0o444)+ monkeypatch.setattr(kernel_install, "_kernel_lock_path", lambda name: str(lock))+ with kernel_install.kernel_install_lock("causal_conv1d", wait_seconds = 0) as held:+ assert held is False+++def test_a_lock_the_filesystem_cannot_take_installs_nothing(monkeypatch, tmp_path):+ import errno++ monkeypatch.setattr(kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / "x.lock"))++ def _no_locks(*args):+ raise OSError(errno.ENOLCK, "No locks available")++ monkeypatch.setattr("fcntl.flock", _no_locks)+ with kernel_install.kernel_install_lock("causal_conv1d", wait_seconds = 0) as held:+ assert held is False+++def test_the_cli_installs_under_the_studio_lock(uv, monkeypatch, capsys):+ import contextlib++ uv(False)+ taken = []++ @contextlib.contextmanager+ def _busy(name, **kwargs):+ taken.append(name)+ yield False++ monkeypatch.setattr(kernel_install, "kernel_install_lock", _busy)+ run = _Runner([], capability = "9 0")+ assert (+ kernel_install.install_kernel("causal_conv1d", _COLAB, run = run, exists = lambda url: True)+ == 0+ )+ assert taken == ["causal_conv1d"] and run.installer_calls == []+ assert "install lock" in capsys.readouterr().outdiff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.pyindex 127b9c49bd7..d016d9ae9ba 100644--- a/studio/backend/tests/test_mlx_training_worker_config.py+++ b/studio/backend/tests/test_mlx_training_worker_config.py@@ -82,12 +82,14 @@ def _load_worker_module():"flash_attn_wheel_url","install_wheel","is_windows_wheel_env",+ "pinned_source_build_supported","probe_torch_wheel_env","url_exists",):setattr(wheel_utils, name, lambda *_args, **_kwargs: None)# Read at import by utils.kernel_install, which the worker imports.for name in (+ "package_installs_offline_env","redact_url_credentials","xformers_wheel_url",):diff --git a/studio/backend/tests/test_ssm_restart_respawn.py b/studio/backend/tests/test_ssm_restart_respawn.pynew file mode 100644index 00000000000..151a3d41857--- /dev/null+++ b/studio/backend/tests/test_ssm_restart_respawn.py@@ -0,0 +1,335 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""An SSM kernel installed after the worker's transformers imported the model's modeling module is+only bound by a fresh worker: the worker asks for one and the orchestrator respawns once."""++import sys+import threading+from types import SimpleNamespace++import pytest++from core.inference import orchestrator as orch_mod+from core.inference import worker as inference_worker+from core.inference.orchestrator import InferenceOrchestrator+++def _orchestrator(+ monkeypatch,+ responses,+ shutdown_results = (),+):+ orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)+ orchestrator._stop_ledger = None+ orchestrator._pending_teardowns = None+ orchestrator.__dict__.update(+ loading_models = set(),+ active_model_name = None,+ models = {},+ _proc = None,+ _resp_queue = object(),+ load_generation = 0,+ _subprocess_shutdown_lock = threading.Lock(),+ )+ spawned = []+ shutdowns = []+ alive = {"value": False}+ monkeypatch.setattr(orch_mod, "prepare_gpu_selection", lambda *_a, **_k: ([], "auto"))+ monkeypatch.setattr(orch_mod, "get_device", lambda: SimpleNamespace(value = "cpu"))+ monkeypatch.setattr("utils.transformers_version.needs_transformers_5", lambda _model: False)+ monkeypatch.setattr("utils.transformers_version.sidecar_swap_kind", lambda: None)+ monkeypatch.setattr(orchestrator, "_ensure_subprocess_alive", lambda: alive["value"])++ def _spawn(config, *args):+ spawned.append(dict(config)) # the orchestrator reuses one sub_config+ alive["value"] = True++ results = list(shutdown_results)++ def _shutdown(timeout = 5):+ shutdowns.append(timeout)+ exited = results.pop(0) if results else True+ if exited:+ alive["value"] = False+ return exited++ def _wait(*_a, **_k):+ response = responses.pop(0)+ if isinstance(response, BaseException):+ raise response+ return response++ monkeypatch.setattr(orchestrator, "_spawn_subprocess", _spawn)+ monkeypatch.setattr(orchestrator, "_shutdown_subprocess", _shutdown)+ monkeypatch.setattr(orchestrator, "_wait_response", _wait)+ return orchestrator, spawned, shutdowns+++def _restart():+ return {+ "type": "loaded",+ "success": False,+ "error_kind": inference_worker.SSM_RESTART_REQUIRED,+ "message": "restart",+ }+++def test_the_orchestrator_respawns_once_when_the_worker_needs_new_kernels(monkeypatch):+ responses = [+ _restart(),+ {"type": "loaded", "success": True, "model_info": {"identifier": "acme/hybrid"}},+ ]+ orchestrator, spawned, shutdowns = _orchestrator(monkeypatch, responses)+ assert orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid")) is True+ assert len(spawned) == 2+ assert shutdowns+ assert orchestrator.active_model_name == "acme/hybrid"+++def test_a_second_restart_request_fails_instead_of_looping(monkeypatch):+ orchestrator, spawned, _ = _orchestrator(monkeypatch, [_restart(), _restart()])+ try:+ orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid"))+ except Exception as exc:+ assert "restart" in str(exc)+ else:+ raise AssertionError("a second restart request must fail the load")+ assert len(spawned) == 2+++def _loaded():+ return {"type": "loaded", "success": True, "model_info": {"identifier": "acme/hybrid"}}+++def _stall():+ from utils.hf_xet_fallback import DownloadStallError+ return DownloadStallError("stalled")+++def test_a_worker_that_survives_shutdown_is_not_respawned_over(monkeypatch):+ # The worker outlived terminate/kill: its handle stays, and no second worker is spawned over+ # one still holding GPU memory (the model-switch guard's rule).+ orchestrator, spawned, _ = _orchestrator(+ monkeypatch, [_restart(), _loaded()], shutdown_results = [False]+ )+ try:+ orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid"))+ except RuntimeError as exc:+ assert "did not exit" in str(exc)+ else:+ raise AssertionError("the load must fail while the old worker survives")+ assert len(spawned) == 1+ assert orchestrator.active_model_name is None+++def test_a_survivor_also_blocks_the_xet_retry(monkeypatch):+ orchestrator, spawned, _ = _orchestrator(+ monkeypatch, [_stall(), _loaded()], shutdown_results = [False]+ )+ try:+ orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid"))+ except RuntimeError as exc:+ assert "did not exit" in str(exc)+ else:+ raise AssertionError("the load must fail while the old worker survives")+ assert len(spawned) == 1+++def test_an_ssm_restart_keeps_the_xet_retry(monkeypatch):+ orchestrator, spawned, _ = _orchestrator(monkeypatch, [_restart(), _stall(), _loaded()])+ assert orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid")) is True+ assert [config["disable_xet"] for config in spawned] == [False, False, True]+++def test_a_xet_retry_keeps_the_ssm_restart(monkeypatch):+ orchestrator, spawned, _ = _orchestrator(monkeypatch, [_stall(), _restart(), _loaded()])+ assert orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid")) is True+ assert [config["disable_xet"] for config in spawned] == [False, True, True]+++def test_spawns_are_bounded_at_three_and_the_stall_message_is_accurate(monkeypatch):+ orchestrator, spawned, _ = _orchestrator(monkeypatch, [_restart(), _stall(), _stall()])+ try:+ orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid"))+ except RuntimeError as exc:+ assert "even with HF_HUB_DISABLE_XET=1" in str(exc)+ else:+ raise AssertionError("a second stall must fail the load")+ assert len(spawned) == 3+++class _Queue(list):+ def put(self, item):+ self.append(item)+++def _fake_provision(installed, model_types):+ from utils.ssm_runtime import SsmKernels+ def _provision(probe, target, hf_token, **kwargs):+ kernels = SsmKernels(True, True, True, probe, frozenset(model_types))+ return kernels, list(installed)++ return _provision+++def test_the_worker_asks_for_a_restart_when_the_modeling_module_is_already_imported(monkeypatch):+ from utils import ssm_runtime++ monkeypatch.setattr(+ ssm_runtime, "provision_ssm_kernels", _fake_provision(["mamba_ssm"], {"nemotron_h"})+ )+ monkeypatch.setitem(+ inference_worker.sys.modules,+ "transformers.models.nemotron_h.modeling_nemotron_h",+ SimpleNamespace(),+ )+ queue = _Queue()+ assert inference_worker._ensure_ssm_kernels([("acme/hybrid", "acme/hybrid")], queue) is False+ assert queue[-1]["error_kind"] == inference_worker.SSM_RESTART_REQUIRED+++def test_no_restart_when_nothing_new_was_installed_or_nothing_was_imported(monkeypatch):+ from utils import ssm_runtime++ monkeypatch.setitem(+ inference_worker.sys.modules,+ "transformers.models.nemotron_h.modeling_nemotron_h",+ SimpleNamespace(),+ )+ monkeypatch.setattr(ssm_runtime, "provision_ssm_kernels", _fake_provision([], {"nemotron_h"}))+ assert inference_worker._ensure_ssm_kernels([("acme/hybrid", "acme/hybrid")], _Queue())++ monkeypatch.delitem(+ inference_worker.sys.modules, "transformers.models.nemotron_h.modeling_nemotron_h"+ )+ monkeypatch.setattr(+ ssm_runtime, "provision_ssm_kernels", _fake_provision(["mamba_ssm"], {"nemotron_h"})+ )+ assert inference_worker._ensure_ssm_kernels([("acme/hybrid", "acme/hybrid")], _Queue())+++# Only modeling modules capture kernel functions at import; a config or tokenizer import does not.+++def test_a_configuration_import_alone_needs_no_restart(monkeypatch):+ from utils import ssm_runtime++ monkeypatch.setitem(+ sys.modules, "transformers.models.nemotron_h.configuration_nemotron_h", SimpleNamespace()+ )+ monkeypatch.setitem(sys.modules, "transformers.models.nemotron_h", SimpleNamespace())+ monkeypatch.delitem(+ sys.modules, "transformers.models.nemotron_h.modeling_nemotron_h", raising = False+ )+ assert ssm_runtime.modeling_modules_needing_restart({"nemotron_h"}) == []+ queue = _Queue()+ monkeypatch.setattr(+ ssm_runtime, "provision_ssm_kernels", _fake_provision(["mamba_ssm"], {"nemotron_h"})+ )+ assert inference_worker._ensure_ssm_kernels([("acme/hybrid", "acme/hybrid")], queue) is True+++def _remote(checkpoint, auto_map):+ from utils import ssm_runtime+ return ssm_runtime.remote_code_modules(checkpoint, {"auto_map": auto_map})+++ "module",+ [+ # 4.57+ / 5.x sanitize "-" and "." in the repo id, under the commit hash.+ "transformers_modules.nvidia.NVIDIA_hyphen_Nemotron_hyphen_Nano_hyphen_9B_hyphen_v2"+ ".0123abcd.modeling_nemotron_h",+ # Older 4.x kept them.+ "transformers_modules.nvidia.NVIDIA-Nemotron-Nano-9B-v2.0123abcd.modeling_nemotron_h",+ ],+)+def test_an_imported_remote_code_modeling_module_needs_a_restart(monkeypatch, module):+ from utils import ssm_runtime++ monkeypatch.setitem(sys.modules, module, SimpleNamespace())+ model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2"+ remote = _remote(model, {"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"})+ assert ssm_runtime.modeling_modules_needing_restart(set(), remote) == [module]+ monkeypatch.setattr(+ ssm_runtime,+ "provision_ssm_kernels",+ lambda probe, target, hf_token, **k: (+ ssm_runtime.SsmKernels(True, True, True, probe, frozenset(), remote),+ ["mamba_ssm"],+ ),+ )+ queue = _Queue()+ assert inference_worker._ensure_ssm_kernels([(model, model)], queue) is False+ assert queue[-1]["error_kind"] == inference_worker.SSM_RESTART_REQUIRED+++def test_any_auto_map_module_name_counts_not_just_modeling(monkeypatch, tmp_path):+ # transformers loads whatever module auto_map names: `model.CustomForCausalLM` is `model`.+ from utils import ssm_runtime++ checkpoint = tmp_path / "custom-remote-checkpoint"+ checkpoint.mkdir()+ remote = _remote(str(checkpoint), {"AutoModelForCausalLM": "model.CustomForCausalLM"})+ for name in (+ # 5.x: local folder name, then a source-files hash.+ "transformers_modules.custom_hyphen_remote_hyphen_checkpoint.abc123.model",+ # 4.57: the local folder name alone.+ "transformers_modules.custom_hyphen_remote_hyphen_checkpoint.model",+ ):+ monkeypatch.setitem(sys.modules, name, SimpleNamespace())+ assert name in ssm_runtime.modeling_modules_needing_restart(set(), remote)+++def test_a_namesake_from_another_repo_does_not_match():+ from utils import ssm_runtime++ remote = _remote("Qwen/Qwen3-Next", {"AutoModelForCausalLM": "modeling_qwen.QwenForCausalLM"})+ namesakes = [+ "transformers_modules.other.Qwen3_hyphen_Next_hyphen_FineTune.0123.modeling_qwen",+ "transformers_modules.Qwen.Qwen3_hyphen_Next_hyphen_FineTune.0123.modeling_qwen",+ "transformers_modules.Qwen.Qwen3_hyphen_Next.0123.modeling_qwen_extra",+ "transformers_modules.Qwen.Qwen3_hyphen_Next.0123.configuration_qwen",+ ]+ for name in namesakes:+ assert not ssm_runtime._is_remote_code_module(name, remote), name+ assert ssm_runtime._is_remote_code_module(+ "transformers_modules.Qwen.Qwen3_hyphen_Next.0123.modeling_qwen", remote+ )+++def test_code_in_another_repo_is_found_under_that_repo():+ from utils import ssm_runtime++ remote = _remote("acme/finetune", {"AutoModel": "nvidia/Base--model.Custom"})+ assert ssm_runtime._is_remote_code_module("transformers_modules.nvidia.Base.0123.model", remote)+ assert not ssm_runtime._is_remote_code_module(+ "transformers_modules.acme.finetune.0123.model", remote+ )+++def test_config_and_tokenizer_auto_map_entries_are_not_implementations():+ remote = _remote(+ "acme/hybrid",+ {+ "AutoConfig": "configuration_hybrid.HybridConfig",+ "AutoTokenizer": ["tokenization_hybrid.HybridTokenizer", None],+ },+ )+ assert remote == frozenset()+++def test_the_resolved_config_carries_its_remote_code(monkeypatch):+ from utils import ssm_runtime++ monkeypatch.setattr(+ ssm_runtime,+ "_resolved_model_config",+ lambda target, token: {+ "model_type": "nemotron_h",+ "auto_map": {"AutoModelForCausalLM": "model.NemotronHForCausalLM"},+ },+ )+ kernels = ssm_runtime.resolve_ssm_kernels("acme/hybrid", "acme/hybrid", None)+ assert (("acme", "hybrid"), ("model",)) in kernels.remote_codediff --git a/studio/backend/tests/test_ssm_runtime.py b/studio/backend/tests/test_ssm_runtime.pyindex 02f3668666e..4b4f01acaaf 100644--- a/studio/backend/tests/test_ssm_runtime.py+++ b/studio/backend/tests/test_ssm_runtime.py@@ -8,6 +8,7 @@ worker wiring, and that the training worker shares the same tables and pinned ke"""import json+import contextlibimport sysimport typesfrom pathlib import Path@@ -21,18 +22,35 @@ if str(_BACKEND) not in sys.path:from utils import ssm_runtime # noqa: E402+_REAL_MAMBA_SSM_FLOOR_REASON = ssm_runtime._mamba_ssm_floor_reason+@pytest.fixture(autouse = True)-def _clear_offline_environment(monkeypatch):+def _clear_offline_environment(monkeypatch, tmp_path):monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)+ monkeypatch.delenv("UV_OFFLINE", raising = False)+ # Per-process memos: each test is its own process here.+ monkeypatch.setattr(ssm_runtime, "_TORCH_ENV", {})+ monkeypatch.setattr(ssm_runtime, "_FAILED_INSTALLS", set())+ monkeypatch.setattr(ssm_runtime, "_MODEL_CONFIGS", {})+ monkeypatch.setattr(ssm_runtime, "_REMOTE_SOURCES", {})+ monkeypatch.setattr(ssm_runtime, "_NATIVE_MODELING_CACHE", {})+ from utils import kernel_install-[email protected](autouse = True)-def _cuda_host_with_triton(monkeypatch):- # A CUDA torch with Triton, where zoo's Triton conv runs; the platform tests below override it.- monkeypatch.delenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", raising = False)- monkeypatch.setattr(ssm_runtime, "_zoo_triton_platform_usable", lambda: True)+ monkeypatch.setattr(+ kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / f"{name}.lock")+ )+ # Every copy of the module (and any child process) uses the test's folder too.+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", str(tmp_path / "locks"))+ # This host's GPU must not decide the optional-mamba floor; its own tests set it.+ monkeypatch.setattr(ssm_runtime, "_mamba_ssm_floor_reason", lambda: None)+ # Nor its torch build: a CUDA torch unless a test says otherwise.+ monkeypatch.setattr(+ ssm_runtime,+ "probe_torch_wheel_env",+ lambda **k: {"cuda_version": "12.8", "cuda_major": "12"},+ )class _Result:@@ -164,9 +182,10 @@ def test_causal_only_model_skips_mamba(monkeypatch):def test_failure_raises_runtime_error(monkeypatch):- # A true SSM model whose mamba-ssm cannot install is fatal (cryptic mid-load import- # otherwise). "Nemotron-3-Nano-30B-A3B" matches the SSM substrings.+ # A required-family model this transformers has no native implementation of cannot load+ # without mamba-ssm, so a failed install is fatal (cryptic mid-load import otherwise).monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda model_type: False)with pytest.raises(RuntimeError):ssm_runtime.ensure_ssm_runtime("unsloth/Nemotron-3-Nano-30B-A3B")@@ -205,7 +224,10 @@ def test_install_kernel_idempotent_when_present(monkeypatch):assert called == [] # short-circuits before touching the network[email protected]("offline_variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"])+# UV_OFFLINE too: uv refuses the index and install_wheel's pip fallback would not.+ "offline_variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "UV_OFFLINE"]+)def test_install_kernel_skips_all_install_work_offline(monkeypatch, offline_variable):monkeypatch.setenv(offline_variable, "on")monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: False)@@ -239,11 +261,13 @@ def test_install_kernel_skips_all_install_work_offline(monkeypatch, offline_varidef test_install_kernel_uses_prebuilt_wheel(monkeypatch):- # not importable before install, importable after the wheel lands- states = iter([False, True])+ # not importable before install (or once the lock is held), importable after the wheel lands+ states = iter([False, False, True])monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))monkeypatch.setattr(- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {"x": "y"}+ ssm_runtime,+ "probe_torch_wheel_env",+ lambda timeout = 30, include_windows = False: {"x": "y", "cuda_version": "12.8"},)seen = {}monkeypatch.setattr(@@ -284,7 +308,9 @@ def test_install_kernel_heartbeats_during_prebuilt_wheel(monkeypatch):monkeypatch.setattr(ssm_runtime, "_HEARTBEAT_SECONDS", 0.05)monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: False)monkeypatch.setattr(- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}+ ssm_runtime,+ "probe_torch_wheel_env",+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},)monkeypatch.setattr(ssm_runtime,@@ -341,7 +367,9 @@ def test_install_kernel_heartbeats_through_the_import_check(monkeypatch):monkeypatch.setattr(ssm_runtime, "_HEARTBEAT_SECONDS", 0.05)monkeypatch.setattr(- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}+ ssm_runtime,+ "probe_torch_wheel_env",+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},)monkeypatch.setattr(ssm_runtime,@@ -359,7 +387,7 @@ def test_install_kernel_heartbeats_through_the_import_check(monkeypatch):def slow_importable(name):seen["n"] += 1- if seen["n"] == 1:+ if seen["n"] <= 2: # before install, and again once the lock is heldreturn Falseimport_started.set()assert import_released.wait(1.0)@@ -408,10 +436,12 @@ def test_heartbeat_does_not_emit_after_the_block(monkeypatch):def test_install_kernel_falls_back_to_source(monkeypatch):# no wheel -> source build -> importable after install- states = iter([False, True]) # before install, after install+ states = iter([False, False, True]) # before install, after the lock, after installmonkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))monkeypatch.setattr(- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}+ ssm_runtime,+ "probe_torch_wheel_env",+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},)monkeypatch.setattr(ssm_runtime, "direct_wheel_url", lambda **k: None)pip_cmds = []@@ -514,23 +544,35 @@ def _windows_install_kernel(monkeypatch.setattr(ssm_runtime.sys, "platform", "win32")# shutil.which consults sys.platform, and a Linux host cannot answer as Windows.monkeypatch.setattr(ssm_runtime.shutil, "which", lambda *a, **k: None)- states = iter([False, importable_after])+ # Before install, once the lock is held, after the wheel.+ states = iter([False, False, importable_after])monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))probe_kwargs = {}monkeypatch.setattr(ssm_runtime,"probe_torch_wheel_env",- lambda **k: probe_kwargs.update(k) or {"platform_tag": "win_amd64"},+ lambda **k: probe_kwargs.update(k) or {"platform_tag": "win_amd64", "cuda_version": "13.0"},)monkeypatch.setattr(ssm_runtime, "direct_wheel_url", lambda **k: wheel_url)monkeypatch.setattr(ssm_runtime, "url_exists", lambda u: exists)+ present = {"value": False}+ monkeypatch.setattr(ssm_runtime, "_installed_distribution", lambda name: present["value"])installs = []- monkeypatch.setattr(- ssm_runtime,- "install_wheel",- lambda url, **k: installs.append(url) or [("uv", _Result(returncode = outcome_rc))],- )++ def _install_wheel(url, **k):+ installs.append(url)+ present["value"] = outcome_rc == 0+ return [("uv", _Result(returncode = outcome_rc))]++ monkeypatch.setattr(ssm_runtime, "install_wheel", _install_wheel)ran = []++ def _run(cmd, **k):+ ran.append(cmd)+ if "uninstall" in cmd:+ present["value"] = False+ return _Result()+ok = ssm_runtime._install_kernel(import_name = "causal_conv1d",display_name = "causal-conv1d",@@ -539,7 +581,7 @@ def _windows_install_kernel(release_tag = "v1.6.1.post4",release_base_url = "https://github.com/Dao-AILab/causal-conv1d/releases/download",status_cb = None,- run = lambda cmd, **k: ran.append(cmd) or _Result(),+ run = _run,)assert probe_kwargs.get("include_windows") is Truereturn ok, ran, installs@@ -554,9 +596,7 @@ def test_windows_installs_the_published_wheel(monkeypatch):assert ran == []- ("wheel_url", "exists"), [(None, False), ("https://x/w.whl", False), ("https://x/w.whl", None)]-)[email protected](("wheel_url", "exists"), [(None, False), ("https://x/w.whl", False)])def test_windows_never_source_builds_without_a_wheel(monkeypatch, wheel_url, exists):ok, ran, installs = _windows_install_kernel(monkeypatch, wheel_url = wheel_url, exists = exists)assert ok is False@@ -575,12 +615,14 @@ def test_windows_rejected_wheel_is_removed_not_source_built(monkeypatch):def test_wheel_installed_but_not_importable_falls_back_to_source(monkeypatch):- # top: not importable; after wheel: still not importable (ABI mismatch) -> source build;- # after source build: importable.- states = iter([False, False, True])+ # top and under the lock: not importable; after wheel: still not importable (ABI mismatch)+ # -> source build; after source build: importable.+ states = iter([False, False, False, True])monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))monkeypatch.setattr(- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}+ ssm_runtime,+ "probe_torch_wheel_env",+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},)monkeypatch.setattr(ssm_runtime, "direct_wheel_url", lambda **k: "https://x/w.whl")monkeypatch.setattr(ssm_runtime, "url_exists", lambda u: True)@@ -630,10 +672,12 @@ def test_hip_source_build_requires_hipcc(monkeypatch):def test_source_build_reinstalls_to_replace_broken_wheel(monkeypatch):# Reached only when not importable (possibly a broken wheel at the pinned version);# the source build must reinstall so it replaces it instead of no-opping.- states = iter([False, True])+ states = iter([False, False, True])monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))monkeypatch.setattr(- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}+ ssm_runtime,+ "probe_torch_wheel_env",+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},)monkeypatch.setattr(ssm_runtime, "direct_wheel_url", lambda **k: None)cmds = []@@ -652,7 +696,7 @@ def test_source_build_reinstalls_to_replace_broken_wheel(monkeypatch):def test_hip_uv_source_build_uses_no_cache(monkeypatch):# ROCm uv source build must skip the cache to avoid reusing stale partial HIP builds.- states = iter([False, True])+ states = iter([False, False, True])monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))monkeypatch.setattr(ssm_runtime,@@ -682,8 +726,8 @@ def test_hip_uv_source_build_uses_no_cache(monkeypatch):def test_inference_worker_calls_ensure_ssm_runtime():src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8")- assert "from utils.ssm_runtime import (\n ensure_ssm_runtime," in src- assert "ensure_ssm_runtime(" in src+ assert "provision_ssm_kernels" in src+ assert "provision_ssm_kernels(" in srcdef test_inference_worker_skips_ssm_on_mlx_and_checks_lora_base():@@ -883,23 +927,22 @@ def test_nested_text_config_resolves_causal_conv1d(tmp_path):assert ssm_runtime.resolved_model_wants_causal_conv1d(target, target, None) is True-# ── Qwen3.5 / 3.6 / Qwen3-Next run on unsloth-zoo's Triton conv, never causal-conv1d ──+# ── Qwen3.5 / 3.6 / Qwen3-Next get causal-conv1d on demand, on every platform ──-_ZOO_GDN_NAMES = [+_QWEN_GDN_NAMES = ["Qwen/Qwen3-Next-80B-A3B",- "Qwen/Qwen3-Next-80B-A3B-Instruct","unsloth/Qwen3.5-2B",- "unsloth/Qwen3.5-35B-A3B-GGUF","unsloth/Qwen3.6-27B","my-org/qwen3_5-finetune",][email protected]("name", _ZOO_GDN_NAMES)-def test_zoo_gdn_models_request_no_kernels(monkeypatch, name):- monkeypatch.delenv(ssm_runtime.ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, raising = False)- assert ssm_runtime.model_wants_causal_conv1d(name) is False[email protected]("name", _QWEN_GDN_NAMES)+def test_qwen_gdn_models_install_causal_conv1d_on_demand(monkeypatch, name):+ # zoo's Triton conv has no decode update and a sequence-sized dW scratch: keep the native kernel.+ monkeypatch.delenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", raising = False)+ assert ssm_runtime.model_wants_causal_conv1d(name) is Trueassert ssm_runtime.model_is_ssm(name) is Falseinstalled = []monkeypatch.setattr(@@ -908,14 +951,7 @@ def test_zoo_gdn_models_request_no_kernels(monkeypatch, name):lambda *, import_name, **_: installed.append(import_name) or True,)ssm_runtime.ensure_ssm_runtime(name)- assert installed == []--[email protected]("name", _ZOO_GDN_NAMES)-def test_zoo_kill_switch_brings_the_causal_conv1d_request_back(monkeypatch, name):- monkeypatch.setenv(ssm_runtime.ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, "1")- assert ssm_runtime.model_wants_causal_conv1d(name) is True- assert ssm_runtime.model_is_ssm(name) is False+ assert installed == ["causal_conv1d"]@pytest.mark.parametrize(@@ -928,27 +964,23 @@ def test_zoo_kill_switch_brings_the_causal_conv1d_request_back(monkeypatch, name"text_config": {"model_type": "qwen3_5_text"},"vision_config": {"model_type": "qwen3_5_vision"},},- {- "model_type": "qwen3_5_moe",- "architectures": ["Qwen3_5MoeForConditionalGeneration"],- "text_config": {"model_type": "qwen3_5_moe_text"},- "vision_config": {"model_type": "qwen3_5_moe_vision"},- },],- ids = ["qwen3_next", "qwen3_5", "qwen3_5_moe"],+ ids = ["qwen3_next", "qwen3_5"],)-def test_zoo_gdn_configs_resolve_to_no_causal_conv1d(monkeypatch, tmp_path, config):+def test_qwen_gdn_configs_resolve_to_causal_conv1d(monkeypatch, tmp_path, config):+ monkeypatch.delenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", raising = False)target = str(_write_config(tmp_path / "renamed", config))- monkeypatch.delenv(ssm_runtime.ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, raising = False)- assert ssm_runtime.model_config_wants_causal_conv1d(config) is False- assert ssm_runtime.resolved_model_wants_causal_conv1d("acme/x", target, None) is False- monkeypatch.setenv(ssm_runtime.ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, "1")assert ssm_runtime.model_config_wants_causal_conv1d(config) is True+ assert ssm_runtime.resolved_model_wants_causal_conv1d("acme/x", target, None) is True-def test_a_config_mixing_a_zoo_gdn_type_with_another_hybrid_still_requests_it():- config = {"model_type": "qwen3_5", "text_config": {"model_type": "lfm2"}}- assert ssm_runtime.model_config_wants_causal_conv1d(config) is True+def test_the_zoo_exemption_is_gone():+ for name in (+ "ZOO_GDN_MODEL_TYPES",+ "zoo_triton_causal_conv1d_usable",+ "_zoo_triton_platform_usable",+ ):+ assert not hasattr(ssm_runtime, name)@pytest.mark.parametrize(@@ -966,28 +998,6 @@ def test_mamba2_hybrids_still_install_both_on_demand(monkeypatch, name):assert order == ["causal_conv1d", "mamba_ssm"]-def test_zoo_gdn_model_types_match_unsloth_zoo():- """Studio's list must equal the models unsloth-zoo's Triton conv patches."""- import ast- import importlib.util-- spec = importlib.util.find_spec("unsloth_zoo")- if spec is None or spec.origin is None:- pytest.skip("unsloth_zoo not installed")- source_path = Path(spec.origin).parent / "temporary_patches" / "gdn_causal_conv1d.py"- if not source_path.is_file():- pytest.skip("this unsloth_zoo has no Triton causal conv")- tree = ast.parse(source_path.read_text(encoding = "utf-8"))- values = [- node.value- for node in ast.walk(tree)- if isinstance(node, ast.Assign)- and any(isinstance(t, ast.Name) and t.id == "_GDN_MODELING" for t in node.targets)- ]- assert len(values) == 1- assert set(ast.literal_eval(values[0])) == ssm_runtime.ZOO_GDN_MODEL_TYPES--# ── mamba-ssm resolved from the config, not only the name ─────────────────────@@ -1011,12 +1021,6 @@ def test_renamed_or_unlisted_ssm_models_resolve_from_their_config(tmp_path, nameassert ssm_runtime.resolved_model_is_ssm(name, target, None) is True# Every mamba-ssm model also wants causal-conv1d.assert ssm_runtime.resolved_model_wants_causal_conv1d(name, target, None) is True- assert (- ssm_runtime.resolved_model_wants_causal_conv1d(- name, target, None, scan_modeling_source = False- )- is True- )def test_a_nested_ssm_text_config_resolves_too(tmp_path):@@ -1052,92 +1056,11 @@ def test_resolved_answers_override_the_name(monkeypatch):def test_the_mamba_ssm_failure_names_the_offline_remedy(monkeypatch):monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda model_type: False)with pytest.raises(RuntimeError, match = "unsloth install-kernels"):ssm_runtime.ensure_ssm_runtime("tiiuae/Falcon-H1-0.5B")-# ── zoo's Triton conv only counts where it can run ────────────────────────────---# The real (cached) platform check, captured before the autouse fixture replaces it.-_REAL_PLATFORM_USABLE = ssm_runtime._zoo_triton_platform_usable.__wrapped__--- "platform, triton, torch_version, usable",- [- ("linux", True, "2.13.0", True),- ("linux", True, "2.13.0+cu130", True),- ("win32", True, "2.13.0+cu130", True),- ("linux", True, "2.11.0+rocm7.1", False),- ("win32", False, "2.13.0+cu130", False),- ("darwin", True, "2.13.0", False),- ("linux", True, None, False),- ("linux", True, "2.13.0+cpu", False),- ("win32", True, "2.13.0", False),- ],- ids = [- "linux-pypi-cuda",- "linux-cu130",- "windows-triton",- "rocm",- "windows-no-triton",- "macos",- "no-torch",- "cpu-torch",- "windows-pypi-cpu",- ],-)-def test_zoo_triton_conv_platform_check(monkeypatch, platform, triton, torch_version, usable):- monkeypatch.setattr(ssm_runtime.sys, "platform", platform)- monkeypatch.delitem(ssm_runtime.sys.modules, "torch", raising = False)- real_find_spec = ssm_runtime.importlib.util.find_spec- monkeypatch.setattr(- ssm_runtime.importlib.util,- "find_spec",- lambda name, *a: (- (object() if triton else None) if name == "triton" else real_find_spec(name, *a)- ),- )-- def _version(name):- if torch_version is None:- raise ssm_runtime.importlib.metadata.PackageNotFoundError(name)- return torch_version-- monkeypatch.setattr(ssm_runtime.importlib.metadata, "version", _version)- assert _REAL_PLATFORM_USABLE() is usable---def test_zoo_triton_conv_platform_check_reads_an_imported_torch(monkeypatch):- import types-- monkeypatch.setattr(ssm_runtime.sys, "platform", "linux")- monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda name, *a: object())- fake = types.SimpleNamespace(version = types.SimpleNamespace(cuda = None, hip = "7.1"))- monkeypatch.setitem(ssm_runtime.sys.modules, "torch", fake)- assert _REAL_PLATFORM_USABLE() is False- fake.version = types.SimpleNamespace(cuda = "13.0", hip = None)- assert _REAL_PLATFORM_USABLE() is True--[email protected]("name", ["unsloth/Qwen3.5-2B", "Qwen/Qwen3-Next-80B-A3B"])-def test_qwen_keeps_causal_conv1d_where_zoo_triton_conv_cannot_run(monkeypatch, tmp_path, name):- monkeypatch.setattr(ssm_runtime, "_zoo_triton_platform_usable", lambda: False)- assert ssm_runtime.model_wants_causal_conv1d(name) is True- config = {"model_type": "qwen3_5", "text_config": {"model_type": "qwen3_5_text"}}- assert ssm_runtime.model_config_wants_causal_conv1d(config) is True---def test_the_platform_check_is_not_run_for_unrelated_models(monkeypatch):- def _boom():- raise AssertionError("the platform check must not run for a non-Qwen model")-- monkeypatch.setattr(ssm_runtime, "_zoo_triton_platform_usable", _boom)- assert ssm_runtime.model_wants_causal_conv1d("unsloth/Llama-3.2-1B") is False- assert ssm_runtime.model_wants_causal_conv1d("LiquidAI/LFM2-1.2B") is True--# ── in-process installs reach an already-imported transformers ────────────────@@ -1180,26 +1103,841 @@ def test_refresh_is_a_no_op_without_transformers_and_never_raises(monkeypatch):ssm_runtime.refresh_transformers_kernel_gates()+class _Queue(list):+ def put(self, item):+ self.append(item)++def test_inference_worker_resolves_kernels_from_the_base_config(monkeypatch, tmp_path):from core.inference import worker as iwtarget = str(_write_config(tmp_path / "renamed", {"model_type": "nemotron_h"}))seen = []++ def _ensure(probe, **kwargs):+ seen.append(+ (probe, kwargs["wants_causal_conv1d"], kwargs["is_ssm"], kwargs["mamba_ssm_required"])+ )+ return []++ monkeypatch.setattr(ssm_runtime, "ensure_ssm_runtime", _ensure)+ assert iw._ensure_ssm_kernels([("renamed", target)], resp_queue = _Queue(), hf_token = None)+ assert seen == [("renamed", True, True, True)]++ src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8")+ assert "_ensure_ssm_kernels(ssm_targets, resp_queue, hf_token, trust_remote_code)" in src+ assert "_ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token, _trust_remote_code)" in src+++# ── mamba-ssm is fatal only where the model cannot load without it ────────────+++ "name",+ ["state-spaces/mamba-130m-hf", "ai21labs/Jamba-v0.1", "Zyphra/Zamba2-7B", "ibm/Bamba-9B"],+)+def test_a_missing_mamba_ssm_is_not_fatal_for_models_with_a_torch_path(monkeypatch, name):+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ messages = []+ assert ssm_runtime.model_requires_mamba_ssm(name) is False+ ssm_runtime.ensure_ssm_runtime(name, status_cb = messages.append) # no raise+ assert any("torch path" in message for message in messages)+++ "model_type, required",+ [+ ("nemotron_h", True),+ ("falcon_h1", True),+ ("granitemoehybrid", True),+ ("mamba", False),+ ("mamba2", False),+ ("falcon_mamba", False),+ ("jamba", False),+ ("zamba", False),+ ("zamba2", False),+ ("bamba", False),+ ],+)+def test_only_the_old_fatal_set_requires_mamba_ssm(tmp_path, model_type, required):+ target = str(_write_config(tmp_path / "ckpt", {"model_type": model_type}))+ kernels = ssm_runtime.resolve_ssm_kernels("acme/x", target, None)+ assert kernels.mamba_ssm is True and kernels.causal_conv1d is True+ assert kernels.mamba_ssm_required is required+++def test_windows_without_triton_is_fatal_only_for_the_required_set(monkeypatch):+ monkeypatch.setattr(ssm_runtime.sys, "platform", "win32")+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda name, *a: None)+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: True)+ ssm_runtime.ensure_ssm_runtime("ai21labs/Jamba-v0.1") # no raise+ with pytest.raises(RuntimeError, match = "Triton"):+ ssm_runtime.ensure_ssm_runtime("tiiuae/Falcon-H1-0.5B")+++# ── a local adapter's base decides the kernels ───────────────────────────────+++def _local_adapter(tmp_path, base: str) -> str:+ adapter = tmp_path / "my-finetune-v3"+ adapter.mkdir()+ (adapter / "adapter_config.json").write_text(+ json.dumps({"base_model_name_or_path": base, "peft_type": "LORA"}), encoding = "utf-8"+ )+ return str(adapter)+++def test_an_unrelatedly_named_local_adapter_resolves_through_its_falcon_h1_base(tmp_path):+ base = str(_write_config(tmp_path / "base", {"model_type": "falcon_h1"}))+ adapter = _local_adapter(tmp_path, base)+ kernels = ssm_runtime.resolve_ssm_kernels(adapter, adapter, None)+ assert (kernels.causal_conv1d, kernels.mamba_ssm, kernels.mamba_ssm_required) == (+ True,+ True,+ True,+ )+ assert ssm_runtime.resolved_model_is_ssm(adapter, adapter, None) is True+ assert ssm_runtime.resolved_model_wants_causal_conv1d(adapter, adapter, None) is True+++def test_the_authorized_adapter_base_pin_wins(tmp_path):+ llama = str(_write_config(tmp_path / "llama", {"model_type": "llama"}))+ falcon = str(_write_config(tmp_path / "falcon", {"model_type": "falcon_h1"}))+ adapter = _local_adapter(tmp_path, llama)+ assert ssm_runtime.resolved_model_is_ssm(adapter, adapter, None) is False+ assert ssm_runtime.resolved_model_is_ssm(adapter, adapter, None, adapter_base = falcon) is True+++def test_the_training_worker_passes_the_adapter_base_pin():+ src = (_BACKEND / "core" / "training" / "worker.py").read_text(encoding = "utf-8")+ assert 'adapter_base = config.get("adapter_base_pin") or None' in src+++# ── the modeling-source scan works without importing transformers ─────────────+++def test_the_modeling_source_scan_reads_files_without_importing_transformers(monkeypatch, tmp_path):+ package = tmp_path / "transformers"+ model_dir = package / "models" / "convnet_x"+ model_dir.mkdir(parents = True)+ (package / "__init__.py").write_text("raise RuntimeError('must not be imported')\n")+ (model_dir / "modeling_convnet_x.py").write_text(+ "from causal_conv1d import causal_conv1d_fn\n", encoding = "utf-8"+ )+ plain = package / "models" / "plain_x"+ plain.mkdir()+ (plain / "modeling_plain_x.py").write_text("import torch\n", encoding = "utf-8")+ spec = types.SimpleNamespace(+ submodule_search_locations = [str(package)], origin = str(package / "__init__.py")+ )+ real_find_spec = ssm_runtime.importlib.util.find_spec+ monkeypatch.setattr(+ ssm_runtime.importlib.util,+ "find_spec",+ lambda name, *a: spec if name == "transformers" else real_find_spec(name, *a),+ )+ monkeypatch.setattr(ssm_runtime, "_TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE", {})+ monkeypatch.delitem(sys.modules, "transformers", raising = False)++ assert ssm_runtime.model_config_wants_causal_conv1d({"model_type": "convnet_x"}) is True+ assert ssm_runtime.model_config_wants_causal_conv1d({"model_type": "plain_x"}) is False+ assert "transformers" not in sys.modules+++# ── wheel availability, failures, cleanup, locking, backend gate, floor (#13971 audit) ──────────+++def _kernel_kwargs(**overrides):+ kwargs = dict(+ import_name = "causal_conv1d",+ display_name = "causal-conv1d",+ pypi_name = "causal-conv1d",+ package_version = "1.6.1",+ release_tag = "v1.6.1.post4",+ release_base_url = "x",+ status_cb = None,+ )+ kwargs.update(overrides)+ return kwargs+++class _Host:+ """A fake environment: what imports, what is installed, what each step does, per kernel."""++ def __init__(+ self,+ monkeypatch,+ *,+ env,+ wheel_url = True,+ head = True,+ wheel_ok = True,+ build_ok = False,+ platform = "linux",+ ):+ self.env = env+ self.installed: set[str] = set()+ self.imports: set[str] = set()+ self.wheel_ok = wheel_ok+ self.build_ok = build_ok+ self.wheels: list[str] = []+ self.commands: list[list[str]] = []+ self.statuses: list[str] = []+ monkeypatch.setattr(ssm_runtime.sys, "platform", platform)+ monkeypatch.setattr(ssm_runtime.shutil, "which", lambda name: None)+ monkeypatch.setattr(ssm_runtime, "probe_torch_wheel_env", lambda **k: env)+ monkeypatch.setattr(+ ssm_runtime,+ "direct_wheel_url",+ lambda **k: f"https://x/{k['filename_prefix']}.whl" if wheel_url else None,+ )+ monkeypatch.setattr(ssm_runtime, "url_exists", lambda url: head)+ monkeypatch.setattr(ssm_runtime, "install_wheel", self.install_wheel)+ monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: name in self.imports)+ monkeypatch.setattr(+ ssm_runtime,+ "_installed_distribution",+ lambda name: name.replace("-", "_") in self.installed,+ )++ def install_wheel(self, url, **kwargs):+ self.wheels.append(url)+ name = url.rsplit("/", 1)[-1].removesuffix(".whl")+ self.installed.add(name) # a download leaves the distribution behind either way+ if self.wheel_ok:+ self.imports.add(name)+ return [("uv", _Result(returncode = 0))]++ def run(self, cmd, **kwargs):+ self.commands.append(list(cmd))+ joined = " ".join(cmd)+ name = "mamba_ssm" if "mamba-ssm" in joined else "causal_conv1d"+ if "uninstall" in cmd:+ self.installed.discard(name)+ self.imports.discard(name)+ else: # a source build+ self.installed.add(name)+ if self.build_ok:+ self.imports.add(name)+ return _Result(returncode = 0 if self.build_ok else 1)++ def install(self, **overrides):+ return ssm_runtime._install_kernel(+ **_kernel_kwargs(status_cb = self.statuses.append, **overrides), run = self.run+ )++ @property+ def builds(self):+ return [cmd for cmd in self.commands if "uninstall" not in cmd]+++_CUDA = {"cuda_version": "12.8", "cuda_major": "12", "platform_tag": "linux_x86_64"}+_WINDOWS = {"cuda_version": "13.0", "cuda_major": "13", "platform_tag": "win_amd64"}++[email protected]("platform, env", [("linux", _CUDA), ("win32", _WINDOWS)])+def test_an_unanswered_head_still_downloads_the_wheel(monkeypatch, platform, env):+ host = _Host(monkeypatch, env = env, head = None, platform = platform)+ assert host.install() is True+ assert host.wheels == ["https://x/causal_conv1d.whl"]+ assert host.builds == [] # the wheel worked: no source build+++def test_a_required_hybrid_loads_when_only_head_is_refused(monkeypatch):+ host = _Host(monkeypatch, env = _CUDA, head = None)+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ installed = ssm_runtime.ensure_ssm_runtime(+ "nvidia/NVIDIA-Nemotron-Nano-9B-v2",+ run = host.run,+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = True,+ )+ assert installed == ["causal_conv1d", "mamba_ssm"]+ assert len(host.wheels) == 2+++def test_a_failed_download_after_an_unanswered_head_falls_back(monkeypatch):+ host = _Host(monkeypatch, env = _CUDA, head = None, wheel_ok = False, build_ok = True)+ assert host.install() is True+ assert len(host.builds) == 1 # the real download failed: then the source build+++def test_a_failed_install_is_not_retried_in_this_process(monkeypatch):+ host = _Host(monkeypatch, env = _CUDA, head = False, build_ok = False)+ assert host.install() is False+ assert host.install() is False+ assert len(host.builds) == 1+++def test_a_failed_install_is_retried_once_the_toolchain_changes(monkeypatch):+ host = _Host(monkeypatch, env = _CUDA, head = False, build_ok = False)+ assert host.install() is False+ monkeypatch.setenv("CUDA_HOME", "/opt/cuda-new")+ host.install()+ assert len(host.builds) == 2++[email protected]("env", [_CUDA, {"hip_version": "6.4", "platform_tag": "linux_x86_64"}])+def test_optional_mamba_ssm_is_wheel_only(monkeypatch, env):+ host = _Host(monkeypatch, env = env, head = False, build_ok = True)+ monkeypatch.setattr(+ ssm_runtime.shutil, "which", lambda name: "/usr/bin/hipcc" if name == "hipcc" else None+ )+ host_installs = ssm_runtime.ensure_ssm_runtime(+ "ai21labs/Jamba-v0.1",+ run = host.run,+ wants_causal_conv1d = False,+ is_ssm = True,+ mamba_ssm_required = False,+ )+ # causal-conv1d keeps its source fallback; mamba-ssm for an optional family does not.+ assert not any("mamba-ssm==" in " ".join(cmd) for cmd in host.builds)+ assert any("causal-conv1d==" in " ".join(cmd) for cmd in host.builds)+ assert "mamba_ssm" not in host_installs+++def test_a_required_hybrid_keeps_the_mamba_ssm_source_fallback(monkeypatch):+ host = _Host(monkeypatch, env = _CUDA, head = False, build_ok = True)+ ssm_runtime.ensure_ssm_runtime(+ "tiiuae/Falcon-H1-0.5B",+ run = host.run,+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = True,+ )+ assert any("mamba-ssm==" in " ".join(cmd) for cmd in host.builds)+++def test_a_cuda_source_build_has_its_own_looser_deadline(monkeypatch):+ seen = {}++ def run(cmd, **kwargs):+ seen.update(kwargs)+ return _Result(returncode = 1)++ host = _Host(monkeypatch, env = _CUDA, head = False)+ ssm_runtime._install_kernel(**_kernel_kwargs(), run = run)+ from utils import kernel_install++ assert seen.get("timeout") == kernel_install.SOURCE_BUILD_TIMEOUT_CUDA+ assert seen.get("timeout") > kernel_install.SOURCE_BUILD_TIMEOUT # HIP keeps 1800++[email protected]("wheel_ok, head", [(False, True), (False, False)])+def test_a_broken_install_is_removed_not_left_behind(monkeypatch, wheel_ok, head):+ host = _Host(monkeypatch, env = _CUDA, head = head, wheel_ok = wheel_ok, build_ok = False)+ assert host.install() is False+ assert host.installed == set()+ assert any("uninstall" in cmd for cmd in host.commands)+++def test_the_install_runs_under_the_kernel_lock_and_rechecks(monkeypatch):+ from utils import kernel_install++ host = _Host(monkeypatch, env = _CUDA)+ held = {}++ @contextlib.contextmanager+ def _lock(name, **kwargs):+ held["name"] = name+ host.imports.add(name) # a peer installed it while this process waited+ yield True++ monkeypatch.setattr(kernel_install, "kernel_install_lock", _lock)+ assert host.install() is True+ assert held["name"] == "causal_conv1d"+ assert host.wheels == [] and host.commands == []+++def test_a_lock_held_past_the_bound_installs_nothing(monkeypatch):+ from utils import kernel_install++ host = _Host(monkeypatch, env = _CUDA)++ @contextlib.contextmanager+ def _busy(name, **kwargs):+ yield False++ monkeypatch.setattr(kernel_install, "kernel_install_lock", _busy)+ assert host.install() is False+ assert host.wheels == [] and host.commands == []+ assert any("install lock" in message for message in host.statuses)+++ "env",+ [+ None, # the torch probe failed+ {"cuda_version": "", "hip_version": "", "platform_tag": "linux_x86_64"}, # CPU / XPU+ ],+)+def test_no_cuda_or_rocm_torch_installs_nothing(monkeypatch, env):+ host = _Host(monkeypatch, env = env, head = True)+ assert host.install() is False+ assert host.wheels == [] and host.commands == []+ assert any("CUDA or ROCm" in message for message in host.statuses)+++def test_a_required_hybrid_on_an_xpu_torch_is_skipped_not_fatal(monkeypatch):+ host = _Host(monkeypatch, env = {"cuda_version": "", "platform_tag": "linux_x86_64"})+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ assert (+ ssm_runtime.ensure_ssm_runtime(+ "tiiuae/Falcon-H1-0.5B",+ run = host.run,+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = True,+ # Native transformers modeling: required (worth a build) but not fatal. A fatal one+ # raises instead, see test_a_fatal_mamba_model_on_a_cpu_or_xpu_torch_raises.+ mamba_ssm_fatal = False,+ )+ == []+ )+++def test_optional_mamba_ssm_is_skipped_below_the_setup_floor(monkeypatch):+ host = _Host(monkeypatch, env = _CUDA)monkeypatch.setattr(ssm_runtime,- "ensure_ssm_runtime",- lambda probe, **kwargs: seen.append(- (probe, kwargs["wants_causal_conv1d"], kwargs["is_ssm"])- ),+ "_mamba_ssm_floor_reason",+ lambda: "needs sm80, or sm75 with Triton 3.4+ (torch 2.8+), and Triton is 3.3",+ )+ ssm_runtime.ensure_ssm_runtime(+ "ai21labs/Jamba-v0.1",+ status_cb = host.statuses.append,+ run = host.run,+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = False,+ )+ assert host.wheels == ["https://x/causal_conv1d.whl"] # causal-conv1d only+ assert any("Skipping mamba-ssm" in message for message in host.statuses)+++def test_a_required_hybrid_ignores_the_floor_as_before(monkeypatch):+ host = _Host(monkeypatch, env = _CUDA)+ monkeypatch.setattr(+ ssm_runtime, "_mamba_ssm_floor_reason", lambda: pytest.fail("no floor for these"))- refreshed = []+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ ssm_runtime.ensure_ssm_runtime(+ "tiiuae/Falcon-H1-0.5B",+ run = host.run,+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = True,+ )+ assert len(host.wheels) == 2+++ "capability, triton, skipped",+ [((7, 5), (3, 3), True), ((7, 5), (3, 4), False), ((8, 0), None, False), (None, None, True)],+)+def test_the_runtime_floor_is_setups(monkeypatch, capability, triton, skipped):+ from utils import kernel_install++ monkeypatch.delitem(sys.modules, "torch", raising = False)+ monkeypatch.delenv("UNSLOTH_MAMBA_PRE_AMPERE_FAST", raising = False)+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: capability)+ monkeypatch.setattr(kernel_install, "_triton_version", lambda run: triton)+ assert (_REAL_MAMBA_SSM_FLOOR_REASON() is not None) is skipped+++def test_provisioning_reads_the_config_once_per_process(monkeypatch):+ reads = []monkeypatch.setattr(- ssm_runtime, "refresh_transformers_kernel_gates", lambda: refreshed.append(1)+ "utils.transformers_version._load_config_json",+ lambda target, token: reads.append(target) or {"model_type": "llama"},)- assert iw._ensure_ssm_kernels([("renamed", target)], resp_queue = None, hf_token = None)- assert seen == [("renamed", True, True)]- assert refreshed == [1]+ monkeypatch.setattr(ssm_runtime, "ensure_ssm_runtime", lambda *a, **k: [])+ for _ in range(3):+ ssm_runtime.provision_ssm_kernels("acme/llama", "acme/llama", None)+ assert reads == ["acme/llama"]- src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8")- assert "_ensure_ssm_kernels(ssm_targets, resp_queue, hf_token)" in src- assert "_ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token)" in src++def test_a_caller_with_the_config_skips_the_read(monkeypatch):+ monkeypatch.setattr(+ "utils.transformers_version._load_config_json",+ lambda *a, **k: pytest.fail("the caller already has the config"),+ )+ kernels = ssm_runtime.resolve_ssm_kernels(+ "acme/hybrid", "acme/hybrid", None, model_config = {"model_type": "nemotron_h"}+ )+ assert kernels.mamba_ssm_required+++def test_the_preflight_and_handle_load_share_one_failed_attempt(monkeypatch):+ # The inference worker provisions before its transformers import and again in _handle_load:+ # a failed optional build must not run twice (two 30-minute HIP builds before).+ host = _Host(+ monkeypatch, env = {"hip_version": "6.4", "platform_tag": "linux_x86_64"}, head = False+ )+ monkeypatch.setattr(ssm_runtime.shutil, "which", lambda name: "/usr/bin/" + name)+ monkeypatch.setattr(ssm_runtime, "_hipcc_gcc_install_dir", lambda: None)+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ for _stage in ("preflight", "_handle_load"):+ ssm_runtime.ensure_ssm_runtime(+ "ai21labs/Jamba-v0.1",+ run = host.run,+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = False,+ )+ assert len(host.builds) == 1 # one causal-conv1d build; mamba-ssm is wheel-only here++[email protected]("torch_mm", ["2.13", "2.14"])+def test_no_pinned_source_build_on_torch_213_and_later(monkeypatch, torch_mm):+ # causal-conv1d 1.6.1 / mamba-ssm 2.3.1 build at C++17; torch 2.13+ headers need C++20, so the+ # build is a certain multi-minute failure: only the prebuilt wheel serves these.+ host = _Host(monkeypatch, env = {**_CUDA, "torch_mm": torch_mm}, wheel_url = False)+ assert host.install() is False+ assert host.builds == []+ assert any("cannot be built from source" in message for message in host.statuses)+++def test_torch_212_keeps_the_source_fallback(monkeypatch):+ host = _Host(monkeypatch, env = {**_CUDA, "torch_mm": "2.12"}, head = False, build_ok = True)+ assert host.install() is True+ assert len(host.builds) == 1+++# ── when a missing mamba-ssm refuses the load (AMD hardware report, PR #13971) ──────────────────++_NVIDIA_REMOTE_NEMOTRON = """+from transformers.utils.import_utils import is_mamba_2_ssm_available++if is_mamba_2_ssm_available():+ from mamba_ssm.ops.triton.ssd_combined import mamba_chunk_scan_combined+else:+ mamba_chunk_scan_combined = None++try:+ from mamba_ssm.ops.triton.layernorm_gated import rmsnorm_fn+except ImportError:+ raise ImportError("mamba-ssm is required by the Mamba model but cannot be imported")+"""++_GATED_REMOTE = """+try:+ from mamba_ssm.ops.triton.layernorm_gated import rmsnorm_fn+except ImportError:+ rmsnorm_fn = None+if is_mamba_2_ssm_available():+ import mamba_ssm+"""+++def _checkpoint(+ tmp_path,+ model_type,+ auto_map = None,+ modeling = None,+ name = "ckpt",+):+ folder = tmp_path / name+ folder.mkdir()+ config = {"model_type": model_type}+ if auto_map:+ config["auto_map"] = auto_map+ (folder / "config.json").write_text(json.dumps(config))+ for filename, text in (modeling or {}).items():+ (folder / filename).write_text(text)+ return str(folder)+++def _failing_mamba(monkeypatch):+ attempts = []++ def _install(**kwargs):+ attempts.append((kwargs["import_name"], kwargs.get("allow_source_build")))+ return False++ monkeypatch.setattr(ssm_runtime, "_install_kernel", _install)+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ return attempts++[email protected]("model_type", ["nemotron_h", "falcon_h1", "granitemoehybrid"])+def test_a_native_required_hybrid_loads_with_a_warning(monkeypatch, tmp_path, model_type):+ # The native transformers model runs without mamba-ssm (it trains on Windows ROCm with none):+ # still worth a source build, but a failure is a warning, not a refused load.+ attempts = _failing_mamba(monkeypatch)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)+ statuses = []+ kernels, installed = ssm_runtime.provision_ssm_kernels(+ "nvidia/NVIDIA-Nemotron-Nano-9B-v2",+ _checkpoint(tmp_path, model_type),+ None,+ status_cb = statuses.append,+ )+ assert kernels.mamba_ssm_required and not kernels.mamba_ssm_fatal+ assert ("mamba_ssm", True) in attempts # the source fallback is kept for these three+ assert installed == []+ assert any("Warning: could not install mamba-ssm" in message for message in statuses)+++def test_remote_code_that_imports_mamba_ssm_still_refuses(monkeypatch, tmp_path):+ _failing_mamba(monkeypatch)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)+ target = _checkpoint(+ tmp_path,+ "nemotron_h",+ auto_map = {"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"},+ modeling = {"modeling_nemotron_h.py": _NVIDIA_REMOTE_NEMOTRON},+ )+ with pytest.raises(RuntimeError, match = "Could not install mamba-ssm"):+ ssm_runtime.provision_ssm_kernels(target, target, None, trust_remote_code = True)+++def test_the_same_checkpoint_without_trust_loads_natively(monkeypatch, tmp_path):+ # auto_map is only followed under trust_remote_code (or when transformers lacks the model).+ _failing_mamba(monkeypatch)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)+ target = _checkpoint(+ tmp_path,+ "nemotron_h",+ auto_map = {"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"},+ modeling = {"modeling_nemotron_h.py": _NVIDIA_REMOTE_NEMOTRON},+ )+ kernels, _ = ssm_runtime.provision_ssm_kernels(target, target, None, trust_remote_code = False)+ assert not kernels.mamba_ssm_fatal+++def test_remote_code_with_a_guarded_import_loads(monkeypatch, tmp_path):+ _failing_mamba(monkeypatch)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)+ target = _checkpoint(+ tmp_path,+ "nemotron_h",+ auto_map = {"AutoModelForCausalLM": "model.CustomForCausalLM"},+ modeling = {"model.py": _GATED_REMOTE},+ )+ kernels, _ = ssm_runtime.provision_ssm_kernels(target, target, None, trust_remote_code = True)+ assert not kernels.mamba_ssm_fatal+++def test_unreadable_remote_code_is_treated_as_requiring_it(monkeypatch, tmp_path):+ _failing_mamba(monkeypatch)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)+ target = _checkpoint(+ tmp_path,+ "falcon_h1",+ auto_map = {"AutoModelForCausalLM": "modeling_falcon_h1.FalconH1ForCausalLM"},+ )+ with pytest.raises(RuntimeError):+ ssm_runtime.provision_ssm_kernels(target, target, None, trust_remote_code = True)++[email protected]("model_type", ["nemotron_h", "falcon_h1"])+def test_a_transformers_without_the_native_model_still_refuses(monkeypatch, tmp_path, model_type):+ _failing_mamba(monkeypatch)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: False)+ target = _checkpoint(tmp_path, model_type)+ with pytest.raises(RuntimeError, match = "Could not install mamba-ssm"):+ ssm_runtime.provision_ssm_kernels(target, target, None)+++def test_windows_without_triton_refuses_only_a_fatal_load(monkeypatch, tmp_path):+ monkeypatch.setattr(ssm_runtime.sys, "platform", "win32")+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ monkeypatch.setattr(+ ssm_runtime.importlib.util,+ "find_spec",+ lambda name, *a, **k: None if name == "triton" else object(),+ )+ statuses = []+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)+ ssm_runtime.provision_ssm_kernels(+ "x", _checkpoint(tmp_path, "nemotron_h", name = "a"), None, status_cb = statuses.append+ )+ assert any("triton-windows" in message for message in statuses)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: False)+ with pytest.raises(RuntimeError, match = "triton-windows"):+ ssm_runtime.provision_ssm_kernels("x", _checkpoint(tmp_path, "nemotron_h", name = "b"), None)+++def test_native_modeling_is_read_from_the_installed_transformers(monkeypatch, tmp_path):+ models = tmp_path / "transformers" / "models"+ (models / "falcon_h1").mkdir(parents = True)+ (models / "falcon_h1" / "modeling_falcon_h1.py").write_text("")+ (models / "nemotron_h").mkdir()+ (models / "nemotron_h" / "configuration_nemotron_h.py").write_text("")+ monkeypatch.setattr(ssm_runtime, "_transformers_models_dir", lambda: models)+ monkeypatch.setattr(ssm_runtime, "_NATIVE_MODELING_CACHE", {})+ assert ssm_runtime.native_modeling_available("falcon_h1") is True+ assert ssm_runtime.native_modeling_available("nemotron_h") is False # config only+ assert ssm_runtime.native_modeling_available("granitemoehybrid") is False+++ "source, fatal",+ [+ ("import mamba_ssm\n", True),+ ("from mamba_ssm.ops import x\n", True),+ (_NVIDIA_REMOTE_NEMOTRON, True),+ (_GATED_REMOTE, False),+ ("def f():\n import mamba_ssm\n", False),+ ("import torch\n", False),+ ("def broken(:\n", True),+ ],+)+def test_the_unconditional_import_scan(source, fatal):+ assert ssm_runtime._imports_mamba_ssm_unconditionally(source) is fatal+++# ── final-audit items (PR #13971) ────────────────────────────────────────────────────────────────+++def test_a_native_granite_4_tiny_preview_loads_with_a_warning(monkeypatch, tmp_path):+ # ibm-granite/granite-4.0-tiny-preview: model_type granitemoehybrid, native in transformers.+ _failing_mamba(monkeypatch)+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: t == "granitemoehybrid")+ statuses = []+ target = _checkpoint(tmp_path, "granitemoehybrid", name = "granite-4.0-tiny-preview")+ kernels, _ = ssm_runtime.provision_ssm_kernels(+ "ibm-granite/granite-4.0-tiny-preview", target, None, status_cb = statuses.append+ )+ assert kernels.mamba_ssm_required and not kernels.mamba_ssm_fatal+ assert any("Warning: could not install mamba-ssm" in message for message in statuses)+++def test_a_transient_config_miss_is_read_again(monkeypatch):+ reads = iter([None, {"model_type": "qwen3_5"}])+ calls = []++ def _load(target, token):+ calls.append(target)+ return next(reads)++ monkeypatch.setattr("utils.transformers_version._load_config_json", _load)+ assert ssm_runtime.resolve_ssm_kernels("acme/m", "acme/m", None).causal_conv1d is False+ assert ssm_runtime.resolve_ssm_kernels("acme/m", "acme/m", None).causal_conv1d is True+ assert ssm_runtime.resolve_ssm_kernels("acme/m", "acme/m", None).causal_conv1d is True+ assert calls == ["acme/m", "acme/m"] # the successful read is the one remembered+++def test_a_kernel_a_peer_installed_counts_as_new_to_this_worker(monkeypatch):+ from utils import kernel_install++ host = _Host(monkeypatch, env = _CUDA)++ @contextlib.contextmanager+ def _lock(name, **kwargs):+ host.imports.add(name) # a peer installed it while this process waited+ yield True++ monkeypatch.setattr(kernel_install, "kernel_install_lock", _lock)+ newly = []+ assert ssm_runtime._install_kernel(**_kernel_kwargs(), run = host.run, newly_installed = newly)+ assert newly == ["causal_conv1d"] # so inference refreshes or restarts its modeling modules++[email protected]("required, budget", [(False, 120), (True, None)])+def test_optional_accelerations_wait_briefly_for_a_peer(monkeypatch, required, budget):+ from utils import kernel_install++ budget = kernel_install.KERNEL_LOCK_WAIT if budget is None else budget++ seen = {}++ @contextlib.contextmanager+ def _lock(name, **kwargs):+ seen.update(kwargs)+ yield False++ host = _Host(monkeypatch, env = _CUDA)+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _lock)+ host.install(required = required)+ assert seen["wait_seconds"] == budget+++def test_ensure_waits_long_only_for_the_required_families(monkeypatch):+ budgets = {}+ monkeypatch.setattr(+ ssm_runtime,+ "_install_kernel",+ lambda **k: budgets.setdefault((k["import_name"], k["required"]), True),+ )+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ ssm_runtime.ensure_ssm_runtime(+ "q", wants_causal_conv1d = True, is_ssm = False, mamba_ssm_required = False+ )+ ssm_runtime.ensure_ssm_runtime(+ "f", wants_causal_conv1d = True, is_ssm = True, mamba_ssm_required = True+ )+ assert set(budgets) == {("causal_conv1d", False), ("causal_conv1d", True), ("mamba_ssm", True)}+++def test_an_installer_that_cannot_start_still_cleans_up_and_is_remembered(monkeypatch):+ host = _Host(monkeypatch, env = _CUDA, head = True)+ host.installed.add("causal_conv1d") # a broken copy left by an earlier attempt++ def _cannot_start(url, **kwargs):+ raise OSError("uv: No such file or directory")++ monkeypatch.setattr(ssm_runtime, "install_wheel", _cannot_start)+ assert host.install() is False+ assert any("uninstall" in cmd for cmd in host.commands) # cleanup ran+ assert host.install() is False+ assert len([cmd for cmd in host.commands if "uninstall" in cmd]) == 1 # not retried+++# ── A fatal mamba-ssm model on a torch with no kernel backend fails clearly ───++[email protected]("env", [{"cuda_version": "", "platform_tag": "linux_x86_64"}, {}])+def test_a_fatal_mamba_model_on_a_cpu_or_xpu_torch_raises(monkeypatch, env):+ # Its remote code imports mamba_ssm, so the torch path cannot load it: refuse with the reason.+ monkeypatch.setattr(ssm_runtime, "probe_torch_wheel_env", lambda **k: dict(env))+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ messages = []+ with pytest.raises(RuntimeError) as raised:+ ssm_runtime.ensure_ssm_runtime(+ "acme/remote-hybrid",+ status_cb = messages.append,+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = True,+ mamba_ssm_fatal = True,+ )+ assert "requires mamba-ssm, which needs a CUDA or ROCm torch build" in str(raised.value)+ assert "offline" not in str(raised.value)+ assert not any("torch path" in message for message in messages)+++def test_a_fatal_mamba_model_on_a_cuda_torch_keeps_the_offline_hint(monkeypatch):+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ with pytest.raises(RuntimeError) as raised:+ ssm_runtime.ensure_ssm_runtime(+ "acme/remote-hybrid",+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = True,+ mamba_ssm_fatal = True,+ )+ assert "Could not install mamba-ssm" in str(raised.value)+ assert "unsloth install-kernels" in str(raised.value)+++def test_a_non_fatal_required_model_on_a_cpu_torch_still_warns_and_loads(monkeypatch):+ monkeypatch.setattr(+ ssm_runtime, "probe_torch_wheel_env", lambda **k: {"cuda_version": "", "platform_tag": "x"}+ )+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())+ messages = []+ ssm_runtime.ensure_ssm_runtime(+ "acme/native-hybrid",+ status_cb = messages.append,+ wants_causal_conv1d = True,+ is_ssm = True,+ mamba_ssm_required = True,+ mamba_ssm_fatal = False,+ ) # no raise+ assert any("torch path" in message for message in messages)diff --git a/studio/backend/tests/test_training_force_terminate_builds.py b/studio/backend/tests/test_training_force_terminate_builds.pynew file mode 100644index 00000000000..fc2db6cf4b0--- /dev/null+++ b/studio/backend/tests/test_training_force_terminate_builds.py@@ -0,0 +1,91 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Force-terminating a training run also stops the processes its worker started: a kernel source+build runs in its own session (so a timeout can stop its compilers together) and would otherwise+outlive the worker, keep compiling, and leave its install lock free for a peer to race."""++import os+import subprocess+import sys+import threading+import time+from types import SimpleNamespace++import pytest++from core.training import training+++def _alive(pid: int) -> bool:+ try:+ os.kill(pid, 0)+ except ProcessLookupError:+ return False+ try:+ with open(f"/proc/{pid}/stat") as stat:+ return stat.read().split()[2] != "Z"+ except OSError:+ return True++[email protected](os.name == "nt", reason = "POSIX sessions")+def test_a_build_in_its_own_session_dies_with_the_run(tmp_path):+ pid_file = tmp_path / "build.pid"+ worker_script = (+ "import subprocess, sys, time; "+ "build = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'], "+ "start_new_session = True); "+ f"open({str(pid_file)!r}, 'w').write(str(build.pid)); "+ "time.sleep(120)"+ )+ worker = subprocess.Popen([sys.executable, "-c", worker_script])+ try:+ for _ in range(100):+ if pid_file.exists() and pid_file.read_text():+ break+ time.sleep(0.05)+ build = int(pid_file.read_text())+ assert _alive(build)+ descendants = training._terminate_descendants(worker.pid)+ assert build in {child.pid for child in descendants}+ worker.terminate()+ worker.wait(timeout = 10)+ training._reap_descendants(descendants)+ for _ in range(50):+ if not _alive(build):+ break+ time.sleep(0.1)+ assert not _alive(build), "the build outlived the force-terminated worker"+ finally:+ if worker.poll() is None:+ worker.kill()+ if pid_file.exists() and pid_file.read_text() and _alive(int(pid_file.read_text())):+ os.kill(int(pid_file.read_text()), 9)+++def test_force_terminate_stops_descendants_before_the_worker(monkeypatch):+ order = []+ proc = SimpleNamespace(+ pid = 4242,+ is_alive = lambda: True,+ terminate = lambda: order.append("worker"),+ join = lambda timeout = None: None,+ kill = lambda: order.append("kill"),+ )+ backend = training.TrainingBackend.__new__(training.TrainingBackend)+ backend.__dict__.update(+ _lock = threading.Lock(),+ _proc = proc,+ _cancel_requested = False,+ _cancel_cleanup_output_dir = None,+ _output_dir = None,+ _pump_thread = None,+ )+ monkeypatch.setattr(+ training, "_terminate_descendants", lambda pid: order.append(("descendants", pid)) or ["c"]+ )+ monkeypatch.setattr(training, "_reap_descendants", lambda d: order.append(("reap", tuple(d))))+ backend.force_terminate()+ assert order[0] == ("descendants", 4242) and order[1] == "worker"+ assert ("reap", ("c",)) in orderdiff --git a/studio/backend/tests/test_training_worker_flash_attn.py b/studio/backend/tests/test_training_worker_flash_attn.pyindex 08993cf03ea..280269b5054 100644--- a/studio/backend/tests/test_training_worker_flash_attn.py+++ b/studio/backend/tests/test_training_worker_flash_attn.py@@ -4,6 +4,7 @@from __future__ import annotationsimport ast+from types import SimpleNamespaceimport builtinsimport importlib.metadataimport inspect@@ -33,9 +34,37 @@ not_on_windows = pytest.mark.skipif(@pytest.fixture(autouse = True)-def _clear_offline_environment(monkeypatch):+def _clear_offline_environment(monkeypatch, tmp_path):monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)+ monkeypatch.delenv("UV_OFFLINE", raising = False)+ # A CUDA torch with nothing published for it unless a test says otherwise: the host's own+ # torch and the network must not decide the tests.+ monkeypatch.setattr(+ worker,+ "probe_torch_wheel_env",+ lambda timeout = 30, include_windows = False: {+ "python_tag": "cp313",+ "abi_tag": "cp313",+ "torch_mm": "0.0",+ "torch_version": "0.0.0",+ "cuda_version": "12.8",+ "cuda_major": "12",+ "hip_version": "",+ "cxx11abi": "TRUE",+ "platform_tag": "linux_x86_64",+ },+ )+ monkeypatch.setattr(worker, "url_exists", lambda url: False)+ # Per-process memo and the environment's lock file: each test is its own process here.+ from utils import kernel_install, ssm_runtime++ monkeypatch.setattr(ssm_runtime, "_FAILED_INSTALLS", set())+ monkeypatch.setattr(+ kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / f"{name}.lock")+ )+ # Every copy of the module (and any child process) uses the test's folder too.+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", str(tmp_path / "locks"))@pytest.fixture(autouse = True)@@ -530,7 +559,10 @@ def test_runtime_flash_attn_skip_env_avoids_all_install_work(monkeypatch):worker._sp.run.assert_not_called()[email protected]("offline_variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"])+# UV_OFFLINE too: uv refuses the index and install_wheel's pip fallback would not.+ "offline_variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "UV_OFFLINE"]+)def test_wheel_first_install_skips_all_install_work_offline(monkeypatch, offline_variable):monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)@@ -603,22 +635,11 @@ def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch)@not_on_windows-def test_causal_conv1d_fast_path_skips_qwen3_6_variants_on_zoo_triton_conv(monkeypatch):- # unsloth-zoo runs Qwen3.5 / 3.6 / Qwen3-Next on its own Triton causal conv (on a CUDA torch- # with Triton, pinned here so a CPU-only CI host answers the same).- from utils import ssm_runtime-+def test_causal_conv1d_fast_path_includes_qwen3_6_variants(monkeypatch):+ # The native kernel: zoo's Triton conv has no decode update and a sequence-sized dW scratch.install_mock = mock.Mock(return_value = True)monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)- monkeypatch.setattr(ssm_runtime, "_zoo_triton_platform_usable", lambda: True)- monkeypatch.delenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", raising = False)-- for model_name in ("unsloth/Qwen3.6-4B", "unsloth/Qwen3_6-4B"):- worker._ensure_causal_conv1d_fast_path(event_queue = [], model_name = model_name)- assert install_mock.call_count == 0- # zoo's kill switch puts them back on the prebuilt causal-conv1d.- monkeypatch.setenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", "1")for model_name in ("unsloth/Qwen3.6-4B", "unsloth/Qwen3_6-4B"):worker._ensure_causal_conv1d_fast_path(event_queue = [], model_name = model_name)assert install_mock.call_count == 2@@ -627,6 +648,9 @@ def test_causal_conv1d_fast_path_skips_qwen3_6_variants_on_zoo_triton_conv(monkedef test_mamba_ssm_follows_the_resolved_answer_not_the_name(monkeypatch):install_mock = mock.Mock(return_value = True)monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)+ # Not the sm80 floor under test: keep a GPU-less runner from skipping the optional family.+ monkeypatch.setattr("utils.ssm_runtime._mamba_ssm_floor_reason", lambda: None)+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)# A real hybrid whose id lacks the substrings (resolved from its config) still installs.worker._ensure_mamba_ssm(event_queue = [], model_name = "nvidia/NVIDIA-Nemotron-Nano-9B-v2", required = True@@ -658,6 +682,8 @@ def test_mamba_ssm_path_preserves_wheel_first_install_args(monkeypatch):filename_prefix = "mamba_ssm",release_tag = worker.MAMBA_SSM.release_tag,release_base_url = "https://github.com/state-spaces/mamba/releases/download",+ allow_source_build = True,+ required = True,)@@ -1005,7 +1031,9 @@ def test_no_install_path_pips_flash_linear_attention_or_tilelang(monkeypatch, uvmonkeypatch.setattr(worker, "_is_importable", lambda name: False)monkeypatch.setattr(worker, "_is_importable_isolated", lambda name: True)monkeypatch.setattr(- worker, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}+ worker,+ "probe_torch_wheel_env",+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},)monkeypatch.setattr(worker, "url_exists", lambda url: False)monkeypatch.setattr(worker, "install_wheel", mock.Mock())@@ -1530,7 +1558,10 @@ def test_windows_mamba_ssm_needs_triton(monkeypatch):install_mock = mock.Mock(return_value = True)monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)monkeypatch.setattr(worker.importlib.util, "find_spec", lambda name, *a: None)- worker._ensure_mamba_ssm(event_queue = [], model_name = "tiiuae/Falcon-H1-0.5B-Instruct")+ # A family with a torch path skips it; one that needs it stops (its own test below).+ worker._ensure_mamba_ssm(+ event_queue = [], model_name = "ai21labs/Jamba-v0.1", required = True, essential = False+ )install_mock.assert_not_called()assert any("triton-windows" in message for message in statuses)@@ -1596,7 +1627,7 @@ def test_windows_installs_a_published_wheel(monkeypatch):@pytest.mark.parametrize(("wheel_url", "exists"),- [(None, False), ("https://x/fa.whl", False), ("https://x/fa.whl", None)],+ [(None, False), ("https://x/fa.whl", False)],)def test_windows_never_falls_back_to_pypi(monkeypatch, wheel_url, exists):ok, runs, installed = _windows_attempt(@@ -1612,3 +1643,427 @@ def test_windows_rejected_wheel_does_not_fall_back_to_pypi(monkeypatch):)assert ok is False and installed is Trueassert runs == []++[email protected]("capability, installs", [((7, 5), False), ((8, 0), True), ((12, 0), True)])+def test_long_context_flash_attn_is_not_downloaded_below_sm80(monkeypatch, capability, installs):+ from utils import kernel_install++ statuses: list[str] = []+ install_mock = mock.Mock(return_value = True)+ monkeypatch.setattr(worker, "_should_try_runtime_flash_attn_install", lambda length: True)+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ # No torch in this worker yet: the child probe answers.+ monkeypatch.delitem(worker.sys.modules, "torch", raising = False)+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: capability)+ monkeypatch.setattr(kernel_install, "_torch_is_hip", lambda run: False)+ monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))+ worker._ensure_flash_attn_for_long_context(event_queue = [], max_seq_length = 65536)+ assert install_mock.called is installs+ if not installs:+ assert any("sm80" in message for message in statuses)+++def _fake_torch(capabilities, hip = None):+ cuda = SimpleNamespace(+ is_available = lambda: bool(capabilities),+ device_count = lambda: len(capabilities),+ get_device_capability = lambda i: capabilities[i],+ )+ return SimpleNamespace(version = SimpleNamespace(hip = hip), cuda = cuda)+++def _no_child_probe(*args, **kwargs):+ raise AssertionError("the worker's own torch answers; no child probe")+++def test_an_importable_flash_attn_skips_the_capability_probe(monkeypatch):+ from utils import kernel_install++ install_mock = mock.Mock(return_value = True)+ monkeypatch.setattr(worker, "_should_try_runtime_flash_attn_install", lambda length: True)+ monkeypatch.setattr(worker, "_is_importable", lambda name: name == "flash_attn")+ monkeypatch.setattr(kernel_install, "_gpu_capability", _no_child_probe)+ monkeypatch.setattr(worker, "_flash_attn_gpu_below_floor", _no_child_probe)+ monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)+ worker._ensure_flash_attn_for_long_context(event_queue = [], max_seq_length = 65536)+ assert not install_mock.called+++ "capabilities, hip, below",+ [+ ([(7, 5)], None, True),+ ([(7, 5), (8, 6)], None, False),+ ([(9, 0)], None, False),+ ([], None, False),+ # ROCm reports a gfx version: never the NVIDIA floor.+ ([(7, 5)], "6.4.0", False),+ ],+)+def test_the_floor_reuses_the_workers_own_torch(monkeypatch, capabilities, hip, below):+ from utils import kernel_install++ monkeypatch.setitem(worker.sys.modules, "torch", _fake_torch(capabilities, hip))+ monkeypatch.setattr(kernel_install, "_gpu_capability", _no_child_probe)+ assert worker._flash_attn_gpu_below_floor() is below+++def test_rocm_long_context_still_installs_flash_attn(monkeypatch):+ install_mock = mock.Mock(return_value = True)+ monkeypatch.setattr(worker, "_should_try_runtime_flash_attn_install", lambda length: True)+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ monkeypatch.setitem(worker.sys.modules, "torch", _fake_torch([(7, 5)], "6.4.0"))+ monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)+ worker._ensure_flash_attn_for_long_context(event_queue = [], max_seq_length = 65536)+ assert install_mock.called+++# An unanswered HEAD (a proxy refusing HEAD but serving the download) is not "unpublished": setup+# always tried the wheel then, and only a real download / import failure falls back.+++def test_windows_an_unanswered_head_still_downloads_the_wheel(monkeypatch):+ ok, runs, installed = _windows_attempt(+ monkeypatch,+ wheel_url = "https://x/fa-win_amd64.whl",+ exists = None,+ importable_after_install = True,+ )+ assert ok is True and installed is True+ assert runs == []+++def test_linux_an_unanswered_head_installs_the_wheel_not_a_source_build(monkeypatch):+ monkeypatch.setattr(worker.sys, "platform", "linux")+ # A cell upstream publishes, so the resolver names a wheel.+ monkeypatch.setattr(+ worker,+ "probe_torch_wheel_env",+ lambda **k: {+ "python_tag": "cp313",+ "torch_mm": "2.10",+ "cuda_version": "12.8",+ "cuda_major": "12",+ "cxx11abi": "TRUE",+ "platform_tag": "linux_x86_64",+ },+ )+ monkeypatch.setattr(worker, "url_exists", lambda url: None)+ installs = []+ monkeypatch.setattr(+ worker,+ "install_wheel",+ lambda url, **k: installs.append(url)+ or [("uv", subprocess.CompletedProcess(["uv"], 0, ""))],+ )+ monkeypatch.setattr(worker, "_is_importable_isolated", lambda name: bool(installs))+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: None)+ builds = []+ monkeypatch.setattr(+ worker._sp,+ "run",+ lambda cmd, **k: builds.append(cmd) or subprocess.CompletedProcess(cmd, 0, ""),+ )+ ok = worker._attempt_package_install(+ event_queue = [],+ **worker._pinned_kernel_kwargs(worker.MAMBA_SSM),+ )+ assert ok is True+ assert len(installs) == 1 and builds == []+++# Optional mamba-ssm (Mamba, Jamba, Zamba, Bamba...): a prebuilt wheel or the torch path, never a+# source build; the families that cannot load without it keep the source fallback.++[email protected]("essential", [False, True])+def test_optional_mamba_ssm_never_source_builds(monkeypatch, essential):+ monkeypatch.setattr(worker.sys, "platform", "linux")+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ monkeypatch.setattr(worker, "_is_importable_isolated", lambda name: False)+ monkeypatch.setattr("utils.ssm_runtime._mamba_ssm_floor_reason", lambda: None)+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: None)+ monkeypatch.setattr(worker, "_reject_install", lambda *a, **k: None)+ monkeypatch.setattr(worker.shutil, "which", lambda name: None)+ builds = []+ monkeypatch.setattr(+ worker._sp,+ "run",+ lambda cmd, **k: builds.append(cmd) or subprocess.CompletedProcess(cmd, 1, ""),+ )+ worker._ensure_mamba_ssm([], "ai21labs/Jamba-v0.1", required = True, essential = essential)+ assert bool([cmd for cmd in builds if "mamba-ssm==" in " ".join(cmd)]) is essential+++def test_optional_mamba_ssm_is_skipped_below_the_setup_floor(monkeypatch):+ statuses = []+ install = mock.Mock()+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ monkeypatch.setattr(+ "utils.ssm_runtime._mamba_ssm_floor_reason",+ lambda: "needs sm80, or sm75 with Triton 3.4+ (torch 2.8+), and Triton is 3.3",+ )+ monkeypatch.setattr(worker, "_install_package_wheel_first", install)+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))+ worker._ensure_mamba_ssm([], "ai21labs/Jamba-v0.1", required = True, essential = False)+ assert not install.called+ assert any("sm75 with Triton 3.4" in message for message in statuses)+++def test_a_required_hybrid_on_a_t4_still_installs_mamba_ssm(monkeypatch):+ # As before setup stopped provisioning: Nemotron-H / Falcon-H1 / Granite-4.0-H install it on+ # any GPU (the runtime never applied setup's floor to them).+ install = mock.Mock()+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ monkeypatch.setattr(+ "utils.ssm_runtime._mamba_ssm_floor_reason", lambda: pytest.fail("no floor for these")+ )+ monkeypatch.setattr(worker, "_install_package_wheel_first", install)+ worker._ensure_mamba_ssm([], "tiiuae/Falcon-H1-0.5B", required = True, essential = True)+ assert install.called+++ "env",+ [+ None, # the probe failed+ {"cuda_version": "", "hip_version": "", "platform_tag": "linux_x86_64"}, # CPU / XPU torch+ ],+)+def test_no_cuda_or_rocm_torch_installs_nothing(monkeypatch, env):+ monkeypatch.setattr(worker, "probe_torch_wheel_env", lambda **k: env)+ statuses = []+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))+ install = mock.Mock()+ monkeypatch.setattr(worker, "install_wheel", install)+ builds = []+ monkeypatch.setattr(worker._sp, "run", lambda cmd, **k: builds.append(cmd))+ assert (+ worker._attempt_package_install(+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)+ )+ is False+ )+ assert not install.called and builds == []+ assert any("CUDA or ROCm" in message for message in statuses)+++def test_the_training_install_runs_under_the_kernel_lock(monkeypatch):+ from utils import kernel_install++ entered = []++ class _Lock:+ def __init__(self, name, **kwargs):+ entered.append(name)++ def __enter__(self):+ return False # another process kept it++ def __exit__(self, *exc):+ return False++ monkeypatch.setattr(kernel_install, "kernel_install_lock", _Lock)+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ attempt = mock.Mock()+ monkeypatch.setattr(worker, "_attempt_package_install", attempt)+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: None)+ assert (+ worker._install_package_wheel_first(+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)+ )+ is False+ )+ assert entered == ["causal_conv1d"] and not attempt.called+++def test_a_peer_finishing_the_install_while_waiting_is_success(monkeypatch):+ states = iter([False, True]) # before the lock, then after it+ monkeypatch.setattr(worker, "_is_importable", lambda name: next(states))+ attempt = mock.Mock()+ monkeypatch.setattr(worker, "_attempt_package_install", attempt)+ assert worker._install_package_wheel_first(+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)+ )+ assert not attempt.called++[email protected]("torch_mm", ["2.13", "2.14"])+def test_no_pinned_source_build_on_torch_213_and_later(monkeypatch, torch_mm):+ monkeypatch.setattr(worker.sys, "platform", "linux")+ monkeypatch.setattr(+ worker,+ "probe_torch_wheel_env",+ lambda **k: {"cuda_version": "12.8", "cuda_major": "12", "torch_mm": torch_mm},+ )+ monkeypatch.setattr(worker, "direct_wheel_url", lambda **k: None)+ statuses = []+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))+ builds = []+ monkeypatch.setattr(worker._sp, "run", lambda cmd, **k: builds.append(cmd))+ assert (+ worker._attempt_package_install(+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)+ )+ is False+ )+ assert builds == []+ assert any("cannot be built from source" in message for message in statuses)+++# Training stays best effort as at 9f5b13325, even for the hybrids that need mamba-ssm: their+# native transformers path can run without it, so a failed install never stops the run here.+++def test_a_required_hybrid_without_mamba_ssm_does_not_stop_training(monkeypatch):+ monkeypatch.setattr(worker.sys, "platform", "linux")+ monkeypatch.setattr(worker, "_install_package_wheel_first", lambda **k: False)+ monkeypatch.setattr(worker, "probe_torch_wheel_env", lambda **k: {"cuda_version": "12.8"})+ worker._ensure_mamba_ssm([], "tiiuae/Falcon-H1-0.5B", required = True, essential = True)+++def test_a_required_hybrid_on_windows_without_triton_does_not_stop_training(monkeypatch):+ monkeypatch.setattr(worker.sys, "platform", "win32")+ monkeypatch.setattr(+ worker.importlib.util, "find_spec", lambda name, *a: None if name == "triton" else object()+ )+ install = mock.Mock()+ monkeypatch.setattr(worker, "_install_package_wheel_first", install)+ statuses = []+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))+ worker._ensure_mamba_ssm([], "tiiuae/Falcon-H1-0.5B", required = True, essential = True)+ assert not install.called+ assert any("triton-windows" in message for message in statuses)+++ "essential, env",+ [+ (False, {"cuda_version": "12.8"}), # optional: the torch path+ (True, {"cuda_version": "", "hip_version": ""}), # CPU / XPU torch: nothing to install+ ],+)+def test_no_stop_for_optional_families_or_a_non_cuda_torch(monkeypatch, essential, env):+ monkeypatch.setattr(worker.sys, "platform", "linux")+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ monkeypatch.setattr("utils.ssm_runtime._mamba_ssm_floor_reason", lambda: None)+ monkeypatch.setattr(worker, "_install_package_wheel_first", lambda **k: False)+ monkeypatch.setattr(worker, "probe_torch_wheel_env", lambda **k: env)+ worker._ensure_mamba_ssm([], "ai21labs/Jamba-v0.1", required = True, essential = essential)+++def test_the_eager_install_and_the_hook_share_one_failed_attempt(monkeypatch):+ # A failed (possibly 30-minute HIP) build is never repeated in one worker, nor after a chat+ # load in the same process already failed it.+ attempts = []+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ monkeypatch.setattr(+ worker, "_attempt_package_install", lambda **k: attempts.append(k["import_name"]) or False+ )+ monkeypatch.setattr(worker, "_reject_install", lambda *a, **k: None)+ for _caller in ("eager", "hook"):+ assert (+ worker._install_package_wheel_first(+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)+ )+ is False+ )+ assert attempts == ["causal_conv1d"]++ from utils import ssm_runtime++ ssm_runtime._FAILED_INSTALLS.add(ssm_runtime._failure_key("mamba_ssm"))+ worker._install_package_wheel_first(+ event_queue = [], **worker._pinned_kernel_kwargs(worker.MAMBA_SSM)+ )+ assert attempts == ["causal_conv1d"]++[email protected]("required, budget", [(False, 120), (True, None)])+def test_training_waits_long_only_for_a_required_kernel(monkeypatch, required, budget):+ import contextlib++ from utils import kernel_install++ if budget is None:+ budget = kernel_install.KERNEL_LOCK_WAIT++ seen = {}++ @contextlib.contextmanager+ def _lock(name, **kwargs):+ seen.update(kwargs)+ yield False++ monkeypatch.setattr(kernel_install, "kernel_install_lock", _lock)+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: None)+ worker._install_package_wheel_first(+ event_queue = [], required = required, **worker._pinned_kernel_kwargs(worker.MAMBA_SSM)+ )+ assert seen["wait_seconds"] == budget+++# Real ROCm gfx1151: the training block compiled mamba-ssm from source for a Mamba2 model (60 s)+# where the inference path stayed wheel-only. Driven as the hardware harness drives it.+++def _hip_mamba2_host(monkeypatch, tmp_path):+ import json++ model = tmp_path / "tiny-mamba2"+ model.mkdir()+ (model / "config.json").write_text(json.dumps({"model_type": "mamba2"}))+ monkeypatch.setattr(worker.sys, "platform", "linux")+ monkeypatch.setattr(+ worker,+ "probe_torch_wheel_env",+ lambda **k: {+ "hip_version": "7.1",+ "torch_mm": "2.10",+ "python_tag": "cp312",+ "cxx11abi": "TRUE",+ "platform_tag": "linux_x86_64",+ },+ )+ monkeypatch.setattr(worker.shutil, "which", lambda name: f"/usr/bin/{name}")+ monkeypatch.setattr(worker, "_hipcc_gcc_install_dir", lambda: None)+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)+ monkeypatch.setattr(worker, "_is_importable_isolated", lambda name: False)+ monkeypatch.setattr("utils.ssm_runtime._mamba_ssm_floor_reason", lambda: None)+ statuses = []+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))+ monkeypatch.setattr(worker, "_reject_install", lambda *a, **k: None)+ builds = []+ monkeypatch.setattr(+ worker._sp,+ "run",+ lambda cmd, **k: builds.append(list(cmd)) or subprocess.CompletedProcess(cmd, 1, ""),+ )+ return str(model), builds, statuses++[email protected]("caller", ["harness", "worker"])+def test_a_mamba2_model_on_rocm_never_builds_mamba_ssm_in_training(monkeypatch, tmp_path, caller):+ from utils import ssm_runtime++ model, builds, statuses = _hip_mamba2_host(monkeypatch, tmp_path)+ kernels = ssm_runtime.resolve_ssm_kernels(model, model, None, adapter_base = None)+ assert kernels.mamba_ssm and not kernels.mamba_ssm_required+ if caller == "harness":+ # scripts/v4_amd/ondemand_probe.py: no `essential`, as an older caller would.+ worker._ensure_mamba_ssm([], model, required = kernels.mamba_ssm)+ else:+ worker._ensure_mamba_ssm(+ [], model, required = kernels.mamba_ssm, essential = kernels.mamba_ssm_required+ )+ assert not [cmd for cmd in builds if "mamba-ssm==" in " ".join(cmd)]+ assert not any("Compiling mamba-ssm" in message for message in statuses)+++def test_a_required_hybrid_on_rocm_keeps_its_mamba_ssm_source_build(monkeypatch, tmp_path):+ _, builds, statuses = _hip_mamba2_host(monkeypatch, tmp_path)+ worker._ensure_mamba_ssm([], "tiiuae/Falcon-H1-0.5B", required = True)+ assert [cmd for cmd in builds if "mamba-ssm==" in " ".join(cmd)]diff --git a/studio/backend/tests/test_wheel_utils_windows_kernels.py b/studio/backend/tests/test_wheel_utils_windows_kernels.pyindex 648f7cd38a8..2e9861deb0e 100644--- a/studio/backend/tests/test_wheel_utils_windows_kernels.py+++ b/studio/backend/tests/test_wheel_utils_windows_kernels.py@@ -167,7 +167,17 @@ def woa_env(**overrides):)[email protected]("python_tag", ["cp311", "cp312", "cp313"])[email protected]("python_tag", ["cp311", "cp312"])+def test_win_arm64_unpublished_interpreters_resolve_to_nothing(python_tag):+ # The release carries cp313 only for win_arm64; a cp311 / cp312 URL would be a 404.+ e = woa_env(python_tag = python_tag, abi_tag = python_tag)+ assert kernel_install.CAUSAL_CONV1D.wheel_url(e) is None+ assert kernel_install.MAMBA_SSM.wheel_url(e) is None+ # flash-attn stays NVIDIA's, which publishes every interpreter.+ assert wheel_utils.flash_attn_wheel_url(e).startswith(NVIDIA)++[email protected]("python_tag", ["cp313"])def test_win_arm64_causal_conv1d_and_mamba_are_ours(python_tag):e = woa_env(python_tag = python_tag, abi_tag = python_tag)assert kernel_install.CAUSAL_CONV1D.wheel_url(e) == (@@ -215,10 +225,12 @@ def test_an_env_without_abi_tag_is_read_as_a_gil_build():"cp313",f"{OURS}flash_attn-2.8.4+cu13torch2.13cxx11abiTRUE-cp313-cp313-linux_x86_64.whl",),+ # Built by the workflow on request, but not published: no URL rather than a 404.+ ("2.14", "cp312", None),("2.14",- "cp312",- f"{OURS}flash_attn-2.8.4+cu13torch2.14cxx11abiTRUE-cp312-cp312-linux_x86_64.whl",+ "cp313",+ f"{OURS}flash_attn-2.8.4+cu13torch2.14cxx11abiTRUE-cp313-cp313-linux_x86_64.whl",),("2.11",@@ -239,9 +251,11 @@ def test_linux_flash_attn_urls_unchanged(torch_mm, python_tag, expected):assert wheel_utils.flash_attn_wheel_url(e) == expected-def test_linux_free_threaded_keeps_resolving_as_before():- # Linux has never looked at abi_tag; adding it must not change a Linux answer.+def test_linux_free_threaded_is_not_served_the_gil_wheel():+ # The release carries cp313-cp313 only; a free-threaded cp313t interpreter cannot install it.e = env(platform_tag = "linux_x86_64", cxx11abi = "TRUE", abi_tag = "cp313t")+ assert wheel_utils.flash_attn_wheel_url(e) is None+ e = env(platform_tag = "linux_x86_64", cxx11abi = "TRUE", abi_tag = "cp313")assert wheel_utils.flash_attn_wheel_url(e) == (f"{OURS}flash_attn-2.8.4+cu13torch2.13cxx11abiTRUE-cp313-cp313-linux_x86_64.whl")diff --git a/studio/backend/utils/kernel_install.py b/studio/backend/utils/kernel_install.pyindex f3bb2ee6039..90fa5a64eff 100644--- a/studio/backend/utils/kernel_install.py+++ b/studio/backend/utils/kernel_install.py@@ -11,13 +11,16 @@ The CLI is wheel-only (never a source build) and uses --no-deps so torch is nevefrom __future__ import annotationsimport argparse+import errno+import importlib.metadataimport osimport platformimport shutilimport subprocessimport sys+from contextlib import contextmanagerfrom dataclasses import dataclass-from typing import Any, Callable+from typing import Any, Callable, Iteratorfrom utils.child_stdio import utf8_child_envfrom utils.wheel_utils import (@@ -30,6 +33,7 @@ from utils.wheel_utils import (direct_wheel_url,flash_attn_wheel_url,install_wheel,+ package_installs_offline_env,probe_torch_wheel_env,redact_url_credentials,url_exists,@@ -142,12 +146,204 @@ def source_build_command(spec: str, *, use_uv: bool, is_hip: bool, reinstall: boreturn cmd+# Overall deadline for a ROCm / HIP source build (10-30 min).+SOURCE_BUILD_TIMEOUT = 1800+++def _cuda_source_build_timeout() -> int:+ """UNSLOTH_SOURCE_BUILD_TIMEOUT (seconds, > 0) when set, else 7200."""+ raw = os.environ.get("UNSLOTH_SOURCE_BUILD_TIMEOUT", "").strip()+ try:+ value = int(float(raw)) if raw else 0+ except ValueError:+ value = 0+ return value if value > 0 else 7200+++# Overall deadline for a CUDA source build. Far looser than HIP's: before the shared deadline a+# CUDA build had none, and a slow one (causal-conv1d on aarch64 can compile for over an hour) must+# not be aborted; this only stops a build that is truly stuck. Read once at import, so the lock+# wait below covers it.+SOURCE_BUILD_TIMEOUT_CUDA = _cuda_source_build_timeout()+# The real runner, captured at import: a caller handing in anything else (a test double, or+# subprocess.run patched over) gets exactly that called.+_SUBPROCESS_RUN = subprocess.run+++def run_process_tree(+ cmd: list[str],+ *,+ run: Callable[..., Any] = _SUBPROCESS_RUN,+ **kwargs: Any,+) -> subprocess.CompletedProcess:+ """``run(cmd, **kwargs)``, but when *run* is the real ``subprocess.run`` a ``timeout`` stops the+ whole process tree (pip / uv and the nvcc / hipcc / ninja they start), not just its leader,+ which ``subprocess.run`` alone would leave compiling. Raises ``TimeoutExpired`` as run does."""+ timeout = kwargs.pop("timeout", None)+ if run is not _SUBPROCESS_RUN or timeout is None:+ if timeout is not None:+ kwargs["timeout"] = timeout+ return run(cmd, **kwargs)+ if os.name == "nt":+ kwargs.setdefault("creationflags", getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0))+ else:+ kwargs["start_new_session"] = True+ with subprocess.Popen(cmd, **kwargs) as process:+ try:+ stdout, stderr = process.communicate(timeout = timeout)+ except subprocess.TimeoutExpired:+ _kill_process_tree(process)+ process.communicate()+ raise subprocess.TimeoutExpired(cmd, timeout)+ except BaseException:+ _kill_process_tree(process)+ raise+ return subprocess.CompletedProcess(cmd, process.returncode, stdout, stderr)+++def _kill_process_tree(process: subprocess.Popen) -> None:+ try:+ if os.name == "nt":+ subprocess.run(+ ["taskkill", "/F", "/T", "/PID", str(process.pid)],+ stdout = subprocess.DEVNULL,+ stderr = subprocess.DEVNULL,+ timeout = 60,+ )+ else:+ import signal+ os.killpg(process.pid, signal.SIGKILL)+ except Exception:+ pass+ try:+ process.kill()+ except Exception:+ pass+++# Long enough for one source build (the longer of the HIP and CUDA deadlines) under another+# process's lock, then give up rather than block.+KERNEL_LOCK_WAIT = max(SOURCE_BUILD_TIMEOUT, SOURCE_BUILD_TIMEOUT_CUDA) + 600+_LOCK_CONTENDED = frozenset(+ code+ for code in (+ getattr(errno, "EACCES", None),+ getattr(errno, "EAGAIN", None),+ getattr(errno, "EWOULDBLOCK", None),+ getattr(errno, "EDEADLK", None),+ getattr(errno, "EDEADLOCK", None),+ )+ if code is not None+)+++# How long an optional acceleration (causal-conv1d, an optional mamba-ssm wheel, flash-attn) waits+# for another process's install of it before the load goes on without it.+OPTIONAL_KERNEL_LOCK_WAIT = 120+++def _kernel_lock_path(name: str) -> str:+ """The one lock file for installing *name* into this environment, shared by every entry point+ that installs or removes it (Studio workers, setup, ``unsloth install-kernels``): beside the+ environment when it is writable, else one per environment in the temp folder."""+ import hashlib+ import tempfile++ override = os.environ.get("UNSLOTH_KERNEL_LOCK_DIR")+ if override:+ os.makedirs(override, exist_ok = True)+ return os.path.join(override, f"{name}.lock")+ folder = os.path.join(sys.prefix, ".unsloth-kernel-locks")+ try:+ os.makedirs(folder, exist_ok = True)+ if os.access(folder, os.W_OK):+ return os.path.join(folder, f"{name}.lock")+ except OSError:+ pass+ env = hashlib.sha256(os.path.realpath(sys.prefix).encode("utf-8")).hexdigest()[:16]+ folder = os.path.join(tempfile.gettempdir(), "unsloth-kernel-locks")+ os.makedirs(folder, exist_ok = True)+ return os.path.join(folder, f"{env}-{name}.lock")+++@contextmanager+def kernel_install_lock(+ name: str,+ *,+ on_wait: Callable[[], None] | None = None,+ wait_seconds: float = KERNEL_LOCK_WAIT,+ beat_seconds: float = 60.0,+) -> Iterator[bool]:+ """Hold the cross-process lock for installing kernel *name* into this environment, covering the+ whole precheck / install / verify / cleanup transaction. Yields True when held, False when it+ is not: another process kept it past *wait_seconds*, or the lock cannot be opened or taken at+ all. Either way the caller installs nothing rather than race a peer unguarded. *on_wait* is+ called when the wait starts and every *beat_seconds* while it lasts."""+ import time++ try:+ handle = open(_kernel_lock_path(name), "a+b")+ except OSError as exc:+ logger_message = f"Unsloth: cannot open the {name} install lock ({exc}); not installing it."+ print(logger_message, file = sys.stderr)+ yield False+ return+ held = False+ try:+ deadline = time.monotonic() + wait_seconds+ next_beat = 0.0+ while True:+ try:+ if os.name == "nt":+ import msvcrt+ handle.seek(0)+ msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)+ else:+ import fcntl+ fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)+ held = True+ break+ except (OSError, ImportError) as exc:+ if not isinstance(exc, OSError) or exc.errno not in _LOCK_CONTENDED:+ print(+ f"Unsloth: cannot take the {name} install lock ({exc}); not installing it.",+ file = sys.stderr,+ )+ break+ now = time.monotonic()+ if now >= deadline:+ break+ if on_wait is not None and now >= next_beat:+ next_beat = now + beat_seconds+ try:+ on_wait()+ except Exception:+ pass+ time.sleep(0.5)+ yield held+ finally:+ if held:+ try:+ if os.name == "nt":+ import msvcrt+ handle.seek(0)+ msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)+ else:+ import fcntl+ fcntl.flock(handle.fileno(), fcntl.LOCK_UN)+ except (OSError, ImportError):+ pass+ if not handle.closed:+ handle.close()++def source_build_run_kwargs(*, is_hip: bool, gcc_install_dir: Callable[[], str | None]) -> tuple[dict[str, Any], str | None]:"""subprocess.run kwargs for a source build, and the gcc dir added to HIPCC_COMPILE_FLAGS_APPEND- (None when nothing was added). Non-HIP builds have no timeout so slow builds (causal-conv1d on- aarch64) are not aborted; ROCm builds take 10-30 min, bounded at 1800 s."""+ (None when nothing was added). ROCm builds take 10-30 min and are bounded at+ SOURCE_BUILD_TIMEOUT (1800 s); CUDA builds at SOURCE_BUILD_TIMEOUT_CUDA (7200 s, or+ UNSLOTH_SOURCE_BUILD_TIMEOUT), loose enough for a slow aarch64 causal-conv1d build."""kwargs: dict[str, Any] = {"stdout": subprocess.PIPE,"stderr": subprocess.STDOUT,@@ -157,9 +353,11 @@ def source_build_run_kwargs("errors": "replace","env": utf8_child_env(),}+ # An overall deadline for every source build (run it through run_process_tree so a timeout+ # also stops the compilers pip / uv started): HIP's, or the far looser CUDA one.+ kwargs["timeout"] = SOURCE_BUILD_TIMEOUT if is_hip else SOURCE_BUILD_TIMEOUT_CUDAif not is_hip:return kwargs, None- kwargs["timeout"] = 1800existing = os.environ.get("HIPCC_COMPILE_FLAGS_APPEND", "")if "--gcc-install-dir" in existing:return kwargs, None@@ -202,6 +400,7 @@ KERNELS = {}_PINNED = {"causal_conv1d": CAUSAL_CONV1D, "mamba_ssm": MAMBA_SSM}+_WITHOUT_GPU_ENV = "UNSLOTH_INSTALL_KERNELS_WITHOUT_GPU"def resolve_wheel_url(name: str, env: dict[str, str] | None) -> str | None:@@ -276,6 +475,39 @@ def _gpu_capability(run: Callable[..., subprocess.CompletedProcess]) -> tuple[inreturn _CAPABILITY[run]+def gpu_below_flash_attn_floor(+ run: Callable[..., subprocess.CompletedProcess] = subprocess.run,+) -> bool:+ """Whether the best visible CUDA GPU is below sm80, where Unsloth leaves FlashAttention 2 off+ (a T4 is sm75), so a flash-attn download would go unused. No visible GPU answers False: an+ image build installs ahead of the GPU it will run on."""+ capability = _gpu_capability(run)+ # ROCm reports a gfx version, not an sm, as its capability: the floor is NVIDIA's.+ return capability is not None and capability < (8, 0) and not _torch_is_hip(run)+++_HIP: dict = {}+++def _torch_is_hip(run: Callable[..., subprocess.CompletedProcess]) -> bool:+ """Whether torch is a ROCm build. Probed once per runner, and only below sm80."""+ if run not in _HIP:+ try:+ result = run(+ [sys.executable, "-c", "import torch; print(bool(torch.version.hip))"],+ stdout = subprocess.PIPE,+ stderr = subprocess.DEVNULL,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 120,+ )+ _HIP[run] = result.stdout.strip() == "True"+ except Exception:+ _HIP[run] = False+ return _HIP[run]++def _probe_gpu_capability(run: Callable[..., subprocess.CompletedProcess],) -> tuple[int, int] | None:@@ -309,6 +541,13 @@ def _loads(check: str, run: Callable[..., subprocess.CompletedProcess]) -> bool:return result.returncode == 0+def _installed_version(distribution: str) -> str | None:+ try:+ return importlib.metadata.version(distribution)+ except Exception:+ return None++def _outside_venv() -> bool:# Colab and other system interpreters: uv refuses them without --system.return sys.prefix == sys.base_prefix@@ -328,70 +567,100 @@ def install_kernel(dry_run: bool = False,run: Callable[..., subprocess.CompletedProcess] = subprocess.run,exists: Callable[[str], bool | None] = url_exists,+ install_without_gpu: bool = False,+ skipped: list[str] | None = None,) -> int:+ # install_without_gpu: no visible GPU yet (an image build for the GPU it will run on) still+ # installs. Each kernel left uninstalled for this machine is appended to *skipped*.distribution, check = KERNELS[name]++ def skip(message: str) -> int:+ print(message)+ if skipped is not None:+ skipped.append(name)+ return 0++ # A copy that already loads (a ROCm source build, a GPU-hidden image builder's earlier install)+ # is success whatever the wheel, GPU and network checks below would say.+ if not dry_run and _installed_version(distribution) is not None and _loads(check, run):+ print(f"Unsloth: {name} already installed and loads.")+ return 0+ # A dry run only prints the URL it would install, so it runs offline too.+ if not dry_run and package_installs_offline_env():+ return skip(+ f"Unsloth: not installing {name} while offline "+ "(UV_OFFLINE / HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE)."+ )url = resolve_wheel_url(name, env)if name in _GPU_GATED and url is not None:capability = _gpu_capability(run)- if capability is None:- print(f"Unsloth: skipping {name}, which needs sm80 or newer (no CUDA GPU is visible).")- return 0+ if capability is None and not install_without_gpu:+ return skip(+ f"Unsloth: skipping {name}, which needs sm80 or newer (no CUDA GPU is visible; "+ "pass --without-gpu to install it for the GPU this image will run on)."+ )if capability is not None and capability < (8, 0):reason = _pre_ampere_skip_reason(name, capability, run)if reason is not None:- print(+ return skip(f"Unsloth: skipping {name}, which {reason} "f"(the best GPU is sm{capability[0]}{capability[1]}).")- return 0torch_desc = f"torch {env.get('torch_version')}" if env else "this environment"# Only a 404 proves nothing is published; an unreachable check falls through to the install.if url is None or exists(url) is False:- print(f"Unsloth: no prebuilt {name} for {torch_desc}; using the torch fallback.")- return 0+ return skip(f"Unsloth: no prebuilt {name} for {torch_desc}; using the torch fallback.")# mamba_ssm imports triton at load; on Windows that is triton-windows, which its metadata cannot name.if name == "mamba_ssm" and sys.platform == "win32" and not _loads("import triton", run):- print(+ return skip("Unsloth: skipping mamba_ssm, which needs Triton on Windows; ""install triton-windows first (Studio setup does), then re-run.")- return 0# UNSLOTH_PYTORCH_MIRROR may carry credentials, and notebook output gets shared.shown = redact_url_credentials(url)if dry_run:print(shown)return 0- if _loads(check, run):- print(f"Unsloth: {name} already installed and loads.")- return 0- failures: list[subprocess.CompletedProcess] = []- outcome = install_prebuilt(- url,- install = install_wheel,- verify = lambda: _loads(check, run),- on_failed = lambda _installer, result: failures.append(result),- use_uv = True,- uv_needs_system = _outside_venv(),- reinstall = True,- run = run,- )- if outcome == "failed":- print(- f"Unsloth: installing {name} from {shown} failed; using the torch fallback.\n"- + failures[-1].stdout[-2000:].replace(url, shown)- )- return 1- if outcome == "rejected":- # A wheel that imports but whose extension cannot load would be picked up and crash later.- removed = (- "removed it" if _uninstall(distribution, run) else f"run `pip uninstall {distribution}`"- )- print(- f"Unsloth: {name} from {shown} does not load with {torch_desc}; {removed}, using the torch fallback."+ # The lock Studio's workers take for the same kernel: never install or remove it under a+ # running load's precheck / verify.+ with kernel_install_lock(+ name, on_wait = lambda: print(f"Unsloth: waiting for another process installing {name}...")+ ) as held:+ if not held:+ return skip(f"Unsloth: could not take the {name} install lock; not installing it.")+ if _loads(check, run):+ print(f"Unsloth: {name} already installed and loads.")+ return 0+ failures: list[subprocess.CompletedProcess] = []+ outcome = install_prebuilt(+ url,+ install = install_wheel,+ verify = lambda: _loads(check, run),+ on_failed = lambda _installer, result: failures.append(result),+ use_uv = True,+ uv_needs_system = _outside_venv(),+ reinstall = True,+ run = run,)- return 1- print(f"Unsloth: installed {name} ({shown.rsplit('/', 1)[-1]}).")- return 0+ if outcome == "failed":+ print(+ f"Unsloth: installing {name} from {shown} failed; using the torch fallback.\n"+ + failures[-1].stdout[-2000:].replace(url, shown)+ )+ return 1+ if outcome == "rejected":+ # A wheel that imports but whose extension cannot load would be picked up and crash later.+ removed = (+ "removed it"+ if _uninstall(distribution, run)+ else f"run `pip uninstall {distribution}`"+ )+ print(+ f"Unsloth: {name} from {shown} does not load with {torch_desc}; {removed}, using the torch fallback."+ )+ return 1+ print(f"Unsloth: installed {name} ({shown.rsplit('/', 1)[-1]}).")+ return 0def main(argv: list[str] | None = None) -> int:@@ -408,11 +677,45 @@ def main(argv: list[str] | None = None) -> int:parser.add_argument("--dry-run", action = "store_true", help = "print the resolved wheel URL and install nothing")+ parser.add_argument(+ "--without-gpu",+ action = "store_true",+ help = (+ "install the GPU-gated kernels (flash_attn, mamba_ssm) even though no CUDA GPU is "+ f"visible, e.g. in an image build; also {_WITHOUT_GPU_ENV}=1"+ ),+ )+ parser.add_argument(+ "--strict",+ action = "store_true",+ help = "exit 2 when a requested kernel was skipped (no GPU, no wheel, offline)",+ )args = parser.parse_args(argv)+ without_gpu = args.without_gpu or os.environ.get(_WITHOUT_GPU_ENV, "").strip().lower() in (+ "1",+ "true",+ "yes",+ "on",+ )env = probe_torch_wheel_env(timeout = 120, include_windows = True)status = 0+ skipped: list[str] = []for name in [name for name in KERNELS if not args.kernels or name in args.kernels]:- status = max(status, install_kernel(name, env, dry_run = args.dry_run))+ status = max(+ status,+ install_kernel(+ name,+ env,+ dry_run = args.dry_run,+ install_without_gpu = without_gpu,+ skipped = skipped,+ ),+ )+ if skipped:+ print(f"Unsloth: not installed on this machine: {', '.join(skipped)}.")+ # A skip stays exit 0, as scripts running under `set -e` expect; --strict opts in.+ if args.strict and status == 0:+ status = 2return statusdiff --git a/studio/backend/utils/ssm_runtime.py b/studio/backend/utils/ssm_runtime.pyindex e85bcecfd8c..ae05fd30cc7 100644--- a/studio/backend/utils/ssm_runtime.py+++ b/studio/backend/utils/ssm_runtime.py@@ -7,9 +7,7 @@ the inference-path counterpart of the training worker's install.from __future__ import annotations-import functoolsimport importlib-import importlib.metadataimport importlib.utilimport osimport shutil@@ -18,7 +16,7 @@ import sysimport threadingfrom contextlib import contextmanagerfrom pathlib import Path-from typing import Any, Callable, Iterator, Optional+from typing import Any, Callable, Iterator, NamedTuple, Optionalfrom loggers import get_loggerfrom utils.kernel_install import (@@ -35,6 +33,7 @@ from utils.wheel_utils import (direct_wheel_url,install_wheel,is_windows_wheel_env,+ pinned_source_build_supported,probe_torch_wheel_env,url_exists,)@@ -44,7 +43,8 @@ logger = get_logger(__name__)StatusCb = Optional[Callable[[str], None]]# Lowercased-id substring matches, shared with the training worker. mamba-ssm models are a subset of the causal-conv1d set.-SSM_MODEL_SUBSTRINGS = (+# The Mamba-2 hybrids whose modeling (or remote) code cannot load without mamba_ssm: a failed install is fatal for them.+REQUIRED_SSM_MODEL_SUBSTRINGS = ("nemotron_h","nemotron-h","nemotron-3-nano",@@ -52,29 +52,28 @@ SSM_MODEL_SUBSTRINGS = ("falcon-h1","granite-4.0-h","granitemoehybrid",+)+SSM_MODEL_SUBSTRINGS = REQUIRED_SSM_MODEL_SUBSTRINGS + ("mamba","jamba","zamba","bamba",)-# config.json model types whose modeling files run mamba_ssm (and so causal_conv1d): Mamba-2-# hybrids and the Mamba-1 family. Matched on the resolved config, so a renamed checkpoint or an-# id without the substrings above (NVIDIA-Nemotron-Nano-9B-v2, granite-4.0-tiny) still counts.-SSM_MODEL_TYPES = frozenset(- {- "nemotron_h",- "falcon_h1",- "granitemoehybrid",- "bamba",- "jamba",- "zamba",- "zamba2",- "mamba",- "mamba2",- "falcon_mamba",- }+# config.json model types whose modeling files run mamba_ssm (and so causal_conv1d). Matched on the+# resolved config, so a renamed checkpoint or an id without the substrings above+# (NVIDIA-Nemotron-Nano-9B-v2, granite-4.0-tiny) still counts. The Mamba-1 family and the other+# hybrids have a working torch path in transformers, so mamba_ssm is only an acceleration there.+REQUIRED_SSM_MODEL_TYPES = frozenset({"nemotron_h", "falcon_h1", "granitemoehybrid"})+SSM_MODEL_TYPES = REQUIRED_SSM_MODEL_TYPES | frozenset(+ {"bamba", "jamba", "zamba", "zamba2", "mamba", "mamba2", "falcon_mamba"})CAUSAL_CONV1D_MODEL_SUBSTRINGS = (+ "qwen3.5",+ "qwen3_5",+ "qwen3.6",+ "qwen3_6",+ "qwen3-next",+ "qwen3_next","nemotron_h","nemotron-h","nemotron-3-nano",@@ -88,52 +87,28 @@ CAUSAL_CONV1D_MODEL_SUBSTRINGS = ("zamba","bamba",)-# GatedDeltaNet models unsloth-zoo runs without causal_conv1d: its Triton causal conv-# (temporary_patches/gdn_causal_conv1d.py, `_GDN_MODELING`) plus the vendored FLA kernels. Qwen3.6-# reuses the qwen3_5 model types. A usable causal_conv1d still wins inside zoo; this only stops-# Studio from downloading it. Zoo's own kill switch brings the request back.-ZOO_GDN_MODEL_TYPES = frozenset({"qwen3_5", "qwen3_5_moe", "qwen3_next"})-ZOO_GDN_MODEL_SUBSTRINGS = (- "qwen3.5",- "qwen3_5",- "qwen3.6",- "qwen3_6",- "qwen3-next",- "qwen3_next",-)-ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH = "UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D"-_TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE: dict[str, bool | None] = {}---def zoo_triton_causal_conv1d_enabled() -> bool:- return os.environ.get(ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, "0") != "1"--[email protected]_cache(maxsize = 1)-def _zoo_triton_platform_usable() -> bool:- """Whether zoo's Triton conv can engage here: a CUDA torch (not ROCm) with Triton importable.- macOS, ROCm and a Windows install without triton-windows keep the prebuilt causal-conv1d.- Read from torch's distribution version, so no model load pays for importing torch: ROCm and- CPU builds carry `+rocm` / `+cpu`, a Windows CUDA build `+cu`, and a Linux PyPI torch with no- local version is the CUDA build."""- if sys.platform == "darwin" or importlib.util.find_spec("triton") is None:- return False- torch = sys.modules.get("torch")- if torch is not None:- version = getattr(torch, "version", None)- return bool(getattr(version, "cuda", None)) and not getattr(version, "hip", None)- try:- local = importlib.metadata.version("torch").partition("+")[2].lower()- except importlib.metadata.PackageNotFoundError:- return False- if local.startswith(("rocm", "cpu")):- return False- return local.startswith("cu") or (not local and sys.platform.startswith("linux"))---def zoo_triton_causal_conv1d_usable() -> bool:- """zoo runs Qwen3.5 / 3.6 / Qwen3-Next without causal_conv1d on this host."""- return zoo_triton_causal_conv1d_enabled() and _zoo_triton_platform_usable()+_TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE: dict[tuple[str, str], bool | None] = {}+++class SsmKernels(NamedTuple):+ """What one model needs: ``causal_conv1d`` and ``mamba_ssm`` (best effort), and whether a+ missing ``mamba_ssm`` must stop the load. ``probe`` is the identifier the decision was made for+ (an adapter's base when there is one); ``model_types`` the config's model types, if read."""++ causal_conv1d: bool+ mamba_ssm: bool+ mamba_ssm_required: bool+ probe: str+ model_types: frozenset[str] = frozenset()+ # (dynamic-module namespace, implementation module) pairs from the config's auto_map, as+ # transformers names them under transformers_modules (see remote_code_modules).+ remote_code: frozenset = frozenset()+ # Whether the load will import code that cannot run without mamba_ssm, so a failed install+ # must refuse it: remote modeling code importing it unconditionally, or a model type this+ # transformers has no native implementation of. mamba_ssm_required alone (Nemotron-H,+ # Falcon-H1, Granite-4.0-H) only means "worth a source build": their native transformers+ # models run without it.+ mamba_ssm_fatal: bool = Falsedef _base_model_type(model_type: str) -> str:@@ -144,27 +119,22 @@ def _base_model_type(model_type: str) -> str:return candidate-def _is_zoo_gdn_model_type(model_type: str) -> bool:- """The model type or one of its `_text` / `_vision` sub-configs belongs to a zoo-covered model."""- return _base_model_type(model_type) in ZOO_GDN_MODEL_TYPES---def _names_zoo_gdn_model(name: str) -> bool:- return any(sub in name for sub in ZOO_GDN_MODEL_SUBSTRINGS)--def model_is_ssm(model_name: str) -> bool:- """Whether *model_name* is a Mamba/SSM hybrid that needs ``mamba_ssm``."""+ """Whether *model_name* is a Mamba/SSM model whose fast path is ``mamba_ssm``."""name = (model_name or "").lower()return any(sub in name for sub in SSM_MODEL_SUBSTRINGS)+def model_requires_mamba_ssm(model_name: str) -> bool:+ """Whether *model_name* cannot load at all without ``mamba_ssm``."""+ name = (model_name or "").lower()+ return any(sub in name for sub in REQUIRED_SSM_MODEL_SUBSTRINGS)++def model_wants_causal_conv1d(model_name: str) -> bool:- """Whether *model_name* needs ``causal_conv1d`` (the SSM set plus linear-attention hybrids like LFM2 whose modeling files lazy-import it; Qwen3.5 / 3.6 / Qwen3-Next only where zoo's Triton conv cannot run)."""+ """Whether *model_name* needs ``causal_conv1d`` (the SSM set plus linear-attention hybrids like Qwen3.5 / 3.6 / Qwen3-Next / LFM2 whose modeling files lazy-import it)."""name = (model_name or "").lower()- if any(sub in name for sub in CAUSAL_CONV1D_MODEL_SUBSTRINGS):- return True- return _names_zoo_gdn_model(name) and not zoo_triton_causal_conv1d_usable()+ return any(sub in name for sub in CAUSAL_CONV1D_MODEL_SUBSTRINGS)def _normalized_model_identifier(value: str) -> str:@@ -173,6 +143,26 @@ def _normalized_model_identifier(value: str) -> str:)+def _transformers_models_dir() -> Path | None:+ """The active transformers' ``models`` folder, found without importing transformers: the+ inference worker asks before its transformers import, which snapshots the kernel gates."""+ try:+ spec = importlib.util.find_spec("transformers")+ except (ImportError, ValueError):+ return None+ if spec is None:+ return None+ locations = list(getattr(spec, "submodule_search_locations", None) or ())+ origin = getattr(spec, "origin", None)+ if not locations and origin:+ locations = [str(Path(origin).parent)]+ for location in locations:+ models_dir = Path(location) / "models"+ if models_dir.is_dir():+ return models_dir+ return None++def _transformers_model_type_uses_causal_conv1d(model_type: str) -> bool | None:candidate = model_type.strip().lower().replace("-", "_")if not candidate or any(@@ -180,14 +170,15 @@ def _transformers_model_type_uses_causal_conv1d(model_type: str) -> bool | None:for character in candidate):return None- if candidate in _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE:- return _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE[candidate]+ models_dir = _transformers_models_dir()+ cache_key = (str(models_dir), candidate)+ if cache_key in _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE:+ return _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE[cache_key]result: bool | None = Nonetry:- import transformers- model_dir = Path(transformers.__file__).parent / "models" / candidate- if model_dir.is_dir():+ model_dir = models_dir / candidate if models_dir is not None else None+ if model_dir is not None and model_dir.is_dir():for modeling_file in model_dir.glob("modeling_*.py"):try:source = modeling_file.read_text(encoding = "utf-8", errors = "ignore")@@ -200,7 +191,7 @@ def _transformers_model_type_uses_causal_conv1d(model_type: str) -> bool | None:except Exception as exc:logger.debug("causal-conv1d model-type inspection skipped: %s", exc)- _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE[candidate] = result+ _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE[cache_key] = resultreturn result@@ -236,85 +227,355 @@ def model_config_is_ssm(model_config: dict) -> bool | None:return any(_base_model_type(model_type) in SSM_MODEL_TYPES for model_type in model_types)-def model_config_wants_causal_conv1d(- model_config: dict, *, scan_modeling_source: bool = True-) -> bool | None:- """``scan_modeling_source=False`` keeps the answer transformers-free (the inference worker asks- before its transformers import, which snapshots the kernel gates)."""+def model_config_requires_mamba_ssm(model_config: dict) -> bool | None:+ """True when the config's model cannot load without ``mamba_ssm``; None without a model type."""+ model_types, _ = _config_identifiers(model_config)+ if not model_types:+ return None+ return any(+ _base_model_type(model_type) in REQUIRED_SSM_MODEL_TYPES for model_type in model_types+ )+++# The required families by name, for a decision with no config to read.+_REQUIRED_MODEL_TYPE_BY_SUBSTRING = (+ ("nemotron_h", "nemotron_h"),+ ("nemotron-h", "nemotron_h"),+ ("nemotron-3-nano", "nemotron_h"),+ ("falcon_h1", "falcon_h1"),+ ("falcon-h1", "falcon_h1"),+ ("granite-4.0-h", "granitemoehybrid"),+ ("granitemoehybrid", "granitemoehybrid"),+)+_NATIVE_MODELING_CACHE: dict[tuple[str, str], bool] = {}+++def native_modeling_available(model_type: str) -> bool:+ """Whether the active transformers ships a native implementation of *model_type*+ (``models/<type>/modeling_*.py`` on disk), found without importing transformers."""+ models_dir = _transformers_models_dir()+ if models_dir is None:+ return False+ key = (str(models_dir), model_type)+ if key not in _NATIVE_MODELING_CACHE:+ try:+ _NATIVE_MODELING_CACHE[key] = any((models_dir / model_type).glob("modeling_*.py"))+ except OSError:+ _NATIVE_MODELING_CACHE[key] = False+ return _NATIVE_MODELING_CACHE[key]+++def _imports_mamba_ssm_unconditionally(source: str) -> bool:+ """Whether module *source* imports ``mamba_ssm`` at import time with no way around it: a+ top-level import, or one in a ``try`` whose handler raises. One under an ``if`` (an+ ``is_mamba_2_ssm_available()`` gate) or with a fallback handler does not count. Unparseable+ source counts: it cannot be shown safe."""+ import ast++ try:+ tree = ast.parse(source)+ except (SyntaxError, ValueError):+ return True++ def imports_mamba(statements) -> bool:+ for node in statements:+ if isinstance(node, ast.Import) and any(+ alias.name.split(".")[0] == "mamba_ssm" for alias in node.names+ ):+ return True+ if (+ isinstance(node, ast.ImportFrom)+ and (node.module or "").split(".")[0] == "mamba_ssm"+ ):+ return True+ return False++ for node in tree.body:+ if imports_mamba([node]):+ return True+ if isinstance(node, ast.Try) and imports_mamba(node.body):+ if any(+ any(isinstance(inner, ast.Raise) for inner in ast.walk(handler))+ for handler in node.handlers+ ):+ return True+ return False+++_REMOTE_SOURCES: dict[tuple[str, str, str | None], str | None] = {}+++def _remote_source_text(repo: str, filename: str, hf_token: str | None) -> str | None:+ """The text of *filename* in a local checkpoint or a Hub repo (the local HF cache first, then+ the Hub unless offline), or None when it cannot be read."""+ key = (repo, filename, hf_token)+ if key in _REMOTE_SOURCES:+ return _REMOTE_SOURCES[key]+ text = None+ try:+ if os.path.isdir(repo):+ path = Path(repo) / filename+ text = path.read_text(encoding = "utf-8") if path.is_file() else None+ elif repo.count("/") == 1:+ text = _hub_cached_text(repo, filename)+ from utils.utils import hf_env_offline+ if text is None and not hf_env_offline():+ import urllib.request++ from utils.transformers_version import _hf_raw_url, _hf_urlopen++ headers = {"User-Agent": "unsloth-studio"}+ if isinstance(hf_token, str) and hf_token:+ headers["Authorization"] = f"Bearer {hf_token}"+ request = urllib.request.Request(_hf_raw_url(repo, filename), headers = headers)+ with _hf_urlopen(request, timeout = 10) as response:+ text = response.read().decode("utf-8", errors = "replace")+ except Exception as exc:+ logger.debug("Could not read %s from %s: %s", filename, repo, exc)+ text = None+ _REMOTE_SOURCES[key] = text+ return text+++def _hub_cached_text(repo: str, filename: str) -> str | None:+ try:+ from utils.hf_cache_settings import get_hf_cache_paths++ repo_dir = Path(str(get_hf_cache_paths().hub_cache)) / (+ "models--" + repo.replace("/", "--")+ )+ ref = repo_dir / "refs" / "main"+ candidates = []+ if ref.is_file():+ candidates.append(+ repo_dir / "snapshots" / ref.read_text(encoding = "utf-8").strip() / filename+ )+ candidates += sorted(+ repo_dir.glob(f"snapshots/*/{filename}"), key = lambda p: p.stat().st_mtime, reverse = True+ )+ for path in candidates:+ if path.is_file():+ return path.read_text(encoding = "utf-8")+ except Exception as exc:+ logger.debug("HF cache lookup of %s in %s failed: %s", filename, repo, exc)+ return None+++def _remote_model_sources(checkpoint: str, model_config: dict) -> list[tuple[str, str]]:+ """(repo, file) of the model classes the config's ``auto_map`` names."""+ auto_map = model_config.get("auto_map") if isinstance(model_config, dict) else None+ if not isinstance(auto_map, dict):+ return []+ sources = []+ for key, value in auto_map.items():+ if not str(key).startswith("AutoModel"):+ continue+ for reference in value if isinstance(value, (list, tuple)) else (value,):+ if not isinstance(reference, str) or "." not in reference:+ continue+ source, _, reference = reference.rpartition("--")+ module = reference.rsplit(".", 1)[0]+ sources.append((source or checkpoint, module.replace(".", "/") + ".py"))+ return list(dict.fromkeys(sources))+++def mamba_ssm_is_fatal(+ checkpoint: str,+ model_config: dict | None,+ model_types,+ *,+ trust_remote_code: bool | None,+ hf_token: str | None = None,+) -> bool:+ """Whether loading this required-family model (Nemotron-H, Falcon-H1, Granite-4.0-H) imports+ code that cannot run without mamba_ssm. Remote code (auto_map, loaded under trust_remote_code+ or because transformers has no native class) counts when a model module imports it+ unconditionally; otherwise it is fatal only when transformers lacks the native model."""+ required_types = {t for t in model_types if t in REQUIRED_SSM_MODEL_TYPES}+ native = bool(required_types) and all(native_modeling_available(t) for t in required_types)+ sources = _remote_model_sources(checkpoint, model_config or {})+ if sources and (trust_remote_code or not native):+ for repo, filename in sources:+ text = _remote_source_text(repo, filename, hf_token)+ if text is None or _imports_mamba_ssm_unconditionally(text):+ return True+ return False+ return not native+++def _name_mamba_ssm_is_fatal(model_name: str) -> bool:+ """mamba_ssm_is_fatal from the name alone: the family's native model is missing."""+ name = (model_name or "").lower()+ types = {model_type for sub, model_type in _REQUIRED_MODEL_TYPE_BY_SUBSTRING if sub in name}+ return bool(types) and not all(native_modeling_available(t) for t in types)+++def model_config_wants_causal_conv1d(model_config: dict) -> bool | None:model_types, architectures = _config_identifiers(model_config)if model_config_is_ssm(model_config):return True- if model_types and all(_is_zoo_gdn_model_type(model_type) for model_type in model_types):- return not zoo_triton_causal_conv1d_usable()- source_requirements = (- {_transformers_model_type_uses_causal_conv1d(model_type) for model_type in model_types}- if scan_modeling_source- else set()- )+ source_requirements = {+ _transformers_model_type_uses_causal_conv1d(model_type) for model_type in model_types+ }if True in source_requirements:return Trueconfig_identifiers = model_types | architectures- normalized = {_normalized_model_identifier(identifier) for identifier in config_identifiers}-- def _matches(substrings: tuple[str, ...]) -> bool:- needles = {_normalized_model_identifier(value) for value in substrings}- return any(needle in identifier for identifier in normalized for needle in needles)-- if _matches(CAUSAL_CONV1D_MODEL_SUBSTRINGS):- return True- if _matches(ZOO_GDN_MODEL_SUBSTRINGS) and not zoo_triton_causal_conv1d_usable():+ normalized_needles = {+ _normalized_model_identifier(value) for value in CAUSAL_CONV1D_MODEL_SUBSTRINGS+ }+ if any(+ needle in _normalized_model_identifier(identifier)+ for identifier in config_identifiers+ for needle in normalized_needles+ ):return Trueif False in source_requirements:return Falsereturn None+def _local_adapter_base(model_load_target: str) -> str | None:+ """A local PEFT adapter folder's base model (``adapter_config.json`` without a ``config.json``)."""+ try:+ folder = Path(model_load_target)+ if not folder.is_dir() or (folder / "config.json").is_file():+ return None+ adapter_config = folder / "adapter_config.json"+ if not adapter_config.is_file():+ return None+ import json++ base = json.loads(adapter_config.read_text(encoding = "utf-8-sig")).get(+ "base_model_name_or_path"+ )+ except Exception as exc:+ logger.debug("Could not read the adapter base of %s: %s", model_load_target, exc)+ return None+ return base if isinstance(base, str) and base.strip() else None+++def _dependency_target(model_load_target: str, adapter_base: str | None) -> str:+ """What decides the kernels: an adapter's (pinned, else recorded) base, else the target itself.+ The adapter stays the load target; only the dependency decision moves to its base."""+ if adapter_base:+ return adapter_base+ return _local_adapter_base(model_load_target) or model_load_target+++# Configs read for the kernel decision, per process: a load's preflight, its _handle_load and each+# kernel share one read (a cold Hub read can take seconds).+_MODEL_CONFIGS: dict[tuple[str, str | None], dict] = {}++def _resolved_model_config(model_load_target: str, hf_token: str | None) -> dict | None:+ key = (str(model_load_target), hf_token)+ if key in _MODEL_CONFIGS:+ return _MODEL_CONFIGS[key]try:from utils.transformers_version import _load_config_jsonmodel_config = _load_config_json(model_load_target, hf_token)except Exception as exc:logger.debug("Could not inspect model config for SSM kernels: %s", exc)- return None- return model_config if isinstance(model_config, dict) else None+ model_config = None+ if not isinstance(model_config, dict):+ return None # a miss may be transient: read again next time+ _MODEL_CONFIGS[key] = model_config+ return model_config-def resolved_model_wants_causal_conv1d(+def resolve_ssm_kernels(model_name: str,model_load_target: str,hf_token: str | None,*,- scan_modeling_source: bool = True,-) -> bool:- model_config = _resolved_model_config(model_load_target, hf_token)+ adapter_base: str | None = None,+ model_config: dict | None = None,+ trust_remote_code: bool | None = None,+) -> SsmKernels:+ """The kernels the model at *model_load_target* needs, from its config (an adapter's base+ config when it is an adapter); the name lists only when there is no config to read. A caller+ that already read the deciding config passes it as *model_config*."""+ target = _dependency_target(model_load_target, adapter_base)+ probe = target if target != model_load_target else model_name+ if model_config is None:+ model_config = _resolved_model_config(target, hf_token)+ model_types: frozenset[str] = frozenset()+ is_ssm = required = wants = None+ remote_code: frozenset = frozenset()if model_config is not None:- requirement = model_config_wants_causal_conv1d(- model_config, scan_modeling_source = scan_modeling_source+ remote_code = remote_code_modules(target, model_config)+ model_types = frozenset(+ _base_model_type(model_type) for model_type in _config_identifiers(model_config)[0]+ )+ is_ssm = model_config_is_ssm(model_config)+ required = model_config_requires_mamba_ssm(model_config)+ wants = model_config_wants_causal_conv1d(model_config)+ logger.info(+ "SSM kernels resolved from model architecture: causal-conv1d=%s mamba-ssm=%s",+ wants,+ is_ssm,)- if requirement is not None:- logger.info(- "causal-conv1d requirement resolved from model architecture: %s",- requirement,+ if is_ssm is None:+ is_ssm = model_is_ssm(probe)+ if required is None:+ required = model_requires_mamba_ssm(probe)+ if wants is None:+ wants = model_wants_causal_conv1d(probe)+ fatal = False+ if is_ssm and required:+ if model_config is not None and model_types:+ fatal = mamba_ssm_is_fatal(+ target,+ model_config,+ model_types,+ trust_remote_code = trust_remote_code,+ hf_token = hf_token,)- return requirement- return model_wants_causal_conv1d(model_name)+ else:+ fatal = _name_mamba_ssm_is_fatal(probe)+ # Every mamba-ssm model also runs causal-conv1d.+ return SsmKernels(+ causal_conv1d = bool(wants or is_ssm),+ mamba_ssm = bool(is_ssm),+ mamba_ssm_required = bool(is_ssm and required),+ probe = probe,+ model_types = model_types,+ remote_code = remote_code,+ mamba_ssm_fatal = bool(fatal),+ )-def resolved_model_is_ssm(model_name: str, model_load_target: str, hf_token: str | None) -> bool:- """Whether the model at *model_load_target* needs ``mamba_ssm``, from its config; the name list- only when there is no config to read."""- model_config = _resolved_model_config(model_load_target, hf_token)- if model_config is not None:- requirement = model_config_is_ssm(model_config)- if requirement is not None:- logger.info("mamba-ssm requirement resolved from model architecture: %s", requirement)- return requirement- return model_is_ssm(model_name)+def resolved_model_wants_causal_conv1d(+ model_name: str,+ model_load_target: str,+ hf_token: str | None,+ *,+ adapter_base: str | None = None,+) -> bool:+ return resolve_ssm_kernels(+ model_name, model_load_target, hf_token, adapter_base = adapter_base+ ).causal_conv1d+++def resolved_model_is_ssm(+ model_name: str,+ model_load_target: str,+ hf_token: str | None,+ *,+ adapter_base: str | None = None,+) -> bool:+ """Whether the model at *model_load_target* runs ``mamba_ssm``, from its (adapter's base) config;+ the name list only when there is no config to read."""+ return resolve_ssm_kernels(+ model_name, model_load_target, hf_token, adapter_base = adapter_base+ ).mamba_ssmdef refresh_transformers_kernel_gates() -> None:"""After an in-process install, let an already-imported transformers see the new kernels: its- availability gates are lru_cached and the hub-kernel map can hold a cached None. Never raises."""+ availability gates are lru_cached and the hub-kernel map can hold a cached None. Never raises.+ This cannot rebind functions a modeling module already captured; see+ ``modeling_modules_needing_restart``."""if "transformers" not in sys.modules:returntry:@@ -342,6 +603,87 @@ def refresh_transformers_kernel_gates() -> None:logger.debug("Could not refresh transformers' hub kernel map: %s", exc)+def _sanitized_module_name(name: str) -> str:+ # transformers.dynamic_module_utils._sanitize_module_name (4.57+, 5.x).+ name = name.replace(".", "_dot_").replace("-", "_hyphen_")+ return f"_{name}" if name and name[0].isdigit() else name+++def _dynamic_module_namespaces(checkpoint: str) -> set[tuple[str, ...]]:+ """The transformers_modules package(s) transformers puts *checkpoint*'s remote code under: a+ Hub repo by its id (org, name), a local folder by its name. Sanitized as 4.57+ and 5.x spell+ them, and raw as older 4.x did."""+ checkpoint = str(checkpoint).strip()+ if not checkpoint:+ return set()+ if os.path.isdir(checkpoint):+ parts = [os.path.basename(os.path.normpath(checkpoint))]+ else:+ parts = checkpoint.split("/")+ if not all(parts):+ return set()+ return {tuple(parts), tuple(_sanitized_module_name(part) for part in parts)}+++def remote_code_modules(checkpoint: str, model_config: dict) -> frozenset:+ """The (namespace, module) pairs of the model classes a config's ``auto_map`` points at: an+ entry ``"model.CustomForCausalLM"`` is module ``model`` of *checkpoint*, and+ ``"org/repo--model.Custom"`` module ``model`` of ``org/repo``. Any module name counts, not just+ ``modeling_*``; AutoConfig / AutoTokenizer / processor entries capture no kernels."""+ auto_map = model_config.get("auto_map") if isinstance(model_config, dict) else None+ if not isinstance(auto_map, dict):+ return frozenset()+ pairs = set()+ for key, value in auto_map.items():+ if not str(key).startswith("AutoModel"):+ continue+ for reference in value if isinstance(value, (list, tuple)) else (value,):+ if not isinstance(reference, str) or "." not in reference:+ continue+ source, _, reference = reference.rpartition("--")+ module = tuple(reference.rsplit(".", 1)[0].split("."))+ for namespace in _dynamic_module_namespaces(source or checkpoint):+ pairs.add((namespace, module))+ return frozenset(pairs)+++def _is_remote_code_module(name: str, remote_code) -> bool:+ # transformers_modules.<namespace>[.<commit or source hash>].<module>, matched exactly.+ parts = tuple(name.split("."))+ if parts[0] != "transformers_modules":+ return False+ for namespace, module in remote_code:+ if parts[1 : 1 + len(namespace)] != namespace:+ continue+ rest = parts[1 + len(namespace) :]+ if rest == module or (len(rest) == len(module) + 1 and rest[1:] == module):+ return True+ return False+++def modeling_modules_needing_restart(model_types, remote_code = ()) -> list[str]:+ """The already-imported implementation modules of *model_types* (``transformers.models.<type>.+ modeling_*``) and of the selected checkpoint's remote code (*remote_code*, from+ :func:`remote_code_modules`). Each captured its kernel functions (module globals, hub-kernel+ decorators) when it was imported, so a kernel installed after that is only picked up by a fresh+ process. A configuration or tokenizer module captures none, so importing one needs no restart."""+ model_types = {model_type for model_type in model_types if model_type}+ stale = []+ for name in list(sys.modules):+ parts = name.split(".")+ if parts[0] == "transformers":+ if (+ len(parts) == 4+ and parts[1] == "models"+ and parts[2] in model_types+ and parts[3].startswith("modeling_")+ ):+ stale.append(name)+ elif remote_code and _is_remote_code_module(name, remote_code):+ stale.append(name)+ return sorted(stale)++def ssm_probe_identifier(model_name: str, base: str | None = None) -> str:"""The identifier whose architecture decides the SSM kernels. The substring match needs a real model id: a LoRA adapter id or a local checkpoint's parent folders are unrelated to its architecture (a Llama LoRA at ``user/falcon-h1-lora`` is not SSM). Prefer *base*; for a bare local checkpoint use its basename."""probe = base or model_name@@ -412,6 +754,81 @@ def _run_with_heartbeat(run, cmd, status_cb, display_name, **kwargs):return run(cmd, **kwargs)+# The torch build this process installs kernels for, probed once (one torch import in a child).+_TORCH_ENV: dict = {}+# Kernels whose install already failed in this process, keyed by kernel, torch and toolchain, so a+# later load stage (or the next load in the same worker) does not download or compile them again.+_FAILED_INSTALLS: set = set()+++def _torch_env() -> dict[str, str] | None:+ key = probe_torch_wheel_env+ if key not in _TORCH_ENV:+ _TORCH_ENV[key] = probe_torch_wheel_env(timeout = 30, include_windows = True)+ return _TORCH_ENV[key]+++def _kernel_backend(env: dict[str, str] | None) -> str | None:+ """ "cuda" or "hip" for a torch these kernels build / ship for; None for CPU-only, XPU, MPS,+ or a torch the probe could not read, which take the model's torch path instead."""+ if not env:+ return None+ if env.get("hip_version"):+ return "hip"+ if env.get("cuda_version") or env.get("cuda_major"):+ return "cuda"+ return None+++def _failure_key(import_name: str) -> tuple:+ try:+ import importlib.metadata as metadata+ torch_version = metadata.version("torch")+ except Exception:+ torch_version = None+ return (+ import_name,+ sys.executable,+ torch_version,+ shutil.which("nvcc"),+ shutil.which("hipcc"),+ os.environ.get("CUDA_HOME") or os.environ.get("CUDA_PATH"),+ os.environ.get("ROCM_PATH"),+ )+++def _installed_distribution(pypi_name: str) -> bool:+ try:+ import importlib.metadata as metadata+ metadata.version(pypi_name)+ return True+ except Exception:+ return False+++def _remove_broken(pypi_name: str, display_name: str, run: Callable[..., Any]) -> None:+ """Uninstall a distribution whose extension does not import, as the training worker does: a+ later availability check would otherwise find it and fail instead of taking the torch path."""+ if not _installed_distribution(pypi_name):+ return+ try:+ removed = (+ run(+ uninstall_command(pypi_name, use_uv = bool(shutil.which("uv"))),+ stdout = subprocess.DEVNULL,+ stderr = subprocess.DEVNULL,+ ).returncode+ == 0+ )+ except Exception:+ removed = False+ logger.warning(+ "%s does not import; %s",+ display_name,+ "removed it" if removed else f"uninstall {pypi_name} manually",+ )++def _install_kernel(*,import_name: str,@@ -422,19 +839,107 @@ def _install_kernel(release_base_url: str,status_cb: StatusCb,run: Callable[..., Any],+ newly_installed: list[str] | None = None,+ allow_source_build: bool = True,+ required: bool = False,) -> bool:- """Install one kernel wheel-first, then a HIP-aware PyPI source build (never on Windows). Returns True iff importable afterwards; idempotent (no-op when already installed)."""+ """Install one kernel wheel-first, then (when *allow_source_build*) a HIP-aware PyPI source build+ (never on Windows). Returns True iff importable afterwards; idempotent (no-op when already+ installed). An install this call made is appended to *newly_installed*. One install per kernel+ runs at a time across processes; a failed one is not retried in this process."""if _is_importable(import_name):logger.info("%s already installed", display_name)return True- from utils.utils import hf_env_offline+ from utils.utils import package_installs_offline- if hf_env_offline():+ if package_installs_offline():+ # No URL probe, no source build, and no uv-to-pip fallback that would reach the network.logger.info("Skipping %s installation while offline", display_name)return False- env = probe_torch_wheel_env(timeout = 30, include_windows = True)+ failure_key = _failure_key(import_name)+ if failure_key in _FAILED_INSTALLS:+ logger.info("%s already failed to install in this process; not retrying", display_name)+ return False++ from utils.kernel_install import (+ KERNEL_LOCK_WAIT,+ OPTIONAL_KERNEL_LOCK_WAIT,+ kernel_install_lock,+ )++ with kernel_install_lock(+ import_name,+ on_wait = lambda: _emit(+ status_cb, f"Waiting for another process installing {display_name}..."+ ),+ # An optional acceleration does not hold a load for a peer's half-hour build.+ wait_seconds = KERNEL_LOCK_WAIT if required else OPTIONAL_KERNEL_LOCK_WAIT,+ ) as held:+ if not held:+ _emit(+ status_cb,+ f"Could not take the {display_name} install lock (another process is installing "+ "it, or the lock is unavailable); continuing without it.",+ )+ return False+ # Another process may have installed it while this one waited: new to this worker all the+ # same, so its already-imported modeling modules are refreshed or restarted.+ if _is_importable(import_name):+ logger.info("%s installed by another process", display_name)+ if newly_installed is not None:+ newly_installed.append(import_name)+ return True+ installed: bool | None = False+ try:+ installed = _install_kernel_locked(+ import_name = import_name,+ display_name = display_name,+ pypi_name = pypi_name,+ package_version = package_version,+ release_tag = release_tag,+ release_base_url = release_base_url,+ status_cb = status_cb,+ run = run,+ allow_source_build = allow_source_build,+ )+ except Exception as exc:+ # An installer that cannot even start (OSError) fails like any other attempt.+ logger.warning("Installing %s failed: %s", display_name, exc, exc_info = True)+ installed = False+ finally:+ if installed is False:+ _remove_broken(pypi_name, display_name, run)+ _FAILED_INSTALLS.add(failure_key)+ if installed is None:+ return False # nothing attempted (unsupported torch backend)+ if installed and newly_installed is not None:+ newly_installed.append(import_name)+ return bool(installed)+++def _install_kernel_locked(+ *,+ import_name: str,+ display_name: str,+ pypi_name: str,+ package_version: str,+ release_tag: str,+ release_base_url: str,+ status_cb: StatusCb,+ run: Callable[..., Any],+ allow_source_build: bool,+) -> bool | None:+ """The install itself, under the kernel's lock. None when nothing was attempted."""+ env = _torch_env()+ backend = _kernel_backend(env)+ if backend is None:+ _emit(+ status_cb,+ f"{display_name} needs a CUDA or ROCm build of torch; using the model's torch path.",+ )+ return Nonewheel_only = sys.platform == "win32" or is_windows_wheel_env(env)wheel_url = direct_wheel_url(filename_prefix = import_name,@@ -444,7 +949,14 @@ def _install_kernel(env = env,)wheel_available = url_exists(wheel_url) if wheel_url else False- if wheel_available:+ # Only a 404 proves nothing is published: an unanswered HEAD (a proxy refusing HEAD but serving+ # the download) still tries the wheel, as setup always did.+ if wheel_url and wheel_available is not False:+ if wheel_available is None:+ _emit(+ status_cb,+ f"Could not check the {display_name} prebuilt wheel; trying to download it.",+ )_emit(status_cb, f"Installing {display_name} (prebuilt kernel) for this model...")# Keep quiet downloads and unpacks within the inactivity deadline (#9398).with _heartbeat(@@ -465,53 +977,43 @@ def _install_kernel(use_uv = bool(shutil.which("uv")),run = run,)- if outcome == "installed":- logger.info("Installed prebuilt %s wheel", display_name)- return True- if outcome == "rejected":- if wheel_only:- removed = (- run(- uninstall_command(pypi_name, use_uv = bool(shutil.which("uv"))),- stdout = subprocess.DEVNULL,- stderr = subprocess.DEVNULL,- ).returncode- == 0- )- logger.warning(- "%s wheel installed but not importable; %s",- display_name,- "removed it" if removed else f"uninstall {pypi_name} manually",- )- return False- logger.warning(- "%s wheel installed but not importable; building from source",- display_name,- )+ if outcome == "installed":+ logger.info("Installed prebuilt %s wheel", display_name)+ return True+ if outcome == "rejected":+ logger.warning("%s wheel installed but not importable", display_name)+ # Remove it before any rebuild: pip / uv would call the broken copy "satisfied".+ _remove_broken(pypi_name, display_name, run)elif wheel_only:logger.info("No prebuilt %s wheel for this Windows environment (%s); using the torch fallback",display_name,wheel_url,)- elif wheel_available is None:+ else:+ logger.info("No prebuilt %s wheel for this environment (%s)", display_name, wheel_url)++ if wheel_only:+ return False+ if not allow_source_build:_emit(status_cb,- f"Could not check the {display_name} prebuilt wheel; building it from source.",+ f"No usable prebuilt {display_name} for this torch; using the model's torch path.",)- else:- logger.info(- "No prebuilt %s wheel for this environment (%s); building from source",- display_name,- wheel_url,+ return False+ if not pinned_source_build_supported(env):+ # The pinned sources build at C++17 and torch 2.13+ needs C++20: a certain failure after+ # minutes of compiling, so only the prebuilt wheel serves this torch.+ _emit(+ status_cb,+ f"No prebuilt {display_name} for torch {env.get('torch_mm')} on this machine, and it "+ "cannot be built from source for it; using the model's torch path.",)-- if wheel_only:return False# Source build (slow). ROCm has no prebuilt wheel and needs hipcc + a gcc-install-dir shim.spec = f"{pypi_name}=={package_version}"- is_hip = bool((env or {}).get("hip_version"))+ is_hip = backend == "hip"if is_hip and not shutil.which("hipcc"):_emit(status_cb, f"{display_name}: hipcc not found; install the ROCm HIP SDK to build it.")return False@@ -522,8 +1024,16 @@ def _install_kernel(# Reinstall so the source build replaces a broken wheel instead of no-opping as "already satisfied".cmd = source_build_command(spec, use_uv = bool(shutil.which("uv")), is_hip = is_hip, reinstall = True)run_kwargs, _ = source_build_run_kwargs(is_hip = is_hip, gcc_install_dir = _hipcc_gcc_install_dir)+ from utils.kernel_install import run_process_tree+try:- result = _run_with_heartbeat(run, cmd, status_cb, display_name, **run_kwargs)+ result = _run_with_heartbeat(+ lambda command, **kwargs: run_process_tree(command, run = run, **kwargs),+ cmd,+ status_cb,+ display_name,+ **run_kwargs,+ )except subprocess.TimeoutExpired:logger.error("%s source build timed out", display_name)_emit(status_cb, f"{display_name} source build timed out.")@@ -552,32 +1062,148 @@ def ensure_ssm_runtime(run: Callable[..., Any] = subprocess.run,wants_causal_conv1d: bool | None = None,is_ssm: bool | None = None,-) -> None:- """Install the SSM kernels *model_name* needs before load, wheel-first; a no-op for non-SSM models and idempotent. Only a true SSM hybrid's ``mamba_ssm`` is fatal (raises ``RuntimeError`` instead of a cryptic mid-load failure); ``causal_conv1d`` is best-effort, with LFM2 falling back to torch and Qwen3.5 / 3.6 / Qwen3-Next on zoo's Triton conv wherever it can run."""+ mamba_ssm_required: bool | None = None,+ mamba_ssm_fatal: bool | None = None,+) -> list[str]:+ """Install the SSM kernels *model_name* needs before load, wheel-first; a no-op for non-SSM models and idempotent. Returns the kernels newly installed by this call. Nemotron-H, Falcon-H1 and Granite-4.0-H (*mamba_ssm_required*) are worth a source build; a load fails on a missing ``mamba_ssm`` (``RuntimeError`` instead of a cryptic mid-load failure) only when *mamba_ssm_fatal*: it imports code that cannot run without it. Everything else, and ``causal_conv1d``, leaves transformers' torch path."""# The caller's config-resolved answers win; the name match is the default.if is_ssm is None:is_ssm = model_is_ssm(model_name)+ if mamba_ssm_required is None:+ mamba_ssm_required = model_requires_mamba_ssm(model_name)+ mamba_ssm_required = bool(is_ssm and mamba_ssm_required)+ if mamba_ssm_fatal is None:+ mamba_ssm_fatal = mamba_ssm_required and _name_mamba_ssm_is_fatal(model_name)+ mamba_ssm_fatal = bool(mamba_ssm_required and mamba_ssm_fatal)if wants_causal_conv1d is None:wants_causal_conv1d = model_wants_causal_conv1d(model_name)# Every mamba-ssm model also runs causal-conv1d.wants_causal_conv1d = wants_causal_conv1d or is_ssm+ installed: list[str] = []if not (wants_causal_conv1d or is_ssm):- return+ return installed++ def _install(kernel: PinnedKernel, *, allow_source_build: bool = True) -> bool:+ return _install_kernel(+ **_pinned_kwargs(kernel),+ status_cb = status_cb,+ run = run,+ newly_installed = installed,+ allow_source_build = allow_source_build,+ # Nemotron-H, Falcon-H1, Granite-4.0-H wait out a peer's install; the rest move on.+ required = mamba_ssm_required,+ )# causal-conv1d first: SSM modeling files lazy-import it, and mamba-ssm's fast path uses it.- if wants_causal_conv1d and not _install_kernel(- **_pinned_kwargs(CAUSAL_CONV1D), status_cb = status_cb, run = run- ):+ if wants_causal_conv1d and not _install(CAUSAL_CONV1D):logger.warning("causal-conv1d unavailable; continuing on the model's torch fallback")- if is_ssm and sys.platform == "win32" and importlib.util.find_spec("triton") is None:+ if not is_ssm:+ return installed+ offline_hint = (+ "If this machine is offline, run `unsloth install-kernels` once while online, "+ "then reload the model."+ )+ if sys.platform == "win32" and importlib.util.find_spec("triton") is None:# mamba_ssm imports triton at load and the wheel goes in --no-deps.- raise RuntimeError(- "mamba-ssm, required by this Mamba model, needs Triton on Windows "- "(the triton-windows package); install it and reload the model."- )- if is_ssm and not _install_kernel(**_pinned_kwargs(MAMBA_SSM), status_cb = status_cb, run = run):- raise RuntimeError(- "Could not install mamba-ssm, required by this Mamba model. If this machine is offline, "- "run `unsloth install-kernels` once while online, then reload the model."+ if mamba_ssm_fatal:+ raise RuntimeError(+ "mamba-ssm, required by this Mamba model, needs Triton on Windows "+ "(the triton-windows package); install it and reload the model."+ )+ _emit(+ status_cb, "mamba-ssm needs Triton on Windows (triton-windows); using the torch path.")+ return installed+ if not mamba_ssm_required:+ # The families with a torch path get mamba-ssm only as a prebuilt wheel, and only where+ # unsloth_zoo keeps its fast path on (setup's sm80 / sm75 + Triton 3.4 floor): never a+ # source build, as before setup stopped installing it for every model.+ reason = None if _is_importable(MAMBA_SSM.import_name) else _mamba_ssm_floor_reason()+ if reason is not None:+ _emit(status_cb, f"Skipping mamba-ssm, which {reason}; using the model's torch path.")+ return installed+ if not _install(MAMBA_SSM, allow_source_build = mamba_ssm_required):+ if mamba_ssm_fatal:+ # The model cannot load without mamba-ssm (its remote code imports it), so no torch+ # path exists to fall back on. On a CPU / XPU torch no install could have helped, so+ # say what is missing instead of pointing at the network.+ if _kernel_backend(_torch_env()) is None:+ raise RuntimeError(+ "This Mamba model requires mamba-ssm, which needs a CUDA or ROCm torch build."+ )+ raise RuntimeError(+ f"Could not install mamba-ssm, required by this Mamba model. {offline_hint}"+ )+ if mamba_ssm_required:+ _emit(+ status_cb,+ "Warning: could not install mamba-ssm; loading this model on transformers' torch "+ "path, which is slower and uses more memory.",+ )+ else:+ _emit(status_cb, "mamba-ssm is not available; continuing on the model's torch path.")+ return installed+++def _mamba_ssm_floor_reason() -> str | None:+ """Why an optional mamba-ssm stays off this machine's GPU, or None when unsloth_zoo would use+ it: setup's rule (kernel_install), from the torch already imported here when there is one."""+ from utils import kernel_install++ capability = None+ torch = sys.modules.get("torch")+ if torch is not None:+ try:+ if getattr(torch.version, "hip", None):+ return None+ if torch.cuda.is_available() and torch.cuda.device_count():+ capability = max(+ tuple(torch.cuda.get_device_capability(i))+ for i in range(torch.cuda.device_count())+ )+ except Exception:+ capability = None+ if capability is None:+ capability = kernel_install._gpu_capability(subprocess.run)+ if capability is None:+ return "needs a visible GPU"+ if capability >= (8, 0):+ return None+ return kernel_install._pre_ampere_skip_reason("mamba_ssm", capability, subprocess.run)+++def provision_ssm_kernels(+ model_name: str,+ model_load_target: str,+ hf_token: str | None,+ *,+ adapter_base: str | None = None,+ model_config: dict | None = None,+ trust_remote_code: bool | None = None,+ status_cb: StatusCb = None,+ run: Callable[..., Any] = subprocess.run,+) -> tuple[SsmKernels, list[str]]:+ """Resolve and install the kernels a load needs (training, inference, export): returns what was+ resolved and what this call newly installed. Raises ``RuntimeError`` only for a model that+ cannot load without ``mamba_ssm``."""+ kernels = resolve_ssm_kernels(+ model_name,+ model_load_target,+ hf_token,+ adapter_base = adapter_base,+ model_config = model_config,+ trust_remote_code = trust_remote_code,+ )+ installed = ensure_ssm_runtime(+ kernels.probe,+ status_cb = status_cb,+ run = run,+ wants_causal_conv1d = kernels.causal_conv1d,+ is_ssm = kernels.mamba_ssm,+ mamba_ssm_required = kernels.mamba_ssm_required,+ mamba_ssm_fatal = kernels.mamba_ssm_fatal,+ )+ if installed:+ refresh_transformers_kernel_gates()+ return kernels, installeddiff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.pyindex e364806771e..f4be7df84ff 100644--- a/studio/backend/utils/utils.py+++ b/studio/backend/utils/utils.py@@ -40,6 +40,22 @@ def hf_env_offline() -> bool:return False+def package_installs_offline() -> bool:+ """True when an on-demand package install (a kernel wheel, a source build) must not reach the+ network: the HF offline switches above, or UV_OFFLINE, the operator's no-network switch for+ installs. Under UV_OFFLINE uv refuses the index, and install_wheel's pip fallback would then+ reach it anyway, so such an install is skipped outright.++ An HF offline that Studio set itself only because the Hub was unreachable (export's offline+ window marks it, wheel_utils.hf_offline_auto_set) says nothing about GitHub / PyPI, so then+ only UV_OFFLINE counts: an operator-set HF offline still blocks every install."""+ from utils.wheel_utils import hf_offline_auto_set, uv_offline++ if hf_offline_auto_set():+ return uv_offline()+ return hf_env_offline() or uv_offline()++def anonymous_and_offline(hf_token, *, repo_id: Optional[str] = None) -> bool:"""The one condition under which a Hub-reaching request can only be answered by disk.diff --git a/studio/backend/utils/wheel_utils.py b/studio/backend/utils/wheel_utils.pyindex a1988d2b300..3c09dee3287 100644--- a/studio/backend/utils/wheel_utils.py+++ b/studio/backend/utils/wheel_utils.py@@ -131,12 +131,12 @@ _UNSLOTH_PREBUILT_VERSIONS = {"mamba_ssm": "2.3.2.post1",}-# platform_tag -> (cxx11abi field, published interpreters or None for any). The Windows cu130 torch also reports cxx11abi True; Windows publishes cp313 GIL builds only, and so does linux_aarch64 (upstream's aarch64 wheels stop at torch 2.10).+# platform_tag -> (cxx11abi field, published interpreters or None for any). The Windows cu130 torch also reports cxx11abi True. Only what the release actually carries: torch 2.13 / 2.14 are published for cp313 GIL builds on every platform (the workflow can build cp311 / cp312 cells, but none has been published, and a URL for one is a 404 in front of the fallback)._UNSLOTH_PREBUILT_PLATFORMS: dict[str, tuple[str, frozenset[str] | None]] = {- "linux_x86_64": ("TRUE", None),+ "linux_x86_64": ("TRUE", frozenset({"cp313"})),"win_amd64": ("TRUE", frozenset({"cp313"})),"linux_aarch64": ("TRUE", frozenset({"cp313"})),- "win_arm64": ("TRUE", frozenset({"cp311", "cp312", "cp313"})),+ "win_arm64": ("TRUE", frozenset({"cp313"})),}# (platform, torch minor) cells outside _UNSLOTH_PREBUILT_TORCH_MM, with their interpreters. Upstream's torch2.10 wheels (and their 2.11 / 2.12 reuse above) have no Windows asset, so win_amd64 torch 2.10, 2.11 and 2.12 are ours; an existing install keeps its minor (a fresh one takes 2.13 on cp313), hence three interpreters.@@ -250,6 +250,56 @@ def unsloth_prebuilt_wheel_url(*, filename_prefix: str, env: dict[str, str] | Noreturn f"{UNSLOTH_PREBUILT_RELEASE_BASE_URL}/{UNSLOTH_PREBUILT_RELEASE_TAG}/{filename}"+# The cells the pinned upstream releases publish (causal-conv1d v1.6.1.post4, mamba-ssm v2.3.1,+# identical for both): (platform, CUDA major, cxx11abi) -> torch minors, each for cp310-cp313.+# Read from the two release asset lists; anything else (CUDA 13 before torch 2.9, aarch64 before+# 2.7, an abiFALSE torch after 2.6, any interpreter outside cp310-cp313) is a URL that 404s in+# front of the source fallback, so it resolves to nothing instead.+_UPSTREAM_SSM_PINS = {+ "causal_conv1d": CAUSAL_CONV1D_PACKAGE_VERSION,+ "mamba_ssm": MAMBA_SSM_PACKAGE_VERSION,+}+_UPSTREAM_SSM_CELLS: dict[tuple[str, str, str], frozenset[str]] = {+ ("linux_x86_64", "11", "FALSE"): frozenset({"2.6"}),+ ("linux_x86_64", "11", "TRUE"): frozenset({"2.6", "2.7"}),+ ("linux_x86_64", "12", "FALSE"): frozenset({"2.6"}),+ ("linux_x86_64", "12", "TRUE"): frozenset({"2.6", "2.7", "2.8", "2.9", "2.10"}),+ ("linux_x86_64", "13", "TRUE"): frozenset({"2.9", "2.10"}),+ ("linux_aarch64", "12", "TRUE"): frozenset({"2.7", "2.8", "2.9", "2.10"}),+ ("linux_aarch64", "13", "TRUE"): frozenset({"2.9", "2.10"}),+}+_UPSTREAM_SSM_PYTHONS = frozenset({"cp310", "cp311", "cp312", "cp313"})+++def _upstream_ssm_cell_published(filename_prefix: str, package_version: str, env: dict) -> bool:+ if _UPSTREAM_SSM_PINS.get(filename_prefix) != package_version:+ return True # not a pinned release this table describes: unchanged+ torch_mm = prebuilt_wheel_torch_mm(str(env.get("torch_mm")))+ cell = (str(env.get("platform_tag")), str(env.get("cuda_major")), str(env.get("cxx11abi")))+ python_tag = env.get("python_tag")+ return (+ torch_mm in _UPSTREAM_SSM_CELLS.get(cell, frozenset())+ and python_tag in _UPSTREAM_SSM_PYTHONS+ and (env.get("abi_tag") or python_tag) == python_tag+ )+++# The newest torch the pinned causal-conv1d 1.6.1 / mamba-ssm 2.3.1 compile against: torch 2.13+# raised its headers to C++20 and both build at C++17 (the prebuilt workflow moved to+# causal-conv1d 1.7.0 and patches mamba-ssm for it, see .github/scripts/patch_mamba_cxx20.py).+PINNED_SOURCE_BUILD_MAX_TORCH = (2, 12)+++def pinned_source_build_supported(env: dict[str, str] | None) -> bool:+ """Whether a source build of the pinned causal-conv1d / mamba-ssm (or their upstream wheels)+ can work with this torch; from torch 2.13 on only our prebuilt wheels can."""+ try:+ major, minor = (int(part) for part in str((env or {}).get("torch_mm") or "").split(".")[:2])+ except ValueError:+ return True # unknown: leave the decision to the build, as before+ return (major, minor) <= PINNED_SOURCE_BUILD_MAX_TORCH++def direct_wheel_url(*,filename_prefix: str,@@ -265,8 +315,10 @@ def direct_wheel_url(ours = unsloth_prebuilt_wheel_url(filename_prefix = filename_prefix, env = env)if ours is not None:return ours- # No upstream release has a Windows asset.- if is_windows_wheel_env(env):+ # No upstream release has a Windows asset, nor one for torch 2.13 or later (see above).+ if is_windows_wheel_env(env) or not pinned_source_build_supported(env):+ return None+ if not _upstream_ssm_cell_published(filename_prefix, package_version, env):return Nonefilename = (@@ -478,6 +530,40 @@ def flash_attn_wheel_url(env: dict[str, str] | None) -> str | None:)+_UV_OFFLINE_TRUE_VALUES = frozenset({"1", "true", "t", "yes", "y", "on"})+_HF_OFFLINE_TRUE_VALUES = frozenset({"1", "true", "yes", "on"})+++def uv_offline() -> bool:+ """UV_OFFLINE, the operator's no-network switch for installs, in uv's boolish spellings."""+ return os.environ.get("UV_OFFLINE", "").strip().lower() in _UV_OFFLINE_TRUE_VALUES+++# Set (to "1") while Studio itself forced HF offline only because the Hugging Face endpoint was+# unreachable (export's _offline_window_if_unreachable), and restored when that window closes.+# HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE then say nothing about GitHub or PyPI, so a kernel install+# may still reach them; only UV_OFFLINE keeps it off the network. Children inherit it.+HF_OFFLINE_AUTO_ENV = "_UNSLOTH_HF_OFFLINE_AUTO"+++def hf_offline_auto_set() -> bool:+ """True while HF offline was set by Studio for an unreachable Hub, not by the operator."""+ return os.environ.get(HF_OFFLINE_AUTO_ENV, "") == "1"+++def package_installs_offline_env() -> bool:+ """The environment half of utils.utils.package_installs_offline, for callers that cannot import+ the backend (`unsloth install-kernels` runs under --no-deps): HF_HUB_OFFLINE,+ TRANSFORMERS_OFFLINE or UV_OFFLINE. While HF offline was only auto-set for an unreachable Hub+ (hf_offline_auto_set), only UV_OFFLINE counts."""+ if hf_offline_auto_set():+ return uv_offline()+ return uv_offline() or any(+ os.environ.get(name, "").strip().lower() in _HF_OFFLINE_TRUE_VALUES+ for name in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE")+ )++def install_wheel(wheel_url: str,*,@@ -510,6 +596,9 @@ def install_wheel(attempts.append(("uv", result))if result.returncode == 0:return attempts+ # uv refused the network under UV_OFFLINE; pip has no such switch and would reach it.+ if uv_offline():+ return attemptspip_cmd = [python_executable, "-m", "pip", "install", "--no-deps"]if reinstall:diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex 18bcf926963..8b615d0d7b7 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -52,7 +52,10 @@ def _nvidia_library_inventory():from backend.utils.kernel_install import (+ KERNELS,+ gpu_below_flash_attn_floor,install_prebuilt,+ kernel_install_lock,uninstall_command,)from backend.utils.wheel_utils import (@@ -8580,6 +8583,12 @@ def _remove_stale_flash_attn() -> None:)+def _flash_attn_gpu_below_floor() -> bool:+ """A visible CUDA GPU below sm80 (a T4), where Unsloth leaves FlashAttention 2 off. No visible+ GPU is False: an image build installs ahead of the GPU it will run on."""+ return gpu_below_flash_attn_floor()++def _ensure_flash_attn() -> None:if _flash_attn_install_disabled():return@@ -8598,6 +8607,12 @@ def _ensure_flash_attn() -> None:# a failed flash-attn probe proves staleness only when torch still imports.stale = env is not None and _installed_distribution_version("flash-attn") is not Nonewheel_url = _build_flash_attn_wheel_url(env) if env else None+ if wheel_url and not (env or {}).get("hip_version") and _flash_attn_gpu_below_floor():+ # Unsloth only turns FlashAttention 2 on from sm80; a T4 would download it for nothing.+ _note("flash-attn needs an sm80 or newer GPU; skipped")+ if stale:+ _remove_stale_flash_attn()+ returnwheel_available = url_exists(wheel_url) if wheel_url else Falseif wheel_available:# installing a distribution invalidates caches keyed on the action counter.@@ -8654,6 +8669,120 @@ def _ensure_flash_attn() -> None:_remove_stale_flash_attn()+# The compiled extension each SSM kernel loads (kernel_install.KERNELS checks the same import).+_SSM_KERNEL_EXTENSIONS = {"causal_conv1d": "causal_conv1d_cuda", "mamba_ssm": "selective_scan_cuda"}+# Exit code of the probe below when torch imported and the extension raised ImportError / OSError.+_SSM_EXTENSION_BROKEN_EXIT = 87+_SSM_EXTENSION_PROBE = (+ "import sys\n"+ "import torch\n"+ "try:\n"+ " import {extension}\n"+ "except (ImportError, OSError) as exc:\n"+ " print('{extension}: %s: %s' % (type(exc).__name__, exc), file = sys.stderr)\n"+ " sys.exit({code})\n"+)+++def _probe_tail(stderr) -> str:+ lines = [line for line in str(stderr or "").strip().splitlines() if line.strip()]+ return " | ".join(lines[-3:])[-400:]+++def _run_import_probe(code: str):+ """Run *code* in a child; None when it hung or could not start (inconclusive)."""+ try:+ return subprocess.run(+ [sys.executable, "-c", code],+ stdout = subprocess.DEVNULL,+ stderr = subprocess.PIPE,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = _FLASH_ATTN_IMPORT_PROBE_TIMEOUT,+ )+ except (OSError, subprocess.TimeoutExpired):+ return None+++def _remove_stale_ssm_kernels() -> None:+ """Remove an installed causal-conv1d / mamba-ssm whose compiled extension no longer imports,+ e.g. built for the torch a repair just replaced. Never installs: a model that needs them+ installs them when it loads, and a broken native module left behind can take that load down.++ Removal needs proof: torch itself imports, and the extension import then completes with an+ ImportError / OSError. A torch that does not import, a probe killed by a signal, a hang or a+ probe that cannot start says nothing about the kernel, and an offline machine could not get+ it back, so each keeps it."""+ if NO_TORCH or IS_MACOS:+ return+ installed = [+ name+ for name in _SSM_KERNEL_EXTENSIONS+ if _installed_distribution_version(KERNELS[name][0]) is not None+ ]+ if not installed:+ return+ torch_probe = _run_import_probe("import torch")+ if torch_probe is None or torch_probe.returncode != 0:+ detail = "" if torch_probe is None else f": {_probe_tail(torch_probe.stderr)}"+ _note(f"torch did not import cleanly; kept {', '.join(installed)}{detail}")+ return+ for name in installed:+ distribution = KERNELS[name][0]+ extension = _SSM_KERNEL_EXTENSIONS[name]+ probe = _run_import_probe(+ _SSM_EXTENSION_PROBE.format(extension = extension, code = _SSM_EXTENSION_BROKEN_EXIT)+ )+ if probe is None or probe.returncode == 0:+ continue+ tail = _probe_tail(probe.stderr)+ if probe.returncode != _SSM_EXTENSION_BROKEN_EXIT or extension not in tail:+ # A signal (negative code) or a crash outside the extension import: inconclusive.+ _note(f"could not check {name} (exit {probe.returncode}); kept it: {tail}")+ continue+ # The verdict above was reached before this lock, which can wait up to KERNEL_LOCK_WAIT+ # while a Studio worker or `unsloth install-kernels` (re)installs this kernel. So decide+ # again while holding it: remove only when the extension still fails the same way, never+ # a fresh install that loads now.+ with kernel_install_lock(name) as held:+ if not held:+ _note(+ f"{name} does not load with this torch, but its install lock is busy; kept it"+ )+ continue+ recheck = _run_import_probe(+ _SSM_EXTENSION_PROBE.format(extension = extension, code = _SSM_EXTENSION_BROKEN_EXIT)+ )+ if recheck is not None and recheck.returncode == 0:+ _note(f"{name} now loads with this torch; kept it")+ continue+ recheck_tail = "" if recheck is None else _probe_tail(recheck.stderr)+ if (+ recheck is None+ or recheck.returncode != _SSM_EXTENSION_BROKEN_EXIT+ or extension not in recheck_tail+ ):+ code = "hung" if recheck is None else f"exit {recheck.returncode}"+ _note(f"could not re-check {name} ({code}); kept it: {recheck_tail}")+ continue+ tail = recheck_tail+ _count_install_action()+ try:+ removed = _uninstall_distribution(distribution)+ except OSError:+ removed = False+ if removed:+ _step("warning", f"{name} does not load with this torch; removed it ({tail})", _warn)+ else:+ _step(+ "warning",+ f"{name} does not load with this torch and could not be removed; "+ f"uninstall {distribution} manually ({tail})",+ _warn,+ )++# -- uv bootstrap ------------------------------------------------------USE_UV = False # Set by _bootstrap_uv() at the start of install_python_stack()@@ -12893,6 +13022,7 @@ def install_python_stack() -> int:if not IS_MACOS and not NO_TORCH:_progress("flash-attn")_ensure_flash_attn()+ _remove_stale_ssm_kernels()# # 6. Patch: override llama_cpp.py with fix from unsloth-zoo feature/llama-cpp-windows-support branch# patch_package_file(@@ -13078,6 +13208,7 @@ def install_python_stack() -> int:_install_torchao_for_torch(_torch_after_repair)# resync flash-attn because the repair can invalidate the wheel selected earlier._ensure_flash_attn()+ _remove_stale_ssm_kernels()# Unguarded: torch==2.10.0 accepts 2.10.0+rocm7.1, and an earlier run may have moved it._evict_xformers_built_for_another_torch(scope = "linux torch repair", family_only = True)_evict_xformers_requiring_another_torch()@@ -13094,6 +13225,7 @@ def install_python_stack() -> int:if _torch_after_flavor and _torch_after_flavor != _torch_before_flavor:# flash-attn was chosen for the torch the invariant just replaced, as in step 13._ensure_flash_attn()+ _remove_stale_ssm_kernels()# A direct run has no setup.ps1 postlude to swap triton back. After the invariant,# because the swap keys off the installed +xpu label._ensure_xpu_triton()diff --git a/tests/conftest.py b/tests/conftest.pyindex 8b1826af955..f7a25d952cf 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -199,3 +199,14 @@ def _apply_upstream_import_fixes_for_tests() -> None:_apply_upstream_import_fixes_for_tests()++[email protected](scope = "session")+def _kernel_lock_dir(tmp_path_factory):+ return str(tmp_path_factory.mktemp("kernel-locks"))++[email protected](autouse = True)+def _isolated_kernel_install_locks(monkeypatch, _kernel_lock_dir):+ # Tests never take (or create) the environment's real kernel install locks.+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", _kernel_lock_dir)diff --git a/tests/python/test_flash_attn_install_python_stack.py b/tests/python/test_flash_attn_install_python_stack.pyindex d945f8afcdf..f99768495cb 100644--- a/tests/python/test_flash_attn_install_python_stack.py+++ b/tests/python/test_flash_attn_install_python_stack.py@@ -46,9 +46,16 @@ sys.path.insert(0, str(STUDIO_DIR))sys.path.insert(0, str(STUDIO_DIR / "backend"))import install_python_stack as ips+import pytestfrom utils import wheel_utils[email protected](autouse = True)+def _an_sm80_gpu(monkeypatch):+ # The host's real GPU must not decide these tests; the capability cases below override it.+ monkeypatch.setattr(ips, "_flash_attn_gpu_below_floor", lambda: False)++class TestPrebuiltWheelTorchMapping:def test_torch_211_maps_to_torch210(self):assert wheel_utils.prebuilt_wheel_torch_mm("2.11") == "2.10"@@ -827,3 +834,64 @@ class TestWarningColor:out = capsys.readouterr().outassert f"\033[{code}mflash-attn install failed (exit code 1)\033[0m" in outassert "\033[96m" not in out+++class TestFlashAttnCapabilityFloor:+ """Unsloth only enables FlashAttention 2 from sm80: below it the wheel is never downloaded."""++ def _run(self, monkeypatch, capability):+ from utils import kernel_install++ monkeypatch.setattr(+ ips,+ "_flash_attn_gpu_below_floor",+ lambda: kernel_install.gpu_below_flash_attn_floor(lambda *a, **k: None),+ )+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: capability)+ calls = {"head": 0, "install": 0}+ notes = []+ with (+ mock.patch.object(ips, "NO_TORCH", False),+ mock.patch.object(ips, "IS_WINDOWS", True),+ mock.patch.object(ips, "IS_MACOS", False),+ mock.patch.object(ips, "_flash_attn_importable", return_value = False),+ mock.patch.object(ips, "_installed_distribution_version", return_value = None),+ mock.patch.object(+ ips,+ "probe_torch_wheel_env",+ return_value = {+ "python_tag": "cp313",+ "abi_tag": "cp313",+ "torch_mm": "2.13",+ "cuda_major": "13",+ "cxx11abi": "TRUE",+ "platform_tag": "win_amd64",+ "hip_version": "",+ },+ ),+ mock.patch.object(+ ips,+ "url_exists",+ side_effect = lambda url: calls.__setitem__("head", calls["head"] + 1) or True,+ ),+ mock.patch.object(+ ips,+ "install_prebuilt",+ side_effect = lambda *a, **k: calls.__setitem__("install", calls["install"] + 1)+ or "installed",+ ),+ mock.patch.object(ips, "_note", side_effect = notes.append),+ mock.patch.object(ips, "_count_install_action"),+ ):+ ips._ensure_flash_attn()+ return calls, notes++ def test_a_t4_does_not_download_flash_attn(self, monkeypatch):+ calls, notes = self._run(monkeypatch, (7, 5))+ assert calls == {"head": 0, "install": 0}+ assert any("sm80" in note for note in notes)++ @pytest.mark.parametrize("capability", [(8, 0), (12, 0), None])+ def test_sm80_and_newer_or_no_visible_gpu_still_install(self, monkeypatch, capability):+ calls, _ = self._run(monkeypatch, capability)+ assert calls == {"head": 1, "install": 1}diff --git a/tests/python/test_prebuilt_cuda_wheels.py b/tests/python/test_prebuilt_cuda_wheels.pyindex 80b28588a60..4e39c9c7056 100644--- a/tests/python/test_prebuilt_cuda_wheels.py+++ b/tests/python/test_prebuilt_cuda_wheels.py@@ -67,7 +67,8 @@ def triggers(doc: dict) -> dict:class TestWorkflowAndResolverAgree:def test_every_cell_the_workflow_builds_is_what_the_resolver_asks_for(self):"""Nothing else checks the workflow's wheel name matches the installer's download; a- mismatch is a 404."""+ mismatch is a 404. The resolver asks only for the interpreters published (cp313: the+ workflow's default), not every one the workflow could be dispatched to build."""checked = 0for package, spec in prebuilt_wheels.SPECS.items():for torch_version in prebuilt_wheels.TORCH_VERSIONS:@@ -80,12 +81,122 @@ class TestWorkflowAndResolverAgree:python_tag = prebuilt_wheels.python_tag(python_version),),)+ if python_version not in prebuilt_wheels.DEFAULT_PYTHON:+ assert url is None, (package, torch_version, python_version)+ continueassert url is not None, (package, torch_version, python_version)assert url.rsplit("/", 1)[1] == builtassert url.startswith(OUR_BASE)checked += 1- # 3 packages x 2 torch minors x 3 interpreters; guards against an emptied table.- assert checked == 18+ # 3 packages x 2 torch minors x cp313; guards against an emptied table.+ assert checked == 6++ def test_every_url_the_resolver_can_emit_is_a_published_cell(self):+ """Whatever the probe reports, a URL names a cell the publish matrix builds and the release+ carries: (platform, torch, interpreter, package). Anything else is a 404 in front of the+ fallback."""+ published = set()+ for platform in prebuilt_wheels.PLATFORMS:+ packages = prebuilt_wheels._platform(platform, "packages") or tuple(+ prebuilt_wheels.SPECS+ )+ for torch_version in prebuilt_wheels._platform_torches(platform):+ torch_mm = prebuilt_wheels.torch_minor(torch_version)+ extra = wheel_utils._UNSLOTH_PREBUILT_EXTRA_CELLS.get((platform, torch_mm))+ pythons = (+ prebuilt_wheels.DEFAULT_PYTHON+ if extra is None+ else tuple(+ v+ for v in prebuilt_wheels.PYTHON_VERSIONS+ if prebuilt_wheels.python_tag(v) in extra+ )+ )+ for python_version in pythons:+ for package in packages:+ published.add(+ prebuilt_wheels.wheel_name(+ package, torch_version, python_version, platform+ )+ )+ emitted = set()+ for platform in (+ "linux_x86_64",+ "linux_aarch64",+ "win_amd64",+ "win_arm64",+ "macosx_14_0_arm64",+ ):+ for torch_mm in ("2.6", "2.9", "2.10", "2.11", "2.12", "2.13", "2.14", "2.15"):+ for python_tag in ("cp310", "cp311", "cp312", "cp313", "cp314"):+ for cuda_version in ("12.8", "13.0", "13.4"):+ for spec in prebuilt_wheels.SPECS.values():+ probe = env(+ torch_mm = torch_mm,+ cuda_major = cuda_version.split(".")[0],+ python_tag = python_tag,+ platform_tag = platform,+ )+ probe["cuda_version"] = cuda_version+ url = wheel_utils.unsloth_prebuilt_wheel_url(+ filename_prefix = spec["dist"], env = probe+ )+ if url is not None:+ assert url.startswith(OUR_BASE)+ emitted.add(url.rsplit("/", 1)[1])+ assert emitted, "the resolver emitted nothing"+ assert emitted <= published, sorted(emitted - published)++ @pytest.mark.parametrize(+ "platform, torch_mm, python_tag, cuda_version",+ [+ ("linux_x86_64", "2.13", "cp312", "13.0"),+ ("linux_x86_64", "2.14", "cp310", "13.0"),+ ("linux_x86_64", "2.13", "cp311", "13.0"),+ ("win_arm64", "2.14", "cp311", "13.4"),+ ("win_arm64", "2.14", "cp312", "13.4"),+ ],+ )+ def test_unpublished_interpreters_resolve_to_nothing(+ self, platform, torch_mm, python_tag, cuda_version+ ):+ probe = env(torch_mm = torch_mm, python_tag = python_tag, platform_tag = platform)+ probe["cuda_version"] = cuda_version+ for prefix in ("causal_conv1d", "mamba_ssm"):+ assert wheel_utils.unsloth_prebuilt_wheel_url(filename_prefix = prefix, env = probe) is None+ # Nor an upstream URL: there is no upstream asset for torch 2.13+ either.+ assert (+ wheel_utils.direct_wheel_url(+ filename_prefix = prefix,+ package_version = "1.6.1",+ release_tag = "v1.6.1.post4",+ release_base_url = "https://github.com/Dao-AILab/causal-conv1d/releases/download",+ env = probe,+ )+ is None+ )++ @pytest.mark.parametrize("cuda_major", ["12", "13"])+ def test_no_upstream_url_for_torch_213_and_later(self, cuda_major):+ probe = env(torch_mm = "2.13", cuda_major = cuda_major, python_tag = "cp312")+ assert (+ wheel_utils.direct_wheel_url(+ filename_prefix = "mamba_ssm",+ package_version = "2.3.1",+ release_tag = "v2.3.1",+ release_base_url = "https://github.com/state-spaces/mamba/releases/download",+ env = probe,+ )+ is None+ )+ # torch 2.12 keeps its upstream (2.10 reuse) URL.+ assert wheel_utils.direct_wheel_url(+ filename_prefix = "mamba_ssm",+ package_version = "2.3.1",+ release_tag = "v2.3.1",+ release_base_url = "https://github.com/state-spaces/mamba/releases/download",+ env = env(torch_mm = "2.12", cuda_major = cuda_major, python_tag = "cp312"),+ ).endswith("torch2.10cxx11abiTRUE-cp312-cp312-linux_x86_64.whl")def test_the_versions_published_are_the_versions_resolved(self):for package, spec in prebuilt_wheels.SPECS.items():@@ -198,12 +309,17 @@ class TestOverrideScope:wheel_utils.unsloth_prebuilt_wheel_url(filename_prefix = "flash_attn", env = None) is None)- @pytest.mark.parametrize("python_tag", ["cp311", "cp312", "cp313"])- def test_every_interpreter_we_build(self, python_tag):+ @pytest.mark.parametrize(+ "python_tag, published", [("cp311", False), ("cp312", False), ("cp313", True)]+ )+ def test_only_the_interpreters_published(self, python_tag, published):url = wheel_utils.unsloth_prebuilt_wheel_url(filename_prefix = "flash_attn", env = env(python_tag = python_tag))- assert url is not None and f"-{python_tag}-{python_tag}-" in url+ if published:+ assert url is not None and f"-{python_tag}-{python_tag}-" in url+ else:+ assert url is Noneclass TestBackwardsCompatible:@@ -859,3 +975,76 @@ def test_release_lookups_do_not_pass_arg_to_gh_jq():)assert "--jq --arg" not in workflowassert workflow.count("== env.TAG") == 2+++# The wheels on causal-conv1d v1.6.1.post4 and mamba v2.3.1 (identical cells for both), read from+# the two release asset lists: (platform, CUDA major, cxx11abi, torch minor), each for cp310-cp313.+_UPSTREAM_PUBLISHED = {+ ("linux_x86_64", "11", "FALSE", "2.6"),+ ("linux_x86_64", "11", "TRUE", "2.6"),+ ("linux_x86_64", "11", "TRUE", "2.7"),+ ("linux_x86_64", "12", "FALSE", "2.6"),+ *(("linux_x86_64", "12", "TRUE", t) for t in ("2.6", "2.7", "2.8", "2.9", "2.10")),+ *(("linux_x86_64", "13", "TRUE", t) for t in ("2.9", "2.10")),+ *(("linux_aarch64", "12", "TRUE", t) for t in ("2.7", "2.8", "2.9", "2.10")),+ *(("linux_aarch64", "13", "TRUE", t) for t in ("2.9", "2.10")),+}+_UPSTREAM_PINS = {+ "causal_conv1d": (+ "1.6.1",+ "v1.6.1.post4",+ "https://github.com/Dao-AILab/causal-conv1d/releases/download",+ ),+ "mamba_ssm": ("2.3.1", "v2.3.1", "https://github.com/state-spaces/mamba/releases/download"),+}+++def test_every_upstream_url_the_resolver_can_emit_is_published():+ """CUDA 13 before torch 2.9, aarch64 before 2.7, abiFALSE after 2.6, torch 2.13+ and any+ interpreter outside cp310-cp313 have no upstream asset: no URL, not a 404."""+ published = {+ f"{prefix}-{version}+cu{cuda}torch{torch_mm}cxx11abi{abi}-{py}-{py}-{platform}.whl"+ for platform, cuda, abi, torch_mm in _UPSTREAM_PUBLISHED+ for prefix, (version, _, _) in _UPSTREAM_PINS.items()+ for py in ("cp310", "cp311", "cp312", "cp313")+ }+ emitted = set()+ for platform in ("linux_x86_64", "linux_aarch64"):+ for cuda in ("11", "12", "13"):+ for abi in ("TRUE", "FALSE"):+ for torch_mm in (+ "2.5",+ "2.6",+ "2.7",+ "2.8",+ "2.9",+ "2.10",+ "2.11",+ "2.12",+ "2.13",+ "2.14",+ ):+ for py in ("cp39", "cp310", "cp311", "cp312", "cp313", "cp314"):+ probe = env(+ torch_mm = torch_mm,+ cuda_major = cuda,+ cxx11abi = abi,+ python_tag = py,+ platform_tag = platform,+ )+ probe["cuda_version"] = f"{cuda}.0"+ for prefix, (version, tag, base) in _UPSTREAM_PINS.items():+ url = wheel_utils.direct_wheel_url(+ filename_prefix = prefix,+ package_version = version,+ release_tag = tag,+ release_base_url = base,+ env = probe,+ )+ if url is None or url.startswith(OUR_BASE):+ continue+ assert url.startswith(f"{base}/{tag}/"), url+ emitted.add(url.rsplit("/", 1)[1])+ assert emitted <= published, sorted(emitted - published)+ # And nothing published is lost (torch 2.11 / 2.12 reuse the 2.10 wheels).+ assert emitted == publisheddiff --git a/tests/python/test_ssm_kernels_install_python_stack.py b/tests/python/test_ssm_kernels_install_python_stack.pyindex a25c192fb83..79cd2133b87 100644--- a/tests/python/test_ssm_kernels_install_python_stack.py+++ b/tests/python/test_ssm_kernels_install_python_stack.py@@ -3,15 +3,20 @@"""Setup installs flash-attn but not causal-conv1d or mamba-ssm: those (~550 MB) install when amodel that needs them is trained or loaded (core/training/worker.py, utils/ssm_runtime.py), and-`unsloth install-kernels` still provisions them ahead of time for offline images."""+`unsloth install-kernels` still provisions them ahead of time for offline images. After torch+moves, setup still removes an installed copy whose compiled extension no longer imports."""from __future__ import annotationsimport ast+import contextlibimport re+import subprocessimport sysfrom pathlib import Path+import pytest+STUDIO_DIR = Path(__file__).resolve().parents[2] / "studio"sys.path.insert(0, str(STUDIO_DIR))sys.path.insert(0, str(STUDIO_DIR / "backend"))@@ -37,16 +42,191 @@ def test_setup_has_no_ssm_kernel_step():assert "causal-conv1d + mamba-ssm" not in SOURCE-def test_flash_attn_is_still_installed_and_resynced_at_all_three_sites():+def test_flash_attn_and_the_stale_kernel_check_run_at_all_three_sites():calls = [m.start() for m in re.finditer(r"\n\s+_ensure_flash_attn\(\)\n", SOURCE)]assert len(calls) == 3for start in calls:- following = SOURCE[start : start + 200]- assert "ssm" not in following.lower(), following+ following = SOURCE[start : start + 120]+ assert "_remove_stale_ssm_kernels()" in following, followingassert "base_total += 1 # flash-attn, Linux and Windows" in SOURCE+class _Kernels:+ """Which SSM distributions are installed and how their extension import ends: True loads,+ False raises ImportError, None hangs, an int is a raw exit code (negative = signal)."""++ def __init__(+ self,+ installed: dict,+ torch_imports = True,+ ):+ self.installed = dict(installed)+ self.torch_imports = torch_imports+ self.probes: list[str] = []+ self.removed: list[str] = []++ def version(self, distribution):+ return "1.0" if distribution in self.installed else None++ def run(self, cmd, *args, **kwargs):+ code = cmd[-1]+ self.probes.append(code)+ if code == "import torch":+ if self.torch_imports is None:+ raise subprocess.TimeoutExpired(cmd, 1)+ if self.torch_imports:+ return subprocess.CompletedProcess(cmd, 0, stderr = "")+ return subprocess.CompletedProcess(+ cmd, 1, stderr = "OSError: libtorch_cuda.so: cannot open shared object file"+ )+ assert "import torch" in code # the extension probe proves torch imports too+ extension = "causal_conv1d_cuda" if "causal_conv1d_cuda" in code else "selective_scan_cuda"+ distribution = "causal-conv1d" if extension == "causal_conv1d_cuda" else "mamba-ssm"+ state = self.installed[distribution]+ if state is None:+ raise subprocess.TimeoutExpired(cmd, 1)+ if state is True:+ return subprocess.CompletedProcess(cmd, 0, stderr = "")+ if state is False:+ return subprocess.CompletedProcess(+ cmd,+ ips._SSM_EXTENSION_BROKEN_EXIT,+ stderr = f"{extension}: ImportError: {extension}.so: undefined symbol: _ZN3c10",+ )+ return subprocess.CompletedProcess(cmd, state, stderr = "Killed")++ def uninstall(self, distribution):+ self.removed.append(distribution)+ self.installed.pop(distribution, None)+ return True+++def _free_lock(name, **kwargs):+ yield True+++def _run_stale_check(monkeypatch, host: _Kernels, **flags):+ monkeypatch.setattr(ips, "NO_TORCH", flags.get("no_torch", False))+ monkeypatch.setattr(ips, "IS_MACOS", flags.get("macos", False))+ monkeypatch.setattr(ips, "_installed_distribution_version", host.version)+ monkeypatch.setattr(ips, "_uninstall_distribution", host.uninstall)+ monkeypatch.setattr(ips, "_count_install_action", lambda: None)+ monkeypatch.setattr(ips, "kernel_install_lock", flags.get("lock", _free_lock))+ warnings = []+ monkeypatch.setattr(ips, "_step", lambda label, value, color_fn = None: warnings.append(value))+ monkeypatch.setattr(ips, "_note", lambda message, color_fn = None: warnings.append(message))+ monkeypatch.setattr(ips.subprocess, "run", host.run)+ installs = []+ monkeypatch.setattr(ips, "install_prebuilt", lambda *a, **k: installs.append(a) or "installed")+ ips._remove_stale_ssm_kernels()+ assert installs == [] # never installs anything+ return warnings+++def test_stale_kernels_that_no_longer_import_are_removed(monkeypatch):+ host = _Kernels({"causal-conv1d": False, "mamba-ssm": False})+ warnings = _run_stale_check(monkeypatch, host)+ assert host.removed == ["causal-conv1d", "mamba-ssm"]+ assert len(warnings) == 2+ # The probe's stderr tail says why.+ assert "undefined symbol" in warnings[0] and "selective_scan_cuda" in warnings[1]+++def test_kernels_that_still_import_are_kept(monkeypatch):+ host = _Kernels({"causal-conv1d": True, "mamba-ssm": False})+ _run_stale_check(monkeypatch, host)+ assert host.removed == ["mamba-ssm"]+++def test_absent_kernels_are_not_probed_or_installed(monkeypatch):+ host = _Kernels({})+ _run_stale_check(monkeypatch, host)+ assert host.probes == [] and host.removed == []+++def test_a_hung_probe_is_inconclusive_and_keeps_the_kernel(monkeypatch):+ host = _Kernels({"causal-conv1d": None})+ _run_stale_check(monkeypatch, host)+ assert host.removed == []+++def test_no_torch_or_macos_skips_the_check(monkeypatch):+ for flags in ({"no_torch": True}, {"macos": True}):+ host = _Kernels({"causal-conv1d": False})+ _run_stale_check(monkeypatch, host, **flags)+ assert host.probes == []++[email protected]("torch_imports", [False, None])+def test_a_torch_that_does_not_import_keeps_every_kernel(monkeypatch, torch_imports):+ # A half-finished repair or a transient failure in torch says nothing about the kernels, and an+ # offline machine could not reinstall them.+ host = _Kernels({"causal-conv1d": False, "mamba-ssm": False}, torch_imports = torch_imports)+ warnings = _run_stale_check(monkeypatch, host)+ assert host.removed == []+ assert host.probes == ["import torch"]+ if torch_imports is False:+ assert "libtorch_cuda.so" in warnings[0]++[email protected]("exit_code", [-9, -11, 1, 2])+def test_a_killed_or_unrelated_failure_is_inconclusive(monkeypatch, exit_code):+ # A signal (OOM killer, segfault) or an exit that is not the extension's ImportError / OSError.+ host = _Kernels({"causal-conv1d": exit_code, "mamba-ssm": exit_code})+ warnings = _run_stale_check(monkeypatch, host)+ assert host.removed == []+ assert all("kept it" in message for message in warnings)+++def test_the_probed_extensions_match_the_kernel_checks():+ for name, extension in ips._SSM_KERNEL_EXTENSIONS.items():+ assert ips.KERNELS[name][1] == f"import torch, {extension}"++def test_the_kernel_cli_still_provisions_both_ahead_of_time():from utils import kernel_installassert {"causal_conv1d", "mamba_ssm"} <= set(kernel_install.KERNELS)assert callable(kernel_install.install_kernel)+++def test_a_busy_install_lock_keeps_the_kernel(monkeypatch):+ # A Studio load verifying a fresh install holds this lock: never remove it underneath that.+ @contextlib.contextmanager+ def _busy(name, **kwargs):+ yield False++ host = _Kernels({"causal-conv1d": False})+ warnings = _run_stale_check(monkeypatch, host, lock = _busy)+ assert host.removed == []+ assert any("install lock is busy" in message for message in warnings)+++def test_a_kernel_reinstalled_while_waiting_on_the_lock_is_kept(monkeypatch):+ # Broken when first probed, but a Studio load / `unsloth install-kernels` held the lock and+ # reinstalled it: the re-probe under the lock loads, so nothing is removed.+ host = _Kernels({"causal-conv1d": False})++ @contextlib.contextmanager+ def _lock_after_reinstall(name, **kwargs):+ host.installed["causal-conv1d"] = True+ yield True++ warnings = _run_stale_check(monkeypatch, host, lock = _lock_after_reinstall)+ assert host.removed == []+ assert any("now loads" in message and "kept it" in message for message in warnings)+ extension_probes = [code for code in host.probes if code != "import torch"]+ assert len(extension_probes) == 2 # probed again under the lock+++def test_an_inconclusive_re_probe_under_the_lock_keeps_the_kernel(monkeypatch):+ host = _Kernels({"causal-conv1d": False})++ @contextlib.contextmanager+ def _lock_then_hang(name, **kwargs):+ host.installed["causal-conv1d"] = None+ yield True++ warnings = _run_stale_check(monkeypatch, host, lock = _lock_then_hang)+ assert host.removed == []+ assert any("kept it" in message for message in warnings)diff --git a/tests/studio/install/test_windows_torch_flavor_invariant.py b/tests/studio/install/test_windows_torch_flavor_invariant.pyindex 413c66bdc8e..7aaf5eaf50f 100644--- a/tests/studio/install/test_windows_torch_flavor_invariant.py+++ b/tests/studio/install/test_windows_torch_flavor_invariant.py@@ -258,7 +258,7 @@ class TestStepThirteenWiring:["_progress", "_torch_step_label", "_probe_installed_torch_version"]+ repairs+ ["_probe_installed_torch_version", "_note", "_install_torchao_for_torch"]- + ["_ensure_flash_attn"]+ + ["_ensure_flash_attn", "_remove_stale_ssm_kernels"]+ ["_evict_xformers_built_for_another_torch", "_evict_xformers_requiring_another_torch"]), step13assert "_install_torchao_for_torch" not in _calls_in(guards[0])