Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: restore Qwen causal-conv1d on demand, narrow fatal mamba-ssm, provision SSM kernels in export (#13971)

Daniel Han7729c37fd09e
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.json
index d31bf604ced..8cd6e9a1e6d 100644
--- a/scripts/risky_loaders_baseline.json
+++ b/scripts/risky_loaders_baseline.json
@@ -38,7 +38,7 @@
"file": "studio/backend/core/export/worker.py",
"rule": "trust-remote-code",
"sink": "keyword",
- "digest": "58d5d0f2c979e2ae",
+ "digest": "35a68c33b5a8b4ad",
"count": 1,
"reason": "Argument to the consent evaluator inside `if trust_remote_code:`; it scans the code and blocks unless approved."
},
diff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.py
index 53754e39189..6f54406d5cd 100644
--- a/studio/backend/core/export/worker.py
+++ b/studio/backend/core/export/worker.py
@@ -192,7 +192,10 @@ def _offline_window_if_unreachable(step = "loading"):
offline constant and cached sessions, so env alone is too late: defer to the loader's
_force_hf_offline (env + in-process flags + session reset). Pre-import (activation),
huggingface_hub is not loaded yet, so setting the env vars suffices for its urllib probes."""
+ from utils.wheel_utils import HF_OFFLINE_AUTO_ENV
+
saved: dict[str, str | None] = {}
+ saved_marker: list = [] # [prior value] once this window marked HF offline as auto-set
force_ctx = None
try:
from utils.transformers_version import _env_offline, hf_endpoint_unreachable
@@ -204,6 +207,10 @@ def _offline_window_if_unreachable(step = "loading"):
)
if not _env_offline() and probe_enabled and hf_endpoint_unreachable():
logger.warning("Hugging Face endpoint unreachable; %s offline", step)
+ # Only the Hub is known to be down: mark this offline as ours so a required kernel
+ # install (GitHub / PyPI) is still allowed; an operator's HF offline never enters here.
+ saved_marker.append(os.environ.get(HF_OFFLINE_AUTO_ENV))
+ os.environ[HF_OFFLINE_AUTO_ENV] = "1"
if "huggingface_hub" in sys.modules:
try:
from unsloth.models.loader_utils import _force_hf_offline
@@ -230,6 +237,11 @@ def _offline_window_if_unreachable(step = "loading"):
os.environ.pop(k, None)
else:
os.environ[k] = v
+ if saved_marker:
+ if saved_marker[0] is None:
+ os.environ.pop(HF_OFFLINE_AUTO_ENV, None)
+ else:
+ os.environ[HF_OFFLINE_AUTO_ENV] = saved_marker[0]
def _send_response(resp_queue: Any, response: dict) -> None:
@@ -239,70 +251,49 @@ def _send_response(resp_queue: Any, response: dict) -> None:
logger.error("Failed to send response: %s", exc)
-def _handle_load(backend, cmd: dict, resp_queue: Any) -> None:
- from hub.utils.hf_tokens import hf_token_arg
-
- checkpoint_path = cmd["checkpoint_path"]
- # The preflight helpers read the policy off the token, so rebuild it once here.
- hf_token = hf_token_arg(cmd.get("hf_token"), allow_ambient_token = cmd.get("allow_ambient", True))
- max_seq_length = cmd.get("max_seq_length", 2048)
- load_in_4bit = cmd.get("load_in_4bit", True)
- # Latest-sidecar checkpoints load 16-bit here too: bnb 4-bit feeds quantized
- # expert weights into unvalidated paths (same flip as the chat worker).
- if load_in_4bit:
- from utils.transformers_version import latest_tier_active_for
-
- # Plain token, like the activation below: the sentinel reads no cache offline,
- # misses the sidecar and leaves 4-bit on.
- if latest_tier_active_for(checkpoint_path, hf_token or None):
- load_in_4bit = False
- logger.info(
- "Latest-transformers sidecar active for %s - forcing a 16-bit "
- "export load (4-bit is disabled for brand-new architectures)",
- checkpoint_path,
- )
- trust_remote_code = cmd.get("trust_remote_code", False)
- base_model = cmd.get("base_model") or None
-
- # Auto-enable trust_remote_code for NemotronH/Nano models.
- if not trust_remote_code:
- from utils.security.trusted_org import is_trusted_org_repo
-
- _NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano")
- _cp_lower = checkpoint_path.lower()
- if (
- any(sub in _cp_lower for sub in _NEMOTRON_TRUST_SUBSTRINGS)
- and (_cp_lower.startswith("unsloth/") or _cp_lower.startswith("nvidia/"))
- # Genuine first-party Hub repo only (not a local/spoof name starting
- # with "unsloth/"); authenticated so private repos resolve.
- and is_trusted_org_repo(checkpoint_path, hf_token = hf_token)
- ):
- trust_remote_code = True
- logger.info(
- "Auto-enabled trust_remote_code for Nemotron model: %s",
- checkpoint_path,
- )
-
- # Malware gate: a poisoned pickle deserializes on load even with
- # trust_remote_code False, so check HF's security scan (metadata-only) every
- # load. Local checkpoints have no Hub scan and are skipped in the helper; a
- # LoRA merges its base weights, so gate that repo too.
- from utils.security import evaluate_file_security, load_scan_target, security_load_subdirs
-
- requested_security_targets = [checkpoint_path]
- try:
- from utils.models.model_config import get_base_model_from_lora_identifier
+def _nemotron_auto_trust(checkpoint_path: str, hf_token) -> bool:
+ """Whether a NemotronH/Nano checkpoint gets trust_remote_code without the user asking: a
+ genuine first-party Hub repo only (not a local/spoof name starting with "unsloth/"),
+ authenticated so private repos resolve."""
+ _NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano")
+ _cp_lower = str(checkpoint_path).lower()
+ if not (
+ any(sub in _cp_lower for sub in _NEMOTRON_TRUST_SUBSTRINGS)
+ and (_cp_lower.startswith("unsloth/") or _cp_lower.startswith("nvidia/"))
+ ):
+ return False
+ from utils.security.trusted_org import is_trusted_org_repo
+
+ return is_trusted_org_repo(checkpoint_path, hf_token = hf_token)
+
+
+def _run_export_security_gates(
+ targets: list,
+ *,
+ trust_remote_code: bool,
+ hf_token,
+ approved_fingerprint,
+ resp_queue: Any,
+ compute_subdirs: bool = True,
+ subject = None,
+) -> bool:
+ """Malware + (when *trust_remote_code*) remote-code consent gates over *targets* (checkpoint +
+ LoRA base). Sends the matching 'loaded' failure and returns False if blocked, True when every
+ target is clear.
+
+ ``compute_subdirs=False`` keeps the gate transformers-free (``security_load_subdirs`` imports
+ ``model_config`` -> ``transformers``, which would snapshot optional-backend availability before
+ the SSM kernels are installed): used for the preflight before kernel provisioning, where
+ ``_handle_load`` re-runs the authoritative gate with full subdir scoping."""
+ from utils.security import evaluate_file_security, load_scan_target
+
+ if compute_subdirs:
+ from utils.security import security_load_subdirs
- # Resolve a LOCAL or REMOTE adapter's base so a remote LoRA base is gated too.
- _base = base_model or get_base_model_from_lora_identifier(checkpoint_path, hf_token)
- if _base:
- requested_security_targets.append(_base)
- except Exception as exc:
- logger.debug("Could not resolve LoRA base for malware scan: %s", exc)
security_targets: list[str] = []
consent_load_subdirs: dict[str, tuple] = {}
- for requested_target in dict.fromkeys(requested_security_targets):
- load_subdirs = security_load_subdirs(requested_target, hf_token)
+ for requested_target in dict.fromkeys(t for t in targets if t):
+ load_subdirs = security_load_subdirs(requested_target, hf_token) if compute_subdirs else ()
target, load_subdirs = load_scan_target(requested_target, load_subdirs)
if target not in consent_load_subdirs:
security_targets.append(target)
@@ -329,20 +320,19 @@ def _handle_load(backend, cmd: dict, resp_queue: Any) -> None:
"ts": time.time(),
},
)
- return
+ return False
- # Consent gate: scan auto_map code before it runs; block CRITICAL/HIGH unless
- # pinned-approved. A LoRA merges its base model, whose code runs, so gate it too.
+ # Consent gate: scan auto_map code before it runs; block CRITICAL/HIGH unless pinned-approved.
+ # A LoRA merges its base model, whose code runs, so gate it too. Adapter + base are scanned as
+ # one combined unit, pinned by a single fingerprint.
if trust_remote_code:
from utils.security import evaluate_remote_code_consent_for_targets
-
- # Scan adapter + base as one combined unit, pinned by a single fingerprint.
_rc = evaluate_remote_code_consent_for_targets(
security_targets,
hf_token = hf_token,
trust_remote_code = True,
- approved_fingerprint = cmd.get("approved_remote_code_fingerprint"),
- subject = cmd.get("subject"),
+ approved_fingerprint = approved_fingerprint,
+ subject = subject,
load_subdirs_by_target = consent_load_subdirs,
)
if _rc.blocked:
@@ -361,7 +351,175 @@ def _handle_load(backend, cmd: dict, resp_queue: Any) -> None:
"ts": time.time(),
},
)
- return
+ return False
+
+ return True
+
+
+def _local_adapter_base(checkpoint_path: str):
+ """``base_model_name_or_path`` of a local adapter checkpoint, read straight from its
+ adapter_config.json (no ML import), else None."""
+ try:
+ import json
+ cfg = Path(checkpoint_path) / "adapter_config.json"
+ if cfg.is_file():
+ return (
+ json.loads(cfg.read_text(encoding = "utf-8-sig")).get("base_model_name_or_path")
+ or None
+ )
+ except Exception:
+ pass
+ return None
+
+
+def _preflight_security_gates(cmd: dict, resp_queue: Any) -> bool:
+ """The security gates, transformers-free, BEFORE any SSM kernel is provisioned: a blocked
+ checkpoint (or LoRA base) must not trigger a native build or a wheel install on its behalf.
+ Gates the checkpoint plus its LoRA base (the explicit one, a local adapter_config.json, or the
+ Hub adapter config), under the trust the load will use. ``_handle_load`` re-runs the
+ authoritative gates afterwards. Returns False after sending the 'loaded' failure."""
+ checkpoint_path = cmd["checkpoint_path"]
+ hf_token = cmd.get("hf_token") or None
+ base = cmd.get("base_model") or _local_adapter_base(checkpoint_path)
+ if not base:
+ try:
+ from utils.transformers_version import _remote_lora_base
+ base = _remote_lora_base(checkpoint_path, hf_token)
+ except Exception as exc:
+ logger.debug("Could not resolve the adapter base for the preflight gates: %s", exc)
+ trust_remote_code = bool(cmd.get("trust_remote_code")) or _nemotron_auto_trust(
+ checkpoint_path, hf_token
+ )
+ return _run_export_security_gates(
+ [checkpoint_path, base],
+ trust_remote_code = trust_remote_code,
+ hf_token = hf_token,
+ approved_fingerprint = cmd.get("approved_remote_code_fingerprint"),
+ resp_queue = resp_queue,
+ compute_subdirs = False, # stay transformers-free until the SSM kernels are installed
+ subject = cmd.get("subject"),
+ )
+
+
+def _export_uses_cuda_kernels(device_backend) -> bool:
+ """causal-conv1d / mamba-ssm are CUDA / ROCm extensions, so only a CUDA export (ROCm reports as
+ "cuda" too) provisions them: MLX, CPU, XPU, an unspecified backend and any macOS export never
+ load them. ssm_runtime then checks the torch build itself."""
+ return sys.platform != "darwin" and str(device_backend or "") in ("cuda", "rocm")
+
+
+def _provision_ssm_kernels(
+ checkpoint_path: str,
+ base_model: str | None,
+ hf_token,
+ resp_queue: Any,
+ device_backend = None,
+ trust_remote_code = None,
+) -> bool:
+ """Install the causal-conv1d / mamba-ssm a checkpoint needs before Unsloth and its modeling
+ import, decided from its config (a LoRA's base config), as a training or chat load does.
+ Returns False after reporting a failure only for a model that cannot load without mamba-ssm."""
+ if not _export_uses_cuda_kernels(device_backend):
+ return True
+ try:
+ from utils.ssm_runtime import provision_ssm_kernels, ssm_probe_identifier
+ from utils.transformers_version import _remote_lora_base
+ except Exception as exc:
+ logger.debug("ssm_runtime unavailable (%s); skipping SSM kernel install", exc)
+ return True
+ base = base_model
+ if not base:
+ try:
+ # Raw HTTP, no ML import; a local adapter's base is read by the provisioner itself.
+ base = _remote_lora_base(checkpoint_path, hf_token or None)
+ except Exception as exc:
+ logger.debug("Could not resolve the adapter base for SSM kernels: %s", exc)
+ status = lambda message: _send_response(
+ resp_queue, {"type": "status", "message": message, "ts": time.time()}
+ )
+ try:
+ # The trust the load will use decides whether an auto_map model runs its remote code.
+ trusted = bool(trust_remote_code) or _nemotron_auto_trust(checkpoint_path, hf_token or None)
+ provision_ssm_kernels(
+ ssm_probe_identifier(checkpoint_path, base),
+ checkpoint_path,
+ hf_token or None,
+ adapter_base = base,
+ trust_remote_code = trusted,
+ status_cb = status,
+ )
+ return True
+ except Exception as exc:
+ _send_response(
+ resp_queue,
+ {
+ "type": "error",
+ "error": (
+ "This model needs SSM kernel libraries (causal-conv1d / mamba-ssm) "
+ f"that could not be installed: {exc}"
+ ),
+ "ts": time.time(),
+ },
+ )
+ return False
+
+
+def _handle_load(backend, cmd: dict, resp_queue: Any) -> None:
+ from hub.utils.hf_tokens import hf_token_arg
+
+ checkpoint_path = cmd["checkpoint_path"]
+ # The preflight helpers read the policy off the token, so rebuild it once here.
+ hf_token = hf_token_arg(cmd.get("hf_token"), allow_ambient_token = cmd.get("allow_ambient", True))
+ max_seq_length = cmd.get("max_seq_length", 2048)
+ load_in_4bit = cmd.get("load_in_4bit", True)
+ # Latest-sidecar checkpoints load 16-bit here too: bnb 4-bit feeds quantized
+ # expert weights into unvalidated paths (same flip as the chat worker).
+ if load_in_4bit:
+ from utils.transformers_version import latest_tier_active_for
+
+ # Plain token, like the activation below: the sentinel reads no cache offline,
+ # misses the sidecar and leaves 4-bit on.
+ if latest_tier_active_for(checkpoint_path, hf_token or None):
+ load_in_4bit = False
+ logger.info(
+ "Latest-transformers sidecar active for %s - forcing a 16-bit "
+ "export load (4-bit is disabled for brand-new architectures)",
+ checkpoint_path,
+ )
+ trust_remote_code = cmd.get("trust_remote_code", False)
+ base_model = cmd.get("base_model") or None
+
+ # Auto-enable trust_remote_code for NemotronH/Nano models.
+ if not trust_remote_code and _nemotron_auto_trust(checkpoint_path, hf_token):
+ trust_remote_code = True
+ logger.info(
+ "Auto-enabled trust_remote_code for Nemotron model: %s",
+ checkpoint_path,
+ )
+
+ # Malware gate: a poisoned pickle deserializes on load even with trust_remote_code False, so
+ # check HF's security scan (metadata-only) every load. Local checkpoints have no Hub scan and
+ # are skipped in the helper; a LoRA merges its base weights, so gate that repo too. This is
+ # the authoritative check (full subdir scoping); the pre-provisioning preflight is a subset.
+ requested_security_targets = [checkpoint_path]
+ try:
+ from utils.models.model_config import get_base_model_from_lora_identifier
+
+ # Resolve a LOCAL or REMOTE adapter's base so a remote LoRA base is gated too.
+ _base = base_model or get_base_model_from_lora_identifier(checkpoint_path, hf_token)
+ if _base:
+ requested_security_targets.append(_base)
+ except Exception as exc:
+ logger.debug("Could not resolve LoRA base for malware scan: %s", exc)
+ if not _run_export_security_gates(
+ requested_security_targets,
+ trust_remote_code = trust_remote_code,
+ hf_token = hf_token,
+ approved_fingerprint = cmd.get("approved_remote_code_fingerprint"),
+ resp_queue = resp_queue,
+ subject = cmd.get("subject"),
+ ):
+ return
try:
_send_response(
@@ -598,6 +756,23 @@ def run_export_process(*, cmd_queue: Any, resp_queue: Any, config: dict) -> None
)
return
+ # Before Unsloth and the modeling import: transformers captures its kernel gates at import,
+ # and some remote modeling code (Nemotron-H) cannot import at all without mamba-ssm. Inside
+ # the activation window, so it adds no Hub reachability probe of its own; an MLX / CPU /
+ # macOS export installs nothing. The security gates run first, so a blocked checkpoint
+ # never triggers a kernel build on its behalf.
+ if not _preflight_security_gates(config, resp_queue):
+ return
+ if not _provision_ssm_kernels(
+ checkpoint_path,
+ config.get("base_model") or None,
+ config.get("hf_token"),
+ resp_queue,
+ config.get("device_backend"),
+ trust_remote_code = config.get("trust_remote_code"),
+ ):
+ return
+
# Importable Triton isn't enough on AMD: its clang-cl JIT also needs the MSVC CRT headers (#7595).
if sys.platform == "win32":
from core._msvc_env import gate_torch_compile_on_windows
@@ -686,8 +861,20 @@ def run_export_process(*, cmd_queue: Any, resp_queue: Any, config: dict) -> None
if cmd_type == "load":
# Load a new checkpoint, reusing this subprocess.
backend.cleanup_memory()
- # Offline window also covers this load's Hub preflights (re-probed per load).
+ # Offline window covers this load's Hub preflights (re-probed per load). The
+ # security gates run before any SSM kernel is provisioned for the checkpoint.
with _offline_window_if_unreachable():
+ if not _preflight_security_gates(cmd, resp_queue):
+ continue
+ if not _provision_ssm_kernels(
+ cmd["checkpoint_path"],
+ cmd.get("base_model") or None,
+ cmd.get("hf_token"),
+ resp_queue,
+ config.get("device_backend"),
+ trust_remote_code = cmd.get("trust_remote_code"),
+ ):
+ continue
_handle_load(backend, cmd, resp_queue)
elif cmd_type == "export":
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 799eca015a8..cf9600e9459 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -873,6 +873,16 @@ class InferenceOrchestrator:
with self._dispatcher_lifecycle_lock:
self._worker_reserved_for = None
+ def _shutdown_before_respawn(self) -> None:
+ """Tear the current worker down before a retry spawns its replacement. A worker that
+ survives terminate/kill keeps its handle (see the model-switch guard in load_model), and a
+ second worker must not be spawned over it."""
+ if self._shutdown_subprocess(timeout = 5) is False:
+ raise RuntimeError(
+ "The inference worker did not exit and still holds GPU memory; "
+ "not starting a new one over it. Retry shortly."
+ )
+
def _shutdown_subprocess(self, timeout: float = 10.0) -> bool:
with self._subprocess_shutdown_lock:
managed = getattr(self, "_managed_engine", None)
@@ -2251,7 +2261,11 @@ class InferenceOrchestrator:
os.environ.get("HF_HUB_DISABLE_XET") == "1"
)
- for attempt in range(2):
+ # One spawn, plus at most one Xet-off retry after a stalled download and one respawn
+ # to bind SSM kernels the worker just installed: each is used once, in either order.
+ xet_retry_used = False
+ ssm_restart_used = False
+ for attempt in range(3):
if model_name not in self.loading_models or (
load_cancel_event is not None and load_cancel_event.is_set()
):
@@ -2265,7 +2279,7 @@ class InferenceOrchestrator:
return False
logger.info(
"Spawning fresh inference subprocess for '%s' "
- "(transformers %s.x, attempt %d/2%s)",
+ "(transformers %s.x, attempt %d/3%s)",
model_name,
needed_major,
attempt + 1,
@@ -2312,18 +2326,23 @@ class InferenceOrchestrator:
return False
except DownloadStallError:
# First stall with Xet on -> retry with Xet disabled
- if attempt == 0 and not disable_xet:
+ if not xet_retry_used and not disable_xet:
+ xet_retry_used = True
logger.warning(
"Download stalled for '%s' -- retrying with HF_HUB_DISABLE_XET=1",
model_name,
)
- self._shutdown_subprocess(timeout = 5)
+ self._shutdown_before_respawn()
disable_xet = True
continue
self._shutdown_subprocess(timeout = 5)
+ if disable_xet:
+ raise RuntimeError(
+ f"Download stalled for '{model_name}' even with "
+ f"HF_HUB_DISABLE_XET=1 -- check your network connection"
+ )
raise RuntimeError(
- f"Download stalled for '{model_name}' even with "
- f"HF_HUB_DISABLE_XET=1 -- check your network connection"
+ f"Download stalled for '{model_name}' -- check your network connection"
)
if resp.get("success"):
@@ -2395,6 +2414,19 @@ class InferenceOrchestrator:
logger.info("Model '%s' loaded successfully in subprocess", model_name)
return True
else:
+ if (
+ resp.get("error_kind") == "ssm_runtime_restart_required"
+ and not ssm_restart_used
+ ):
+ # The worker installed an SSM kernel after its transformers had already
+ # imported the model's modeling module; a fresh worker binds it.
+ ssm_restart_used = True
+ logger.info(
+ "Respawning the inference worker for '%s' to pick up new SSM kernels",
+ model_name,
+ )
+ self._shutdown_before_respawn()
+ continue
# Worker reports failures (consent gate included) under "message".
error = resp.get("message") or resp.get("error") or "Failed to load model"
self.active_model_name = None
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 06e165a8cd1..a663a746b73 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -386,25 +386,28 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool:
return load_in_4bit
+SSM_RESTART_REQUIRED = "ssm_runtime_restart_required"
+
+
def _ensure_ssm_kernels(
targets: list,
resp_queue: Any,
hf_token: str | None = None,
+ trust_remote_code: bool | None = None,
) -> bool:
"""Install the SSM kernels the given model(s) lazy-import in from_pretrained; no-op for
non-SSM models, idempotent. *targets* are ``(probe_identifier, config_target)`` pairs: the
- kernels are decided from the config at ``config_target`` (the resolved base), the name only
- when there is none. Returns True on success; on a fatal mamba-ssm failure sends a 'loaded'
- failure response and returns False. Call BEFORE importing transformers, which snapshots its
- optional-backend gates at import; an in-process reload clears them after installing.
+ kernels are decided from the config at ``config_target`` (the resolved base, or a local
+ adapter's recorded base), the name only when there is none. Call BEFORE importing
+ transformers, which snapshots its optional-backend gates at import.
+
+ Returns True on success. Returns False after sending a 'loaded' failure: on a fatal mamba-ssm
+ failure, or when a kernel was newly installed after this process already imported the model's
+ modeling module, whose captured kernel functions only a fresh worker picks up (the
+ orchestrator respawns once on ``SSM_RESTART_REQUIRED``).
"""
try:
- from utils.ssm_runtime import (
- ensure_ssm_runtime,
- refresh_transformers_kernel_gates,
- resolved_model_is_ssm,
- resolved_model_wants_causal_conv1d,
- )
+ from utils.ssm_runtime import modeling_modules_needing_restart, provision_ssm_kernels
except Exception as exc:
logger.debug("ssm_runtime unavailable (%s); skipping SSM kernel pre-install", exc)
return True
@@ -412,17 +415,39 @@ def _ensure_ssm_kernels(
_ssm_status = lambda m: _send_response(resp_queue, {"type": "status", "message": m})
try:
for probe, config_target in dict.fromkeys(t for t in targets if t and t[0]):
- config_target = config_target or probe
- ensure_ssm_runtime(
+ # trust_remote_code decides whether an auto_map model loads its remote code, and so
+ # whether a missing mamba-ssm refuses the load (see ssm_runtime.mamba_ssm_is_fatal).
+ kernels, installed = provision_ssm_kernels(
probe,
+ config_target or probe,
+ hf_token,
+ trust_remote_code = trust_remote_code,
status_cb = _ssm_status,
- # transformers-free: this runs before the worker's transformers import.
- wants_causal_conv1d = resolved_model_wants_causal_conv1d(
- probe, config_target, hf_token, scan_modeling_source = False
- ),
- is_ssm = resolved_model_is_ssm(probe, config_target, hf_token),
)
- refresh_transformers_kernel_gates()
+ stale = (
+ modeling_modules_needing_restart(kernels.model_types, kernels.remote_code)
+ if installed
+ else []
+ )
+ if stale:
+ logger.info(
+ "Installed %s after %s was imported; restarting the worker to pick it up",
+ ", ".join(installed),
+ ", ".join(stale),
+ )
+ _send_response(
+ resp_queue,
+ {
+ "type": "loaded",
+ "success": False,
+ "message": (
+ f"Installed {', '.join(installed)} for this model; "
+ "restarting the model worker to use it."
+ ),
+ "error_kind": SSM_RESTART_REQUIRED,
+ },
+ )
+ return False
return True
except Exception as exc:
_send_response(
@@ -717,7 +742,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
_ssm_base or config["model_name"],
)
]
- if not _ensure_ssm_kernels(ssm_targets, resp_queue, hf_token):
+ if not _ensure_ssm_kernels(ssm_targets, resp_queue, hf_token, trust_remote_code):
return
# Heartbeat keeps the orchestrator's inactivity deadline alive during slow
@@ -2490,7 +2515,7 @@ def run_inference_process(
from utils.ssm_runtime import ssm_probe_identifier
_ssm_targets = [(ssm_probe_identifier(model_name, _base), _base or model_name)]
- if not _ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token):
+ if not _ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token, _trust_remote_code):
return
try:
diff --git a/studio/backend/core/systemone/clef_runtime.py b/studio/backend/core/systemone/clef_runtime.py
index 4a6cb10596f..547a6414e10 100644
--- a/studio/backend/core/systemone/clef_runtime.py
+++ b/studio/backend/core/systemone/clef_runtime.py
@@ -46,19 +46,10 @@ class _Events:
def _fast_paths(folder: str) -> None:
- # Clef is a Qwen3.5 backbone: unsloth_zoo ships its gated delta kernels and a Triton causal conv,
- # so nothing is installed. Where that conv cannot run (switched off, ROCm, no Triton) it gets the
- # prebuilt causal-conv1d a training run would; a failed install leaves transformers' torch path.
+ # The same prebuilt causal-conv1d a Qwen3.5 training run gets; the gated delta kernels ship with
+ # unsloth_zoo. A failed install leaves transformers' torch path, which is slower but right.
try:
- from utils.ssm_runtime import zoo_triton_causal_conv1d_usable
-
- if zoo_triton_causal_conv1d_usable():
- # The hooks below also steer FLA off TileLang where it faults; keep that part.
- from core.training.worker import _guard_fla_tilelang
- _guard_fla_tilelang()
- return
from core.training.worker import _ensure_causal_conv1d_fast_path, _install_fast_path_hooks
-
_ensure_causal_conv1d_fast_path(_Events(), folder, required = True)
_install_fast_path_hooks(_Events(), folder, install_causal_conv1d = True)
except Exception as exc:
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index 0204ab91d61..8e95901766a 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -105,6 +105,38 @@ _pyplot = None
_pyplot_failed = False
+def _terminate_descendants(pid: int) -> list:
+ """SIGTERM every process *pid* started (recursively) and return them for _reap_descendants.
+ Collected before the worker dies: once it is gone they are re-parented and lost."""
+ try:
+ import psutil
+ descendants = psutil.Process(pid).children(recursive = True)
+ except Exception:
+ return []
+ for child in descendants:
+ try:
+ child.terminate()
+ except Exception:
+ pass
+ return descendants
+
+
+def _reap_descendants(descendants: list, timeout: float = 5.0) -> None:
+ """Wait for terminated descendants and SIGKILL any still running."""
+ if not descendants:
+ return
+ try:
+ import psutil
+ _, alive = psutil.wait_procs(descendants, timeout = timeout)
+ except Exception:
+ alive = descendants
+ for child in alive:
+ try:
+ child.kill()
+ except Exception:
+ pass
+
+
@dataclass(frozen = True)
class TrainingStartRequestRecord:
start_request_id: str
@@ -2473,8 +2505,12 @@ class TrainingBackend:
proc = self._proc
if target_proc is not None and proc is not target_proc:
return
+ descendants = []
if proc is not None and proc.is_alive():
logger.info("Force-terminating training subprocess (pid=%s)", proc.pid)
+ # First the processes it started: a kernel source build runs in its own session
+ # (so its compilers can be stopped together) and would outlive the worker.
+ descendants = _terminate_descendants(proc.pid)
proc.terminate()
cancelled = self._cancel_requested
output_dir = self._cancel_cleanup_output_dir or self._output_dir
@@ -2484,6 +2520,7 @@ class TrainingBackend:
if proc.is_alive():
proc.kill()
proc.join(timeout = 2.0)
+ _reap_descendants(descendants)
# Wait for pump thread to finish DB finalization (8s covers SQLite's 5s lock timeout).
if self._pump_thread is not None and self._pump_thread.is_alive():
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 5bf50dbcb82..d7955ec596c 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -85,6 +85,7 @@ from utils.wheel_utils import (
flash_attn_wheel_url,
install_wheel,
is_windows_wheel_env,
+ pinned_source_build_supported,
probe_torch_wheel_env,
url_exists,
)
@@ -306,6 +307,22 @@ def _is_model_cache_artifact_error(error: BaseException | None) -> bool:
return False
+def _package_installs_offline() -> bool:
+ # UV_OFFLINE too: uv would refuse the index and install_wheel's pip fallback would not.
+ try:
+ from utils.utils import package_installs_offline
+ return package_installs_offline()
+ except Exception:
+ return os.environ.get("UV_OFFLINE", "").strip().lower() in {
+ "1",
+ "true",
+ "t",
+ "yes",
+ "y",
+ "on",
+ }
+
+
def _model_offline_mode_enabled() -> bool:
try:
from utils.utils import hf_env_offline
@@ -1421,7 +1438,13 @@ def _reject_install(event_queue: Any, pypi_name: str, display_name: str, reason:
def _install_package_wheel_first(
- *, event_queue: Any, import_name: str, display_name: str, pypi_name: str, **kwargs: Any
+ *,
+ event_queue: Any,
+ import_name: str,
+ display_name: str,
+ pypi_name: str,
+ required: bool = False,
+ **kwargs: Any,
) -> bool:
"""Install a fast-path package, wheel first, and never leave an unusable one behind. The two
"touch nothing" guards run here, outside the cleanup: an already-working package returns
@@ -1433,23 +1456,59 @@ def _install_package_wheel_first(
logger.info("%s already installed", display_name)
return True
- if _model_offline_mode_enabled():
+ if _model_offline_mode_enabled() or _package_installs_offline():
logger.info("Skipping %s installation while offline", display_name)
return False
- installed = False
- try:
- installed = _attempt_package_install(
- event_queue = event_queue,
- import_name = import_name,
- display_name = display_name,
- pypi_name = pypi_name,
- **kwargs,
- )
- return installed
- finally:
- if not installed:
- _reject_install(event_queue, pypi_name, display_name, "will not import")
+ # The runtime's failed-attempt memo (same key): the eager install and the fast-path hook, or a
+ # chat load in the same process, never repeat a failed (possibly half-hour) build.
+ from utils.ssm_runtime import _FAILED_INSTALLS, _failure_key
+
+ failure_key = _failure_key(import_name)
+ if failure_key in _FAILED_INSTALLS:
+ logger.info("%s already failed to install in this process; not retrying", display_name)
+ return False
+
+ from utils.kernel_install import (
+ KERNEL_LOCK_WAIT,
+ OPTIONAL_KERNEL_LOCK_WAIT,
+ kernel_install_lock,
+ )
+
+ # One install per package and environment at a time, across Studio workers: precheck, install,
+ # verify and cleanup are one transaction, or a failed peer could remove what another verified.
+ # An optional acceleration does not hold a run for a peer's half-hour build.
+ with kernel_install_lock(
+ import_name,
+ on_wait = lambda: _send_status(
+ event_queue, f"Waiting for another process installing {display_name}..."
+ ),
+ wait_seconds = KERNEL_LOCK_WAIT if required else OPTIONAL_KERNEL_LOCK_WAIT,
+ ) as held:
+ if not held:
+ _send_status(
+ event_queue,
+ f"Could not take the {display_name} install lock (another process is installing "
+ "it, or the lock is unavailable); continuing without it",
+ )
+ return False
+ if _is_importable(import_name):
+ logger.info("%s installed by another process", display_name)
+ return True
+ installed = False
+ try:
+ installed = _attempt_package_install(
+ event_queue = event_queue,
+ import_name = import_name,
+ display_name = display_name,
+ pypi_name = pypi_name,
+ **kwargs,
+ )
+ return installed
+ finally:
+ if not installed:
+ _FAILED_INSTALLS.add(failure_key)
+ _reject_install(event_queue, pypi_name, display_name, "will not import")
def _attempt_package_install(
@@ -1465,12 +1524,19 @@ def _attempt_package_install(
wheel_url_builder: Callable[[dict[str, str] | None], str | None] | None = None,
pypi_spec: str | None = None,
pypi_status_message: str | None = None,
+ allow_source_build: bool = True,
) -> bool:
"""The install itself. Call it through _install_package_wheel_first, never directly."""
# Set when a wheel installed but would not import; see the uninstall before the fallback.
wheel_rejected = False
env = probe_torch_wheel_env(timeout = 30, include_windows = True)
+ if not env or not (env.get("cuda_version") or env.get("cuda_major") or env.get("hip_version")):
+ # These are CUDA / ROCm extensions: CPU-only, XPU, MPS or an unreadable torch builds none.
+ _send_status(
+ event_queue, f"{display_name} needs a CUDA or ROCm build of torch; skipping it"
+ )
+ return False
# Windows is wheel-only: a PyPI fallback there is a source build needing MSVC + the CUDA toolkit.
wheel_only = sys.platform == "win32" or is_windows_wheel_env(env)
if wheel_url_builder is not None:
@@ -1487,7 +1553,14 @@ def _attempt_package_install(
wheel_available = url_exists(wheel_url) if wheel_url else False
if wheel_url is None:
logger.info("No compatible %s wheel candidate", display_name)
- elif wheel_available:
+ elif wheel_available is not False:
+ # Only a 404 proves nothing is published: an unanswered HEAD (a proxy refusing HEAD but
+ # serving the download) still tries the wheel, and falls back only if that fails.
+ if wheel_available is None:
+ _send_status(
+ event_queue,
+ f"Could not check the {display_name} prebuilt wheel; trying to download it.",
+ )
_send_status(event_queue, f"Installing {display_name} for faster training...")
outcome = install_prebuilt(
wheel_url,
@@ -1513,18 +1586,24 @@ def _attempt_package_install(
"removing it" if wheel_only else "falling back to PyPI",
)
wheel_rejected = True
- elif wheel_available is None:
- _send_status(
- event_queue,
- f"Could not check the {display_name} prebuilt wheel; "
- + ("skipping it." if wheel_only else "installing from PyPI."),
- )
else:
logger.info("No published %s wheel found: %s", display_name, wheel_url)
if wheel_only:
logger.info("%s: no usable prebuilt wheel for Windows; skipping", display_name)
return False
+ if not allow_source_build:
+ _send_status(event_queue, f"No usable prebuilt {display_name} for this torch; skipping it")
+ return False
+ if pypi_version is not None and not pinned_source_build_supported(env):
+ # The pinned causal-conv1d / mamba-ssm sources build at C++17 and torch 2.13+ needs
+ # C++20: a certain failure after minutes of compiling.
+ _send_status(
+ event_queue,
+ f"No prebuilt {display_name} for torch {env.get('torch_mm')}, and it cannot be built "
+ "from source for it; skipping it",
+ )
+ return False
is_hip = env and env.get("hip_version")
if is_hip and not shutil.which("hipcc"):
@@ -1587,8 +1666,11 @@ def _attempt_package_install(
_gcc_dir,
)
+ from utils.kernel_install import run_process_tree
+
try:
- result = _sp.run(pypi_cmd, **_run_kwargs)
+ # Bounded, and a timeout stops the compilers pip / uv started too.
+ result = run_process_tree(pypi_cmd, run = _sp.run, **_run_kwargs)
except _sp.TimeoutExpired:
logger.error(
"%s installation timed out after %ds",
@@ -1683,20 +1765,43 @@ def _ensure_mamba_ssm(
model_name: str,
*,
required: bool | None = None,
+ essential: bool | None = None,
) -> None:
# `required` is the config-resolved answer (resolved_model_is_ssm); the name is the fallback.
+ # `essential`: Nemotron-H, Falcon-H1, Granite-4.0-H, which may build it from source. Training
+ # stays best effort as at 9f5b13325: their native transformers path can run without it, so a
+ # missing mamba-ssm is left to the model load. The others have a torch path and get mamba-ssm
+ # only as a prebuilt wheel where unsloth_zoo keeps its fast path on, never as a source build.
if required is None:
required = _model_is_ssm(model_name)
if not required:
return
+ if essential is None:
+ # Unspecified (the worker passes the config's answer): only the families named for it
+ # (Nemotron-H, Falcon-H1, Granite-4.0-H) may build from source, never Mamba / Mamba2 /
+ # Jamba / Zamba / Bamba, which a caller leaving it out must not turn into a source build.
+ from utils.ssm_runtime import model_requires_mamba_ssm
+ essential = model_requires_mamba_ssm(model_name)
if sys.platform == "win32" and importlib.util.find_spec("triton") is None:
# mamba_ssm imports triton at load and the wheel goes in --no-deps.
_send_status(event_queue, "mamba-ssm needs Triton on Windows (triton-windows); skipping it")
return
+ if not essential and not _is_importable(MAMBA_SSM.import_name):
+ from utils.ssm_runtime import _mamba_ssm_floor_reason
+ reason = _mamba_ssm_floor_reason()
+ if reason is not None:
+ _send_status(event_queue, f"Skipping mamba-ssm, which {reason}")
+ return
+
logger.info("SSM model detected; setting up mamba-ssm after causal-conv1d")
- _install_package_wheel_first(event_queue = event_queue, **_pinned_kernel_kwargs(MAMBA_SSM))
+ _install_package_wheel_first(
+ event_queue = event_queue,
+ **_pinned_kernel_kwargs(MAMBA_SSM),
+ allow_source_build = essential,
+ required = essential,
+ )
def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
@@ -2144,9 +2249,40 @@ def _should_try_runtime_flash_attn_install(max_seq_length: int) -> bool:
return sys.platform.startswith("linux") or sys.platform == "win32"
+def _flash_attn_gpu_below_floor() -> bool:
+ """Whether the best visible GPU is below sm80, where Unsloth leaves FlashAttention 2 off. Asks
+ the torch this worker already imported when there is one (no child process); ROCm never takes
+ the NVIDIA floor; no visible GPU is not below it."""
+ torch = sys.modules.get("torch")
+ if torch is not None:
+ try:
+ if getattr(torch.version, "hip", None):
+ return False
+ if not torch.cuda.is_available():
+ return False
+ count = torch.cuda.device_count()
+ return count > 0 and max(
+ tuple(torch.cuda.get_device_capability(i)) for i in range(count)
+ ) < (8, 0)
+ except Exception as exc:
+ logger.debug("In-process GPU capability check failed (%s); probing a child", exc)
+ from utils.kernel_install import gpu_below_flash_attn_floor
+
+ return gpu_below_flash_attn_floor()
+
+
def _ensure_flash_attn_for_long_context(event_queue: Any, max_seq_length: int) -> None:
if not _should_try_runtime_flash_attn_install(max_seq_length):
return
+ if _is_importable("flash_attn"):
+ # Already usable: nothing to download, so no capability probe either.
+ logger.info("flash-attn already installed")
+ return
+
+ if _flash_attn_gpu_below_floor():
+ # Unsloth only turns FlashAttention 2 on from sm80 (a T4 is sm75): nothing to download.
+ _send_status(event_queue, "flash-attn needs an sm80 or newer GPU; continuing without it")
+ return
installed = _install_package_wheel_first(
event_queue = event_queue,
@@ -4028,25 +4164,25 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
# 2) mamba-ssm + flash-attn keep their substring / size gates.
# 3) FLA gated-delta kernels: vendored by unsloth_zoo, nothing to install.
# Laya decision models are ModernBERT encoders: none of these apply.
- # Clef decision models are Qwen3.5 backbones: zoo's gated-delta and Triton conv kernels cover them.
+ # Clef decision models are Qwen3.5 backbones and need the same gated-delta / conv kernels.
if (
not config.get("is_decision")
or config.get("decision_layout") in ("clef", "llm")
or _decision_has_llm_backbone(model_load_target, _worker_hf_token(config))
):
try:
- from utils.ssm_runtime import (
- resolved_model_is_ssm,
- resolved_model_wants_causal_conv1d,
- )
+ from utils.ssm_runtime import resolve_ssm_kernels
- is_ssm = resolved_model_is_ssm(model_name, model_load_target, _worker_hf_token(config))
- # Every mamba-ssm model also runs causal-conv1d.
- wants_causal_conv1d = is_ssm or resolved_model_wants_causal_conv1d(
+ # A continued local adapter loads from its folder, but its base decides the kernels:
+ # the authorized pin when the run was checked, else the adapter's recorded base.
+ kernels = resolve_ssm_kernels(
model_name,
model_load_target,
_worker_hf_token(config),
+ adapter_base = config.get("adapter_base_pin") or None,
)
+ is_ssm = kernels.mamba_ssm
+ wants_causal_conv1d = kernels.causal_conv1d
_ensure_causal_conv1d_fast_path(
event_queue,
model_name,
@@ -4057,7 +4193,12 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
model_name,
install_causal_conv1d = wants_causal_conv1d,
)
- _ensure_mamba_ssm(event_queue, model_name, required = is_ssm)
+ _ensure_mamba_ssm(
+ event_queue,
+ model_name,
+ required = is_ssm,
+ essential = kernels.mamba_ssm_required,
+ )
_ensure_flash_attn_for_long_context(
event_queue,
int(config.get("max_seq_length", 2048)),
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 649906732a1..43d214cdb5a 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -1465,3 +1465,14 @@ def traced_offload_hooks(monkeypatch):
orig = getattr(fn, "_unsloth_orig", None)
if orig is not None:
monkeypatch.setattr(cls, name, orig)
+
+
[email protected](scope = "session")
+def _kernel_lock_dir(tmp_path_factory):
+ return str(tmp_path_factory.mktemp("kernel-locks"))
+
+
[email protected](autouse = True)
+def _isolated_kernel_install_locks(monkeypatch, _kernel_lock_dir):
+ # Tests never take (or create) the environment's real kernel install locks.
+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", _kernel_lock_dir)
diff --git a/studio/backend/tests/test_clef_fast_paths.py b/studio/backend/tests/test_clef_fast_paths.py
index 3105cb5dcc5..26838a370f1 100644
--- a/studio/backend/tests/test_clef_fast_paths.py
+++ b/studio/backend/tests/test_clef_fast_paths.py
@@ -1,8 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Clef's fast-path setup: nothing is installed where zoo's Triton conv runs, but the FLA TileLang
-guard still applies; elsewhere the prebuilt causal-conv1d is installed as before."""
+"""Clef (a Qwen3.5 backbone) installs the prebuilt causal-conv1d and its availability hook, whose
+setup also applies the FLA TileLang guard."""
import sys
from pathlib import Path
@@ -13,33 +13,27 @@ if str(_BACKEND) not in sys.path:
from core.systemone import clef_runtime # noqa: E402
from core.training import worker # noqa: E402
-from utils import ssm_runtime # noqa: E402
-def _record(monkeypatch, usable: bool) -> list[str]:
- calls: list[str] = []
- monkeypatch.setattr(ssm_runtime, "zoo_triton_causal_conv1d_usable", lambda: usable)
- monkeypatch.setattr(worker, "_guard_fla_tilelang", lambda: calls.append("guard"))
+def test_clef_installs_causal_conv1d_and_its_hooks(monkeypatch):
+ calls: list = []
monkeypatch.setattr(
worker,
"_ensure_causal_conv1d_fast_path",
- lambda *args, **kwargs: calls.append("install"),
+ lambda *args, **kwargs: calls.append(("install", kwargs.get("required"))),
)
monkeypatch.setattr(
worker,
"_install_fast_path_hooks",
- lambda *args, **kwargs: calls.append("hooks"),
+ lambda *args, **kwargs: calls.append(("hooks", kwargs.get("install_causal_conv1d"))),
)
- return calls
-
-
-def test_clef_installs_nothing_but_keeps_the_tilelang_guard(monkeypatch):
- calls = _record(monkeypatch, usable = True)
clef_runtime._fast_paths("/models/clef")
- assert calls == ["guard"]
+ assert calls == [("install", True), ("hooks", True)]
-def test_clef_installs_causal_conv1d_where_zoo_conv_cannot_run(monkeypatch):
- calls = _record(monkeypatch, usable = False)
- clef_runtime._fast_paths("/models/clef")
- assert calls == ["install", "hooks"]
+def test_the_fast_path_hooks_apply_the_tilelang_guard(monkeypatch):
+ guarded = []
+ monkeypatch.setattr(worker, "_guard_fla_tilelang", lambda: guarded.append(True))
+ monkeypatch.setenv(worker._FAST_PATH_HOOKS_SKIP_ENV, "1")
+ worker._install_fast_path_hooks(event_queue = [], model_name = "/models/clef")
+ assert guarded == [True]
diff --git a/studio/backend/tests/test_export_offline_window.py b/studio/backend/tests/test_export_offline_window.py
index b5bde893fea..875660fe3f6 100644
--- a/studio/backend/tests/test_export_offline_window.py
+++ b/studio/backend/tests/test_export_offline_window.py
@@ -100,3 +100,132 @@ def test_push_export_is_not_pinned_offline(monkeypatch):
export = {"type": "export", "export_type": "merged", "push_to_hub": True}
calls, _ = _run_worker(monkeypatch, [export], lambda: True)
assert calls == [(False, True)]
+
+
+# ── An auto-set (Hub unreachable) offline does not block kernel installs ──────
+
+import contextlib # noqa: E402
+
+import pytest # noqa: E402
+
+
+def _isolated_offline_env(monkeypatch, **preset):
+ for name in (
+ "HF_HUB_OFFLINE",
+ "TRANSFORMERS_OFFLINE",
+ "UV_OFFLINE",
+ "_UNSLOTH_HF_OFFLINE_AUTO",
+ ):
+ monkeypatch.delenv(name, raising = False)
+ for name, value in preset.items():
+ monkeypatch.setenv(name, value)
+ monkeypatch.setenv("UNSLOTH_OFFLINE_PROBE", "1")
+
+
[email protected](params = ["env", "force"])
+def window_branch(request, monkeypatch):
+ """Both ways the window goes offline: env vars only (huggingface_hub not yet usable) or the
+ loader's _force_hf_offline (post-import)."""
+ from utils.utils import force_hf_offline
+
+ if request.param == "env":
+ monkeypatch.setitem(sys.modules, "unsloth.models.loader_utils", None) # import fails
+ else:
+ fake_loader = types.ModuleType("unsloth.models.loader_utils")
+ fake_loader._force_hf_offline = force_hf_offline
+ monkeypatch.setitem(sys.modules, "unsloth.models.loader_utils", fake_loader)
+ return request.param
+
+
+def _install_kernel_reaches_the_lock(monkeypatch):
+ """Whether ssm_runtime._install_kernel gets past its offline guard (to the kernel lock, which
+ is stubbed busy so nothing installs)."""
+ from utils import kernel_install, ssm_runtime
+
+ reached = []
+
+ @contextlib.contextmanager
+ def _busy(name, **kwargs):
+ reached.append(name)
+ yield False
+
+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _busy)
+ monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: False)
+ monkeypatch.setattr(ssm_runtime, "_FAILED_INSTALLS", set())
+ monkeypatch.setattr(
+ ssm_runtime, "install_wheel", lambda *a, **k: pytest.fail("no install in this test")
+ )
+ ssm_runtime._install_kernel(
+ import_name = "causal_conv1d",
+ display_name = "causal-conv1d",
+ pypi_name = "causal-conv1d",
+ package_version = "1.6.1",
+ release_tag = "v1.6.1.post4",
+ release_base_url = "x",
+ status_cb = None,
+ run = lambda *a, **k: pytest.fail("no subprocess in this test"),
+ )
+ return bool(reached)
+
+
+def test_an_unreachable_hub_still_allows_kernel_installs_inside_the_window(
+ monkeypatch, window_branch
+):
+ from core.export import worker
+ from utils import transformers_version
+ from utils.utils import hf_env_offline, package_installs_offline
+ from utils.wheel_utils import package_installs_offline_env
+
+ _isolated_offline_env(monkeypatch)
+ monkeypatch.setattr(transformers_version, "hf_endpoint_unreachable", lambda: True)
+ with worker._offline_window_if_unreachable(step = "activating transformers"):
+ assert hf_env_offline() # the Hub side is offline
+ assert os.environ.get("HF_HUB_OFFLINE") == "1"
+ assert package_installs_offline() is False
+ assert package_installs_offline_env() is False
+ assert _install_kernel_reaches_the_lock(monkeypatch)
+ # Marker and env restored on exit.
+ assert "_UNSLOTH_HF_OFFLINE_AUTO" not in os.environ
+ assert "HF_HUB_OFFLINE" not in os.environ
+
+
+def test_uv_offline_still_blocks_installs_inside_the_window(monkeypatch, window_branch):
+ from core.export import worker
+ from utils import transformers_version
+ from utils.utils import package_installs_offline
+ from utils.wheel_utils import package_installs_offline_env
+
+ _isolated_offline_env(monkeypatch, UV_OFFLINE = "1")
+ monkeypatch.setattr(transformers_version, "hf_endpoint_unreachable", lambda: True)
+ with worker._offline_window_if_unreachable():
+ assert package_installs_offline() is True
+ assert package_installs_offline_env() is True
+ assert not _install_kernel_reaches_the_lock(monkeypatch)
+
+
[email protected]("variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"])
+def test_an_operator_set_hf_offline_still_blocks_installs(monkeypatch, variable):
+ from core.export import worker
+ from utils import transformers_version
+ from utils.utils import package_installs_offline
+ from utils.wheel_utils import package_installs_offline_env
+
+ _isolated_offline_env(monkeypatch, **{variable: "1"})
+ monkeypatch.setattr(transformers_version, "hf_endpoint_unreachable", lambda: True)
+ with worker._offline_window_if_unreachable():
+ assert "_UNSLOTH_HF_OFFLINE_AUTO" not in os.environ
+ assert package_installs_offline() is True
+ assert package_installs_offline_env() is True
+ assert not _install_kernel_reaches_the_lock(monkeypatch)
+ assert package_installs_offline() is True
+
+
+def test_a_reachable_hub_sets_no_marker(monkeypatch):
+ from core.export import worker
+ from utils import transformers_version
+
+ _isolated_offline_env(monkeypatch)
+ monkeypatch.setattr(transformers_version, "hf_endpoint_unreachable", lambda: False)
+ with worker._offline_window_if_unreachable():
+ assert "_UNSLOTH_HF_OFFLINE_AUTO" not in os.environ
+ assert "HF_HUB_OFFLINE" not in os.environ
diff --git a/studio/backend/tests/test_export_ssm_kernels.py b/studio/backend/tests/test_export_ssm_kernels.py
new file mode 100644
index 00000000000..3685fe45fd3
--- /dev/null
+++ b/studio/backend/tests/test_export_ssm_kernels.py
@@ -0,0 +1,417 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Export installs the SSM kernels a checkpoint needs before Unsloth and its modeling import, as a
+training or chat load does: setup no longer installs them, and Nemotron-H's remote modeling code
+cannot even import without mamba-ssm."""
+
+import json
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+from core.export import worker as export_worker # noqa: E402
+from utils import ssm_runtime # noqa: E402
+
+
+class _Queue(list):
+ def put(self, item):
+ self.append(item)
+
+
+class _ClosedQueue:
+ def get(self, timeout = None):
+ raise EOFError
+
+
+def test_a_nemotron_h_export_provisions_kernels_before_the_backend_import(monkeypatch, tmp_path):
+ order = [
+ step
+ for step in _run_export(monkeypatch, tmp_path, {"device_backend": "cuda"})
+ if step[0] != "window"
+ ]
+ assert order[0] == ("provision", True, True)
+ assert order.index(("provision", True, True)) < order.index(("import",))
+ assert ("load",) in order
+
+
[email protected]("device_backend", ["cpu", "mlx"])
+def test_a_cpu_or_mlx_export_makes_no_extra_connectivity_probe(
+ monkeypatch, tmp_path, device_backend
+):
+ # 9f5b13325 entered no "installing SSM kernels" window; an export that installs nothing must not.
+ order = _run_export(monkeypatch, tmp_path, {"device_backend": device_backend})
+ assert ("window", "installing SSM kernels") not in order
+ assert not any(step[0] == "provision" for step in order)
+ assert ("load",) in order
+
+
[email protected]("device_backend", ["cuda", "cpu", "mlx"])
+def test_provisioning_adds_no_connectivity_window(monkeypatch, tmp_path, device_backend):
+ # Same window count as 9f5b13325: provisioning rides the activation window.
+ order = _run_export(monkeypatch, tmp_path, {"device_backend": device_backend})
+ windows = [step for step in order if step[0] == "window"]
+ assert windows == [("window", "activating transformers"), ("window", None)]
+
+
+def test_a_cuda_export_installs_inside_the_activation_window(monkeypatch, tmp_path):
+ order = _run_export(monkeypatch, tmp_path, {"device_backend": "cuda"})
+ activation = order.index(("window", "activating transformers"))
+ provision = order.index(("provision", True, True))
+ assert activation < provision < order.index(("import",))
+ assert not any(step[0] == "window" for step in order[activation + 1 : provision])
+
+
+def _run_export(monkeypatch, tmp_path, extra_config):
+ import contextlib
+
+ monkeypatch.setattr(export_worker.sys, "platform", "linux")
+ checkpoint = tmp_path / "nemotron-export"
+ checkpoint.mkdir()
+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "nemotron_h"}))
+ order: list = []
+
+ @contextlib.contextmanager
+ def _window(step = None):
+ order.append(("window", step))
+ yield
+
+ monkeypatch.setattr(export_worker, "_offline_window_if_unreachable", _window)
+
+ def _ensure(probe, **kwargs):
+ order.append(("provision", kwargs["is_ssm"], kwargs["mamba_ssm_required"]))
+ return []
+
+ monkeypatch.setattr(ssm_runtime, "ensure_ssm_runtime", _ensure)
+ monkeypatch.setattr(export_worker, "_setup_log_capture", lambda queue: None)
+ monkeypatch.setattr(export_worker, "_activate_transformers_version", lambda *a, **k: None)
+ monkeypatch.setattr(export_worker, "apply_gpu_ids", lambda *a, **k: None)
+ monkeypatch.setattr(
+ "core._torchao_stub.install_torchao_windows_rocm_real_or_stub", lambda: None
+ )
+ monkeypatch.setattr("core.import_guards.ensure_real_packages", lambda *names: None)
+
+ class _Backend:
+ def __init__(self):
+ order.append(("backend",))
+
+ fake_export = types.ModuleType("core.export.export")
+
+ def _getattr(name):
+ if name == "ExportBackend":
+ order.append(("import",))
+ return _Backend
+ raise AttributeError(name)
+
+ fake_export.__getattr__ = _getattr
+ monkeypatch.setitem(sys.modules, "core.export.export", fake_export)
+ monkeypatch.setattr(export_worker, "_handle_load", lambda *a, **k: order.append(("load",)))
+ # run_export_process configures its own process; keep that from leaking into this one.
+ for key in ("TQDM_MININTERVAL", "TOKENIZERS_PARALLELISM", "PYTHONWARNINGS", "PYTHONUNBUFFERED"):
+ monkeypatch.setenv(key, "")
+ monkeypatch.delenv(key)
+ monkeypatch.setenv("ENVIRONMENT_TYPE", "development")
+ monkeypatch.setattr("loggers.config.LogConfig.setup_logging", lambda *a, **k: None)
+ monkeypatch.setattr("loggers.config.allow_progress_bars", lambda: None)
+
+ queue = _Queue()
+ export_worker.run_export_process(
+ cmd_queue = _ClosedQueue(),
+ resp_queue = queue,
+ config = {"checkpoint_path": str(checkpoint), "allow_ambient": True, **extra_config},
+ )
+ return order
+
+
+def test_a_failed_required_install_stops_the_export_before_the_import(monkeypatch, tmp_path):
+ checkpoint = tmp_path / "falcon"
+ checkpoint.mkdir()
+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "falcon_h1"}))
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ monkeypatch.setattr(ssm_runtime, "_torch_env", lambda: {"cuda_version": "12.8"})
+ # A transformers without the native Falcon-H1 model cannot load it without mamba-ssm.
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda model_type: False)
+ queue = _Queue()
+ assert export_worker._provision_ssm_kernels(str(checkpoint), None, None, queue, "cuda") is False
+ assert queue[-1]["type"] == "error"
+ assert "mamba-ssm" in queue[-1]["error"]
+
+
+def test_a_native_required_hybrid_exports_without_mamba_ssm(monkeypatch, tmp_path):
+ checkpoint = tmp_path / "falcon"
+ checkpoint.mkdir()
+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "falcon_h1"}))
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ monkeypatch.setattr(ssm_runtime, "_torch_env", lambda: {"cuda_version": "12.8"})
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda model_type: True)
+ queue = _Queue()
+ assert export_worker._provision_ssm_kernels(str(checkpoint), None, None, queue, "cuda") is True
+ assert not any(item.get("type") == "error" for item in queue)
+ assert any("Warning: could not install mamba-ssm" in item.get("message", "") for item in queue)
+
+
+def test_an_optional_ssm_family_exports_on_the_torch_path(monkeypatch, tmp_path):
+ checkpoint = tmp_path / "jamba"
+ checkpoint.mkdir()
+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "jamba"}))
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ queue = _Queue()
+ assert export_worker._provision_ssm_kernels(str(checkpoint), None, None, queue, "cuda") is True
+ assert not any(item.get("type") == "error" for item in queue)
+
+
[email protected]("pinned", [True, False])
+def test_a_lora_export_resolves_its_base(monkeypatch, tmp_path, pinned):
+ base = tmp_path / "base"
+ base.mkdir()
+ (base / "config.json").write_text(json.dumps({"model_type": "falcon_h1"}))
+ adapter = tmp_path / "my-adapter"
+ adapter.mkdir()
+ (adapter / "adapter_config.json").write_text(json.dumps({"base_model_name_or_path": str(base)}))
+ seen = []
+ monkeypatch.setattr(
+ ssm_runtime, "ensure_ssm_runtime", lambda probe, **k: seen.append(k["is_ssm"]) or []
+ )
+ monkeypatch.setattr("utils.transformers_version._remote_lora_base", lambda *a, **k: None)
+ assert export_worker._provision_ssm_kernels(
+ str(adapter), str(base) if pinned else None, None, _Queue(), "cuda"
+ )
+ assert seen == [True]
+
+
+ "platform, device_backend",
+ [("darwin", None), ("darwin", "mlx"), ("linux", "mlx"), ("linux", "cpu")],
+)
+def test_mlx_cpu_and_macos_exports_install_no_cuda_kernels(
+ monkeypatch, tmp_path, platform, device_backend
+):
+ # causal-conv1d / mamba-ssm are CUDA / ROCm extensions: an MLX or CPU export never loads them,
+ # and a source build there would only delay (or fail) a valid export.
+ checkpoint = tmp_path / "qwen"
+ checkpoint.mkdir()
+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "qwen3_5"}))
+ monkeypatch.setattr(export_worker.sys, "platform", platform)
+ monkeypatch.setattr(
+ ssm_runtime,
+ "provision_ssm_kernels",
+ lambda *a, **k: pytest.fail("no SSM kernel provisioning on MLX / CPU / macOS"),
+ )
+ queue = _Queue()
+ assert (
+ export_worker._provision_ssm_kernels(str(checkpoint), None, None, queue, device_backend)
+ is True
+ )
+ assert queue == []
+
+
[email protected]("device_backend", ["cuda", "rocm"])
+def test_cuda_exports_still_provision(monkeypatch, tmp_path, device_backend):
+ checkpoint = tmp_path / "qwen"
+ checkpoint.mkdir()
+ (checkpoint / "config.json").write_text(json.dumps({"model_type": "qwen3_5"}))
+ monkeypatch.setattr(export_worker.sys, "platform", "linux")
+ seen = []
+ monkeypatch.setattr(
+ ssm_runtime, "provision_ssm_kernels", lambda *a, **k: seen.append(a) or (None, [])
+ )
+ assert export_worker._provision_ssm_kernels(
+ str(checkpoint), None, None, _Queue(), device_backend
+ )
+ assert seen
+
+
+def test_both_export_provisioning_sites_pass_the_device_backend():
+ import ast
+ import inspect
+
+ tree = ast.parse(inspect.getsource(export_worker.run_export_process))
+ calls = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "_provision_ssm_kernels"
+ ]
+ assert len(calls) == 2
+ for call in calls:
+ assert ast.unparse(call.args[4]) == "config.get('device_backend')"
+
+
[email protected]("device_backend", ["xpu", None, ""])
+def test_xpu_or_unspecified_exports_install_no_cuda_kernels(monkeypatch, tmp_path, device_backend):
+ # Only a CUDA / ROCm export provisions: XPU builds none, and an unknown backend is not CUDA.
+ monkeypatch.setattr(export_worker.sys, "platform", "linux")
+ monkeypatch.setattr(
+ ssm_runtime,
+ "provision_ssm_kernels",
+ lambda *a, **k: pytest.fail("no SSM kernel provisioning off CUDA / ROCm"),
+ )
+ assert export_worker._provision_ssm_kernels("acme/hybrid", None, None, _Queue(), device_backend)
+
+
+# ── Security gates run before any SSM kernel is provisioned ──────────────────
+
+
+class _CmdQueue:
+ def __init__(self, cmds):
+ self._cmds = list(cmds)
+
+ def get(self, timeout = None):
+ if self._cmds:
+ return self._cmds.pop(0)
+ raise EOFError
+
+
+class _Verdict:
+ def __init__(self, blocked):
+ self.blocked = blocked
+ self.reason = "flagged pickle"
+ self.model_name = "acme/evil"
+ self.max_severity = "CRITICAL"
+
+ def response_payload(self):
+ return {"blocked": self.blocked}
+
+
+def _gated_export(
+ monkeypatch,
+ tmp_path,
+ *,
+ malware,
+ consent,
+ cmds = (),
+ trust = True,
+):
+ """Run the export worker with the Hub security verdicts stubbed: *malware* / *consent* say
+ whether each gate blocks, *cmds* are fed to the command loop after startup."""
+ import contextlib
+
+ monkeypatch.setattr(export_worker.sys, "platform", "linux")
+ order: list = []
+ monkeypatch.setattr(
+ export_worker, "_offline_window_if_unreachable", lambda step = None: contextlib.nullcontext()
+ )
+ monkeypatch.setattr(
+ export_worker,
+ "_provision_ssm_kernels",
+ lambda *a, **k: order.append(("provision", a[0])) or True,
+ )
+ monkeypatch.setattr("utils.transformers_version._remote_lora_base", lambda *a, **k: None)
+ monkeypatch.setattr(export_worker, "_nemotron_auto_trust", lambda *a, **k: False)
+
+ def _file_security(target, **kwargs):
+ order.append(("malware_scan", target, kwargs["load_subdirs"]))
+ return _Verdict(target in malware if isinstance(malware, set) else malware)
+
+ def _consent(targets, **kwargs):
+ order.append(("consent_scan", tuple(targets)))
+ return _Verdict(consent)
+
+ monkeypatch.setattr("utils.security.evaluate_file_security", _file_security)
+ monkeypatch.setattr("utils.security.evaluate_remote_code_consent_for_targets", _consent)
+ # The preflight must stay transformers-free: no load-subdir computation.
+ monkeypatch.setattr(
+ "utils.security.security_load_subdirs",
+ lambda *a, **k: pytest.fail("preflight computed load subdirs (imports transformers)"),
+ )
+ monkeypatch.setattr(export_worker, "_setup_log_capture", lambda queue: None)
+ monkeypatch.setattr(export_worker, "_activate_transformers_version", lambda *a, **k: None)
+ monkeypatch.setattr(export_worker, "apply_gpu_ids", lambda *a, **k: None)
+ monkeypatch.setattr(
+ "core._torchao_stub.install_torchao_windows_rocm_real_or_stub", lambda: None
+ )
+ monkeypatch.setattr("core.import_guards.ensure_real_packages", lambda *names: None)
+
+ class _Backend:
+ def cleanup_memory(self):
+ pass
+
+ fake_export = types.ModuleType("core.export.export")
+ fake_export.ExportBackend = _Backend
+ monkeypatch.setitem(sys.modules, "core.export.export", fake_export)
+ monkeypatch.setattr(
+ export_worker,
+ "_handle_load",
+ lambda b, cmd, q: order.append(("load", cmd["checkpoint_path"])),
+ )
+ for key in ("TQDM_MININTERVAL", "TOKENIZERS_PARALLELISM", "PYTHONWARNINGS", "PYTHONUNBUFFERED"):
+ monkeypatch.setenv(key, "")
+ monkeypatch.delenv(key)
+ monkeypatch.setenv("ENVIRONMENT_TYPE", "development")
+ monkeypatch.setattr("loggers.config.LogConfig.setup_logging", lambda *a, **k: None)
+ monkeypatch.setattr("loggers.config.allow_progress_bars", lambda: None)
+
+ queue = _Queue()
+ export_worker.run_export_process(
+ cmd_queue = _CmdQueue(cmds),
+ resp_queue = queue,
+ config = {
+ "checkpoint_path": "acme/evil",
+ "base_model": "acme/evil-base",
+ "trust_remote_code": trust,
+ "allow_ambient": True,
+ "device_backend": "cuda",
+ },
+ )
+ return order, queue
+
+
+def test_a_malware_blocked_export_provisions_no_kernels(monkeypatch, tmp_path):
+ order, queue = _gated_export(monkeypatch, tmp_path, malware = True, consent = False)
+ assert not any(step[0] == "provision" for step in order)
+ assert not any(step[0] == "load" for step in order)
+ loaded = [r for r in queue if r.get("type") == "loaded"]
+ assert loaded and loaded[0]["error_kind"] == "malware_blocked"
+ assert loaded[0]["success"] is False
+ # Transformers-free preflight: no subdir scoping.
+ assert order[0] == ("malware_scan", "acme/evil", ())
+
+
+def test_a_consent_blocked_trusted_export_provisions_no_kernels(monkeypatch, tmp_path):
+ order, queue = _gated_export(monkeypatch, tmp_path, malware = False, consent = True)
+ assert ("consent_scan", ("acme/evil", "acme/evil-base")) in order
+ assert not any(step[0] == "provision" for step in order)
+ loaded = [r for r in queue if r.get("type") == "loaded"]
+ assert loaded and loaded[0]["error_kind"] == "remote_code_blocked"
+ assert "remote_code" in loaded[0]
+
+
+def test_an_untrusted_export_skips_the_consent_scan_and_provisions(monkeypatch, tmp_path):
+ order, queue = _gated_export(monkeypatch, tmp_path, malware = False, consent = True, trust = False)
+ assert not any(step[0] == "consent_scan" for step in order)
+ assert ("provision", "acme/evil") in order
+ assert order.index(("malware_scan", "acme/evil", ())) < order.index(("provision", "acme/evil"))
+
+
+def test_a_blocked_load_command_provisions_no_kernels(monkeypatch, tmp_path):
+ # Startup is clear; the later "load" command names a flagged checkpoint.
+ order, queue = _gated_export(
+ monkeypatch,
+ tmp_path,
+ malware = {"acme/evil2"},
+ consent = False,
+ cmds = [{"type": "load", "checkpoint_path": "acme/evil2", "trust_remote_code": True}],
+ )
+ assert ("provision", "acme/evil") in order # startup was clear
+ assert ("load", "acme/evil") in order
+ assert ("provision", "acme/evil2") not in order
+ assert ("load", "acme/evil2") not in order
+ loaded = [r for r in queue if r.get("type") == "loaded"]
+ assert [r["error_kind"] for r in loaded] == ["malware_blocked"]
+
+
+def test_a_clear_load_command_gates_then_provisions_then_loads(monkeypatch, tmp_path):
+ order, queue = _gated_export(
+ monkeypatch,
+ tmp_path,
+ malware = False,
+ consent = False,
+ cmds = [{"type": "load", "checkpoint_path": "acme/ok2"}],
+ )
+ scan = order.index(("malware_scan", "acme/ok2", ()))
+ assert scan < order.index(("provision", "acme/ok2")) < order.index(("load", "acme/ok2"))
diff --git a/studio/backend/tests/test_kernel_install.py b/studio/backend/tests/test_kernel_install.py
index 9f6b7c02135..6848275995a 100644
--- a/studio/backend/tests/test_kernel_install.py
+++ b/studio/backend/tests/test_kernel_install.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import os
import subprocess
+import time
import sys
from pathlib import Path
from types import SimpleNamespace
@@ -19,6 +20,8 @@ if str(_BACKEND) not in sys.path:
from utils import kernel_install # noqa: E402
+_REAL_LOCK_PATH = kernel_install._kernel_lock_path
+
_REPO = _BACKEND.parent.parent
_PT = "https://download.pytorch.org/whl"
_CC1D = "https://github.com/Dao-AILab/causal-conv1d/releases/download/v1.6.1.post4"
@@ -139,6 +142,20 @@ _COLAB = _env("2.11.0+cu130", "13.0")
_COLAB_XFORMERS = f"{_PT}/cu130/xformers-0.0.35-py39-none-manylinux_2_28_x86_64.whl"
[email protected](autouse = True)
+def _nothing_installed_and_online(monkeypatch, tmp_path):
+ monkeypatch.setattr(
+ kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / f"{name}.lock")
+ )
+ # Every copy of the module (and any child process) uses the test's folder too.
+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", str(tmp_path / "locks"))
+ # Tests that need an installed distribution say so; the host's own packages and offline
+ # switches must not decide them.
+ monkeypatch.setattr(kernel_install, "_installed_version", lambda distribution: None)
+ for name in ("UV_OFFLINE", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"):
+ monkeypatch.delenv(name, raising = False)
+
+
@pytest.fixture
def uv(monkeypatch):
def use(available, outside_venv = False):
@@ -513,12 +530,150 @@ def test_main_installs_all_by_default_in_dependency_order(monkeypatch, argv, exp
order = []
monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: _COLAB)
monkeypatch.setattr(
- kernel_install, "install_kernel", lambda name, env, dry_run = False: order.append(name) or 0
+ kernel_install, "install_kernel", lambda name, env, **kwargs: order.append(name) or 0
)
assert kernel_install.main(argv) == 0
assert order == expected
+# A GPU-less image build (Docker, CI) provisions for the GPU it will run on only when asked to.
+
+
[email protected]("opt_in", ["flag", "env"])
+def test_install_kernels_without_gpu_installs_mamba_ssm_headless(uv, monkeypatch, opt_in):
+ uv(False)
+ monkeypatch.setattr(kernel_install.sys, "platform", "linux")
+ monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: _COLAB)
+ monkeypatch.setattr(kernel_install, "url_exists", lambda url: True)
+ run = _Runner([False, True], capability = "None")
+ real = kernel_install.install_kernel
+ monkeypatch.setattr(
+ kernel_install,
+ "install_kernel",
+ lambda name, env, **kwargs: real(name, env, run = run, exists = lambda url: True, **kwargs),
+ )
+ argv = ["mamba_ssm"]
+ monkeypatch.delenv("UNSLOTH_INSTALL_KERNELS_WITHOUT_GPU", raising = False)
+ if opt_in == "flag":
+ argv.append("--without-gpu")
+ else:
+ monkeypatch.setenv("UNSLOTH_INSTALL_KERNELS_WITHOUT_GPU", "1")
+ assert kernel_install.main(argv) == 0
+ assert run.installer_calls[0][-1].endswith(
+ "mamba_ssm-2.3.1+cu13torch2.10cxx11abiTRUE-cp313-cp313-linux_x86_64.whl"
+ )
+
+
[email protected]("strict, code", [(False, 0), (True, 2)])
+def test_a_requested_kernel_skipped_without_a_gpu_is_listed_and_fails_only_with_strict(
+ monkeypatch, capsys, strict, code
+):
+ # 9f5b13325 and 4a6429a65 exit 0 here, and `set -e` install scripts rely on it.
+ monkeypatch.delenv("UNSLOTH_INSTALL_KERNELS_WITHOUT_GPU", raising = False)
+ monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: _COLAB)
+ run = _Runner([], capability = "None")
+ real = kernel_install.install_kernel
+ monkeypatch.setattr(
+ kernel_install,
+ "install_kernel",
+ lambda name, env, **kwargs: real(name, env, run = run, exists = lambda url: True, **kwargs),
+ )
+ assert kernel_install.main(["mamba_ssm"] + (["--strict"] if strict else [])) == code
+ assert run.installer_calls == []
+ out = capsys.readouterr().out
+ assert "--without-gpu" in out
+ assert "not installed on this machine: mamba_ssm" in out
+
+
+ "name, env, capability",
+ [
+ # A ROCm source build: no CUDA wheel resolves.
+ ("causal_conv1d", None, "9 0"),
+ # A GPU-hidden builder with an earlier install.
+ ("mamba_ssm", _COLAB, "None"),
+ ("flash_attn", _COLAB, "None"),
+ ],
+)
+def test_an_already_working_kernel_is_success_before_any_skip(
+ monkeypatch, capsys, name, env, capability
+):
+ monkeypatch.setattr(
+ kernel_install,
+ "_installed_version",
+ lambda distribution: "1.0" if distribution == kernel_install.KERNELS[name][0] else None,
+ )
+ monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: env)
+ run = _Runner([True], capability = capability)
+ real = kernel_install.install_kernel
+ monkeypatch.setattr(
+ kernel_install,
+ "install_kernel",
+ lambda n, e, **kwargs: real(n, e, run = run, exists = lambda url: True, **kwargs),
+ )
+ assert kernel_install.main([name, "--strict"]) == 0
+ assert run.installer_calls == []
+ assert not any("get_device_capability" in " ".join(c) for c in run.calls)
+ out = capsys.readouterr().out
+ assert f"{name} already installed and loads" in out
+ assert "not installed on this machine" not in out
+
+
+def test_an_installed_kernel_that_does_not_load_still_installs(uv, monkeypatch):
+ uv(False)
+ run = _Runner([False, False, True], capability = "9 0")
+ monkeypatch.setattr(kernel_install, "_installed_version", lambda distribution: "1.0")
+ assert (
+ kernel_install.install_kernel("causal_conv1d", _COLAB, run = run, exists = lambda url: True)
+ == 0
+ )
+ assert run.installer_calls
+
+
[email protected]("variable", ["UV_OFFLINE", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"])
+def test_the_cli_installs_nothing_offline(monkeypatch, capsys, variable):
+ monkeypatch.setenv(variable, "1")
+ run = _Runner([], capability = "9 0")
+ url_checks = []
+ assert (
+ kernel_install.install_kernel(
+ "causal_conv1d", _COLAB, run = run, exists = lambda url: url_checks.append(url)
+ )
+ == 0
+ )
+ assert run.calls == [] and url_checks == []
+ assert "offline" in capsys.readouterr().out
+
+
+def test_offline_an_already_working_kernel_still_succeeds(monkeypatch, capsys):
+ monkeypatch.setenv("UV_OFFLINE", "1")
+ monkeypatch.setattr(kernel_install, "_installed_version", lambda distribution: "1.0")
+ run = _Runner([True], capability = "9 0")
+ assert (
+ kernel_install.install_kernel("causal_conv1d", _COLAB, run = run, exists = lambda url: True)
+ == 0
+ )
+ assert "already installed and loads" in capsys.readouterr().out
+
+
+def test_the_default_run_lists_skipped_kernels_and_keeps_its_exit_code(monkeypatch, capsys):
+ monkeypatch.setattr(kernel_install, "probe_torch_wheel_env", lambda **kwargs: _COLAB)
+
+ def _install(
+ name,
+ env,
+ skipped = None,
+ **kwargs,
+ ):
+ if name == "mamba_ssm":
+ skipped.append(name)
+ return 0
+
+ monkeypatch.setattr(kernel_install, "install_kernel", _install)
+ assert kernel_install.main([]) == 0
+ assert "not installed on this machine: mamba_ssm" in capsys.readouterr().out
+
+
def test_flash_attn_without_a_wheel_never_probes_the_gpu():
run = _Runner([])
assert kernel_install.install_kernel("flash_attn", None, run = run, exists = lambda url: True) == 0
@@ -676,7 +831,8 @@ def test_source_build_run_kwargs(monkeypatch):
kwargs, gcc = kernel_install.source_build_run_kwargs(
is_hip = False, gcc_install_dir = lambda: pytest.fail("non-HIP never looks for gcc")
)
- assert gcc is None and "timeout" not in kwargs
+ # CUDA builds get their own, far looser deadline (a slow aarch64 build must not be aborted).
+ assert gcc is None and kwargs["timeout"] == kernel_install.SOURCE_BUILD_TIMEOUT_CUDA
assert kwargs["encoding"] == "utf-8" and kwargs["stderr"] == subprocess.STDOUT
kwargs, gcc = kernel_install.source_build_run_kwargs(
@@ -693,6 +849,73 @@ def test_source_build_run_kwargs(monkeypatch):
assert gcc is None and kwargs["env"]["HIPCC_COMPILE_FLAGS_APPEND"] == "--gcc-install-dir=/mine"
+ "raw, expected",
+ [
+ (None, 7200),
+ ("", 7200),
+ ("10800", 10800),
+ ("3600.5", 3600),
+ ("0", 7200),
+ ("-5", 7200),
+ ("x", 7200),
+ ],
+)
+def test_the_cuda_source_build_timeout_is_7200_unless_overridden(monkeypatch, raw, expected):
+ if raw is None:
+ monkeypatch.delenv("UNSLOTH_SOURCE_BUILD_TIMEOUT", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_SOURCE_BUILD_TIMEOUT", raw)
+ assert kernel_install._cuda_source_build_timeout() == expected
+
+
+def test_hip_keeps_1800_and_the_lock_wait_outlasts_both_deadlines():
+ assert kernel_install.SOURCE_BUILD_TIMEOUT == 1800
+ assert kernel_install.SOURCE_BUILD_TIMEOUT_CUDA >= 1800
+ assert kernel_install.KERNEL_LOCK_WAIT > kernel_install.SOURCE_BUILD_TIMEOUT
+ assert kernel_install.KERNEL_LOCK_WAIT > kernel_install.SOURCE_BUILD_TIMEOUT_CUDA
+
+
[email protected]("raw, cuda", [(None, 7200), ("14400", 14400), ("600", 600)])
+def test_the_env_override_is_read_at_import_and_the_lock_wait_follows(raw, cuda):
+ import json as _json
+
+ env = {k: v for k, v in os.environ.items() if k != "UNSLOTH_SOURCE_BUILD_TIMEOUT"}
+ if raw is not None:
+ env["UNSLOTH_SOURCE_BUILD_TIMEOUT"] = raw
+ code = (
+ "import json, sys; sys.path.insert(0, sys.argv[1]); from utils import kernel_install as k; "
+ "print(json.dumps([k.SOURCE_BUILD_TIMEOUT, k.SOURCE_BUILD_TIMEOUT_CUDA, k.KERNEL_LOCK_WAIT]))"
+ )
+ out = subprocess.run(
+ [sys.executable, "-c", code, str(_BACKEND)],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 120,
+ check = True,
+ ).stdout
+ hip, cuda_seen, lock_wait = _json.loads(out.strip().splitlines()[-1])
+ assert (hip, cuda_seen) == (1800, cuda)
+ assert lock_wait == max(1800, cuda) + 600
+ assert lock_wait > hip and lock_wait > cuda_seen
+
+
+def test_a_dry_run_prints_the_url_even_offline(monkeypatch, capsys):
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ run = _Runner([])
+ assert (
+ kernel_install.install_kernel(
+ "xformers", _COLAB, dry_run = True, run = run, exists = lambda url: True
+ )
+ == 0
+ )
+ assert run.calls == []
+ out = capsys.readouterr().out
+ assert "offline" not in out
+ assert out.strip().endswith("xformers-0.0.35-py39-none-manylinux_2_28_x86_64.whl")
+
+
@pytest.mark.parametrize(
"use_uv, system, expected",
[
@@ -737,3 +960,195 @@ def test_windows_mamba_ssm_installs_with_triton(uv, monkeypatch):
assert run.installer_calls[0][-1].endswith(
"mamba_ssm-2.3.2.post1+cu13torch2.13cxx11abiTRUE-cp313-cp313-win_amd64.whl"
)
+
+
+ "capability, below",
+ [((7, 5), True), ((7, 0), True), ((8, 0), False), ((12, 0), False), (None, False)],
+)
+def test_flash_attn_floor_is_sm80_and_no_visible_gpu_is_not_below(monkeypatch, capability, below):
+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: capability)
+ monkeypatch.setattr(kernel_install, "_torch_is_hip", lambda run: False)
+ assert kernel_install.gpu_below_flash_attn_floor() is below
+
+
+def test_rocm_never_takes_the_sm80_floor(monkeypatch):
+ # ROCm reports a gfx version as its capability; the sm80 floor is an NVIDIA rule.
+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: (7, 5))
+ monkeypatch.setattr(kernel_install, "_torch_is_hip", lambda run: True)
+ assert kernel_install.gpu_below_flash_attn_floor() is False
+
+
+def test_the_hip_probe_reads_torch_version_hip():
+ def run(cmd, **kwargs):
+ assert "torch.version.hip" in cmd[-1]
+ return SimpleNamespace(returncode = 0, stdout = "True\n")
+
+ assert kernel_install._torch_is_hip(run) is True
+
+
+def test_install_wheel_never_falls_back_to_pip_under_uv_offline(monkeypatch):
+ # uv refuses the index under UV_OFFLINE; pip has no such switch and would reach it.
+ from utils import wheel_utils
+
+ monkeypatch.setattr(wheel_utils.shutil, "which", lambda name: "/usr/bin/uv")
+ calls = []
+
+ def run(cmd, **kwargs):
+ calls.append(cmd)
+ return subprocess.CompletedProcess(cmd, 1, stdout = "offline")
+
+ monkeypatch.setenv("UV_OFFLINE", "1")
+ attempts = wheel_utils.install_wheel(
+ "https://example.invalid/k.whl", python_executable = sys.executable, use_uv = True, run = run
+ )
+ assert [installer for installer, _ in attempts] == ["uv"]
+ assert all(cmd[0] == "uv" for cmd in calls)
+
+ monkeypatch.delenv("UV_OFFLINE")
+ calls.clear()
+ attempts = wheel_utils.install_wheel(
+ "https://example.invalid/k.whl", python_executable = sys.executable, use_uv = True, run = run
+ )
+ assert [installer for installer, _ in attempts] == ["uv", "pip"]
+
+
[email protected](os.name == "nt", reason = "POSIX process groups")
+def test_a_timed_out_build_stops_its_whole_process_tree(tmp_path):
+ # pip / uv start nvcc / hipcc / ninja; a timeout must stop those too, not just the leader.
+ pid_file = tmp_path / "grandchild.pid"
+ script = (
+ "import subprocess, sys, time; "
+ "p = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)']); "
+ f"open({str(pid_file)!r}, 'w').write(str(p.pid)); "
+ "time.sleep(60)"
+ )
+ # A generous timeout: on a loaded host the child can take seconds to start its grandchild.
+ with pytest.raises(subprocess.TimeoutExpired):
+ kernel_install.run_process_tree(
+ [sys.executable, "-c", script], timeout = 15, stdout = subprocess.PIPE
+ )
+ assert pid_file.exists(), "the build never started its grandchild within the timeout"
+ grandchild = int(pid_file.read_text())
+ for _ in range(100):
+ try:
+ os.kill(grandchild, 0)
+ except ProcessLookupError:
+ break
+ # Reaped by init once killed; a zombie still answers kill(0) briefly.
+ try:
+ with open(f"/proc/{grandchild}/stat") as stat:
+ if stat.read().split()[2] == "Z":
+ break
+ except FileNotFoundError:
+ break
+ time.sleep(0.1)
+ else:
+ os.kill(grandchild, 9)
+ pytest.fail("the build's grandchild outlived the timeout")
+
+
+def test_a_test_double_runner_is_called_as_given():
+ calls = []
+
+ def run(cmd, **kwargs):
+ calls.append(kwargs)
+ return subprocess.CompletedProcess(cmd, 0)
+
+ kernel_install.run_process_tree(["x"], run = run, timeout = 5, text = True)
+ assert calls == [{"timeout": 5, "text": True}]
+
+
+def test_the_kernel_lock_is_exclusive_bounded_and_beats_while_waiting(tmp_path, monkeypatch):
+ monkeypatch.setattr(kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / name))
+ beats = []
+ with kernel_install.kernel_install_lock("causal_conv1d") as first:
+ assert first is True
+ with kernel_install.kernel_install_lock(
+ "causal_conv1d", on_wait = lambda: beats.append(1), wait_seconds = 1.2, beat_seconds = 0.4
+ ) as second:
+ assert second is False # held elsewhere past the bound: install nothing
+ with kernel_install.kernel_install_lock("mamba_ssm", wait_seconds = 0) as other:
+ assert other is True # one lock per kernel
+ assert len(beats) >= 2
+ with kernel_install.kernel_install_lock("causal_conv1d", wait_seconds = 0) as again:
+ assert again is True # released
+
+
+def test_the_kernel_lock_is_per_environment(monkeypatch, tmp_path):
+ monkeypatch.delenv("UNSLOTH_KERNEL_LOCK_DIR", raising = False)
+ first_env, other_env = tmp_path / "a", tmp_path / "b"
+ first_env.mkdir()
+ other_env.mkdir()
+ monkeypatch.setattr(kernel_install.sys, "prefix", str(first_env))
+ first = _REAL_LOCK_PATH("causal_conv1d")
+ monkeypatch.setattr(kernel_install.sys, "prefix", str(other_env))
+ assert _REAL_LOCK_PATH("causal_conv1d") != first
+
+
+def test_the_lock_lives_beside_the_environment_when_writable(monkeypatch, tmp_path):
+ monkeypatch.delenv("UNSLOTH_KERNEL_LOCK_DIR", raising = False)
+ env = tmp_path / "venv"
+ env.mkdir()
+ monkeypatch.setattr(kernel_install.sys, "prefix", str(env))
+ assert _REAL_LOCK_PATH("mamba_ssm") == str(env / ".unsloth-kernel-locks" / "mamba_ssm.lock")
+
+
[email protected](os.name == "nt" or os.geteuid() == 0, reason = "POSIX permissions, non-root")
+def test_a_read_only_environment_falls_back_to_one_temp_lock(monkeypatch, tmp_path):
+ monkeypatch.delenv("UNSLOTH_KERNEL_LOCK_DIR", raising = False)
+ env = tmp_path / "venv"
+ env.mkdir()
+ env.chmod(0o555)
+ try:
+ monkeypatch.setattr(kernel_install.sys, "prefix", str(env))
+ monkeypatch.setattr("tempfile.tempdir", str(tmp_path / "tmp"))
+ (tmp_path / "tmp").mkdir()
+ first = _REAL_LOCK_PATH("mamba_ssm")
+ assert first.startswith(str(tmp_path / "tmp")) and first == _REAL_LOCK_PATH("mamba_ssm")
+ finally:
+ env.chmod(0o755)
+
+
[email protected](os.name == "nt" or os.geteuid() == 0, reason = "POSIX permissions, non-root")
+def test_a_lock_that_cannot_be_opened_installs_nothing(monkeypatch, tmp_path):
+ lock = tmp_path / "ro.lock"
+ lock.write_text("")
+ lock.chmod(0o444)
+ monkeypatch.setattr(kernel_install, "_kernel_lock_path", lambda name: str(lock))
+ with kernel_install.kernel_install_lock("causal_conv1d", wait_seconds = 0) as held:
+ assert held is False
+
+
+def test_a_lock_the_filesystem_cannot_take_installs_nothing(monkeypatch, tmp_path):
+ import errno
+
+ monkeypatch.setattr(kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / "x.lock"))
+
+ def _no_locks(*args):
+ raise OSError(errno.ENOLCK, "No locks available")
+
+ monkeypatch.setattr("fcntl.flock", _no_locks)
+ with kernel_install.kernel_install_lock("causal_conv1d", wait_seconds = 0) as held:
+ assert held is False
+
+
+def test_the_cli_installs_under_the_studio_lock(uv, monkeypatch, capsys):
+ import contextlib
+
+ uv(False)
+ taken = []
+
+ @contextlib.contextmanager
+ def _busy(name, **kwargs):
+ taken.append(name)
+ yield False
+
+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _busy)
+ run = _Runner([], capability = "9 0")
+ assert (
+ kernel_install.install_kernel("causal_conv1d", _COLAB, run = run, exists = lambda url: True)
+ == 0
+ )
+ assert taken == ["causal_conv1d"] and run.installer_calls == []
+ assert "install lock" in capsys.readouterr().out
diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.py
index 127b9c49bd7..d016d9ae9ba 100644
--- a/studio/backend/tests/test_mlx_training_worker_config.py
+++ b/studio/backend/tests/test_mlx_training_worker_config.py
@@ -82,12 +82,14 @@ def _load_worker_module():
"flash_attn_wheel_url",
"install_wheel",
"is_windows_wheel_env",
+ "pinned_source_build_supported",
"probe_torch_wheel_env",
"url_exists",
):
setattr(wheel_utils, name, lambda *_args, **_kwargs: None)
# Read at import by utils.kernel_install, which the worker imports.
for name in (
+ "package_installs_offline_env",
"redact_url_credentials",
"xformers_wheel_url",
):
diff --git a/studio/backend/tests/test_ssm_restart_respawn.py b/studio/backend/tests/test_ssm_restart_respawn.py
new file mode 100644
index 00000000000..151a3d41857
--- /dev/null
+++ b/studio/backend/tests/test_ssm_restart_respawn.py
@@ -0,0 +1,335 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An SSM kernel installed after the worker's transformers imported the model's modeling module is
+only bound by a fresh worker: the worker asks for one and the orchestrator respawns once."""
+
+import sys
+import threading
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference import orchestrator as orch_mod
+from core.inference import worker as inference_worker
+from core.inference.orchestrator import InferenceOrchestrator
+
+
+def _orchestrator(
+ monkeypatch,
+ responses,
+ shutdown_results = (),
+):
+ orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ orchestrator._stop_ledger = None
+ orchestrator._pending_teardowns = None
+ orchestrator.__dict__.update(
+ loading_models = set(),
+ active_model_name = None,
+ models = {},
+ _proc = None,
+ _resp_queue = object(),
+ load_generation = 0,
+ _subprocess_shutdown_lock = threading.Lock(),
+ )
+ spawned = []
+ shutdowns = []
+ alive = {"value": False}
+ monkeypatch.setattr(orch_mod, "prepare_gpu_selection", lambda *_a, **_k: ([], "auto"))
+ monkeypatch.setattr(orch_mod, "get_device", lambda: SimpleNamespace(value = "cpu"))
+ monkeypatch.setattr("utils.transformers_version.needs_transformers_5", lambda _model: False)
+ monkeypatch.setattr("utils.transformers_version.sidecar_swap_kind", lambda: None)
+ monkeypatch.setattr(orchestrator, "_ensure_subprocess_alive", lambda: alive["value"])
+
+ def _spawn(config, *args):
+ spawned.append(dict(config)) # the orchestrator reuses one sub_config
+ alive["value"] = True
+
+ results = list(shutdown_results)
+
+ def _shutdown(timeout = 5):
+ shutdowns.append(timeout)
+ exited = results.pop(0) if results else True
+ if exited:
+ alive["value"] = False
+ return exited
+
+ def _wait(*_a, **_k):
+ response = responses.pop(0)
+ if isinstance(response, BaseException):
+ raise response
+ return response
+
+ monkeypatch.setattr(orchestrator, "_spawn_subprocess", _spawn)
+ monkeypatch.setattr(orchestrator, "_shutdown_subprocess", _shutdown)
+ monkeypatch.setattr(orchestrator, "_wait_response", _wait)
+ return orchestrator, spawned, shutdowns
+
+
+def _restart():
+ return {
+ "type": "loaded",
+ "success": False,
+ "error_kind": inference_worker.SSM_RESTART_REQUIRED,
+ "message": "restart",
+ }
+
+
+def test_the_orchestrator_respawns_once_when_the_worker_needs_new_kernels(monkeypatch):
+ responses = [
+ _restart(),
+ {"type": "loaded", "success": True, "model_info": {"identifier": "acme/hybrid"}},
+ ]
+ orchestrator, spawned, shutdowns = _orchestrator(monkeypatch, responses)
+ assert orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid")) is True
+ assert len(spawned) == 2
+ assert shutdowns
+ assert orchestrator.active_model_name == "acme/hybrid"
+
+
+def test_a_second_restart_request_fails_instead_of_looping(monkeypatch):
+ orchestrator, spawned, _ = _orchestrator(monkeypatch, [_restart(), _restart()])
+ try:
+ orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid"))
+ except Exception as exc:
+ assert "restart" in str(exc)
+ else:
+ raise AssertionError("a second restart request must fail the load")
+ assert len(spawned) == 2
+
+
+def _loaded():
+ return {"type": "loaded", "success": True, "model_info": {"identifier": "acme/hybrid"}}
+
+
+def _stall():
+ from utils.hf_xet_fallback import DownloadStallError
+ return DownloadStallError("stalled")
+
+
+def test_a_worker_that_survives_shutdown_is_not_respawned_over(monkeypatch):
+ # The worker outlived terminate/kill: its handle stays, and no second worker is spawned over
+ # one still holding GPU memory (the model-switch guard's rule).
+ orchestrator, spawned, _ = _orchestrator(
+ monkeypatch, [_restart(), _loaded()], shutdown_results = [False]
+ )
+ try:
+ orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid"))
+ except RuntimeError as exc:
+ assert "did not exit" in str(exc)
+ else:
+ raise AssertionError("the load must fail while the old worker survives")
+ assert len(spawned) == 1
+ assert orchestrator.active_model_name is None
+
+
+def test_a_survivor_also_blocks_the_xet_retry(monkeypatch):
+ orchestrator, spawned, _ = _orchestrator(
+ monkeypatch, [_stall(), _loaded()], shutdown_results = [False]
+ )
+ try:
+ orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid"))
+ except RuntimeError as exc:
+ assert "did not exit" in str(exc)
+ else:
+ raise AssertionError("the load must fail while the old worker survives")
+ assert len(spawned) == 1
+
+
+def test_an_ssm_restart_keeps_the_xet_retry(monkeypatch):
+ orchestrator, spawned, _ = _orchestrator(monkeypatch, [_restart(), _stall(), _loaded()])
+ assert orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid")) is True
+ assert [config["disable_xet"] for config in spawned] == [False, False, True]
+
+
+def test_a_xet_retry_keeps_the_ssm_restart(monkeypatch):
+ orchestrator, spawned, _ = _orchestrator(monkeypatch, [_stall(), _restart(), _loaded()])
+ assert orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid")) is True
+ assert [config["disable_xet"] for config in spawned] == [False, True, True]
+
+
+def test_spawns_are_bounded_at_three_and_the_stall_message_is_accurate(monkeypatch):
+ orchestrator, spawned, _ = _orchestrator(monkeypatch, [_restart(), _stall(), _stall()])
+ try:
+ orchestrator.load_model(SimpleNamespace(identifier = "acme/hybrid"))
+ except RuntimeError as exc:
+ assert "even with HF_HUB_DISABLE_XET=1" in str(exc)
+ else:
+ raise AssertionError("a second stall must fail the load")
+ assert len(spawned) == 3
+
+
+class _Queue(list):
+ def put(self, item):
+ self.append(item)
+
+
+def _fake_provision(installed, model_types):
+ from utils.ssm_runtime import SsmKernels
+ def _provision(probe, target, hf_token, **kwargs):
+ kernels = SsmKernels(True, True, True, probe, frozenset(model_types))
+ return kernels, list(installed)
+
+ return _provision
+
+
+def test_the_worker_asks_for_a_restart_when_the_modeling_module_is_already_imported(monkeypatch):
+ from utils import ssm_runtime
+
+ monkeypatch.setattr(
+ ssm_runtime, "provision_ssm_kernels", _fake_provision(["mamba_ssm"], {"nemotron_h"})
+ )
+ monkeypatch.setitem(
+ inference_worker.sys.modules,
+ "transformers.models.nemotron_h.modeling_nemotron_h",
+ SimpleNamespace(),
+ )
+ queue = _Queue()
+ assert inference_worker._ensure_ssm_kernels([("acme/hybrid", "acme/hybrid")], queue) is False
+ assert queue[-1]["error_kind"] == inference_worker.SSM_RESTART_REQUIRED
+
+
+def test_no_restart_when_nothing_new_was_installed_or_nothing_was_imported(monkeypatch):
+ from utils import ssm_runtime
+
+ monkeypatch.setitem(
+ inference_worker.sys.modules,
+ "transformers.models.nemotron_h.modeling_nemotron_h",
+ SimpleNamespace(),
+ )
+ monkeypatch.setattr(ssm_runtime, "provision_ssm_kernels", _fake_provision([], {"nemotron_h"}))
+ assert inference_worker._ensure_ssm_kernels([("acme/hybrid", "acme/hybrid")], _Queue())
+
+ monkeypatch.delitem(
+ inference_worker.sys.modules, "transformers.models.nemotron_h.modeling_nemotron_h"
+ )
+ monkeypatch.setattr(
+ ssm_runtime, "provision_ssm_kernels", _fake_provision(["mamba_ssm"], {"nemotron_h"})
+ )
+ assert inference_worker._ensure_ssm_kernels([("acme/hybrid", "acme/hybrid")], _Queue())
+
+
+# Only modeling modules capture kernel functions at import; a config or tokenizer import does not.
+
+
+def test_a_configuration_import_alone_needs_no_restart(monkeypatch):
+ from utils import ssm_runtime
+
+ monkeypatch.setitem(
+ sys.modules, "transformers.models.nemotron_h.configuration_nemotron_h", SimpleNamespace()
+ )
+ monkeypatch.setitem(sys.modules, "transformers.models.nemotron_h", SimpleNamespace())
+ monkeypatch.delitem(
+ sys.modules, "transformers.models.nemotron_h.modeling_nemotron_h", raising = False
+ )
+ assert ssm_runtime.modeling_modules_needing_restart({"nemotron_h"}) == []
+ queue = _Queue()
+ monkeypatch.setattr(
+ ssm_runtime, "provision_ssm_kernels", _fake_provision(["mamba_ssm"], {"nemotron_h"})
+ )
+ assert inference_worker._ensure_ssm_kernels([("acme/hybrid", "acme/hybrid")], queue) is True
+
+
+def _remote(checkpoint, auto_map):
+ from utils import ssm_runtime
+ return ssm_runtime.remote_code_modules(checkpoint, {"auto_map": auto_map})
+
+
+ "module",
+ [
+ # 4.57+ / 5.x sanitize "-" and "." in the repo id, under the commit hash.
+ "transformers_modules.nvidia.NVIDIA_hyphen_Nemotron_hyphen_Nano_hyphen_9B_hyphen_v2"
+ ".0123abcd.modeling_nemotron_h",
+ # Older 4.x kept them.
+ "transformers_modules.nvidia.NVIDIA-Nemotron-Nano-9B-v2.0123abcd.modeling_nemotron_h",
+ ],
+)
+def test_an_imported_remote_code_modeling_module_needs_a_restart(monkeypatch, module):
+ from utils import ssm_runtime
+
+ monkeypatch.setitem(sys.modules, module, SimpleNamespace())
+ model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
+ remote = _remote(model, {"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"})
+ assert ssm_runtime.modeling_modules_needing_restart(set(), remote) == [module]
+ monkeypatch.setattr(
+ ssm_runtime,
+ "provision_ssm_kernels",
+ lambda probe, target, hf_token, **k: (
+ ssm_runtime.SsmKernels(True, True, True, probe, frozenset(), remote),
+ ["mamba_ssm"],
+ ),
+ )
+ queue = _Queue()
+ assert inference_worker._ensure_ssm_kernels([(model, model)], queue) is False
+ assert queue[-1]["error_kind"] == inference_worker.SSM_RESTART_REQUIRED
+
+
+def test_any_auto_map_module_name_counts_not_just_modeling(monkeypatch, tmp_path):
+ # transformers loads whatever module auto_map names: `model.CustomForCausalLM` is `model`.
+ from utils import ssm_runtime
+
+ checkpoint = tmp_path / "custom-remote-checkpoint"
+ checkpoint.mkdir()
+ remote = _remote(str(checkpoint), {"AutoModelForCausalLM": "model.CustomForCausalLM"})
+ for name in (
+ # 5.x: local folder name, then a source-files hash.
+ "transformers_modules.custom_hyphen_remote_hyphen_checkpoint.abc123.model",
+ # 4.57: the local folder name alone.
+ "transformers_modules.custom_hyphen_remote_hyphen_checkpoint.model",
+ ):
+ monkeypatch.setitem(sys.modules, name, SimpleNamespace())
+ assert name in ssm_runtime.modeling_modules_needing_restart(set(), remote)
+
+
+def test_a_namesake_from_another_repo_does_not_match():
+ from utils import ssm_runtime
+
+ remote = _remote("Qwen/Qwen3-Next", {"AutoModelForCausalLM": "modeling_qwen.QwenForCausalLM"})
+ namesakes = [
+ "transformers_modules.other.Qwen3_hyphen_Next_hyphen_FineTune.0123.modeling_qwen",
+ "transformers_modules.Qwen.Qwen3_hyphen_Next_hyphen_FineTune.0123.modeling_qwen",
+ "transformers_modules.Qwen.Qwen3_hyphen_Next.0123.modeling_qwen_extra",
+ "transformers_modules.Qwen.Qwen3_hyphen_Next.0123.configuration_qwen",
+ ]
+ for name in namesakes:
+ assert not ssm_runtime._is_remote_code_module(name, remote), name
+ assert ssm_runtime._is_remote_code_module(
+ "transformers_modules.Qwen.Qwen3_hyphen_Next.0123.modeling_qwen", remote
+ )
+
+
+def test_code_in_another_repo_is_found_under_that_repo():
+ from utils import ssm_runtime
+
+ remote = _remote("acme/finetune", {"AutoModel": "nvidia/Base--model.Custom"})
+ assert ssm_runtime._is_remote_code_module("transformers_modules.nvidia.Base.0123.model", remote)
+ assert not ssm_runtime._is_remote_code_module(
+ "transformers_modules.acme.finetune.0123.model", remote
+ )
+
+
+def test_config_and_tokenizer_auto_map_entries_are_not_implementations():
+ remote = _remote(
+ "acme/hybrid",
+ {
+ "AutoConfig": "configuration_hybrid.HybridConfig",
+ "AutoTokenizer": ["tokenization_hybrid.HybridTokenizer", None],
+ },
+ )
+ assert remote == frozenset()
+
+
+def test_the_resolved_config_carries_its_remote_code(monkeypatch):
+ from utils import ssm_runtime
+
+ monkeypatch.setattr(
+ ssm_runtime,
+ "_resolved_model_config",
+ lambda target, token: {
+ "model_type": "nemotron_h",
+ "auto_map": {"AutoModelForCausalLM": "model.NemotronHForCausalLM"},
+ },
+ )
+ kernels = ssm_runtime.resolve_ssm_kernels("acme/hybrid", "acme/hybrid", None)
+ assert (("acme", "hybrid"), ("model",)) in kernels.remote_code
diff --git a/studio/backend/tests/test_ssm_runtime.py b/studio/backend/tests/test_ssm_runtime.py
index 02f3668666e..4b4f01acaaf 100644
--- a/studio/backend/tests/test_ssm_runtime.py
+++ b/studio/backend/tests/test_ssm_runtime.py
@@ -8,6 +8,7 @@ worker wiring, and that the training worker shares the same tables and pinned ke
"""
import json
+import contextlib
import sys
import types
from pathlib import Path
@@ -21,18 +22,35 @@ if str(_BACKEND) not in sys.path:
from utils import ssm_runtime # noqa: E402
+_REAL_MAMBA_SSM_FLOOR_REASON = ssm_runtime._mamba_ssm_floor_reason
+
@pytest.fixture(autouse = True)
-def _clear_offline_environment(monkeypatch):
+def _clear_offline_environment(monkeypatch, tmp_path):
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
+ monkeypatch.delenv("UV_OFFLINE", raising = False)
+ # Per-process memos: each test is its own process here.
+ monkeypatch.setattr(ssm_runtime, "_TORCH_ENV", {})
+ monkeypatch.setattr(ssm_runtime, "_FAILED_INSTALLS", set())
+ monkeypatch.setattr(ssm_runtime, "_MODEL_CONFIGS", {})
+ monkeypatch.setattr(ssm_runtime, "_REMOTE_SOURCES", {})
+ monkeypatch.setattr(ssm_runtime, "_NATIVE_MODELING_CACHE", {})
+ from utils import kernel_install
-
[email protected](autouse = True)
-def _cuda_host_with_triton(monkeypatch):
- # A CUDA torch with Triton, where zoo's Triton conv runs; the platform tests below override it.
- monkeypatch.delenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", raising = False)
- monkeypatch.setattr(ssm_runtime, "_zoo_triton_platform_usable", lambda: True)
+ monkeypatch.setattr(
+ kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / f"{name}.lock")
+ )
+ # Every copy of the module (and any child process) uses the test's folder too.
+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", str(tmp_path / "locks"))
+ # This host's GPU must not decide the optional-mamba floor; its own tests set it.
+ monkeypatch.setattr(ssm_runtime, "_mamba_ssm_floor_reason", lambda: None)
+ # Nor its torch build: a CUDA torch unless a test says otherwise.
+ monkeypatch.setattr(
+ ssm_runtime,
+ "probe_torch_wheel_env",
+ lambda **k: {"cuda_version": "12.8", "cuda_major": "12"},
+ )
class _Result:
@@ -164,9 +182,10 @@ def test_causal_only_model_skips_mamba(monkeypatch):
def test_failure_raises_runtime_error(monkeypatch):
- # A true SSM model whose mamba-ssm cannot install is fatal (cryptic mid-load import
- # otherwise). "Nemotron-3-Nano-30B-A3B" matches the SSM substrings.
+ # A required-family model this transformers has no native implementation of cannot load
+ # without mamba-ssm, so a failed install is fatal (cryptic mid-load import otherwise).
monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda model_type: False)
with pytest.raises(RuntimeError):
ssm_runtime.ensure_ssm_runtime("unsloth/Nemotron-3-Nano-30B-A3B")
@@ -205,7 +224,10 @@ def test_install_kernel_idempotent_when_present(monkeypatch):
assert called == [] # short-circuits before touching the network
[email protected]("offline_variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"])
+# UV_OFFLINE too: uv refuses the index and install_wheel's pip fallback would not.
+ "offline_variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "UV_OFFLINE"]
+)
def test_install_kernel_skips_all_install_work_offline(monkeypatch, offline_variable):
monkeypatch.setenv(offline_variable, "on")
monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: False)
@@ -239,11 +261,13 @@ def test_install_kernel_skips_all_install_work_offline(monkeypatch, offline_vari
def test_install_kernel_uses_prebuilt_wheel(monkeypatch):
- # not importable before install, importable after the wheel lands
- states = iter([False, True])
+ # not importable before install (or once the lock is held), importable after the wheel lands
+ states = iter([False, False, True])
monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))
monkeypatch.setattr(
- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {"x": "y"}
+ ssm_runtime,
+ "probe_torch_wheel_env",
+ lambda timeout = 30, include_windows = False: {"x": "y", "cuda_version": "12.8"},
)
seen = {}
monkeypatch.setattr(
@@ -284,7 +308,9 @@ def test_install_kernel_heartbeats_during_prebuilt_wheel(monkeypatch):
monkeypatch.setattr(ssm_runtime, "_HEARTBEAT_SECONDS", 0.05)
monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: False)
monkeypatch.setattr(
- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}
+ ssm_runtime,
+ "probe_torch_wheel_env",
+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},
)
monkeypatch.setattr(
ssm_runtime,
@@ -341,7 +367,9 @@ def test_install_kernel_heartbeats_through_the_import_check(monkeypatch):
monkeypatch.setattr(ssm_runtime, "_HEARTBEAT_SECONDS", 0.05)
monkeypatch.setattr(
- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}
+ ssm_runtime,
+ "probe_torch_wheel_env",
+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},
)
monkeypatch.setattr(
ssm_runtime,
@@ -359,7 +387,7 @@ def test_install_kernel_heartbeats_through_the_import_check(monkeypatch):
def slow_importable(name):
seen["n"] += 1
- if seen["n"] == 1:
+ if seen["n"] <= 2: # before install, and again once the lock is held
return False
import_started.set()
assert import_released.wait(1.0)
@@ -408,10 +436,12 @@ def test_heartbeat_does_not_emit_after_the_block(monkeypatch):
def test_install_kernel_falls_back_to_source(monkeypatch):
# no wheel -> source build -> importable after install
- states = iter([False, True]) # before install, after install
+ states = iter([False, False, True]) # before install, after the lock, after install
monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))
monkeypatch.setattr(
- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}
+ ssm_runtime,
+ "probe_torch_wheel_env",
+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},
)
monkeypatch.setattr(ssm_runtime, "direct_wheel_url", lambda **k: None)
pip_cmds = []
@@ -514,23 +544,35 @@ def _windows_install_kernel(
monkeypatch.setattr(ssm_runtime.sys, "platform", "win32")
# shutil.which consults sys.platform, and a Linux host cannot answer as Windows.
monkeypatch.setattr(ssm_runtime.shutil, "which", lambda *a, **k: None)
- states = iter([False, importable_after])
+ # Before install, once the lock is held, after the wheel.
+ states = iter([False, False, importable_after])
monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))
probe_kwargs = {}
monkeypatch.setattr(
ssm_runtime,
"probe_torch_wheel_env",
- lambda **k: probe_kwargs.update(k) or {"platform_tag": "win_amd64"},
+ lambda **k: probe_kwargs.update(k) or {"platform_tag": "win_amd64", "cuda_version": "13.0"},
)
monkeypatch.setattr(ssm_runtime, "direct_wheel_url", lambda **k: wheel_url)
monkeypatch.setattr(ssm_runtime, "url_exists", lambda u: exists)
+ present = {"value": False}
+ monkeypatch.setattr(ssm_runtime, "_installed_distribution", lambda name: present["value"])
installs = []
- monkeypatch.setattr(
- ssm_runtime,
- "install_wheel",
- lambda url, **k: installs.append(url) or [("uv", _Result(returncode = outcome_rc))],
- )
+
+ def _install_wheel(url, **k):
+ installs.append(url)
+ present["value"] = outcome_rc == 0
+ return [("uv", _Result(returncode = outcome_rc))]
+
+ monkeypatch.setattr(ssm_runtime, "install_wheel", _install_wheel)
ran = []
+
+ def _run(cmd, **k):
+ ran.append(cmd)
+ if "uninstall" in cmd:
+ present["value"] = False
+ return _Result()
+
ok = ssm_runtime._install_kernel(
import_name = "causal_conv1d",
display_name = "causal-conv1d",
@@ -539,7 +581,7 @@ def _windows_install_kernel(
release_tag = "v1.6.1.post4",
release_base_url = "https://github.com/Dao-AILab/causal-conv1d/releases/download",
status_cb = None,
- run = lambda cmd, **k: ran.append(cmd) or _Result(),
+ run = _run,
)
assert probe_kwargs.get("include_windows") is True
return ok, ran, installs
@@ -554,9 +596,7 @@ def test_windows_installs_the_published_wheel(monkeypatch):
assert ran == []
- ("wheel_url", "exists"), [(None, False), ("https://x/w.whl", False), ("https://x/w.whl", None)]
-)
[email protected](("wheel_url", "exists"), [(None, False), ("https://x/w.whl", False)])
def test_windows_never_source_builds_without_a_wheel(monkeypatch, wheel_url, exists):
ok, ran, installs = _windows_install_kernel(monkeypatch, wheel_url = wheel_url, exists = exists)
assert ok is False
@@ -575,12 +615,14 @@ def test_windows_rejected_wheel_is_removed_not_source_built(monkeypatch):
def test_wheel_installed_but_not_importable_falls_back_to_source(monkeypatch):
- # top: not importable; after wheel: still not importable (ABI mismatch) -> source build;
- # after source build: importable.
- states = iter([False, False, True])
+ # top and under the lock: not importable; after wheel: still not importable (ABI mismatch)
+ # -> source build; after source build: importable.
+ states = iter([False, False, False, True])
monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))
monkeypatch.setattr(
- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}
+ ssm_runtime,
+ "probe_torch_wheel_env",
+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},
)
monkeypatch.setattr(ssm_runtime, "direct_wheel_url", lambda **k: "https://x/w.whl")
monkeypatch.setattr(ssm_runtime, "url_exists", lambda u: True)
@@ -630,10 +672,12 @@ def test_hip_source_build_requires_hipcc(monkeypatch):
def test_source_build_reinstalls_to_replace_broken_wheel(monkeypatch):
# Reached only when not importable (possibly a broken wheel at the pinned version);
# the source build must reinstall so it replaces it instead of no-opping.
- states = iter([False, True])
+ states = iter([False, False, True])
monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))
monkeypatch.setattr(
- ssm_runtime, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}
+ ssm_runtime,
+ "probe_torch_wheel_env",
+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},
)
monkeypatch.setattr(ssm_runtime, "direct_wheel_url", lambda **k: None)
cmds = []
@@ -652,7 +696,7 @@ def test_source_build_reinstalls_to_replace_broken_wheel(monkeypatch):
def test_hip_uv_source_build_uses_no_cache(monkeypatch):
# ROCm uv source build must skip the cache to avoid reusing stale partial HIP builds.
- states = iter([False, True])
+ states = iter([False, False, True])
monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: next(states))
monkeypatch.setattr(
ssm_runtime,
@@ -682,8 +726,8 @@ def test_hip_uv_source_build_uses_no_cache(monkeypatch):
def test_inference_worker_calls_ensure_ssm_runtime():
src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8")
- assert "from utils.ssm_runtime import (\n ensure_ssm_runtime," in src
- assert "ensure_ssm_runtime(" in src
+ assert "provision_ssm_kernels" in src
+ assert "provision_ssm_kernels(" in src
def test_inference_worker_skips_ssm_on_mlx_and_checks_lora_base():
@@ -883,23 +927,22 @@ def test_nested_text_config_resolves_causal_conv1d(tmp_path):
assert ssm_runtime.resolved_model_wants_causal_conv1d(target, target, None) is True
-# ── Qwen3.5 / 3.6 / Qwen3-Next run on unsloth-zoo's Triton conv, never causal-conv1d ──
+# ── Qwen3.5 / 3.6 / Qwen3-Next get causal-conv1d on demand, on every platform ──
-_ZOO_GDN_NAMES = [
+_QWEN_GDN_NAMES = [
"Qwen/Qwen3-Next-80B-A3B",
- "Qwen/Qwen3-Next-80B-A3B-Instruct",
"unsloth/Qwen3.5-2B",
- "unsloth/Qwen3.5-35B-A3B-GGUF",
"unsloth/Qwen3.6-27B",
"my-org/qwen3_5-finetune",
]
[email protected]("name", _ZOO_GDN_NAMES)
-def test_zoo_gdn_models_request_no_kernels(monkeypatch, name):
- monkeypatch.delenv(ssm_runtime.ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, raising = False)
- assert ssm_runtime.model_wants_causal_conv1d(name) is False
[email protected]("name", _QWEN_GDN_NAMES)
+def test_qwen_gdn_models_install_causal_conv1d_on_demand(monkeypatch, name):
+ # zoo's Triton conv has no decode update and a sequence-sized dW scratch: keep the native kernel.
+ monkeypatch.delenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", raising = False)
+ assert ssm_runtime.model_wants_causal_conv1d(name) is True
assert ssm_runtime.model_is_ssm(name) is False
installed = []
monkeypatch.setattr(
@@ -908,14 +951,7 @@ def test_zoo_gdn_models_request_no_kernels(monkeypatch, name):
lambda *, import_name, **_: installed.append(import_name) or True,
)
ssm_runtime.ensure_ssm_runtime(name)
- assert installed == []
-
-
[email protected]("name", _ZOO_GDN_NAMES)
-def test_zoo_kill_switch_brings_the_causal_conv1d_request_back(monkeypatch, name):
- monkeypatch.setenv(ssm_runtime.ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, "1")
- assert ssm_runtime.model_wants_causal_conv1d(name) is True
- assert ssm_runtime.model_is_ssm(name) is False
+ assert installed == ["causal_conv1d"]
@pytest.mark.parametrize(
@@ -928,27 +964,23 @@ def test_zoo_kill_switch_brings_the_causal_conv1d_request_back(monkeypatch, name
"text_config": {"model_type": "qwen3_5_text"},
"vision_config": {"model_type": "qwen3_5_vision"},
},
- {
- "model_type": "qwen3_5_moe",
- "architectures": ["Qwen3_5MoeForConditionalGeneration"],
- "text_config": {"model_type": "qwen3_5_moe_text"},
- "vision_config": {"model_type": "qwen3_5_moe_vision"},
- },
],
- ids = ["qwen3_next", "qwen3_5", "qwen3_5_moe"],
+ ids = ["qwen3_next", "qwen3_5"],
)
-def test_zoo_gdn_configs_resolve_to_no_causal_conv1d(monkeypatch, tmp_path, config):
+def test_qwen_gdn_configs_resolve_to_causal_conv1d(monkeypatch, tmp_path, config):
+ monkeypatch.delenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", raising = False)
target = str(_write_config(tmp_path / "renamed", config))
- monkeypatch.delenv(ssm_runtime.ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, raising = False)
- assert ssm_runtime.model_config_wants_causal_conv1d(config) is False
- assert ssm_runtime.resolved_model_wants_causal_conv1d("acme/x", target, None) is False
- monkeypatch.setenv(ssm_runtime.ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, "1")
assert ssm_runtime.model_config_wants_causal_conv1d(config) is True
+ assert ssm_runtime.resolved_model_wants_causal_conv1d("acme/x", target, None) is True
-def test_a_config_mixing_a_zoo_gdn_type_with_another_hybrid_still_requests_it():
- config = {"model_type": "qwen3_5", "text_config": {"model_type": "lfm2"}}
- assert ssm_runtime.model_config_wants_causal_conv1d(config) is True
+def test_the_zoo_exemption_is_gone():
+ for name in (
+ "ZOO_GDN_MODEL_TYPES",
+ "zoo_triton_causal_conv1d_usable",
+ "_zoo_triton_platform_usable",
+ ):
+ assert not hasattr(ssm_runtime, name)
@pytest.mark.parametrize(
@@ -966,28 +998,6 @@ def test_mamba2_hybrids_still_install_both_on_demand(monkeypatch, name):
assert order == ["causal_conv1d", "mamba_ssm"]
-def test_zoo_gdn_model_types_match_unsloth_zoo():
- """Studio's list must equal the models unsloth-zoo's Triton conv patches."""
- import ast
- import importlib.util
-
- spec = importlib.util.find_spec("unsloth_zoo")
- if spec is None or spec.origin is None:
- pytest.skip("unsloth_zoo not installed")
- source_path = Path(spec.origin).parent / "temporary_patches" / "gdn_causal_conv1d.py"
- if not source_path.is_file():
- pytest.skip("this unsloth_zoo has no Triton causal conv")
- tree = ast.parse(source_path.read_text(encoding = "utf-8"))
- values = [
- node.value
- for node in ast.walk(tree)
- if isinstance(node, ast.Assign)
- and any(isinstance(t, ast.Name) and t.id == "_GDN_MODELING" for t in node.targets)
- ]
- assert len(values) == 1
- assert set(ast.literal_eval(values[0])) == ssm_runtime.ZOO_GDN_MODEL_TYPES
-
-
# ── mamba-ssm resolved from the config, not only the name ─────────────────────
@@ -1011,12 +1021,6 @@ def test_renamed_or_unlisted_ssm_models_resolve_from_their_config(tmp_path, name
assert ssm_runtime.resolved_model_is_ssm(name, target, None) is True
# Every mamba-ssm model also wants causal-conv1d.
assert ssm_runtime.resolved_model_wants_causal_conv1d(name, target, None) is True
- assert (
- ssm_runtime.resolved_model_wants_causal_conv1d(
- name, target, None, scan_modeling_source = False
- )
- is True
- )
def test_a_nested_ssm_text_config_resolves_too(tmp_path):
@@ -1052,92 +1056,11 @@ def test_resolved_answers_override_the_name(monkeypatch):
def test_the_mamba_ssm_failure_names_the_offline_remedy(monkeypatch):
monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda model_type: False)
with pytest.raises(RuntimeError, match = "unsloth install-kernels"):
ssm_runtime.ensure_ssm_runtime("tiiuae/Falcon-H1-0.5B")
-# ── zoo's Triton conv only counts where it can run ────────────────────────────
-
-
-# The real (cached) platform check, captured before the autouse fixture replaces it.
-_REAL_PLATFORM_USABLE = ssm_runtime._zoo_triton_platform_usable.__wrapped__
-
-
- "platform, triton, torch_version, usable",
- [
- ("linux", True, "2.13.0", True),
- ("linux", True, "2.13.0+cu130", True),
- ("win32", True, "2.13.0+cu130", True),
- ("linux", True, "2.11.0+rocm7.1", False),
- ("win32", False, "2.13.0+cu130", False),
- ("darwin", True, "2.13.0", False),
- ("linux", True, None, False),
- ("linux", True, "2.13.0+cpu", False),
- ("win32", True, "2.13.0", False),
- ],
- ids = [
- "linux-pypi-cuda",
- "linux-cu130",
- "windows-triton",
- "rocm",
- "windows-no-triton",
- "macos",
- "no-torch",
- "cpu-torch",
- "windows-pypi-cpu",
- ],
-)
-def test_zoo_triton_conv_platform_check(monkeypatch, platform, triton, torch_version, usable):
- monkeypatch.setattr(ssm_runtime.sys, "platform", platform)
- monkeypatch.delitem(ssm_runtime.sys.modules, "torch", raising = False)
- real_find_spec = ssm_runtime.importlib.util.find_spec
- monkeypatch.setattr(
- ssm_runtime.importlib.util,
- "find_spec",
- lambda name, *a: (
- (object() if triton else None) if name == "triton" else real_find_spec(name, *a)
- ),
- )
-
- def _version(name):
- if torch_version is None:
- raise ssm_runtime.importlib.metadata.PackageNotFoundError(name)
- return torch_version
-
- monkeypatch.setattr(ssm_runtime.importlib.metadata, "version", _version)
- assert _REAL_PLATFORM_USABLE() is usable
-
-
-def test_zoo_triton_conv_platform_check_reads_an_imported_torch(monkeypatch):
- import types
-
- monkeypatch.setattr(ssm_runtime.sys, "platform", "linux")
- monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda name, *a: object())
- fake = types.SimpleNamespace(version = types.SimpleNamespace(cuda = None, hip = "7.1"))
- monkeypatch.setitem(ssm_runtime.sys.modules, "torch", fake)
- assert _REAL_PLATFORM_USABLE() is False
- fake.version = types.SimpleNamespace(cuda = "13.0", hip = None)
- assert _REAL_PLATFORM_USABLE() is True
-
-
[email protected]("name", ["unsloth/Qwen3.5-2B", "Qwen/Qwen3-Next-80B-A3B"])
-def test_qwen_keeps_causal_conv1d_where_zoo_triton_conv_cannot_run(monkeypatch, tmp_path, name):
- monkeypatch.setattr(ssm_runtime, "_zoo_triton_platform_usable", lambda: False)
- assert ssm_runtime.model_wants_causal_conv1d(name) is True
- config = {"model_type": "qwen3_5", "text_config": {"model_type": "qwen3_5_text"}}
- assert ssm_runtime.model_config_wants_causal_conv1d(config) is True
-
-
-def test_the_platform_check_is_not_run_for_unrelated_models(monkeypatch):
- def _boom():
- raise AssertionError("the platform check must not run for a non-Qwen model")
-
- monkeypatch.setattr(ssm_runtime, "_zoo_triton_platform_usable", _boom)
- assert ssm_runtime.model_wants_causal_conv1d("unsloth/Llama-3.2-1B") is False
- assert ssm_runtime.model_wants_causal_conv1d("LiquidAI/LFM2-1.2B") is True
-
-
# ── in-process installs reach an already-imported transformers ────────────────
@@ -1180,26 +1103,841 @@ def test_refresh_is_a_no_op_without_transformers_and_never_raises(monkeypatch):
ssm_runtime.refresh_transformers_kernel_gates()
+class _Queue(list):
+ def put(self, item):
+ self.append(item)
+
+
def test_inference_worker_resolves_kernels_from_the_base_config(monkeypatch, tmp_path):
from core.inference import worker as iw
target = str(_write_config(tmp_path / "renamed", {"model_type": "nemotron_h"}))
seen = []
+
+ def _ensure(probe, **kwargs):
+ seen.append(
+ (probe, kwargs["wants_causal_conv1d"], kwargs["is_ssm"], kwargs["mamba_ssm_required"])
+ )
+ return []
+
+ monkeypatch.setattr(ssm_runtime, "ensure_ssm_runtime", _ensure)
+ assert iw._ensure_ssm_kernels([("renamed", target)], resp_queue = _Queue(), hf_token = None)
+ assert seen == [("renamed", True, True, True)]
+
+ src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8")
+ assert "_ensure_ssm_kernels(ssm_targets, resp_queue, hf_token, trust_remote_code)" in src
+ assert "_ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token, _trust_remote_code)" in src
+
+
+# ── mamba-ssm is fatal only where the model cannot load without it ────────────
+
+
+ "name",
+ ["state-spaces/mamba-130m-hf", "ai21labs/Jamba-v0.1", "Zyphra/Zamba2-7B", "ibm/Bamba-9B"],
+)
+def test_a_missing_mamba_ssm_is_not_fatal_for_models_with_a_torch_path(monkeypatch, name):
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ messages = []
+ assert ssm_runtime.model_requires_mamba_ssm(name) is False
+ ssm_runtime.ensure_ssm_runtime(name, status_cb = messages.append) # no raise
+ assert any("torch path" in message for message in messages)
+
+
+ "model_type, required",
+ [
+ ("nemotron_h", True),
+ ("falcon_h1", True),
+ ("granitemoehybrid", True),
+ ("mamba", False),
+ ("mamba2", False),
+ ("falcon_mamba", False),
+ ("jamba", False),
+ ("zamba", False),
+ ("zamba2", False),
+ ("bamba", False),
+ ],
+)
+def test_only_the_old_fatal_set_requires_mamba_ssm(tmp_path, model_type, required):
+ target = str(_write_config(tmp_path / "ckpt", {"model_type": model_type}))
+ kernels = ssm_runtime.resolve_ssm_kernels("acme/x", target, None)
+ assert kernels.mamba_ssm is True and kernels.causal_conv1d is True
+ assert kernels.mamba_ssm_required is required
+
+
+def test_windows_without_triton_is_fatal_only_for_the_required_set(monkeypatch):
+ monkeypatch.setattr(ssm_runtime.sys, "platform", "win32")
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda name, *a: None)
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: True)
+ ssm_runtime.ensure_ssm_runtime("ai21labs/Jamba-v0.1") # no raise
+ with pytest.raises(RuntimeError, match = "Triton"):
+ ssm_runtime.ensure_ssm_runtime("tiiuae/Falcon-H1-0.5B")
+
+
+# ── a local adapter's base decides the kernels ───────────────────────────────
+
+
+def _local_adapter(tmp_path, base: str) -> str:
+ adapter = tmp_path / "my-finetune-v3"
+ adapter.mkdir()
+ (adapter / "adapter_config.json").write_text(
+ json.dumps({"base_model_name_or_path": base, "peft_type": "LORA"}), encoding = "utf-8"
+ )
+ return str(adapter)
+
+
+def test_an_unrelatedly_named_local_adapter_resolves_through_its_falcon_h1_base(tmp_path):
+ base = str(_write_config(tmp_path / "base", {"model_type": "falcon_h1"}))
+ adapter = _local_adapter(tmp_path, base)
+ kernels = ssm_runtime.resolve_ssm_kernels(adapter, adapter, None)
+ assert (kernels.causal_conv1d, kernels.mamba_ssm, kernels.mamba_ssm_required) == (
+ True,
+ True,
+ True,
+ )
+ assert ssm_runtime.resolved_model_is_ssm(adapter, adapter, None) is True
+ assert ssm_runtime.resolved_model_wants_causal_conv1d(adapter, adapter, None) is True
+
+
+def test_the_authorized_adapter_base_pin_wins(tmp_path):
+ llama = str(_write_config(tmp_path / "llama", {"model_type": "llama"}))
+ falcon = str(_write_config(tmp_path / "falcon", {"model_type": "falcon_h1"}))
+ adapter = _local_adapter(tmp_path, llama)
+ assert ssm_runtime.resolved_model_is_ssm(adapter, adapter, None) is False
+ assert ssm_runtime.resolved_model_is_ssm(adapter, adapter, None, adapter_base = falcon) is True
+
+
+def test_the_training_worker_passes_the_adapter_base_pin():
+ src = (_BACKEND / "core" / "training" / "worker.py").read_text(encoding = "utf-8")
+ assert 'adapter_base = config.get("adapter_base_pin") or None' in src
+
+
+# ── the modeling-source scan works without importing transformers ─────────────
+
+
+def test_the_modeling_source_scan_reads_files_without_importing_transformers(monkeypatch, tmp_path):
+ package = tmp_path / "transformers"
+ model_dir = package / "models" / "convnet_x"
+ model_dir.mkdir(parents = True)
+ (package / "__init__.py").write_text("raise RuntimeError('must not be imported')\n")
+ (model_dir / "modeling_convnet_x.py").write_text(
+ "from causal_conv1d import causal_conv1d_fn\n", encoding = "utf-8"
+ )
+ plain = package / "models" / "plain_x"
+ plain.mkdir()
+ (plain / "modeling_plain_x.py").write_text("import torch\n", encoding = "utf-8")
+ spec = types.SimpleNamespace(
+ submodule_search_locations = [str(package)], origin = str(package / "__init__.py")
+ )
+ real_find_spec = ssm_runtime.importlib.util.find_spec
+ monkeypatch.setattr(
+ ssm_runtime.importlib.util,
+ "find_spec",
+ lambda name, *a: spec if name == "transformers" else real_find_spec(name, *a),
+ )
+ monkeypatch.setattr(ssm_runtime, "_TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE", {})
+ monkeypatch.delitem(sys.modules, "transformers", raising = False)
+
+ assert ssm_runtime.model_config_wants_causal_conv1d({"model_type": "convnet_x"}) is True
+ assert ssm_runtime.model_config_wants_causal_conv1d({"model_type": "plain_x"}) is False
+ assert "transformers" not in sys.modules
+
+
+# ── wheel availability, failures, cleanup, locking, backend gate, floor (#13971 audit) ──────────
+
+
+def _kernel_kwargs(**overrides):
+ kwargs = dict(
+ import_name = "causal_conv1d",
+ display_name = "causal-conv1d",
+ pypi_name = "causal-conv1d",
+ package_version = "1.6.1",
+ release_tag = "v1.6.1.post4",
+ release_base_url = "x",
+ status_cb = None,
+ )
+ kwargs.update(overrides)
+ return kwargs
+
+
+class _Host:
+ """A fake environment: what imports, what is installed, what each step does, per kernel."""
+
+ def __init__(
+ self,
+ monkeypatch,
+ *,
+ env,
+ wheel_url = True,
+ head = True,
+ wheel_ok = True,
+ build_ok = False,
+ platform = "linux",
+ ):
+ self.env = env
+ self.installed: set[str] = set()
+ self.imports: set[str] = set()
+ self.wheel_ok = wheel_ok
+ self.build_ok = build_ok
+ self.wheels: list[str] = []
+ self.commands: list[list[str]] = []
+ self.statuses: list[str] = []
+ monkeypatch.setattr(ssm_runtime.sys, "platform", platform)
+ monkeypatch.setattr(ssm_runtime.shutil, "which", lambda name: None)
+ monkeypatch.setattr(ssm_runtime, "probe_torch_wheel_env", lambda **k: env)
+ monkeypatch.setattr(
+ ssm_runtime,
+ "direct_wheel_url",
+ lambda **k: f"https://x/{k['filename_prefix']}.whl" if wheel_url else None,
+ )
+ monkeypatch.setattr(ssm_runtime, "url_exists", lambda url: head)
+ monkeypatch.setattr(ssm_runtime, "install_wheel", self.install_wheel)
+ monkeypatch.setattr(ssm_runtime, "_is_importable", lambda name: name in self.imports)
+ monkeypatch.setattr(
+ ssm_runtime,
+ "_installed_distribution",
+ lambda name: name.replace("-", "_") in self.installed,
+ )
+
+ def install_wheel(self, url, **kwargs):
+ self.wheels.append(url)
+ name = url.rsplit("/", 1)[-1].removesuffix(".whl")
+ self.installed.add(name) # a download leaves the distribution behind either way
+ if self.wheel_ok:
+ self.imports.add(name)
+ return [("uv", _Result(returncode = 0))]
+
+ def run(self, cmd, **kwargs):
+ self.commands.append(list(cmd))
+ joined = " ".join(cmd)
+ name = "mamba_ssm" if "mamba-ssm" in joined else "causal_conv1d"
+ if "uninstall" in cmd:
+ self.installed.discard(name)
+ self.imports.discard(name)
+ else: # a source build
+ self.installed.add(name)
+ if self.build_ok:
+ self.imports.add(name)
+ return _Result(returncode = 0 if self.build_ok else 1)
+
+ def install(self, **overrides):
+ return ssm_runtime._install_kernel(
+ **_kernel_kwargs(status_cb = self.statuses.append, **overrides), run = self.run
+ )
+
+ @property
+ def builds(self):
+ return [cmd for cmd in self.commands if "uninstall" not in cmd]
+
+
+_CUDA = {"cuda_version": "12.8", "cuda_major": "12", "platform_tag": "linux_x86_64"}
+_WINDOWS = {"cuda_version": "13.0", "cuda_major": "13", "platform_tag": "win_amd64"}
+
+
[email protected]("platform, env", [("linux", _CUDA), ("win32", _WINDOWS)])
+def test_an_unanswered_head_still_downloads_the_wheel(monkeypatch, platform, env):
+ host = _Host(monkeypatch, env = env, head = None, platform = platform)
+ assert host.install() is True
+ assert host.wheels == ["https://x/causal_conv1d.whl"]
+ assert host.builds == [] # the wheel worked: no source build
+
+
+def test_a_required_hybrid_loads_when_only_head_is_refused(monkeypatch):
+ host = _Host(monkeypatch, env = _CUDA, head = None)
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ installed = ssm_runtime.ensure_ssm_runtime(
+ "nvidia/NVIDIA-Nemotron-Nano-9B-v2",
+ run = host.run,
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = True,
+ )
+ assert installed == ["causal_conv1d", "mamba_ssm"]
+ assert len(host.wheels) == 2
+
+
+def test_a_failed_download_after_an_unanswered_head_falls_back(monkeypatch):
+ host = _Host(monkeypatch, env = _CUDA, head = None, wheel_ok = False, build_ok = True)
+ assert host.install() is True
+ assert len(host.builds) == 1 # the real download failed: then the source build
+
+
+def test_a_failed_install_is_not_retried_in_this_process(monkeypatch):
+ host = _Host(monkeypatch, env = _CUDA, head = False, build_ok = False)
+ assert host.install() is False
+ assert host.install() is False
+ assert len(host.builds) == 1
+
+
+def test_a_failed_install_is_retried_once_the_toolchain_changes(monkeypatch):
+ host = _Host(monkeypatch, env = _CUDA, head = False, build_ok = False)
+ assert host.install() is False
+ monkeypatch.setenv("CUDA_HOME", "/opt/cuda-new")
+ host.install()
+ assert len(host.builds) == 2
+
+
[email protected]("env", [_CUDA, {"hip_version": "6.4", "platform_tag": "linux_x86_64"}])
+def test_optional_mamba_ssm_is_wheel_only(monkeypatch, env):
+ host = _Host(monkeypatch, env = env, head = False, build_ok = True)
+ monkeypatch.setattr(
+ ssm_runtime.shutil, "which", lambda name: "/usr/bin/hipcc" if name == "hipcc" else None
+ )
+ host_installs = ssm_runtime.ensure_ssm_runtime(
+ "ai21labs/Jamba-v0.1",
+ run = host.run,
+ wants_causal_conv1d = False,
+ is_ssm = True,
+ mamba_ssm_required = False,
+ )
+ # causal-conv1d keeps its source fallback; mamba-ssm for an optional family does not.
+ assert not any("mamba-ssm==" in " ".join(cmd) for cmd in host.builds)
+ assert any("causal-conv1d==" in " ".join(cmd) for cmd in host.builds)
+ assert "mamba_ssm" not in host_installs
+
+
+def test_a_required_hybrid_keeps_the_mamba_ssm_source_fallback(monkeypatch):
+ host = _Host(monkeypatch, env = _CUDA, head = False, build_ok = True)
+ ssm_runtime.ensure_ssm_runtime(
+ "tiiuae/Falcon-H1-0.5B",
+ run = host.run,
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = True,
+ )
+ assert any("mamba-ssm==" in " ".join(cmd) for cmd in host.builds)
+
+
+def test_a_cuda_source_build_has_its_own_looser_deadline(monkeypatch):
+ seen = {}
+
+ def run(cmd, **kwargs):
+ seen.update(kwargs)
+ return _Result(returncode = 1)
+
+ host = _Host(monkeypatch, env = _CUDA, head = False)
+ ssm_runtime._install_kernel(**_kernel_kwargs(), run = run)
+ from utils import kernel_install
+
+ assert seen.get("timeout") == kernel_install.SOURCE_BUILD_TIMEOUT_CUDA
+ assert seen.get("timeout") > kernel_install.SOURCE_BUILD_TIMEOUT # HIP keeps 1800
+
+
[email protected]("wheel_ok, head", [(False, True), (False, False)])
+def test_a_broken_install_is_removed_not_left_behind(monkeypatch, wheel_ok, head):
+ host = _Host(monkeypatch, env = _CUDA, head = head, wheel_ok = wheel_ok, build_ok = False)
+ assert host.install() is False
+ assert host.installed == set()
+ assert any("uninstall" in cmd for cmd in host.commands)
+
+
+def test_the_install_runs_under_the_kernel_lock_and_rechecks(monkeypatch):
+ from utils import kernel_install
+
+ host = _Host(monkeypatch, env = _CUDA)
+ held = {}
+
+ @contextlib.contextmanager
+ def _lock(name, **kwargs):
+ held["name"] = name
+ host.imports.add(name) # a peer installed it while this process waited
+ yield True
+
+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _lock)
+ assert host.install() is True
+ assert held["name"] == "causal_conv1d"
+ assert host.wheels == [] and host.commands == []
+
+
+def test_a_lock_held_past_the_bound_installs_nothing(monkeypatch):
+ from utils import kernel_install
+
+ host = _Host(monkeypatch, env = _CUDA)
+
+ @contextlib.contextmanager
+ def _busy(name, **kwargs):
+ yield False
+
+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _busy)
+ assert host.install() is False
+ assert host.wheels == [] and host.commands == []
+ assert any("install lock" in message for message in host.statuses)
+
+
+ "env",
+ [
+ None, # the torch probe failed
+ {"cuda_version": "", "hip_version": "", "platform_tag": "linux_x86_64"}, # CPU / XPU
+ ],
+)
+def test_no_cuda_or_rocm_torch_installs_nothing(monkeypatch, env):
+ host = _Host(monkeypatch, env = env, head = True)
+ assert host.install() is False
+ assert host.wheels == [] and host.commands == []
+ assert any("CUDA or ROCm" in message for message in host.statuses)
+
+
+def test_a_required_hybrid_on_an_xpu_torch_is_skipped_not_fatal(monkeypatch):
+ host = _Host(monkeypatch, env = {"cuda_version": "", "platform_tag": "linux_x86_64"})
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ assert (
+ ssm_runtime.ensure_ssm_runtime(
+ "tiiuae/Falcon-H1-0.5B",
+ run = host.run,
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = True,
+ # Native transformers modeling: required (worth a build) but not fatal. A fatal one
+ # raises instead, see test_a_fatal_mamba_model_on_a_cpu_or_xpu_torch_raises.
+ mamba_ssm_fatal = False,
+ )
+ == []
+ )
+
+
+def test_optional_mamba_ssm_is_skipped_below_the_setup_floor(monkeypatch):
+ host = _Host(monkeypatch, env = _CUDA)
monkeypatch.setattr(
ssm_runtime,
- "ensure_ssm_runtime",
- lambda probe, **kwargs: seen.append(
- (probe, kwargs["wants_causal_conv1d"], kwargs["is_ssm"])
- ),
+ "_mamba_ssm_floor_reason",
+ lambda: "needs sm80, or sm75 with Triton 3.4+ (torch 2.8+), and Triton is 3.3",
+ )
+ ssm_runtime.ensure_ssm_runtime(
+ "ai21labs/Jamba-v0.1",
+ status_cb = host.statuses.append,
+ run = host.run,
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = False,
+ )
+ assert host.wheels == ["https://x/causal_conv1d.whl"] # causal-conv1d only
+ assert any("Skipping mamba-ssm" in message for message in host.statuses)
+
+
+def test_a_required_hybrid_ignores_the_floor_as_before(monkeypatch):
+ host = _Host(monkeypatch, env = _CUDA)
+ monkeypatch.setattr(
+ ssm_runtime, "_mamba_ssm_floor_reason", lambda: pytest.fail("no floor for these")
)
- refreshed = []
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ ssm_runtime.ensure_ssm_runtime(
+ "tiiuae/Falcon-H1-0.5B",
+ run = host.run,
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = True,
+ )
+ assert len(host.wheels) == 2
+
+
+ "capability, triton, skipped",
+ [((7, 5), (3, 3), True), ((7, 5), (3, 4), False), ((8, 0), None, False), (None, None, True)],
+)
+def test_the_runtime_floor_is_setups(monkeypatch, capability, triton, skipped):
+ from utils import kernel_install
+
+ monkeypatch.delitem(sys.modules, "torch", raising = False)
+ monkeypatch.delenv("UNSLOTH_MAMBA_PRE_AMPERE_FAST", raising = False)
+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: capability)
+ monkeypatch.setattr(kernel_install, "_triton_version", lambda run: triton)
+ assert (_REAL_MAMBA_SSM_FLOOR_REASON() is not None) is skipped
+
+
+def test_provisioning_reads_the_config_once_per_process(monkeypatch):
+ reads = []
monkeypatch.setattr(
- ssm_runtime, "refresh_transformers_kernel_gates", lambda: refreshed.append(1)
+ "utils.transformers_version._load_config_json",
+ lambda target, token: reads.append(target) or {"model_type": "llama"},
)
- assert iw._ensure_ssm_kernels([("renamed", target)], resp_queue = None, hf_token = None)
- assert seen == [("renamed", True, True)]
- assert refreshed == [1]
+ monkeypatch.setattr(ssm_runtime, "ensure_ssm_runtime", lambda *a, **k: [])
+ for _ in range(3):
+ ssm_runtime.provision_ssm_kernels("acme/llama", "acme/llama", None)
+ assert reads == ["acme/llama"]
- src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8")
- assert "_ensure_ssm_kernels(ssm_targets, resp_queue, hf_token)" in src
- assert "_ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token)" in src
+
+def test_a_caller_with_the_config_skips_the_read(monkeypatch):
+ monkeypatch.setattr(
+ "utils.transformers_version._load_config_json",
+ lambda *a, **k: pytest.fail("the caller already has the config"),
+ )
+ kernels = ssm_runtime.resolve_ssm_kernels(
+ "acme/hybrid", "acme/hybrid", None, model_config = {"model_type": "nemotron_h"}
+ )
+ assert kernels.mamba_ssm_required
+
+
+def test_the_preflight_and_handle_load_share_one_failed_attempt(monkeypatch):
+ # The inference worker provisions before its transformers import and again in _handle_load:
+ # a failed optional build must not run twice (two 30-minute HIP builds before).
+ host = _Host(
+ monkeypatch, env = {"hip_version": "6.4", "platform_tag": "linux_x86_64"}, head = False
+ )
+ monkeypatch.setattr(ssm_runtime.shutil, "which", lambda name: "/usr/bin/" + name)
+ monkeypatch.setattr(ssm_runtime, "_hipcc_gcc_install_dir", lambda: None)
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ for _stage in ("preflight", "_handle_load"):
+ ssm_runtime.ensure_ssm_runtime(
+ "ai21labs/Jamba-v0.1",
+ run = host.run,
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = False,
+ )
+ assert len(host.builds) == 1 # one causal-conv1d build; mamba-ssm is wheel-only here
+
+
[email protected]("torch_mm", ["2.13", "2.14"])
+def test_no_pinned_source_build_on_torch_213_and_later(monkeypatch, torch_mm):
+ # causal-conv1d 1.6.1 / mamba-ssm 2.3.1 build at C++17; torch 2.13+ headers need C++20, so the
+ # build is a certain multi-minute failure: only the prebuilt wheel serves these.
+ host = _Host(monkeypatch, env = {**_CUDA, "torch_mm": torch_mm}, wheel_url = False)
+ assert host.install() is False
+ assert host.builds == []
+ assert any("cannot be built from source" in message for message in host.statuses)
+
+
+def test_torch_212_keeps_the_source_fallback(monkeypatch):
+ host = _Host(monkeypatch, env = {**_CUDA, "torch_mm": "2.12"}, head = False, build_ok = True)
+ assert host.install() is True
+ assert len(host.builds) == 1
+
+
+# ── when a missing mamba-ssm refuses the load (AMD hardware report, PR #13971) ──────────────────
+
+_NVIDIA_REMOTE_NEMOTRON = """
+from transformers.utils.import_utils import is_mamba_2_ssm_available
+
+if is_mamba_2_ssm_available():
+ from mamba_ssm.ops.triton.ssd_combined import mamba_chunk_scan_combined
+else:
+ mamba_chunk_scan_combined = None
+
+try:
+ from mamba_ssm.ops.triton.layernorm_gated import rmsnorm_fn
+except ImportError:
+ raise ImportError("mamba-ssm is required by the Mamba model but cannot be imported")
+"""
+
+_GATED_REMOTE = """
+try:
+ from mamba_ssm.ops.triton.layernorm_gated import rmsnorm_fn
+except ImportError:
+ rmsnorm_fn = None
+if is_mamba_2_ssm_available():
+ import mamba_ssm
+"""
+
+
+def _checkpoint(
+ tmp_path,
+ model_type,
+ auto_map = None,
+ modeling = None,
+ name = "ckpt",
+):
+ folder = tmp_path / name
+ folder.mkdir()
+ config = {"model_type": model_type}
+ if auto_map:
+ config["auto_map"] = auto_map
+ (folder / "config.json").write_text(json.dumps(config))
+ for filename, text in (modeling or {}).items():
+ (folder / filename).write_text(text)
+ return str(folder)
+
+
+def _failing_mamba(monkeypatch):
+ attempts = []
+
+ def _install(**kwargs):
+ attempts.append((kwargs["import_name"], kwargs.get("allow_source_build")))
+ return False
+
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", _install)
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ return attempts
+
+
[email protected]("model_type", ["nemotron_h", "falcon_h1", "granitemoehybrid"])
+def test_a_native_required_hybrid_loads_with_a_warning(monkeypatch, tmp_path, model_type):
+ # The native transformers model runs without mamba-ssm (it trains on Windows ROCm with none):
+ # still worth a source build, but a failure is a warning, not a refused load.
+ attempts = _failing_mamba(monkeypatch)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)
+ statuses = []
+ kernels, installed = ssm_runtime.provision_ssm_kernels(
+ "nvidia/NVIDIA-Nemotron-Nano-9B-v2",
+ _checkpoint(tmp_path, model_type),
+ None,
+ status_cb = statuses.append,
+ )
+ assert kernels.mamba_ssm_required and not kernels.mamba_ssm_fatal
+ assert ("mamba_ssm", True) in attempts # the source fallback is kept for these three
+ assert installed == []
+ assert any("Warning: could not install mamba-ssm" in message for message in statuses)
+
+
+def test_remote_code_that_imports_mamba_ssm_still_refuses(monkeypatch, tmp_path):
+ _failing_mamba(monkeypatch)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)
+ target = _checkpoint(
+ tmp_path,
+ "nemotron_h",
+ auto_map = {"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"},
+ modeling = {"modeling_nemotron_h.py": _NVIDIA_REMOTE_NEMOTRON},
+ )
+ with pytest.raises(RuntimeError, match = "Could not install mamba-ssm"):
+ ssm_runtime.provision_ssm_kernels(target, target, None, trust_remote_code = True)
+
+
+def test_the_same_checkpoint_without_trust_loads_natively(monkeypatch, tmp_path):
+ # auto_map is only followed under trust_remote_code (or when transformers lacks the model).
+ _failing_mamba(monkeypatch)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)
+ target = _checkpoint(
+ tmp_path,
+ "nemotron_h",
+ auto_map = {"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"},
+ modeling = {"modeling_nemotron_h.py": _NVIDIA_REMOTE_NEMOTRON},
+ )
+ kernels, _ = ssm_runtime.provision_ssm_kernels(target, target, None, trust_remote_code = False)
+ assert not kernels.mamba_ssm_fatal
+
+
+def test_remote_code_with_a_guarded_import_loads(monkeypatch, tmp_path):
+ _failing_mamba(monkeypatch)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)
+ target = _checkpoint(
+ tmp_path,
+ "nemotron_h",
+ auto_map = {"AutoModelForCausalLM": "model.CustomForCausalLM"},
+ modeling = {"model.py": _GATED_REMOTE},
+ )
+ kernels, _ = ssm_runtime.provision_ssm_kernels(target, target, None, trust_remote_code = True)
+ assert not kernels.mamba_ssm_fatal
+
+
+def test_unreadable_remote_code_is_treated_as_requiring_it(monkeypatch, tmp_path):
+ _failing_mamba(monkeypatch)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)
+ target = _checkpoint(
+ tmp_path,
+ "falcon_h1",
+ auto_map = {"AutoModelForCausalLM": "modeling_falcon_h1.FalconH1ForCausalLM"},
+ )
+ with pytest.raises(RuntimeError):
+ ssm_runtime.provision_ssm_kernels(target, target, None, trust_remote_code = True)
+
+
[email protected]("model_type", ["nemotron_h", "falcon_h1"])
+def test_a_transformers_without_the_native_model_still_refuses(monkeypatch, tmp_path, model_type):
+ _failing_mamba(monkeypatch)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: False)
+ target = _checkpoint(tmp_path, model_type)
+ with pytest.raises(RuntimeError, match = "Could not install mamba-ssm"):
+ ssm_runtime.provision_ssm_kernels(target, target, None)
+
+
+def test_windows_without_triton_refuses_only_a_fatal_load(monkeypatch, tmp_path):
+ monkeypatch.setattr(ssm_runtime.sys, "platform", "win32")
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ monkeypatch.setattr(
+ ssm_runtime.importlib.util,
+ "find_spec",
+ lambda name, *a, **k: None if name == "triton" else object(),
+ )
+ statuses = []
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: True)
+ ssm_runtime.provision_ssm_kernels(
+ "x", _checkpoint(tmp_path, "nemotron_h", name = "a"), None, status_cb = statuses.append
+ )
+ assert any("triton-windows" in message for message in statuses)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: False)
+ with pytest.raises(RuntimeError, match = "triton-windows"):
+ ssm_runtime.provision_ssm_kernels("x", _checkpoint(tmp_path, "nemotron_h", name = "b"), None)
+
+
+def test_native_modeling_is_read_from_the_installed_transformers(monkeypatch, tmp_path):
+ models = tmp_path / "transformers" / "models"
+ (models / "falcon_h1").mkdir(parents = True)
+ (models / "falcon_h1" / "modeling_falcon_h1.py").write_text("")
+ (models / "nemotron_h").mkdir()
+ (models / "nemotron_h" / "configuration_nemotron_h.py").write_text("")
+ monkeypatch.setattr(ssm_runtime, "_transformers_models_dir", lambda: models)
+ monkeypatch.setattr(ssm_runtime, "_NATIVE_MODELING_CACHE", {})
+ assert ssm_runtime.native_modeling_available("falcon_h1") is True
+ assert ssm_runtime.native_modeling_available("nemotron_h") is False # config only
+ assert ssm_runtime.native_modeling_available("granitemoehybrid") is False
+
+
+ "source, fatal",
+ [
+ ("import mamba_ssm\n", True),
+ ("from mamba_ssm.ops import x\n", True),
+ (_NVIDIA_REMOTE_NEMOTRON, True),
+ (_GATED_REMOTE, False),
+ ("def f():\n import mamba_ssm\n", False),
+ ("import torch\n", False),
+ ("def broken(:\n", True),
+ ],
+)
+def test_the_unconditional_import_scan(source, fatal):
+ assert ssm_runtime._imports_mamba_ssm_unconditionally(source) is fatal
+
+
+# ── final-audit items (PR #13971) ────────────────────────────────────────────────────────────────
+
+
+def test_a_native_granite_4_tiny_preview_loads_with_a_warning(monkeypatch, tmp_path):
+ # ibm-granite/granite-4.0-tiny-preview: model_type granitemoehybrid, native in transformers.
+ _failing_mamba(monkeypatch)
+ monkeypatch.setattr(ssm_runtime, "native_modeling_available", lambda t: t == "granitemoehybrid")
+ statuses = []
+ target = _checkpoint(tmp_path, "granitemoehybrid", name = "granite-4.0-tiny-preview")
+ kernels, _ = ssm_runtime.provision_ssm_kernels(
+ "ibm-granite/granite-4.0-tiny-preview", target, None, status_cb = statuses.append
+ )
+ assert kernels.mamba_ssm_required and not kernels.mamba_ssm_fatal
+ assert any("Warning: could not install mamba-ssm" in message for message in statuses)
+
+
+def test_a_transient_config_miss_is_read_again(monkeypatch):
+ reads = iter([None, {"model_type": "qwen3_5"}])
+ calls = []
+
+ def _load(target, token):
+ calls.append(target)
+ return next(reads)
+
+ monkeypatch.setattr("utils.transformers_version._load_config_json", _load)
+ assert ssm_runtime.resolve_ssm_kernels("acme/m", "acme/m", None).causal_conv1d is False
+ assert ssm_runtime.resolve_ssm_kernels("acme/m", "acme/m", None).causal_conv1d is True
+ assert ssm_runtime.resolve_ssm_kernels("acme/m", "acme/m", None).causal_conv1d is True
+ assert calls == ["acme/m", "acme/m"] # the successful read is the one remembered
+
+
+def test_a_kernel_a_peer_installed_counts_as_new_to_this_worker(monkeypatch):
+ from utils import kernel_install
+
+ host = _Host(monkeypatch, env = _CUDA)
+
+ @contextlib.contextmanager
+ def _lock(name, **kwargs):
+ host.imports.add(name) # a peer installed it while this process waited
+ yield True
+
+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _lock)
+ newly = []
+ assert ssm_runtime._install_kernel(**_kernel_kwargs(), run = host.run, newly_installed = newly)
+ assert newly == ["causal_conv1d"] # so inference refreshes or restarts its modeling modules
+
+
[email protected]("required, budget", [(False, 120), (True, None)])
+def test_optional_accelerations_wait_briefly_for_a_peer(monkeypatch, required, budget):
+ from utils import kernel_install
+
+ budget = kernel_install.KERNEL_LOCK_WAIT if budget is None else budget
+
+ seen = {}
+
+ @contextlib.contextmanager
+ def _lock(name, **kwargs):
+ seen.update(kwargs)
+ yield False
+
+ host = _Host(monkeypatch, env = _CUDA)
+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _lock)
+ host.install(required = required)
+ assert seen["wait_seconds"] == budget
+
+
+def test_ensure_waits_long_only_for_the_required_families(monkeypatch):
+ budgets = {}
+ monkeypatch.setattr(
+ ssm_runtime,
+ "_install_kernel",
+ lambda **k: budgets.setdefault((k["import_name"], k["required"]), True),
+ )
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ ssm_runtime.ensure_ssm_runtime(
+ "q", wants_causal_conv1d = True, is_ssm = False, mamba_ssm_required = False
+ )
+ ssm_runtime.ensure_ssm_runtime(
+ "f", wants_causal_conv1d = True, is_ssm = True, mamba_ssm_required = True
+ )
+ assert set(budgets) == {("causal_conv1d", False), ("causal_conv1d", True), ("mamba_ssm", True)}
+
+
+def test_an_installer_that_cannot_start_still_cleans_up_and_is_remembered(monkeypatch):
+ host = _Host(monkeypatch, env = _CUDA, head = True)
+ host.installed.add("causal_conv1d") # a broken copy left by an earlier attempt
+
+ def _cannot_start(url, **kwargs):
+ raise OSError("uv: No such file or directory")
+
+ monkeypatch.setattr(ssm_runtime, "install_wheel", _cannot_start)
+ assert host.install() is False
+ assert any("uninstall" in cmd for cmd in host.commands) # cleanup ran
+ assert host.install() is False
+ assert len([cmd for cmd in host.commands if "uninstall" in cmd]) == 1 # not retried
+
+
+# ── A fatal mamba-ssm model on a torch with no kernel backend fails clearly ───
+
+
[email protected]("env", [{"cuda_version": "", "platform_tag": "linux_x86_64"}, {}])
+def test_a_fatal_mamba_model_on_a_cpu_or_xpu_torch_raises(monkeypatch, env):
+ # Its remote code imports mamba_ssm, so the torch path cannot load it: refuse with the reason.
+ monkeypatch.setattr(ssm_runtime, "probe_torch_wheel_env", lambda **k: dict(env))
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ messages = []
+ with pytest.raises(RuntimeError) as raised:
+ ssm_runtime.ensure_ssm_runtime(
+ "acme/remote-hybrid",
+ status_cb = messages.append,
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = True,
+ mamba_ssm_fatal = True,
+ )
+ assert "requires mamba-ssm, which needs a CUDA or ROCm torch build" in str(raised.value)
+ assert "offline" not in str(raised.value)
+ assert not any("torch path" in message for message in messages)
+
+
+def test_a_fatal_mamba_model_on_a_cuda_torch_keeps_the_offline_hint(monkeypatch):
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ with pytest.raises(RuntimeError) as raised:
+ ssm_runtime.ensure_ssm_runtime(
+ "acme/remote-hybrid",
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = True,
+ mamba_ssm_fatal = True,
+ )
+ assert "Could not install mamba-ssm" in str(raised.value)
+ assert "unsloth install-kernels" in str(raised.value)
+
+
+def test_a_non_fatal_required_model_on_a_cpu_torch_still_warns_and_loads(monkeypatch):
+ monkeypatch.setattr(
+ ssm_runtime, "probe_torch_wheel_env", lambda **k: {"cuda_version": "", "platform_tag": "x"}
+ )
+ monkeypatch.setattr(ssm_runtime, "_install_kernel", lambda **k: False)
+ monkeypatch.setattr(ssm_runtime.importlib.util, "find_spec", lambda *a, **k: object())
+ messages = []
+ ssm_runtime.ensure_ssm_runtime(
+ "acme/native-hybrid",
+ status_cb = messages.append,
+ wants_causal_conv1d = True,
+ is_ssm = True,
+ mamba_ssm_required = True,
+ mamba_ssm_fatal = False,
+ ) # no raise
+ assert any("torch path" in message for message in messages)
diff --git a/studio/backend/tests/test_training_force_terminate_builds.py b/studio/backend/tests/test_training_force_terminate_builds.py
new file mode 100644
index 00000000000..fc2db6cf4b0
--- /dev/null
+++ b/studio/backend/tests/test_training_force_terminate_builds.py
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Force-terminating a training run also stops the processes its worker started: a kernel source
+build runs in its own session (so a timeout can stop its compilers together) and would otherwise
+outlive the worker, keep compiling, and leave its install lock free for a peer to race."""
+
+import os
+import subprocess
+import sys
+import threading
+import time
+from types import SimpleNamespace
+
+import pytest
+
+from core.training import training
+
+
+def _alive(pid: int) -> bool:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ try:
+ with open(f"/proc/{pid}/stat") as stat:
+ return stat.read().split()[2] != "Z"
+ except OSError:
+ return True
+
+
[email protected](os.name == "nt", reason = "POSIX sessions")
+def test_a_build_in_its_own_session_dies_with_the_run(tmp_path):
+ pid_file = tmp_path / "build.pid"
+ worker_script = (
+ "import subprocess, sys, time; "
+ "build = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'], "
+ "start_new_session = True); "
+ f"open({str(pid_file)!r}, 'w').write(str(build.pid)); "
+ "time.sleep(120)"
+ )
+ worker = subprocess.Popen([sys.executable, "-c", worker_script])
+ try:
+ for _ in range(100):
+ if pid_file.exists() and pid_file.read_text():
+ break
+ time.sleep(0.05)
+ build = int(pid_file.read_text())
+ assert _alive(build)
+ descendants = training._terminate_descendants(worker.pid)
+ assert build in {child.pid for child in descendants}
+ worker.terminate()
+ worker.wait(timeout = 10)
+ training._reap_descendants(descendants)
+ for _ in range(50):
+ if not _alive(build):
+ break
+ time.sleep(0.1)
+ assert not _alive(build), "the build outlived the force-terminated worker"
+ finally:
+ if worker.poll() is None:
+ worker.kill()
+ if pid_file.exists() and pid_file.read_text() and _alive(int(pid_file.read_text())):
+ os.kill(int(pid_file.read_text()), 9)
+
+
+def test_force_terminate_stops_descendants_before_the_worker(monkeypatch):
+ order = []
+ proc = SimpleNamespace(
+ pid = 4242,
+ is_alive = lambda: True,
+ terminate = lambda: order.append("worker"),
+ join = lambda timeout = None: None,
+ kill = lambda: order.append("kill"),
+ )
+ backend = training.TrainingBackend.__new__(training.TrainingBackend)
+ backend.__dict__.update(
+ _lock = threading.Lock(),
+ _proc = proc,
+ _cancel_requested = False,
+ _cancel_cleanup_output_dir = None,
+ _output_dir = None,
+ _pump_thread = None,
+ )
+ monkeypatch.setattr(
+ training, "_terminate_descendants", lambda pid: order.append(("descendants", pid)) or ["c"]
+ )
+ monkeypatch.setattr(training, "_reap_descendants", lambda d: order.append(("reap", tuple(d))))
+ backend.force_terminate()
+ assert order[0] == ("descendants", 4242) and order[1] == "worker"
+ assert ("reap", ("c",)) in order
diff --git a/studio/backend/tests/test_training_worker_flash_attn.py b/studio/backend/tests/test_training_worker_flash_attn.py
index 08993cf03ea..280269b5054 100644
--- a/studio/backend/tests/test_training_worker_flash_attn.py
+++ b/studio/backend/tests/test_training_worker_flash_attn.py
@@ -4,6 +4,7 @@
from __future__ import annotations
import ast
+from types import SimpleNamespace
import builtins
import importlib.metadata
import inspect
@@ -33,9 +34,37 @@ not_on_windows = pytest.mark.skipif(
@pytest.fixture(autouse = True)
-def _clear_offline_environment(monkeypatch):
+def _clear_offline_environment(monkeypatch, tmp_path):
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
+ monkeypatch.delenv("UV_OFFLINE", raising = False)
+ # A CUDA torch with nothing published for it unless a test says otherwise: the host's own
+ # torch and the network must not decide the tests.
+ monkeypatch.setattr(
+ worker,
+ "probe_torch_wheel_env",
+ lambda timeout = 30, include_windows = False: {
+ "python_tag": "cp313",
+ "abi_tag": "cp313",
+ "torch_mm": "0.0",
+ "torch_version": "0.0.0",
+ "cuda_version": "12.8",
+ "cuda_major": "12",
+ "hip_version": "",
+ "cxx11abi": "TRUE",
+ "platform_tag": "linux_x86_64",
+ },
+ )
+ monkeypatch.setattr(worker, "url_exists", lambda url: False)
+ # Per-process memo and the environment's lock file: each test is its own process here.
+ from utils import kernel_install, ssm_runtime
+
+ monkeypatch.setattr(ssm_runtime, "_FAILED_INSTALLS", set())
+ monkeypatch.setattr(
+ kernel_install, "_kernel_lock_path", lambda name: str(tmp_path / f"{name}.lock")
+ )
+ # Every copy of the module (and any child process) uses the test's folder too.
+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", str(tmp_path / "locks"))
@pytest.fixture(autouse = True)
@@ -530,7 +559,10 @@ def test_runtime_flash_attn_skip_env_avoids_all_install_work(monkeypatch):
worker._sp.run.assert_not_called()
[email protected]("offline_variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"])
+# UV_OFFLINE too: uv refuses the index and install_wheel's pip fallback would not.
+ "offline_variable", ["HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "UV_OFFLINE"]
+)
def test_wheel_first_install_skips_all_install_work_offline(monkeypatch, offline_variable):
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
@@ -603,22 +635,11 @@ def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch)
@not_on_windows
-def test_causal_conv1d_fast_path_skips_qwen3_6_variants_on_zoo_triton_conv(monkeypatch):
- # unsloth-zoo runs Qwen3.5 / 3.6 / Qwen3-Next on its own Triton causal conv (on a CUDA torch
- # with Triton, pinned here so a CPU-only CI host answers the same).
- from utils import ssm_runtime
-
+def test_causal_conv1d_fast_path_includes_qwen3_6_variants(monkeypatch):
+ # The native kernel: zoo's Triton conv has no decode update and a sequence-sized dW scratch.
install_mock = mock.Mock(return_value = True)
monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
- monkeypatch.setattr(ssm_runtime, "_zoo_triton_platform_usable", lambda: True)
- monkeypatch.delenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", raising = False)
-
- for model_name in ("unsloth/Qwen3.6-4B", "unsloth/Qwen3_6-4B"):
- worker._ensure_causal_conv1d_fast_path(event_queue = [], model_name = model_name)
- assert install_mock.call_count == 0
- # zoo's kill switch puts them back on the prebuilt causal-conv1d.
- monkeypatch.setenv("UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D", "1")
for model_name in ("unsloth/Qwen3.6-4B", "unsloth/Qwen3_6-4B"):
worker._ensure_causal_conv1d_fast_path(event_queue = [], model_name = model_name)
assert install_mock.call_count == 2
@@ -627,6 +648,9 @@ def test_causal_conv1d_fast_path_skips_qwen3_6_variants_on_zoo_triton_conv(monke
def test_mamba_ssm_follows_the_resolved_answer_not_the_name(monkeypatch):
install_mock = mock.Mock(return_value = True)
monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
+ # Not the sm80 floor under test: keep a GPU-less runner from skipping the optional family.
+ monkeypatch.setattr("utils.ssm_runtime._mamba_ssm_floor_reason", lambda: None)
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
# A real hybrid whose id lacks the substrings (resolved from its config) still installs.
worker._ensure_mamba_ssm(
event_queue = [], model_name = "nvidia/NVIDIA-Nemotron-Nano-9B-v2", required = True
@@ -658,6 +682,8 @@ def test_mamba_ssm_path_preserves_wheel_first_install_args(monkeypatch):
filename_prefix = "mamba_ssm",
release_tag = worker.MAMBA_SSM.release_tag,
release_base_url = "https://github.com/state-spaces/mamba/releases/download",
+ allow_source_build = True,
+ required = True,
)
@@ -1005,7 +1031,9 @@ def test_no_install_path_pips_flash_linear_attention_or_tilelang(monkeypatch, uv
monkeypatch.setattr(worker, "_is_importable", lambda name: False)
monkeypatch.setattr(worker, "_is_importable_isolated", lambda name: True)
monkeypatch.setattr(
- worker, "probe_torch_wheel_env", lambda timeout = 30, include_windows = False: {}
+ worker,
+ "probe_torch_wheel_env",
+ lambda timeout = 30, include_windows = False: {"cuda_version": "12.8"},
)
monkeypatch.setattr(worker, "url_exists", lambda url: False)
monkeypatch.setattr(worker, "install_wheel", mock.Mock())
@@ -1530,7 +1558,10 @@ def test_windows_mamba_ssm_needs_triton(monkeypatch):
install_mock = mock.Mock(return_value = True)
monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
monkeypatch.setattr(worker.importlib.util, "find_spec", lambda name, *a: None)
- worker._ensure_mamba_ssm(event_queue = [], model_name = "tiiuae/Falcon-H1-0.5B-Instruct")
+ # A family with a torch path skips it; one that needs it stops (its own test below).
+ worker._ensure_mamba_ssm(
+ event_queue = [], model_name = "ai21labs/Jamba-v0.1", required = True, essential = False
+ )
install_mock.assert_not_called()
assert any("triton-windows" in message for message in statuses)
@@ -1596,7 +1627,7 @@ def test_windows_installs_a_published_wheel(monkeypatch):
@pytest.mark.parametrize(
("wheel_url", "exists"),
- [(None, False), ("https://x/fa.whl", False), ("https://x/fa.whl", None)],
+ [(None, False), ("https://x/fa.whl", False)],
)
def test_windows_never_falls_back_to_pypi(monkeypatch, wheel_url, exists):
ok, runs, installed = _windows_attempt(
@@ -1612,3 +1643,427 @@ def test_windows_rejected_wheel_does_not_fall_back_to_pypi(monkeypatch):
)
assert ok is False and installed is True
assert runs == []
+
+
[email protected]("capability, installs", [((7, 5), False), ((8, 0), True), ((12, 0), True)])
+def test_long_context_flash_attn_is_not_downloaded_below_sm80(monkeypatch, capability, installs):
+ from utils import kernel_install
+
+ statuses: list[str] = []
+ install_mock = mock.Mock(return_value = True)
+ monkeypatch.setattr(worker, "_should_try_runtime_flash_attn_install", lambda length: True)
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ # No torch in this worker yet: the child probe answers.
+ monkeypatch.delitem(worker.sys.modules, "torch", raising = False)
+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: capability)
+ monkeypatch.setattr(kernel_install, "_torch_is_hip", lambda run: False)
+ monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))
+ worker._ensure_flash_attn_for_long_context(event_queue = [], max_seq_length = 65536)
+ assert install_mock.called is installs
+ if not installs:
+ assert any("sm80" in message for message in statuses)
+
+
+def _fake_torch(capabilities, hip = None):
+ cuda = SimpleNamespace(
+ is_available = lambda: bool(capabilities),
+ device_count = lambda: len(capabilities),
+ get_device_capability = lambda i: capabilities[i],
+ )
+ return SimpleNamespace(version = SimpleNamespace(hip = hip), cuda = cuda)
+
+
+def _no_child_probe(*args, **kwargs):
+ raise AssertionError("the worker's own torch answers; no child probe")
+
+
+def test_an_importable_flash_attn_skips_the_capability_probe(monkeypatch):
+ from utils import kernel_install
+
+ install_mock = mock.Mock(return_value = True)
+ monkeypatch.setattr(worker, "_should_try_runtime_flash_attn_install", lambda length: True)
+ monkeypatch.setattr(worker, "_is_importable", lambda name: name == "flash_attn")
+ monkeypatch.setattr(kernel_install, "_gpu_capability", _no_child_probe)
+ monkeypatch.setattr(worker, "_flash_attn_gpu_below_floor", _no_child_probe)
+ monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
+ worker._ensure_flash_attn_for_long_context(event_queue = [], max_seq_length = 65536)
+ assert not install_mock.called
+
+
+ "capabilities, hip, below",
+ [
+ ([(7, 5)], None, True),
+ ([(7, 5), (8, 6)], None, False),
+ ([(9, 0)], None, False),
+ ([], None, False),
+ # ROCm reports a gfx version: never the NVIDIA floor.
+ ([(7, 5)], "6.4.0", False),
+ ],
+)
+def test_the_floor_reuses_the_workers_own_torch(monkeypatch, capabilities, hip, below):
+ from utils import kernel_install
+
+ monkeypatch.setitem(worker.sys.modules, "torch", _fake_torch(capabilities, hip))
+ monkeypatch.setattr(kernel_install, "_gpu_capability", _no_child_probe)
+ assert worker._flash_attn_gpu_below_floor() is below
+
+
+def test_rocm_long_context_still_installs_flash_attn(monkeypatch):
+ install_mock = mock.Mock(return_value = True)
+ monkeypatch.setattr(worker, "_should_try_runtime_flash_attn_install", lambda length: True)
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ monkeypatch.setitem(worker.sys.modules, "torch", _fake_torch([(7, 5)], "6.4.0"))
+ monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock)
+ worker._ensure_flash_attn_for_long_context(event_queue = [], max_seq_length = 65536)
+ assert install_mock.called
+
+
+# An unanswered HEAD (a proxy refusing HEAD but serving the download) is not "unpublished": setup
+# always tried the wheel then, and only a real download / import failure falls back.
+
+
+def test_windows_an_unanswered_head_still_downloads_the_wheel(monkeypatch):
+ ok, runs, installed = _windows_attempt(
+ monkeypatch,
+ wheel_url = "https://x/fa-win_amd64.whl",
+ exists = None,
+ importable_after_install = True,
+ )
+ assert ok is True and installed is True
+ assert runs == []
+
+
+def test_linux_an_unanswered_head_installs_the_wheel_not_a_source_build(monkeypatch):
+ monkeypatch.setattr(worker.sys, "platform", "linux")
+ # A cell upstream publishes, so the resolver names a wheel.
+ monkeypatch.setattr(
+ worker,
+ "probe_torch_wheel_env",
+ lambda **k: {
+ "python_tag": "cp313",
+ "torch_mm": "2.10",
+ "cuda_version": "12.8",
+ "cuda_major": "12",
+ "cxx11abi": "TRUE",
+ "platform_tag": "linux_x86_64",
+ },
+ )
+ monkeypatch.setattr(worker, "url_exists", lambda url: None)
+ installs = []
+ monkeypatch.setattr(
+ worker,
+ "install_wheel",
+ lambda url, **k: installs.append(url)
+ or [("uv", subprocess.CompletedProcess(["uv"], 0, ""))],
+ )
+ monkeypatch.setattr(worker, "_is_importable_isolated", lambda name: bool(installs))
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: None)
+ builds = []
+ monkeypatch.setattr(
+ worker._sp,
+ "run",
+ lambda cmd, **k: builds.append(cmd) or subprocess.CompletedProcess(cmd, 0, ""),
+ )
+ ok = worker._attempt_package_install(
+ event_queue = [],
+ **worker._pinned_kernel_kwargs(worker.MAMBA_SSM),
+ )
+ assert ok is True
+ assert len(installs) == 1 and builds == []
+
+
+# Optional mamba-ssm (Mamba, Jamba, Zamba, Bamba...): a prebuilt wheel or the torch path, never a
+# source build; the families that cannot load without it keep the source fallback.
+
+
[email protected]("essential", [False, True])
+def test_optional_mamba_ssm_never_source_builds(monkeypatch, essential):
+ monkeypatch.setattr(worker.sys, "platform", "linux")
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ monkeypatch.setattr(worker, "_is_importable_isolated", lambda name: False)
+ monkeypatch.setattr("utils.ssm_runtime._mamba_ssm_floor_reason", lambda: None)
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: None)
+ monkeypatch.setattr(worker, "_reject_install", lambda *a, **k: None)
+ monkeypatch.setattr(worker.shutil, "which", lambda name: None)
+ builds = []
+ monkeypatch.setattr(
+ worker._sp,
+ "run",
+ lambda cmd, **k: builds.append(cmd) or subprocess.CompletedProcess(cmd, 1, ""),
+ )
+ worker._ensure_mamba_ssm([], "ai21labs/Jamba-v0.1", required = True, essential = essential)
+ assert bool([cmd for cmd in builds if "mamba-ssm==" in " ".join(cmd)]) is essential
+
+
+def test_optional_mamba_ssm_is_skipped_below_the_setup_floor(monkeypatch):
+ statuses = []
+ install = mock.Mock()
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ monkeypatch.setattr(
+ "utils.ssm_runtime._mamba_ssm_floor_reason",
+ lambda: "needs sm80, or sm75 with Triton 3.4+ (torch 2.8+), and Triton is 3.3",
+ )
+ monkeypatch.setattr(worker, "_install_package_wheel_first", install)
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))
+ worker._ensure_mamba_ssm([], "ai21labs/Jamba-v0.1", required = True, essential = False)
+ assert not install.called
+ assert any("sm75 with Triton 3.4" in message for message in statuses)
+
+
+def test_a_required_hybrid_on_a_t4_still_installs_mamba_ssm(monkeypatch):
+ # As before setup stopped provisioning: Nemotron-H / Falcon-H1 / Granite-4.0-H install it on
+ # any GPU (the runtime never applied setup's floor to them).
+ install = mock.Mock()
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ monkeypatch.setattr(
+ "utils.ssm_runtime._mamba_ssm_floor_reason", lambda: pytest.fail("no floor for these")
+ )
+ monkeypatch.setattr(worker, "_install_package_wheel_first", install)
+ worker._ensure_mamba_ssm([], "tiiuae/Falcon-H1-0.5B", required = True, essential = True)
+ assert install.called
+
+
+ "env",
+ [
+ None, # the probe failed
+ {"cuda_version": "", "hip_version": "", "platform_tag": "linux_x86_64"}, # CPU / XPU torch
+ ],
+)
+def test_no_cuda_or_rocm_torch_installs_nothing(monkeypatch, env):
+ monkeypatch.setattr(worker, "probe_torch_wheel_env", lambda **k: env)
+ statuses = []
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))
+ install = mock.Mock()
+ monkeypatch.setattr(worker, "install_wheel", install)
+ builds = []
+ monkeypatch.setattr(worker._sp, "run", lambda cmd, **k: builds.append(cmd))
+ assert (
+ worker._attempt_package_install(
+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)
+ )
+ is False
+ )
+ assert not install.called and builds == []
+ assert any("CUDA or ROCm" in message for message in statuses)
+
+
+def test_the_training_install_runs_under_the_kernel_lock(monkeypatch):
+ from utils import kernel_install
+
+ entered = []
+
+ class _Lock:
+ def __init__(self, name, **kwargs):
+ entered.append(name)
+
+ def __enter__(self):
+ return False # another process kept it
+
+ def __exit__(self, *exc):
+ return False
+
+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _Lock)
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ attempt = mock.Mock()
+ monkeypatch.setattr(worker, "_attempt_package_install", attempt)
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: None)
+ assert (
+ worker._install_package_wheel_first(
+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)
+ )
+ is False
+ )
+ assert entered == ["causal_conv1d"] and not attempt.called
+
+
+def test_a_peer_finishing_the_install_while_waiting_is_success(monkeypatch):
+ states = iter([False, True]) # before the lock, then after it
+ monkeypatch.setattr(worker, "_is_importable", lambda name: next(states))
+ attempt = mock.Mock()
+ monkeypatch.setattr(worker, "_attempt_package_install", attempt)
+ assert worker._install_package_wheel_first(
+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)
+ )
+ assert not attempt.called
+
+
[email protected]("torch_mm", ["2.13", "2.14"])
+def test_no_pinned_source_build_on_torch_213_and_later(monkeypatch, torch_mm):
+ monkeypatch.setattr(worker.sys, "platform", "linux")
+ monkeypatch.setattr(
+ worker,
+ "probe_torch_wheel_env",
+ lambda **k: {"cuda_version": "12.8", "cuda_major": "12", "torch_mm": torch_mm},
+ )
+ monkeypatch.setattr(worker, "direct_wheel_url", lambda **k: None)
+ statuses = []
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))
+ builds = []
+ monkeypatch.setattr(worker._sp, "run", lambda cmd, **k: builds.append(cmd))
+ assert (
+ worker._attempt_package_install(
+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)
+ )
+ is False
+ )
+ assert builds == []
+ assert any("cannot be built from source" in message for message in statuses)
+
+
+# Training stays best effort as at 9f5b13325, even for the hybrids that need mamba-ssm: their
+# native transformers path can run without it, so a failed install never stops the run here.
+
+
+def test_a_required_hybrid_without_mamba_ssm_does_not_stop_training(monkeypatch):
+ monkeypatch.setattr(worker.sys, "platform", "linux")
+ monkeypatch.setattr(worker, "_install_package_wheel_first", lambda **k: False)
+ monkeypatch.setattr(worker, "probe_torch_wheel_env", lambda **k: {"cuda_version": "12.8"})
+ worker._ensure_mamba_ssm([], "tiiuae/Falcon-H1-0.5B", required = True, essential = True)
+
+
+def test_a_required_hybrid_on_windows_without_triton_does_not_stop_training(monkeypatch):
+ monkeypatch.setattr(worker.sys, "platform", "win32")
+ monkeypatch.setattr(
+ worker.importlib.util, "find_spec", lambda name, *a: None if name == "triton" else object()
+ )
+ install = mock.Mock()
+ monkeypatch.setattr(worker, "_install_package_wheel_first", install)
+ statuses = []
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))
+ worker._ensure_mamba_ssm([], "tiiuae/Falcon-H1-0.5B", required = True, essential = True)
+ assert not install.called
+ assert any("triton-windows" in message for message in statuses)
+
+
+ "essential, env",
+ [
+ (False, {"cuda_version": "12.8"}), # optional: the torch path
+ (True, {"cuda_version": "", "hip_version": ""}), # CPU / XPU torch: nothing to install
+ ],
+)
+def test_no_stop_for_optional_families_or_a_non_cuda_torch(monkeypatch, essential, env):
+ monkeypatch.setattr(worker.sys, "platform", "linux")
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ monkeypatch.setattr("utils.ssm_runtime._mamba_ssm_floor_reason", lambda: None)
+ monkeypatch.setattr(worker, "_install_package_wheel_first", lambda **k: False)
+ monkeypatch.setattr(worker, "probe_torch_wheel_env", lambda **k: env)
+ worker._ensure_mamba_ssm([], "ai21labs/Jamba-v0.1", required = True, essential = essential)
+
+
+def test_the_eager_install_and_the_hook_share_one_failed_attempt(monkeypatch):
+ # A failed (possibly 30-minute HIP) build is never repeated in one worker, nor after a chat
+ # load in the same process already failed it.
+ attempts = []
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ monkeypatch.setattr(
+ worker, "_attempt_package_install", lambda **k: attempts.append(k["import_name"]) or False
+ )
+ monkeypatch.setattr(worker, "_reject_install", lambda *a, **k: None)
+ for _caller in ("eager", "hook"):
+ assert (
+ worker._install_package_wheel_first(
+ event_queue = [], **worker._pinned_kernel_kwargs(worker.CAUSAL_CONV1D)
+ )
+ is False
+ )
+ assert attempts == ["causal_conv1d"]
+
+ from utils import ssm_runtime
+
+ ssm_runtime._FAILED_INSTALLS.add(ssm_runtime._failure_key("mamba_ssm"))
+ worker._install_package_wheel_first(
+ event_queue = [], **worker._pinned_kernel_kwargs(worker.MAMBA_SSM)
+ )
+ assert attempts == ["causal_conv1d"]
+
+
[email protected]("required, budget", [(False, 120), (True, None)])
+def test_training_waits_long_only_for_a_required_kernel(monkeypatch, required, budget):
+ import contextlib
+
+ from utils import kernel_install
+
+ if budget is None:
+ budget = kernel_install.KERNEL_LOCK_WAIT
+
+ seen = {}
+
+ @contextlib.contextmanager
+ def _lock(name, **kwargs):
+ seen.update(kwargs)
+ yield False
+
+ monkeypatch.setattr(kernel_install, "kernel_install_lock", _lock)
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: None)
+ worker._install_package_wheel_first(
+ event_queue = [], required = required, **worker._pinned_kernel_kwargs(worker.MAMBA_SSM)
+ )
+ assert seen["wait_seconds"] == budget
+
+
+# Real ROCm gfx1151: the training block compiled mamba-ssm from source for a Mamba2 model (60 s)
+# where the inference path stayed wheel-only. Driven as the hardware harness drives it.
+
+
+def _hip_mamba2_host(monkeypatch, tmp_path):
+ import json
+
+ model = tmp_path / "tiny-mamba2"
+ model.mkdir()
+ (model / "config.json").write_text(json.dumps({"model_type": "mamba2"}))
+ monkeypatch.setattr(worker.sys, "platform", "linux")
+ monkeypatch.setattr(
+ worker,
+ "probe_torch_wheel_env",
+ lambda **k: {
+ "hip_version": "7.1",
+ "torch_mm": "2.10",
+ "python_tag": "cp312",
+ "cxx11abi": "TRUE",
+ "platform_tag": "linux_x86_64",
+ },
+ )
+ monkeypatch.setattr(worker.shutil, "which", lambda name: f"/usr/bin/{name}")
+ monkeypatch.setattr(worker, "_hipcc_gcc_install_dir", lambda: None)
+ monkeypatch.setattr(worker, "_is_importable", lambda name: False)
+ monkeypatch.setattr(worker, "_is_importable_isolated", lambda name: False)
+ monkeypatch.setattr("utils.ssm_runtime._mamba_ssm_floor_reason", lambda: None)
+ statuses = []
+ monkeypatch.setattr(worker, "_send_status", lambda queue, message: statuses.append(message))
+ monkeypatch.setattr(worker, "_reject_install", lambda *a, **k: None)
+ builds = []
+ monkeypatch.setattr(
+ worker._sp,
+ "run",
+ lambda cmd, **k: builds.append(list(cmd)) or subprocess.CompletedProcess(cmd, 1, ""),
+ )
+ return str(model), builds, statuses
+
+
[email protected]("caller", ["harness", "worker"])
+def test_a_mamba2_model_on_rocm_never_builds_mamba_ssm_in_training(monkeypatch, tmp_path, caller):
+ from utils import ssm_runtime
+
+ model, builds, statuses = _hip_mamba2_host(monkeypatch, tmp_path)
+ kernels = ssm_runtime.resolve_ssm_kernels(model, model, None, adapter_base = None)
+ assert kernels.mamba_ssm and not kernels.mamba_ssm_required
+ if caller == "harness":
+ # scripts/v4_amd/ondemand_probe.py: no `essential`, as an older caller would.
+ worker._ensure_mamba_ssm([], model, required = kernels.mamba_ssm)
+ else:
+ worker._ensure_mamba_ssm(
+ [], model, required = kernels.mamba_ssm, essential = kernels.mamba_ssm_required
+ )
+ assert not [cmd for cmd in builds if "mamba-ssm==" in " ".join(cmd)]
+ assert not any("Compiling mamba-ssm" in message for message in statuses)
+
+
+def test_a_required_hybrid_on_rocm_keeps_its_mamba_ssm_source_build(monkeypatch, tmp_path):
+ _, builds, statuses = _hip_mamba2_host(monkeypatch, tmp_path)
+ worker._ensure_mamba_ssm([], "tiiuae/Falcon-H1-0.5B", required = True)
+ assert [cmd for cmd in builds if "mamba-ssm==" in " ".join(cmd)]
diff --git a/studio/backend/tests/test_wheel_utils_windows_kernels.py b/studio/backend/tests/test_wheel_utils_windows_kernels.py
index 648f7cd38a8..2e9861deb0e 100644
--- a/studio/backend/tests/test_wheel_utils_windows_kernels.py
+++ b/studio/backend/tests/test_wheel_utils_windows_kernels.py
@@ -167,7 +167,17 @@ def woa_env(**overrides):
)
[email protected]("python_tag", ["cp311", "cp312", "cp313"])
[email protected]("python_tag", ["cp311", "cp312"])
+def test_win_arm64_unpublished_interpreters_resolve_to_nothing(python_tag):
+ # The release carries cp313 only for win_arm64; a cp311 / cp312 URL would be a 404.
+ e = woa_env(python_tag = python_tag, abi_tag = python_tag)
+ assert kernel_install.CAUSAL_CONV1D.wheel_url(e) is None
+ assert kernel_install.MAMBA_SSM.wheel_url(e) is None
+ # flash-attn stays NVIDIA's, which publishes every interpreter.
+ assert wheel_utils.flash_attn_wheel_url(e).startswith(NVIDIA)
+
+
[email protected]("python_tag", ["cp313"])
def test_win_arm64_causal_conv1d_and_mamba_are_ours(python_tag):
e = woa_env(python_tag = python_tag, abi_tag = python_tag)
assert kernel_install.CAUSAL_CONV1D.wheel_url(e) == (
@@ -215,10 +225,12 @@ def test_an_env_without_abi_tag_is_read_as_a_gil_build():
"cp313",
f"{OURS}flash_attn-2.8.4+cu13torch2.13cxx11abiTRUE-cp313-cp313-linux_x86_64.whl",
),
+ # Built by the workflow on request, but not published: no URL rather than a 404.
+ ("2.14", "cp312", None),
(
"2.14",
- "cp312",
- f"{OURS}flash_attn-2.8.4+cu13torch2.14cxx11abiTRUE-cp312-cp312-linux_x86_64.whl",
+ "cp313",
+ f"{OURS}flash_attn-2.8.4+cu13torch2.14cxx11abiTRUE-cp313-cp313-linux_x86_64.whl",
),
(
"2.11",
@@ -239,9 +251,11 @@ def test_linux_flash_attn_urls_unchanged(torch_mm, python_tag, expected):
assert wheel_utils.flash_attn_wheel_url(e) == expected
-def test_linux_free_threaded_keeps_resolving_as_before():
- # Linux has never looked at abi_tag; adding it must not change a Linux answer.
+def test_linux_free_threaded_is_not_served_the_gil_wheel():
+ # The release carries cp313-cp313 only; a free-threaded cp313t interpreter cannot install it.
e = env(platform_tag = "linux_x86_64", cxx11abi = "TRUE", abi_tag = "cp313t")
+ assert wheel_utils.flash_attn_wheel_url(e) is None
+ e = env(platform_tag = "linux_x86_64", cxx11abi = "TRUE", abi_tag = "cp313")
assert wheel_utils.flash_attn_wheel_url(e) == (
f"{OURS}flash_attn-2.8.4+cu13torch2.13cxx11abiTRUE-cp313-cp313-linux_x86_64.whl"
)
diff --git a/studio/backend/utils/kernel_install.py b/studio/backend/utils/kernel_install.py
index f3bb2ee6039..90fa5a64eff 100644
--- a/studio/backend/utils/kernel_install.py
+++ b/studio/backend/utils/kernel_install.py
@@ -11,13 +11,16 @@ The CLI is wheel-only (never a source build) and uses --no-deps so torch is neve
from __future__ import annotations
import argparse
+import errno
+import importlib.metadata
import os
import platform
import shutil
import subprocess
import sys
+from contextlib import contextmanager
from dataclasses import dataclass
-from typing import Any, Callable
+from typing import Any, Callable, Iterator
from utils.child_stdio import utf8_child_env
from utils.wheel_utils import (
@@ -30,6 +33,7 @@ from utils.wheel_utils import (
direct_wheel_url,
flash_attn_wheel_url,
install_wheel,
+ package_installs_offline_env,
probe_torch_wheel_env,
redact_url_credentials,
url_exists,
@@ -142,12 +146,204 @@ def source_build_command(spec: str, *, use_uv: bool, is_hip: bool, reinstall: bo
return cmd
+# Overall deadline for a ROCm / HIP source build (10-30 min).
+SOURCE_BUILD_TIMEOUT = 1800
+
+
+def _cuda_source_build_timeout() -> int:
+ """UNSLOTH_SOURCE_BUILD_TIMEOUT (seconds, > 0) when set, else 7200."""
+ raw = os.environ.get("UNSLOTH_SOURCE_BUILD_TIMEOUT", "").strip()
+ try:
+ value = int(float(raw)) if raw else 0
+ except ValueError:
+ value = 0
+ return value if value > 0 else 7200
+
+
+# Overall deadline for a CUDA source build. Far looser than HIP's: before the shared deadline a
+# CUDA build had none, and a slow one (causal-conv1d on aarch64 can compile for over an hour) must
+# not be aborted; this only stops a build that is truly stuck. Read once at import, so the lock
+# wait below covers it.
+SOURCE_BUILD_TIMEOUT_CUDA = _cuda_source_build_timeout()
+# The real runner, captured at import: a caller handing in anything else (a test double, or
+# subprocess.run patched over) gets exactly that called.
+_SUBPROCESS_RUN = subprocess.run
+
+
+def run_process_tree(
+ cmd: list[str],
+ *,
+ run: Callable[..., Any] = _SUBPROCESS_RUN,
+ **kwargs: Any,
+) -> subprocess.CompletedProcess:
+ """``run(cmd, **kwargs)``, but when *run* is the real ``subprocess.run`` a ``timeout`` stops the
+ whole process tree (pip / uv and the nvcc / hipcc / ninja they start), not just its leader,
+ which ``subprocess.run`` alone would leave compiling. Raises ``TimeoutExpired`` as run does."""
+ timeout = kwargs.pop("timeout", None)
+ if run is not _SUBPROCESS_RUN or timeout is None:
+ if timeout is not None:
+ kwargs["timeout"] = timeout
+ return run(cmd, **kwargs)
+ if os.name == "nt":
+ kwargs.setdefault("creationflags", getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0))
+ else:
+ kwargs["start_new_session"] = True
+ with subprocess.Popen(cmd, **kwargs) as process:
+ try:
+ stdout, stderr = process.communicate(timeout = timeout)
+ except subprocess.TimeoutExpired:
+ _kill_process_tree(process)
+ process.communicate()
+ raise subprocess.TimeoutExpired(cmd, timeout)
+ except BaseException:
+ _kill_process_tree(process)
+ raise
+ return subprocess.CompletedProcess(cmd, process.returncode, stdout, stderr)
+
+
+def _kill_process_tree(process: subprocess.Popen) -> None:
+ try:
+ if os.name == "nt":
+ subprocess.run(
+ ["taskkill", "/F", "/T", "/PID", str(process.pid)],
+ stdout = subprocess.DEVNULL,
+ stderr = subprocess.DEVNULL,
+ timeout = 60,
+ )
+ else:
+ import signal
+ os.killpg(process.pid, signal.SIGKILL)
+ except Exception:
+ pass
+ try:
+ process.kill()
+ except Exception:
+ pass
+
+
+# Long enough for one source build (the longer of the HIP and CUDA deadlines) under another
+# process's lock, then give up rather than block.
+KERNEL_LOCK_WAIT = max(SOURCE_BUILD_TIMEOUT, SOURCE_BUILD_TIMEOUT_CUDA) + 600
+_LOCK_CONTENDED = frozenset(
+ code
+ for code in (
+ getattr(errno, "EACCES", None),
+ getattr(errno, "EAGAIN", None),
+ getattr(errno, "EWOULDBLOCK", None),
+ getattr(errno, "EDEADLK", None),
+ getattr(errno, "EDEADLOCK", None),
+ )
+ if code is not None
+)
+
+
+# How long an optional acceleration (causal-conv1d, an optional mamba-ssm wheel, flash-attn) waits
+# for another process's install of it before the load goes on without it.
+OPTIONAL_KERNEL_LOCK_WAIT = 120
+
+
+def _kernel_lock_path(name: str) -> str:
+ """The one lock file for installing *name* into this environment, shared by every entry point
+ that installs or removes it (Studio workers, setup, ``unsloth install-kernels``): beside the
+ environment when it is writable, else one per environment in the temp folder."""
+ import hashlib
+ import tempfile
+
+ override = os.environ.get("UNSLOTH_KERNEL_LOCK_DIR")
+ if override:
+ os.makedirs(override, exist_ok = True)
+ return os.path.join(override, f"{name}.lock")
+ folder = os.path.join(sys.prefix, ".unsloth-kernel-locks")
+ try:
+ os.makedirs(folder, exist_ok = True)
+ if os.access(folder, os.W_OK):
+ return os.path.join(folder, f"{name}.lock")
+ except OSError:
+ pass
+ env = hashlib.sha256(os.path.realpath(sys.prefix).encode("utf-8")).hexdigest()[:16]
+ folder = os.path.join(tempfile.gettempdir(), "unsloth-kernel-locks")
+ os.makedirs(folder, exist_ok = True)
+ return os.path.join(folder, f"{env}-{name}.lock")
+
+
+@contextmanager
+def kernel_install_lock(
+ name: str,
+ *,
+ on_wait: Callable[[], None] | None = None,
+ wait_seconds: float = KERNEL_LOCK_WAIT,
+ beat_seconds: float = 60.0,
+) -> Iterator[bool]:
+ """Hold the cross-process lock for installing kernel *name* into this environment, covering the
+ whole precheck / install / verify / cleanup transaction. Yields True when held, False when it
+ is not: another process kept it past *wait_seconds*, or the lock cannot be opened or taken at
+ all. Either way the caller installs nothing rather than race a peer unguarded. *on_wait* is
+ called when the wait starts and every *beat_seconds* while it lasts."""
+ import time
+
+ try:
+ handle = open(_kernel_lock_path(name), "a+b")
+ except OSError as exc:
+ logger_message = f"Unsloth: cannot open the {name} install lock ({exc}); not installing it."
+ print(logger_message, file = sys.stderr)
+ yield False
+ return
+ held = False
+ try:
+ deadline = time.monotonic() + wait_seconds
+ next_beat = 0.0
+ while True:
+ try:
+ if os.name == "nt":
+ import msvcrt
+ handle.seek(0)
+ msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+ else:
+ import fcntl
+ fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ held = True
+ break
+ except (OSError, ImportError) as exc:
+ if not isinstance(exc, OSError) or exc.errno not in _LOCK_CONTENDED:
+ print(
+ f"Unsloth: cannot take the {name} install lock ({exc}); not installing it.",
+ file = sys.stderr,
+ )
+ break
+ now = time.monotonic()
+ if now >= deadline:
+ break
+ if on_wait is not None and now >= next_beat:
+ next_beat = now + beat_seconds
+ try:
+ on_wait()
+ except Exception:
+ pass
+ time.sleep(0.5)
+ yield held
+ finally:
+ if held:
+ try:
+ if os.name == "nt":
+ import msvcrt
+ handle.seek(0)
+ msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+ fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+ except (OSError, ImportError):
+ pass
+ if not handle.closed:
+ handle.close()
+
+
def source_build_run_kwargs(
*, is_hip: bool, gcc_install_dir: Callable[[], str | None]
) -> tuple[dict[str, Any], str | None]:
"""subprocess.run kwargs for a source build, and the gcc dir added to HIPCC_COMPILE_FLAGS_APPEND
- (None when nothing was added). Non-HIP builds have no timeout so slow builds (causal-conv1d on
- aarch64) are not aborted; ROCm builds take 10-30 min, bounded at 1800 s."""
+ (None when nothing was added). ROCm builds take 10-30 min and are bounded at
+ SOURCE_BUILD_TIMEOUT (1800 s); CUDA builds at SOURCE_BUILD_TIMEOUT_CUDA (7200 s, or
+ UNSLOTH_SOURCE_BUILD_TIMEOUT), loose enough for a slow aarch64 causal-conv1d build."""
kwargs: dict[str, Any] = {
"stdout": subprocess.PIPE,
"stderr": subprocess.STDOUT,
@@ -157,9 +353,11 @@ def source_build_run_kwargs(
"errors": "replace",
"env": utf8_child_env(),
}
+ # An overall deadline for every source build (run it through run_process_tree so a timeout
+ # also stops the compilers pip / uv started): HIP's, or the far looser CUDA one.
+ kwargs["timeout"] = SOURCE_BUILD_TIMEOUT if is_hip else SOURCE_BUILD_TIMEOUT_CUDA
if not is_hip:
return kwargs, None
- kwargs["timeout"] = 1800
existing = os.environ.get("HIPCC_COMPILE_FLAGS_APPEND", "")
if "--gcc-install-dir" in existing:
return kwargs, None
@@ -202,6 +400,7 @@ KERNELS = {
}
_PINNED = {"causal_conv1d": CAUSAL_CONV1D, "mamba_ssm": MAMBA_SSM}
+_WITHOUT_GPU_ENV = "UNSLOTH_INSTALL_KERNELS_WITHOUT_GPU"
def resolve_wheel_url(name: str, env: dict[str, str] | None) -> str | None:
@@ -276,6 +475,39 @@ def _gpu_capability(run: Callable[..., subprocess.CompletedProcess]) -> tuple[in
return _CAPABILITY[run]
+def gpu_below_flash_attn_floor(
+ run: Callable[..., subprocess.CompletedProcess] = subprocess.run,
+) -> bool:
+ """Whether the best visible CUDA GPU is below sm80, where Unsloth leaves FlashAttention 2 off
+ (a T4 is sm75), so a flash-attn download would go unused. No visible GPU answers False: an
+ image build installs ahead of the GPU it will run on."""
+ capability = _gpu_capability(run)
+ # ROCm reports a gfx version, not an sm, as its capability: the floor is NVIDIA's.
+ return capability is not None and capability < (8, 0) and not _torch_is_hip(run)
+
+
+_HIP: dict = {}
+
+
+def _torch_is_hip(run: Callable[..., subprocess.CompletedProcess]) -> bool:
+ """Whether torch is a ROCm build. Probed once per runner, and only below sm80."""
+ if run not in _HIP:
+ try:
+ result = run(
+ [sys.executable, "-c", "import torch; print(bool(torch.version.hip))"],
+ stdout = subprocess.PIPE,
+ stderr = subprocess.DEVNULL,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 120,
+ )
+ _HIP[run] = result.stdout.strip() == "True"
+ except Exception:
+ _HIP[run] = False
+ return _HIP[run]
+
+
def _probe_gpu_capability(
run: Callable[..., subprocess.CompletedProcess],
) -> tuple[int, int] | None:
@@ -309,6 +541,13 @@ def _loads(check: str, run: Callable[..., subprocess.CompletedProcess]) -> bool:
return result.returncode == 0
+def _installed_version(distribution: str) -> str | None:
+ try:
+ return importlib.metadata.version(distribution)
+ except Exception:
+ return None
+
+
def _outside_venv() -> bool:
# Colab and other system interpreters: uv refuses them without --system.
return sys.prefix == sys.base_prefix
@@ -328,70 +567,100 @@ def install_kernel(
dry_run: bool = False,
run: Callable[..., subprocess.CompletedProcess] = subprocess.run,
exists: Callable[[str], bool | None] = url_exists,
+ install_without_gpu: bool = False,
+ skipped: list[str] | None = None,
) -> int:
+ # install_without_gpu: no visible GPU yet (an image build for the GPU it will run on) still
+ # installs. Each kernel left uninstalled for this machine is appended to *skipped*.
distribution, check = KERNELS[name]
+
+ def skip(message: str) -> int:
+ print(message)
+ if skipped is not None:
+ skipped.append(name)
+ return 0
+
+ # A copy that already loads (a ROCm source build, a GPU-hidden image builder's earlier install)
+ # is success whatever the wheel, GPU and network checks below would say.
+ if not dry_run and _installed_version(distribution) is not None and _loads(check, run):
+ print(f"Unsloth: {name} already installed and loads.")
+ return 0
+ # A dry run only prints the URL it would install, so it runs offline too.
+ if not dry_run and package_installs_offline_env():
+ return skip(
+ f"Unsloth: not installing {name} while offline "
+ "(UV_OFFLINE / HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE)."
+ )
url = resolve_wheel_url(name, env)
if name in _GPU_GATED and url is not None:
capability = _gpu_capability(run)
- if capability is None:
- print(f"Unsloth: skipping {name}, which needs sm80 or newer (no CUDA GPU is visible).")
- return 0
+ if capability is None and not install_without_gpu:
+ return skip(
+ f"Unsloth: skipping {name}, which needs sm80 or newer (no CUDA GPU is visible; "
+ "pass --without-gpu to install it for the GPU this image will run on)."
+ )
if capability is not None and capability < (8, 0):
reason = _pre_ampere_skip_reason(name, capability, run)
if reason is not None:
- print(
+ return skip(
f"Unsloth: skipping {name}, which {reason} "
f"(the best GPU is sm{capability[0]}{capability[1]})."
)
- return 0
torch_desc = f"torch {env.get('torch_version')}" if env else "this environment"
# Only a 404 proves nothing is published; an unreachable check falls through to the install.
if url is None or exists(url) is False:
- print(f"Unsloth: no prebuilt {name} for {torch_desc}; using the torch fallback.")
- return 0
+ return skip(f"Unsloth: no prebuilt {name} for {torch_desc}; using the torch fallback.")
# mamba_ssm imports triton at load; on Windows that is triton-windows, which its metadata cannot name.
if name == "mamba_ssm" and sys.platform == "win32" and not _loads("import triton", run):
- print(
+ return skip(
"Unsloth: skipping mamba_ssm, which needs Triton on Windows; "
"install triton-windows first (Studio setup does), then re-run."
)
- return 0
# UNSLOTH_PYTORCH_MIRROR may carry credentials, and notebook output gets shared.
shown = redact_url_credentials(url)
if dry_run:
print(shown)
return 0
- if _loads(check, run):
- print(f"Unsloth: {name} already installed and loads.")
- return 0
- failures: list[subprocess.CompletedProcess] = []
- outcome = install_prebuilt(
- url,
- install = install_wheel,
- verify = lambda: _loads(check, run),
- on_failed = lambda _installer, result: failures.append(result),
- use_uv = True,
- uv_needs_system = _outside_venv(),
- reinstall = True,
- run = run,
- )
- if outcome == "failed":
- print(
- f"Unsloth: installing {name} from {shown} failed; using the torch fallback.\n"
- + failures[-1].stdout[-2000:].replace(url, shown)
- )
- return 1
- if outcome == "rejected":
- # A wheel that imports but whose extension cannot load would be picked up and crash later.
- removed = (
- "removed it" if _uninstall(distribution, run) else f"run `pip uninstall {distribution}`"
- )
- print(
- f"Unsloth: {name} from {shown} does not load with {torch_desc}; {removed}, using the torch fallback."
+ # The lock Studio's workers take for the same kernel: never install or remove it under a
+ # running load's precheck / verify.
+ with kernel_install_lock(
+ name, on_wait = lambda: print(f"Unsloth: waiting for another process installing {name}...")
+ ) as held:
+ if not held:
+ return skip(f"Unsloth: could not take the {name} install lock; not installing it.")
+ if _loads(check, run):
+ print(f"Unsloth: {name} already installed and loads.")
+ return 0
+ failures: list[subprocess.CompletedProcess] = []
+ outcome = install_prebuilt(
+ url,
+ install = install_wheel,
+ verify = lambda: _loads(check, run),
+ on_failed = lambda _installer, result: failures.append(result),
+ use_uv = True,
+ uv_needs_system = _outside_venv(),
+ reinstall = True,
+ run = run,
)
- return 1
- print(f"Unsloth: installed {name} ({shown.rsplit('/', 1)[-1]}).")
- return 0
+ if outcome == "failed":
+ print(
+ f"Unsloth: installing {name} from {shown} failed; using the torch fallback.\n"
+ + failures[-1].stdout[-2000:].replace(url, shown)
+ )
+ return 1
+ if outcome == "rejected":
+ # A wheel that imports but whose extension cannot load would be picked up and crash later.
+ removed = (
+ "removed it"
+ if _uninstall(distribution, run)
+ else f"run `pip uninstall {distribution}`"
+ )
+ print(
+ f"Unsloth: {name} from {shown} does not load with {torch_desc}; {removed}, using the torch fallback."
+ )
+ return 1
+ print(f"Unsloth: installed {name} ({shown.rsplit('/', 1)[-1]}).")
+ return 0
def main(argv: list[str] | None = None) -> int:
@@ -408,11 +677,45 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument(
"--dry-run", action = "store_true", help = "print the resolved wheel URL and install nothing"
)
+ parser.add_argument(
+ "--without-gpu",
+ action = "store_true",
+ help = (
+ "install the GPU-gated kernels (flash_attn, mamba_ssm) even though no CUDA GPU is "
+ f"visible, e.g. in an image build; also {_WITHOUT_GPU_ENV}=1"
+ ),
+ )
+ parser.add_argument(
+ "--strict",
+ action = "store_true",
+ help = "exit 2 when a requested kernel was skipped (no GPU, no wheel, offline)",
+ )
args = parser.parse_args(argv)
+ without_gpu = args.without_gpu or os.environ.get(_WITHOUT_GPU_ENV, "").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ )
env = probe_torch_wheel_env(timeout = 120, include_windows = True)
status = 0
+ skipped: list[str] = []
for name in [name for name in KERNELS if not args.kernels or name in args.kernels]:
- status = max(status, install_kernel(name, env, dry_run = args.dry_run))
+ status = max(
+ status,
+ install_kernel(
+ name,
+ env,
+ dry_run = args.dry_run,
+ install_without_gpu = without_gpu,
+ skipped = skipped,
+ ),
+ )
+ if skipped:
+ print(f"Unsloth: not installed on this machine: {', '.join(skipped)}.")
+ # A skip stays exit 0, as scripts running under `set -e` expect; --strict opts in.
+ if args.strict and status == 0:
+ status = 2
return status
diff --git a/studio/backend/utils/ssm_runtime.py b/studio/backend/utils/ssm_runtime.py
index e85bcecfd8c..ae05fd30cc7 100644
--- a/studio/backend/utils/ssm_runtime.py
+++ b/studio/backend/utils/ssm_runtime.py
@@ -7,9 +7,7 @@ the inference-path counterpart of the training worker's install.
from __future__ import annotations
-import functools
import importlib
-import importlib.metadata
import importlib.util
import os
import shutil
@@ -18,7 +16,7 @@ import sys
import threading
from contextlib import contextmanager
from pathlib import Path
-from typing import Any, Callable, Iterator, Optional
+from typing import Any, Callable, Iterator, NamedTuple, Optional
from loggers import get_logger
from utils.kernel_install import (
@@ -35,6 +33,7 @@ from utils.wheel_utils import (
direct_wheel_url,
install_wheel,
is_windows_wheel_env,
+ pinned_source_build_supported,
probe_torch_wheel_env,
url_exists,
)
@@ -44,7 +43,8 @@ logger = get_logger(__name__)
StatusCb = Optional[Callable[[str], None]]
# Lowercased-id substring matches, shared with the training worker. mamba-ssm models are a subset of the causal-conv1d set.
-SSM_MODEL_SUBSTRINGS = (
+# The Mamba-2 hybrids whose modeling (or remote) code cannot load without mamba_ssm: a failed install is fatal for them.
+REQUIRED_SSM_MODEL_SUBSTRINGS = (
"nemotron_h",
"nemotron-h",
"nemotron-3-nano",
@@ -52,29 +52,28 @@ SSM_MODEL_SUBSTRINGS = (
"falcon-h1",
"granite-4.0-h",
"granitemoehybrid",
+)
+SSM_MODEL_SUBSTRINGS = REQUIRED_SSM_MODEL_SUBSTRINGS + (
"mamba",
"jamba",
"zamba",
"bamba",
)
-# config.json model types whose modeling files run mamba_ssm (and so causal_conv1d): Mamba-2
-# hybrids and the Mamba-1 family. Matched on the resolved config, so a renamed checkpoint or an
-# id without the substrings above (NVIDIA-Nemotron-Nano-9B-v2, granite-4.0-tiny) still counts.
-SSM_MODEL_TYPES = frozenset(
- {
- "nemotron_h",
- "falcon_h1",
- "granitemoehybrid",
- "bamba",
- "jamba",
- "zamba",
- "zamba2",
- "mamba",
- "mamba2",
- "falcon_mamba",
- }
+# config.json model types whose modeling files run mamba_ssm (and so causal_conv1d). Matched on the
+# resolved config, so a renamed checkpoint or an id without the substrings above
+# (NVIDIA-Nemotron-Nano-9B-v2, granite-4.0-tiny) still counts. The Mamba-1 family and the other
+# hybrids have a working torch path in transformers, so mamba_ssm is only an acceleration there.
+REQUIRED_SSM_MODEL_TYPES = frozenset({"nemotron_h", "falcon_h1", "granitemoehybrid"})
+SSM_MODEL_TYPES = REQUIRED_SSM_MODEL_TYPES | frozenset(
+ {"bamba", "jamba", "zamba", "zamba2", "mamba", "mamba2", "falcon_mamba"}
)
CAUSAL_CONV1D_MODEL_SUBSTRINGS = (
+ "qwen3.5",
+ "qwen3_5",
+ "qwen3.6",
+ "qwen3_6",
+ "qwen3-next",
+ "qwen3_next",
"nemotron_h",
"nemotron-h",
"nemotron-3-nano",
@@ -88,52 +87,28 @@ CAUSAL_CONV1D_MODEL_SUBSTRINGS = (
"zamba",
"bamba",
)
-# GatedDeltaNet models unsloth-zoo runs without causal_conv1d: its Triton causal conv
-# (temporary_patches/gdn_causal_conv1d.py, `_GDN_MODELING`) plus the vendored FLA kernels. Qwen3.6
-# reuses the qwen3_5 model types. A usable causal_conv1d still wins inside zoo; this only stops
-# Studio from downloading it. Zoo's own kill switch brings the request back.
-ZOO_GDN_MODEL_TYPES = frozenset({"qwen3_5", "qwen3_5_moe", "qwen3_next"})
-ZOO_GDN_MODEL_SUBSTRINGS = (
- "qwen3.5",
- "qwen3_5",
- "qwen3.6",
- "qwen3_6",
- "qwen3-next",
- "qwen3_next",
-)
-ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH = "UNSLOTH_DISABLE_TRITON_CAUSAL_CONV1D"
-_TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE: dict[str, bool | None] = {}
-
-
-def zoo_triton_causal_conv1d_enabled() -> bool:
- return os.environ.get(ZOO_TRITON_CAUSAL_CONV1D_KILL_SWITCH, "0") != "1"
-
-
[email protected]_cache(maxsize = 1)
-def _zoo_triton_platform_usable() -> bool:
- """Whether zoo's Triton conv can engage here: a CUDA torch (not ROCm) with Triton importable.
- macOS, ROCm and a Windows install without triton-windows keep the prebuilt causal-conv1d.
- Read from torch's distribution version, so no model load pays for importing torch: ROCm and
- CPU builds carry `+rocm` / `+cpu`, a Windows CUDA build `+cu`, and a Linux PyPI torch with no
- local version is the CUDA build."""
- if sys.platform == "darwin" or importlib.util.find_spec("triton") is None:
- return False
- torch = sys.modules.get("torch")
- if torch is not None:
- version = getattr(torch, "version", None)
- return bool(getattr(version, "cuda", None)) and not getattr(version, "hip", None)
- try:
- local = importlib.metadata.version("torch").partition("+")[2].lower()
- except importlib.metadata.PackageNotFoundError:
- return False
- if local.startswith(("rocm", "cpu")):
- return False
- return local.startswith("cu") or (not local and sys.platform.startswith("linux"))
-
-
-def zoo_triton_causal_conv1d_usable() -> bool:
- """zoo runs Qwen3.5 / 3.6 / Qwen3-Next without causal_conv1d on this host."""
- return zoo_triton_causal_conv1d_enabled() and _zoo_triton_platform_usable()
+_TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE: dict[tuple[str, str], bool | None] = {}
+
+
+class SsmKernels(NamedTuple):
+ """What one model needs: ``causal_conv1d`` and ``mamba_ssm`` (best effort), and whether a
+ missing ``mamba_ssm`` must stop the load. ``probe`` is the identifier the decision was made for
+ (an adapter's base when there is one); ``model_types`` the config's model types, if read."""
+
+ causal_conv1d: bool
+ mamba_ssm: bool
+ mamba_ssm_required: bool
+ probe: str
+ model_types: frozenset[str] = frozenset()
+ # (dynamic-module namespace, implementation module) pairs from the config's auto_map, as
+ # transformers names them under transformers_modules (see remote_code_modules).
+ remote_code: frozenset = frozenset()
+ # Whether the load will import code that cannot run without mamba_ssm, so a failed install
+ # must refuse it: remote modeling code importing it unconditionally, or a model type this
+ # transformers has no native implementation of. mamba_ssm_required alone (Nemotron-H,
+ # Falcon-H1, Granite-4.0-H) only means "worth a source build": their native transformers
+ # models run without it.
+ mamba_ssm_fatal: bool = False
def _base_model_type(model_type: str) -> str:
@@ -144,27 +119,22 @@ def _base_model_type(model_type: str) -> str:
return candidate
-def _is_zoo_gdn_model_type(model_type: str) -> bool:
- """The model type or one of its `_text` / `_vision` sub-configs belongs to a zoo-covered model."""
- return _base_model_type(model_type) in ZOO_GDN_MODEL_TYPES
-
-
-def _names_zoo_gdn_model(name: str) -> bool:
- return any(sub in name for sub in ZOO_GDN_MODEL_SUBSTRINGS)
-
-
def model_is_ssm(model_name: str) -> bool:
- """Whether *model_name* is a Mamba/SSM hybrid that needs ``mamba_ssm``."""
+ """Whether *model_name* is a Mamba/SSM model whose fast path is ``mamba_ssm``."""
name = (model_name or "").lower()
return any(sub in name for sub in SSM_MODEL_SUBSTRINGS)
+def model_requires_mamba_ssm(model_name: str) -> bool:
+ """Whether *model_name* cannot load at all without ``mamba_ssm``."""
+ name = (model_name or "").lower()
+ return any(sub in name for sub in REQUIRED_SSM_MODEL_SUBSTRINGS)
+
+
def model_wants_causal_conv1d(model_name: str) -> bool:
- """Whether *model_name* needs ``causal_conv1d`` (the SSM set plus linear-attention hybrids like LFM2 whose modeling files lazy-import it; Qwen3.5 / 3.6 / Qwen3-Next only where zoo's Triton conv cannot run)."""
+ """Whether *model_name* needs ``causal_conv1d`` (the SSM set plus linear-attention hybrids like Qwen3.5 / 3.6 / Qwen3-Next / LFM2 whose modeling files lazy-import it)."""
name = (model_name or "").lower()
- if any(sub in name for sub in CAUSAL_CONV1D_MODEL_SUBSTRINGS):
- return True
- return _names_zoo_gdn_model(name) and not zoo_triton_causal_conv1d_usable()
+ return any(sub in name for sub in CAUSAL_CONV1D_MODEL_SUBSTRINGS)
def _normalized_model_identifier(value: str) -> str:
@@ -173,6 +143,26 @@ def _normalized_model_identifier(value: str) -> str:
)
+def _transformers_models_dir() -> Path | None:
+ """The active transformers' ``models`` folder, found without importing transformers: the
+ inference worker asks before its transformers import, which snapshots the kernel gates."""
+ try:
+ spec = importlib.util.find_spec("transformers")
+ except (ImportError, ValueError):
+ return None
+ if spec is None:
+ return None
+ locations = list(getattr(spec, "submodule_search_locations", None) or ())
+ origin = getattr(spec, "origin", None)
+ if not locations and origin:
+ locations = [str(Path(origin).parent)]
+ for location in locations:
+ models_dir = Path(location) / "models"
+ if models_dir.is_dir():
+ return models_dir
+ return None
+
+
def _transformers_model_type_uses_causal_conv1d(model_type: str) -> bool | None:
candidate = model_type.strip().lower().replace("-", "_")
if not candidate or any(
@@ -180,14 +170,15 @@ def _transformers_model_type_uses_causal_conv1d(model_type: str) -> bool | None:
for character in candidate
):
return None
- if candidate in _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE:
- return _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE[candidate]
+ models_dir = _transformers_models_dir()
+ cache_key = (str(models_dir), candidate)
+ if cache_key in _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE:
+ return _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE[cache_key]
result: bool | None = None
try:
- import transformers
- model_dir = Path(transformers.__file__).parent / "models" / candidate
- if model_dir.is_dir():
+ model_dir = models_dir / candidate if models_dir is not None else None
+ if model_dir is not None and model_dir.is_dir():
for modeling_file in model_dir.glob("modeling_*.py"):
try:
source = modeling_file.read_text(encoding = "utf-8", errors = "ignore")
@@ -200,7 +191,7 @@ def _transformers_model_type_uses_causal_conv1d(model_type: str) -> bool | None:
except Exception as exc:
logger.debug("causal-conv1d model-type inspection skipped: %s", exc)
- _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE[candidate] = result
+ _TRANSFORMERS_CAUSAL_CONV1D_MODEL_TYPE_CACHE[cache_key] = result
return result
@@ -236,85 +227,355 @@ def model_config_is_ssm(model_config: dict) -> bool | None:
return any(_base_model_type(model_type) in SSM_MODEL_TYPES for model_type in model_types)
-def model_config_wants_causal_conv1d(
- model_config: dict, *, scan_modeling_source: bool = True
-) -> bool | None:
- """``scan_modeling_source=False`` keeps the answer transformers-free (the inference worker asks
- before its transformers import, which snapshots the kernel gates)."""
+def model_config_requires_mamba_ssm(model_config: dict) -> bool | None:
+ """True when the config's model cannot load without ``mamba_ssm``; None without a model type."""
+ model_types, _ = _config_identifiers(model_config)
+ if not model_types:
+ return None
+ return any(
+ _base_model_type(model_type) in REQUIRED_SSM_MODEL_TYPES for model_type in model_types
+ )
+
+
+# The required families by name, for a decision with no config to read.
+_REQUIRED_MODEL_TYPE_BY_SUBSTRING = (
+ ("nemotron_h", "nemotron_h"),
+ ("nemotron-h", "nemotron_h"),
+ ("nemotron-3-nano", "nemotron_h"),
+ ("falcon_h1", "falcon_h1"),
+ ("falcon-h1", "falcon_h1"),
+ ("granite-4.0-h", "granitemoehybrid"),
+ ("granitemoehybrid", "granitemoehybrid"),
+)
+_NATIVE_MODELING_CACHE: dict[tuple[str, str], bool] = {}
+
+
+def native_modeling_available(model_type: str) -> bool:
+ """Whether the active transformers ships a native implementation of *model_type*
+ (``models/<type>/modeling_*.py`` on disk), found without importing transformers."""
+ models_dir = _transformers_models_dir()
+ if models_dir is None:
+ return False
+ key = (str(models_dir), model_type)
+ if key not in _NATIVE_MODELING_CACHE:
+ try:
+ _NATIVE_MODELING_CACHE[key] = any((models_dir / model_type).glob("modeling_*.py"))
+ except OSError:
+ _NATIVE_MODELING_CACHE[key] = False
+ return _NATIVE_MODELING_CACHE[key]
+
+
+def _imports_mamba_ssm_unconditionally(source: str) -> bool:
+ """Whether module *source* imports ``mamba_ssm`` at import time with no way around it: a
+ top-level import, or one in a ``try`` whose handler raises. One under an ``if`` (an
+ ``is_mamba_2_ssm_available()`` gate) or with a fallback handler does not count. Unparseable
+ source counts: it cannot be shown safe."""
+ import ast
+
+ try:
+ tree = ast.parse(source)
+ except (SyntaxError, ValueError):
+ return True
+
+ def imports_mamba(statements) -> bool:
+ for node in statements:
+ if isinstance(node, ast.Import) and any(
+ alias.name.split(".")[0] == "mamba_ssm" for alias in node.names
+ ):
+ return True
+ if (
+ isinstance(node, ast.ImportFrom)
+ and (node.module or "").split(".")[0] == "mamba_ssm"
+ ):
+ return True
+ return False
+
+ for node in tree.body:
+ if imports_mamba([node]):
+ return True
+ if isinstance(node, ast.Try) and imports_mamba(node.body):
+ if any(
+ any(isinstance(inner, ast.Raise) for inner in ast.walk(handler))
+ for handler in node.handlers
+ ):
+ return True
+ return False
+
+
+_REMOTE_SOURCES: dict[tuple[str, str, str | None], str | None] = {}
+
+
+def _remote_source_text(repo: str, filename: str, hf_token: str | None) -> str | None:
+ """The text of *filename* in a local checkpoint or a Hub repo (the local HF cache first, then
+ the Hub unless offline), or None when it cannot be read."""
+ key = (repo, filename, hf_token)
+ if key in _REMOTE_SOURCES:
+ return _REMOTE_SOURCES[key]
+ text = None
+ try:
+ if os.path.isdir(repo):
+ path = Path(repo) / filename
+ text = path.read_text(encoding = "utf-8") if path.is_file() else None
+ elif repo.count("/") == 1:
+ text = _hub_cached_text(repo, filename)
+ from utils.utils import hf_env_offline
+ if text is None and not hf_env_offline():
+ import urllib.request
+
+ from utils.transformers_version import _hf_raw_url, _hf_urlopen
+
+ headers = {"User-Agent": "unsloth-studio"}
+ if isinstance(hf_token, str) and hf_token:
+ headers["Authorization"] = f"Bearer {hf_token}"
+ request = urllib.request.Request(_hf_raw_url(repo, filename), headers = headers)
+ with _hf_urlopen(request, timeout = 10) as response:
+ text = response.read().decode("utf-8", errors = "replace")
+ except Exception as exc:
+ logger.debug("Could not read %s from %s: %s", filename, repo, exc)
+ text = None
+ _REMOTE_SOURCES[key] = text
+ return text
+
+
+def _hub_cached_text(repo: str, filename: str) -> str | None:
+ try:
+ from utils.hf_cache_settings import get_hf_cache_paths
+
+ repo_dir = Path(str(get_hf_cache_paths().hub_cache)) / (
+ "models--" + repo.replace("/", "--")
+ )
+ ref = repo_dir / "refs" / "main"
+ candidates = []
+ if ref.is_file():
+ candidates.append(
+ repo_dir / "snapshots" / ref.read_text(encoding = "utf-8").strip() / filename
+ )
+ candidates += sorted(
+ repo_dir.glob(f"snapshots/*/{filename}"), key = lambda p: p.stat().st_mtime, reverse = True
+ )
+ for path in candidates:
+ if path.is_file():
+ return path.read_text(encoding = "utf-8")
+ except Exception as exc:
+ logger.debug("HF cache lookup of %s in %s failed: %s", filename, repo, exc)
+ return None
+
+
+def _remote_model_sources(checkpoint: str, model_config: dict) -> list[tuple[str, str]]:
+ """(repo, file) of the model classes the config's ``auto_map`` names."""
+ auto_map = model_config.get("auto_map") if isinstance(model_config, dict) else None
+ if not isinstance(auto_map, dict):
+ return []
+ sources = []
+ for key, value in auto_map.items():
+ if not str(key).startswith("AutoModel"):
+ continue
+ for reference in value if isinstance(value, (list, tuple)) else (value,):
+ if not isinstance(reference, str) or "." not in reference:
+ continue
+ source, _, reference = reference.rpartition("--")
+ module = reference.rsplit(".", 1)[0]
+ sources.append((source or checkpoint, module.replace(".", "/") + ".py"))
+ return list(dict.fromkeys(sources))
+
+
+def mamba_ssm_is_fatal(
+ checkpoint: str,
+ model_config: dict | None,
+ model_types,
+ *,
+ trust_remote_code: bool | None,
+ hf_token: str | None = None,
+) -> bool:
+ """Whether loading this required-family model (Nemotron-H, Falcon-H1, Granite-4.0-H) imports
+ code that cannot run without mamba_ssm. Remote code (auto_map, loaded under trust_remote_code
+ or because transformers has no native class) counts when a model module imports it
+ unconditionally; otherwise it is fatal only when transformers lacks the native model."""
+ required_types = {t for t in model_types if t in REQUIRED_SSM_MODEL_TYPES}
+ native = bool(required_types) and all(native_modeling_available(t) for t in required_types)
+ sources = _remote_model_sources(checkpoint, model_config or {})
+ if sources and (trust_remote_code or not native):
+ for repo, filename in sources:
+ text = _remote_source_text(repo, filename, hf_token)
+ if text is None or _imports_mamba_ssm_unconditionally(text):
+ return True
+ return False
+ return not native
+
+
+def _name_mamba_ssm_is_fatal(model_name: str) -> bool:
+ """mamba_ssm_is_fatal from the name alone: the family's native model is missing."""
+ name = (model_name or "").lower()
+ types = {model_type for sub, model_type in _REQUIRED_MODEL_TYPE_BY_SUBSTRING if sub in name}
+ return bool(types) and not all(native_modeling_available(t) for t in types)
+
+
+def model_config_wants_causal_conv1d(model_config: dict) -> bool | None:
model_types, architectures = _config_identifiers(model_config)
if model_config_is_ssm(model_config):
return True
- if model_types and all(_is_zoo_gdn_model_type(model_type) for model_type in model_types):
- return not zoo_triton_causal_conv1d_usable()
- source_requirements = (
- {_transformers_model_type_uses_causal_conv1d(model_type) for model_type in model_types}
- if scan_modeling_source
- else set()
- )
+ source_requirements = {
+ _transformers_model_type_uses_causal_conv1d(model_type) for model_type in model_types
+ }
if True in source_requirements:
return True
config_identifiers = model_types | architectures
- normalized = {_normalized_model_identifier(identifier) for identifier in config_identifiers}
-
- def _matches(substrings: tuple[str, ...]) -> bool:
- needles = {_normalized_model_identifier(value) for value in substrings}
- return any(needle in identifier for identifier in normalized for needle in needles)
-
- if _matches(CAUSAL_CONV1D_MODEL_SUBSTRINGS):
- return True
- if _matches(ZOO_GDN_MODEL_SUBSTRINGS) and not zoo_triton_causal_conv1d_usable():
+ normalized_needles = {
+ _normalized_model_identifier(value) for value in CAUSAL_CONV1D_MODEL_SUBSTRINGS
+ }
+ if any(
+ needle in _normalized_model_identifier(identifier)
+ for identifier in config_identifiers
+ for needle in normalized_needles
+ ):
return True
if False in source_requirements:
return False
return None
+def _local_adapter_base(model_load_target: str) -> str | None:
+ """A local PEFT adapter folder's base model (``adapter_config.json`` without a ``config.json``)."""
+ try:
+ folder = Path(model_load_target)
+ if not folder.is_dir() or (folder / "config.json").is_file():
+ return None
+ adapter_config = folder / "adapter_config.json"
+ if not adapter_config.is_file():
+ return None
+ import json
+
+ base = json.loads(adapter_config.read_text(encoding = "utf-8-sig")).get(
+ "base_model_name_or_path"
+ )
+ except Exception as exc:
+ logger.debug("Could not read the adapter base of %s: %s", model_load_target, exc)
+ return None
+ return base if isinstance(base, str) and base.strip() else None
+
+
+def _dependency_target(model_load_target: str, adapter_base: str | None) -> str:
+ """What decides the kernels: an adapter's (pinned, else recorded) base, else the target itself.
+ The adapter stays the load target; only the dependency decision moves to its base."""
+ if adapter_base:
+ return adapter_base
+ return _local_adapter_base(model_load_target) or model_load_target
+
+
+# Configs read for the kernel decision, per process: a load's preflight, its _handle_load and each
+# kernel share one read (a cold Hub read can take seconds).
+_MODEL_CONFIGS: dict[tuple[str, str | None], dict] = {}
+
+
def _resolved_model_config(model_load_target: str, hf_token: str | None) -> dict | None:
+ key = (str(model_load_target), hf_token)
+ if key in _MODEL_CONFIGS:
+ return _MODEL_CONFIGS[key]
try:
from utils.transformers_version import _load_config_json
model_config = _load_config_json(model_load_target, hf_token)
except Exception as exc:
logger.debug("Could not inspect model config for SSM kernels: %s", exc)
- return None
- return model_config if isinstance(model_config, dict) else None
+ model_config = None
+ if not isinstance(model_config, dict):
+ return None # a miss may be transient: read again next time
+ _MODEL_CONFIGS[key] = model_config
+ return model_config
-def resolved_model_wants_causal_conv1d(
+def resolve_ssm_kernels(
model_name: str,
model_load_target: str,
hf_token: str | None,
*,
- scan_modeling_source: bool = True,
-) -> bool:
- model_config = _resolved_model_config(model_load_target, hf_token)
+ adapter_base: str | None = None,
+ model_config: dict | None = None,
+ trust_remote_code: bool | None = None,
+) -> SsmKernels:
+ """The kernels the model at *model_load_target* needs, from its config (an adapter's base
+ config when it is an adapter); the name lists only when there is no config to read. A caller
+ that already read the deciding config passes it as *model_config*."""
+ target = _dependency_target(model_load_target, adapter_base)
+ probe = target if target != model_load_target else model_name
+ if model_config is None:
+ model_config = _resolved_model_config(target, hf_token)
+ model_types: frozenset[str] = frozenset()
+ is_ssm = required = wants = None
+ remote_code: frozenset = frozenset()
if model_config is not None:
- requirement = model_config_wants_causal_conv1d(
- model_config, scan_modeling_source = scan_modeling_source
+ remote_code = remote_code_modules(target, model_config)
+ model_types = frozenset(
+ _base_model_type(model_type) for model_type in _config_identifiers(model_config)[0]
+ )
+ is_ssm = model_config_is_ssm(model_config)
+ required = model_config_requires_mamba_ssm(model_config)
+ wants = model_config_wants_causal_conv1d(model_config)
+ logger.info(
+ "SSM kernels resolved from model architecture: causal-conv1d=%s mamba-ssm=%s",
+ wants,
+ is_ssm,
)
- if requirement is not None:
- logger.info(
- "causal-conv1d requirement resolved from model architecture: %s",
- requirement,
+ if is_ssm is None:
+ is_ssm = model_is_ssm(probe)
+ if required is None:
+ required = model_requires_mamba_ssm(probe)
+ if wants is None:
+ wants = model_wants_causal_conv1d(probe)
+ fatal = False
+ if is_ssm and required:
+ if model_config is not None and model_types:
+ fatal = mamba_ssm_is_fatal(
+ target,
+ model_config,
+ model_types,
+ trust_remote_code = trust_remote_code,
+ hf_token = hf_token,
)
- return requirement
- return model_wants_causal_conv1d(model_name)
+ else:
+ fatal = _name_mamba_ssm_is_fatal(probe)
+ # Every mamba-ssm model also runs causal-conv1d.
+ return SsmKernels(
+ causal_conv1d = bool(wants or is_ssm),
+ mamba_ssm = bool(is_ssm),
+ mamba_ssm_required = bool(is_ssm and required),
+ probe = probe,
+ model_types = model_types,
+ remote_code = remote_code,
+ mamba_ssm_fatal = bool(fatal),
+ )
-def resolved_model_is_ssm(model_name: str, model_load_target: str, hf_token: str | None) -> bool:
- """Whether the model at *model_load_target* needs ``mamba_ssm``, from its config; the name list
- only when there is no config to read."""
- model_config = _resolved_model_config(model_load_target, hf_token)
- if model_config is not None:
- requirement = model_config_is_ssm(model_config)
- if requirement is not None:
- logger.info("mamba-ssm requirement resolved from model architecture: %s", requirement)
- return requirement
- return model_is_ssm(model_name)
+def resolved_model_wants_causal_conv1d(
+ model_name: str,
+ model_load_target: str,
+ hf_token: str | None,
+ *,
+ adapter_base: str | None = None,
+) -> bool:
+ return resolve_ssm_kernels(
+ model_name, model_load_target, hf_token, adapter_base = adapter_base
+ ).causal_conv1d
+
+
+def resolved_model_is_ssm(
+ model_name: str,
+ model_load_target: str,
+ hf_token: str | None,
+ *,
+ adapter_base: str | None = None,
+) -> bool:
+ """Whether the model at *model_load_target* runs ``mamba_ssm``, from its (adapter's base) config;
+ the name list only when there is no config to read."""
+ return resolve_ssm_kernels(
+ model_name, model_load_target, hf_token, adapter_base = adapter_base
+ ).mamba_ssm
def refresh_transformers_kernel_gates() -> None:
"""After an in-process install, let an already-imported transformers see the new kernels: its
- availability gates are lru_cached and the hub-kernel map can hold a cached None. Never raises."""
+ availability gates are lru_cached and the hub-kernel map can hold a cached None. Never raises.
+ This cannot rebind functions a modeling module already captured; see
+ ``modeling_modules_needing_restart``."""
if "transformers" not in sys.modules:
return
try:
@@ -342,6 +603,87 @@ def refresh_transformers_kernel_gates() -> None:
logger.debug("Could not refresh transformers' hub kernel map: %s", exc)
+def _sanitized_module_name(name: str) -> str:
+ # transformers.dynamic_module_utils._sanitize_module_name (4.57+, 5.x).
+ name = name.replace(".", "_dot_").replace("-", "_hyphen_")
+ return f"_{name}" if name and name[0].isdigit() else name
+
+
+def _dynamic_module_namespaces(checkpoint: str) -> set[tuple[str, ...]]:
+ """The transformers_modules package(s) transformers puts *checkpoint*'s remote code under: a
+ Hub repo by its id (org, name), a local folder by its name. Sanitized as 4.57+ and 5.x spell
+ them, and raw as older 4.x did."""
+ checkpoint = str(checkpoint).strip()
+ if not checkpoint:
+ return set()
+ if os.path.isdir(checkpoint):
+ parts = [os.path.basename(os.path.normpath(checkpoint))]
+ else:
+ parts = checkpoint.split("/")
+ if not all(parts):
+ return set()
+ return {tuple(parts), tuple(_sanitized_module_name(part) for part in parts)}
+
+
+def remote_code_modules(checkpoint: str, model_config: dict) -> frozenset:
+ """The (namespace, module) pairs of the model classes a config's ``auto_map`` points at: an
+ entry ``"model.CustomForCausalLM"`` is module ``model`` of *checkpoint*, and
+ ``"org/repo--model.Custom"`` module ``model`` of ``org/repo``. Any module name counts, not just
+ ``modeling_*``; AutoConfig / AutoTokenizer / processor entries capture no kernels."""
+ auto_map = model_config.get("auto_map") if isinstance(model_config, dict) else None
+ if not isinstance(auto_map, dict):
+ return frozenset()
+ pairs = set()
+ for key, value in auto_map.items():
+ if not str(key).startswith("AutoModel"):
+ continue
+ for reference in value if isinstance(value, (list, tuple)) else (value,):
+ if not isinstance(reference, str) or "." not in reference:
+ continue
+ source, _, reference = reference.rpartition("--")
+ module = tuple(reference.rsplit(".", 1)[0].split("."))
+ for namespace in _dynamic_module_namespaces(source or checkpoint):
+ pairs.add((namespace, module))
+ return frozenset(pairs)
+
+
+def _is_remote_code_module(name: str, remote_code) -> bool:
+ # transformers_modules.<namespace>[.<commit or source hash>].<module>, matched exactly.
+ parts = tuple(name.split("."))
+ if parts[0] != "transformers_modules":
+ return False
+ for namespace, module in remote_code:
+ if parts[1 : 1 + len(namespace)] != namespace:
+ continue
+ rest = parts[1 + len(namespace) :]
+ if rest == module or (len(rest) == len(module) + 1 and rest[1:] == module):
+ return True
+ return False
+
+
+def modeling_modules_needing_restart(model_types, remote_code = ()) -> list[str]:
+ """The already-imported implementation modules of *model_types* (``transformers.models.<type>.
+ modeling_*``) and of the selected checkpoint's remote code (*remote_code*, from
+ :func:`remote_code_modules`). Each captured its kernel functions (module globals, hub-kernel
+ decorators) when it was imported, so a kernel installed after that is only picked up by a fresh
+ process. A configuration or tokenizer module captures none, so importing one needs no restart."""
+ model_types = {model_type for model_type in model_types if model_type}
+ stale = []
+ for name in list(sys.modules):
+ parts = name.split(".")
+ if parts[0] == "transformers":
+ if (
+ len(parts) == 4
+ and parts[1] == "models"
+ and parts[2] in model_types
+ and parts[3].startswith("modeling_")
+ ):
+ stale.append(name)
+ elif remote_code and _is_remote_code_module(name, remote_code):
+ stale.append(name)
+ return sorted(stale)
+
+
def ssm_probe_identifier(model_name: str, base: str | None = None) -> str:
"""The identifier whose architecture decides the SSM kernels. The substring match needs a real model id: a LoRA adapter id or a local checkpoint's parent folders are unrelated to its architecture (a Llama LoRA at ``user/falcon-h1-lora`` is not SSM). Prefer *base*; for a bare local checkpoint use its basename."""
probe = base or model_name
@@ -412,6 +754,81 @@ def _run_with_heartbeat(run, cmd, status_cb, display_name, **kwargs):
return run(cmd, **kwargs)
+# The torch build this process installs kernels for, probed once (one torch import in a child).
+_TORCH_ENV: dict = {}
+# Kernels whose install already failed in this process, keyed by kernel, torch and toolchain, so a
+# later load stage (or the next load in the same worker) does not download or compile them again.
+_FAILED_INSTALLS: set = set()
+
+
+def _torch_env() -> dict[str, str] | None:
+ key = probe_torch_wheel_env
+ if key not in _TORCH_ENV:
+ _TORCH_ENV[key] = probe_torch_wheel_env(timeout = 30, include_windows = True)
+ return _TORCH_ENV[key]
+
+
+def _kernel_backend(env: dict[str, str] | None) -> str | None:
+ """ "cuda" or "hip" for a torch these kernels build / ship for; None for CPU-only, XPU, MPS,
+ or a torch the probe could not read, which take the model's torch path instead."""
+ if not env:
+ return None
+ if env.get("hip_version"):
+ return "hip"
+ if env.get("cuda_version") or env.get("cuda_major"):
+ return "cuda"
+ return None
+
+
+def _failure_key(import_name: str) -> tuple:
+ try:
+ import importlib.metadata as metadata
+ torch_version = metadata.version("torch")
+ except Exception:
+ torch_version = None
+ return (
+ import_name,
+ sys.executable,
+ torch_version,
+ shutil.which("nvcc"),
+ shutil.which("hipcc"),
+ os.environ.get("CUDA_HOME") or os.environ.get("CUDA_PATH"),
+ os.environ.get("ROCM_PATH"),
+ )
+
+
+def _installed_distribution(pypi_name: str) -> bool:
+ try:
+ import importlib.metadata as metadata
+ metadata.version(pypi_name)
+ return True
+ except Exception:
+ return False
+
+
+def _remove_broken(pypi_name: str, display_name: str, run: Callable[..., Any]) -> None:
+ """Uninstall a distribution whose extension does not import, as the training worker does: a
+ later availability check would otherwise find it and fail instead of taking the torch path."""
+ if not _installed_distribution(pypi_name):
+ return
+ try:
+ removed = (
+ run(
+ uninstall_command(pypi_name, use_uv = bool(shutil.which("uv"))),
+ stdout = subprocess.DEVNULL,
+ stderr = subprocess.DEVNULL,
+ ).returncode
+ == 0
+ )
+ except Exception:
+ removed = False
+ logger.warning(
+ "%s does not import; %s",
+ display_name,
+ "removed it" if removed else f"uninstall {pypi_name} manually",
+ )
+
+
def _install_kernel(
*,
import_name: str,
@@ -422,19 +839,107 @@ def _install_kernel(
release_base_url: str,
status_cb: StatusCb,
run: Callable[..., Any],
+ newly_installed: list[str] | None = None,
+ allow_source_build: bool = True,
+ required: bool = False,
) -> bool:
- """Install one kernel wheel-first, then a HIP-aware PyPI source build (never on Windows). Returns True iff importable afterwards; idempotent (no-op when already installed)."""
+ """Install one kernel wheel-first, then (when *allow_source_build*) a HIP-aware PyPI source build
+ (never on Windows). Returns True iff importable afterwards; idempotent (no-op when already
+ installed). An install this call made is appended to *newly_installed*. One install per kernel
+ runs at a time across processes; a failed one is not retried in this process."""
if _is_importable(import_name):
logger.info("%s already installed", display_name)
return True
- from utils.utils import hf_env_offline
+ from utils.utils import package_installs_offline
- if hf_env_offline():
+ if package_installs_offline():
+ # No URL probe, no source build, and no uv-to-pip fallback that would reach the network.
logger.info("Skipping %s installation while offline", display_name)
return False
- env = probe_torch_wheel_env(timeout = 30, include_windows = True)
+ failure_key = _failure_key(import_name)
+ if failure_key in _FAILED_INSTALLS:
+ logger.info("%s already failed to install in this process; not retrying", display_name)
+ return False
+
+ from utils.kernel_install import (
+ KERNEL_LOCK_WAIT,
+ OPTIONAL_KERNEL_LOCK_WAIT,
+ kernel_install_lock,
+ )
+
+ with kernel_install_lock(
+ import_name,
+ on_wait = lambda: _emit(
+ status_cb, f"Waiting for another process installing {display_name}..."
+ ),
+ # An optional acceleration does not hold a load for a peer's half-hour build.
+ wait_seconds = KERNEL_LOCK_WAIT if required else OPTIONAL_KERNEL_LOCK_WAIT,
+ ) as held:
+ if not held:
+ _emit(
+ status_cb,
+ f"Could not take the {display_name} install lock (another process is installing "
+ "it, or the lock is unavailable); continuing without it.",
+ )
+ return False
+ # Another process may have installed it while this one waited: new to this worker all the
+ # same, so its already-imported modeling modules are refreshed or restarted.
+ if _is_importable(import_name):
+ logger.info("%s installed by another process", display_name)
+ if newly_installed is not None:
+ newly_installed.append(import_name)
+ return True
+ installed: bool | None = False
+ try:
+ installed = _install_kernel_locked(
+ import_name = import_name,
+ display_name = display_name,
+ pypi_name = pypi_name,
+ package_version = package_version,
+ release_tag = release_tag,
+ release_base_url = release_base_url,
+ status_cb = status_cb,
+ run = run,
+ allow_source_build = allow_source_build,
+ )
+ except Exception as exc:
+ # An installer that cannot even start (OSError) fails like any other attempt.
+ logger.warning("Installing %s failed: %s", display_name, exc, exc_info = True)
+ installed = False
+ finally:
+ if installed is False:
+ _remove_broken(pypi_name, display_name, run)
+ _FAILED_INSTALLS.add(failure_key)
+ if installed is None:
+ return False # nothing attempted (unsupported torch backend)
+ if installed and newly_installed is not None:
+ newly_installed.append(import_name)
+ return bool(installed)
+
+
+def _install_kernel_locked(
+ *,
+ import_name: str,
+ display_name: str,
+ pypi_name: str,
+ package_version: str,
+ release_tag: str,
+ release_base_url: str,
+ status_cb: StatusCb,
+ run: Callable[..., Any],
+ allow_source_build: bool,
+) -> bool | None:
+ """The install itself, under the kernel's lock. None when nothing was attempted."""
+ env = _torch_env()
+ backend = _kernel_backend(env)
+ if backend is None:
+ _emit(
+ status_cb,
+ f"{display_name} needs a CUDA or ROCm build of torch; using the model's torch path.",
+ )
+ return None
wheel_only = sys.platform == "win32" or is_windows_wheel_env(env)
wheel_url = direct_wheel_url(
filename_prefix = import_name,
@@ -444,7 +949,14 @@ def _install_kernel(
env = env,
)
wheel_available = url_exists(wheel_url) if wheel_url else False
- if wheel_available:
+ # Only a 404 proves nothing is published: an unanswered HEAD (a proxy refusing HEAD but serving
+ # the download) still tries the wheel, as setup always did.
+ if wheel_url and wheel_available is not False:
+ if wheel_available is None:
+ _emit(
+ status_cb,
+ f"Could not check the {display_name} prebuilt wheel; trying to download it.",
+ )
_emit(status_cb, f"Installing {display_name} (prebuilt kernel) for this model...")
# Keep quiet downloads and unpacks within the inactivity deadline (#9398).
with _heartbeat(
@@ -465,53 +977,43 @@ def _install_kernel(
use_uv = bool(shutil.which("uv")),
run = run,
)
- if outcome == "installed":
- logger.info("Installed prebuilt %s wheel", display_name)
- return True
- if outcome == "rejected":
- if wheel_only:
- removed = (
- run(
- uninstall_command(pypi_name, use_uv = bool(shutil.which("uv"))),
- stdout = subprocess.DEVNULL,
- stderr = subprocess.DEVNULL,
- ).returncode
- == 0
- )
- logger.warning(
- "%s wheel installed but not importable; %s",
- display_name,
- "removed it" if removed else f"uninstall {pypi_name} manually",
- )
- return False
- logger.warning(
- "%s wheel installed but not importable; building from source",
- display_name,
- )
+ if outcome == "installed":
+ logger.info("Installed prebuilt %s wheel", display_name)
+ return True
+ if outcome == "rejected":
+ logger.warning("%s wheel installed but not importable", display_name)
+ # Remove it before any rebuild: pip / uv would call the broken copy "satisfied".
+ _remove_broken(pypi_name, display_name, run)
elif wheel_only:
logger.info(
"No prebuilt %s wheel for this Windows environment (%s); using the torch fallback",
display_name,
wheel_url,
)
- elif wheel_available is None:
+ else:
+ logger.info("No prebuilt %s wheel for this environment (%s)", display_name, wheel_url)
+
+ if wheel_only:
+ return False
+ if not allow_source_build:
_emit(
status_cb,
- f"Could not check the {display_name} prebuilt wheel; building it from source.",
+ f"No usable prebuilt {display_name} for this torch; using the model's torch path.",
)
- else:
- logger.info(
- "No prebuilt %s wheel for this environment (%s); building from source",
- display_name,
- wheel_url,
+ return False
+ if not pinned_source_build_supported(env):
+ # The pinned sources build at C++17 and torch 2.13+ needs C++20: a certain failure after
+ # minutes of compiling, so only the prebuilt wheel serves this torch.
+ _emit(
+ status_cb,
+ f"No prebuilt {display_name} for torch {env.get('torch_mm')} on this machine, and it "
+ "cannot be built from source for it; using the model's torch path.",
)
-
- if wheel_only:
return False
# Source build (slow). ROCm has no prebuilt wheel and needs hipcc + a gcc-install-dir shim.
spec = f"{pypi_name}=={package_version}"
- is_hip = bool((env or {}).get("hip_version"))
+ is_hip = backend == "hip"
if is_hip and not shutil.which("hipcc"):
_emit(status_cb, f"{display_name}: hipcc not found; install the ROCm HIP SDK to build it.")
return False
@@ -522,8 +1024,16 @@ def _install_kernel(
# Reinstall so the source build replaces a broken wheel instead of no-opping as "already satisfied".
cmd = source_build_command(spec, use_uv = bool(shutil.which("uv")), is_hip = is_hip, reinstall = True)
run_kwargs, _ = source_build_run_kwargs(is_hip = is_hip, gcc_install_dir = _hipcc_gcc_install_dir)
+ from utils.kernel_install import run_process_tree
+
try:
- result = _run_with_heartbeat(run, cmd, status_cb, display_name, **run_kwargs)
+ result = _run_with_heartbeat(
+ lambda command, **kwargs: run_process_tree(command, run = run, **kwargs),
+ cmd,
+ status_cb,
+ display_name,
+ **run_kwargs,
+ )
except subprocess.TimeoutExpired:
logger.error("%s source build timed out", display_name)
_emit(status_cb, f"{display_name} source build timed out.")
@@ -552,32 +1062,148 @@ def ensure_ssm_runtime(
run: Callable[..., Any] = subprocess.run,
wants_causal_conv1d: bool | None = None,
is_ssm: bool | None = None,
-) -> None:
- """Install the SSM kernels *model_name* needs before load, wheel-first; a no-op for non-SSM models and idempotent. Only a true SSM hybrid's ``mamba_ssm`` is fatal (raises ``RuntimeError`` instead of a cryptic mid-load failure); ``causal_conv1d`` is best-effort, with LFM2 falling back to torch and Qwen3.5 / 3.6 / Qwen3-Next on zoo's Triton conv wherever it can run."""
+ mamba_ssm_required: bool | None = None,
+ mamba_ssm_fatal: bool | None = None,
+) -> list[str]:
+ """Install the SSM kernels *model_name* needs before load, wheel-first; a no-op for non-SSM models and idempotent. Returns the kernels newly installed by this call. Nemotron-H, Falcon-H1 and Granite-4.0-H (*mamba_ssm_required*) are worth a source build; a load fails on a missing ``mamba_ssm`` (``RuntimeError`` instead of a cryptic mid-load failure) only when *mamba_ssm_fatal*: it imports code that cannot run without it. Everything else, and ``causal_conv1d``, leaves transformers' torch path."""
# The caller's config-resolved answers win; the name match is the default.
if is_ssm is None:
is_ssm = model_is_ssm(model_name)
+ if mamba_ssm_required is None:
+ mamba_ssm_required = model_requires_mamba_ssm(model_name)
+ mamba_ssm_required = bool(is_ssm and mamba_ssm_required)
+ if mamba_ssm_fatal is None:
+ mamba_ssm_fatal = mamba_ssm_required and _name_mamba_ssm_is_fatal(model_name)
+ mamba_ssm_fatal = bool(mamba_ssm_required and mamba_ssm_fatal)
if wants_causal_conv1d is None:
wants_causal_conv1d = model_wants_causal_conv1d(model_name)
# Every mamba-ssm model also runs causal-conv1d.
wants_causal_conv1d = wants_causal_conv1d or is_ssm
+ installed: list[str] = []
if not (wants_causal_conv1d or is_ssm):
- return
+ return installed
+
+ def _install(kernel: PinnedKernel, *, allow_source_build: bool = True) -> bool:
+ return _install_kernel(
+ **_pinned_kwargs(kernel),
+ status_cb = status_cb,
+ run = run,
+ newly_installed = installed,
+ allow_source_build = allow_source_build,
+ # Nemotron-H, Falcon-H1, Granite-4.0-H wait out a peer's install; the rest move on.
+ required = mamba_ssm_required,
+ )
# causal-conv1d first: SSM modeling files lazy-import it, and mamba-ssm's fast path uses it.
- if wants_causal_conv1d and not _install_kernel(
- **_pinned_kwargs(CAUSAL_CONV1D), status_cb = status_cb, run = run
- ):
+ if wants_causal_conv1d and not _install(CAUSAL_CONV1D):
logger.warning("causal-conv1d unavailable; continuing on the model's torch fallback")
- if is_ssm and sys.platform == "win32" and importlib.util.find_spec("triton") is None:
+ if not is_ssm:
+ return installed
+ offline_hint = (
+ "If this machine is offline, run `unsloth install-kernels` once while online, "
+ "then reload the model."
+ )
+ if sys.platform == "win32" and importlib.util.find_spec("triton") is None:
# mamba_ssm imports triton at load and the wheel goes in --no-deps.
- raise RuntimeError(
- "mamba-ssm, required by this Mamba model, needs Triton on Windows "
- "(the triton-windows package); install it and reload the model."
- )
- if is_ssm and not _install_kernel(**_pinned_kwargs(MAMBA_SSM), status_cb = status_cb, run = run):
- raise RuntimeError(
- "Could not install mamba-ssm, required by this Mamba model. If this machine is offline, "
- "run `unsloth install-kernels` once while online, then reload the model."
+ if mamba_ssm_fatal:
+ raise RuntimeError(
+ "mamba-ssm, required by this Mamba model, needs Triton on Windows "
+ "(the triton-windows package); install it and reload the model."
+ )
+ _emit(
+ status_cb, "mamba-ssm needs Triton on Windows (triton-windows); using the torch path."
)
+ return installed
+ if not mamba_ssm_required:
+ # The families with a torch path get mamba-ssm only as a prebuilt wheel, and only where
+ # unsloth_zoo keeps its fast path on (setup's sm80 / sm75 + Triton 3.4 floor): never a
+ # source build, as before setup stopped installing it for every model.
+ reason = None if _is_importable(MAMBA_SSM.import_name) else _mamba_ssm_floor_reason()
+ if reason is not None:
+ _emit(status_cb, f"Skipping mamba-ssm, which {reason}; using the model's torch path.")
+ return installed
+ if not _install(MAMBA_SSM, allow_source_build = mamba_ssm_required):
+ if mamba_ssm_fatal:
+ # The model cannot load without mamba-ssm (its remote code imports it), so no torch
+ # path exists to fall back on. On a CPU / XPU torch no install could have helped, so
+ # say what is missing instead of pointing at the network.
+ if _kernel_backend(_torch_env()) is None:
+ raise RuntimeError(
+ "This Mamba model requires mamba-ssm, which needs a CUDA or ROCm torch build."
+ )
+ raise RuntimeError(
+ f"Could not install mamba-ssm, required by this Mamba model. {offline_hint}"
+ )
+ if mamba_ssm_required:
+ _emit(
+ status_cb,
+ "Warning: could not install mamba-ssm; loading this model on transformers' torch "
+ "path, which is slower and uses more memory.",
+ )
+ else:
+ _emit(status_cb, "mamba-ssm is not available; continuing on the model's torch path.")
+ return installed
+
+
+def _mamba_ssm_floor_reason() -> str | None:
+ """Why an optional mamba-ssm stays off this machine's GPU, or None when unsloth_zoo would use
+ it: setup's rule (kernel_install), from the torch already imported here when there is one."""
+ from utils import kernel_install
+
+ capability = None
+ torch = sys.modules.get("torch")
+ if torch is not None:
+ try:
+ if getattr(torch.version, "hip", None):
+ return None
+ if torch.cuda.is_available() and torch.cuda.device_count():
+ capability = max(
+ tuple(torch.cuda.get_device_capability(i))
+ for i in range(torch.cuda.device_count())
+ )
+ except Exception:
+ capability = None
+ if capability is None:
+ capability = kernel_install._gpu_capability(subprocess.run)
+ if capability is None:
+ return "needs a visible GPU"
+ if capability >= (8, 0):
+ return None
+ return kernel_install._pre_ampere_skip_reason("mamba_ssm", capability, subprocess.run)
+
+
+def provision_ssm_kernels(
+ model_name: str,
+ model_load_target: str,
+ hf_token: str | None,
+ *,
+ adapter_base: str | None = None,
+ model_config: dict | None = None,
+ trust_remote_code: bool | None = None,
+ status_cb: StatusCb = None,
+ run: Callable[..., Any] = subprocess.run,
+) -> tuple[SsmKernels, list[str]]:
+ """Resolve and install the kernels a load needs (training, inference, export): returns what was
+ resolved and what this call newly installed. Raises ``RuntimeError`` only for a model that
+ cannot load without ``mamba_ssm``."""
+ kernels = resolve_ssm_kernels(
+ model_name,
+ model_load_target,
+ hf_token,
+ adapter_base = adapter_base,
+ model_config = model_config,
+ trust_remote_code = trust_remote_code,
+ )
+ installed = ensure_ssm_runtime(
+ kernels.probe,
+ status_cb = status_cb,
+ run = run,
+ wants_causal_conv1d = kernels.causal_conv1d,
+ is_ssm = kernels.mamba_ssm,
+ mamba_ssm_required = kernels.mamba_ssm_required,
+ mamba_ssm_fatal = kernels.mamba_ssm_fatal,
+ )
+ if installed:
+ refresh_transformers_kernel_gates()
+ return kernels, installed
diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py
index e364806771e..f4be7df84ff 100644
--- a/studio/backend/utils/utils.py
+++ b/studio/backend/utils/utils.py
@@ -40,6 +40,22 @@ def hf_env_offline() -> bool:
return False
+def package_installs_offline() -> bool:
+ """True when an on-demand package install (a kernel wheel, a source build) must not reach the
+ network: the HF offline switches above, or UV_OFFLINE, the operator's no-network switch for
+ installs. Under UV_OFFLINE uv refuses the index, and install_wheel's pip fallback would then
+ reach it anyway, so such an install is skipped outright.
+
+ An HF offline that Studio set itself only because the Hub was unreachable (export's offline
+ window marks it, wheel_utils.hf_offline_auto_set) says nothing about GitHub / PyPI, so then
+ only UV_OFFLINE counts: an operator-set HF offline still blocks every install."""
+ from utils.wheel_utils import hf_offline_auto_set, uv_offline
+
+ if hf_offline_auto_set():
+ return uv_offline()
+ return hf_env_offline() or uv_offline()
+
+
def anonymous_and_offline(hf_token, *, repo_id: Optional[str] = None) -> bool:
"""The one condition under which a Hub-reaching request can only be answered by disk.
diff --git a/studio/backend/utils/wheel_utils.py b/studio/backend/utils/wheel_utils.py
index a1988d2b300..3c09dee3287 100644
--- a/studio/backend/utils/wheel_utils.py
+++ b/studio/backend/utils/wheel_utils.py
@@ -131,12 +131,12 @@ _UNSLOTH_PREBUILT_VERSIONS = {
"mamba_ssm": "2.3.2.post1",
}
-# platform_tag -> (cxx11abi field, published interpreters or None for any). The Windows cu130 torch also reports cxx11abi True; Windows publishes cp313 GIL builds only, and so does linux_aarch64 (upstream's aarch64 wheels stop at torch 2.10).
+# platform_tag -> (cxx11abi field, published interpreters or None for any). The Windows cu130 torch also reports cxx11abi True. Only what the release actually carries: torch 2.13 / 2.14 are published for cp313 GIL builds on every platform (the workflow can build cp311 / cp312 cells, but none has been published, and a URL for one is a 404 in front of the fallback).
_UNSLOTH_PREBUILT_PLATFORMS: dict[str, tuple[str, frozenset[str] | None]] = {
- "linux_x86_64": ("TRUE", None),
+ "linux_x86_64": ("TRUE", frozenset({"cp313"})),
"win_amd64": ("TRUE", frozenset({"cp313"})),
"linux_aarch64": ("TRUE", frozenset({"cp313"})),
- "win_arm64": ("TRUE", frozenset({"cp311", "cp312", "cp313"})),
+ "win_arm64": ("TRUE", frozenset({"cp313"})),
}
# (platform, torch minor) cells outside _UNSLOTH_PREBUILT_TORCH_MM, with their interpreters. Upstream's torch2.10 wheels (and their 2.11 / 2.12 reuse above) have no Windows asset, so win_amd64 torch 2.10, 2.11 and 2.12 are ours; an existing install keeps its minor (a fresh one takes 2.13 on cp313), hence three interpreters.
@@ -250,6 +250,56 @@ def unsloth_prebuilt_wheel_url(*, filename_prefix: str, env: dict[str, str] | No
return f"{UNSLOTH_PREBUILT_RELEASE_BASE_URL}/{UNSLOTH_PREBUILT_RELEASE_TAG}/{filename}"
+# The cells the pinned upstream releases publish (causal-conv1d v1.6.1.post4, mamba-ssm v2.3.1,
+# identical for both): (platform, CUDA major, cxx11abi) -> torch minors, each for cp310-cp313.
+# Read from the two release asset lists; anything else (CUDA 13 before torch 2.9, aarch64 before
+# 2.7, an abiFALSE torch after 2.6, any interpreter outside cp310-cp313) is a URL that 404s in
+# front of the source fallback, so it resolves to nothing instead.
+_UPSTREAM_SSM_PINS = {
+ "causal_conv1d": CAUSAL_CONV1D_PACKAGE_VERSION,
+ "mamba_ssm": MAMBA_SSM_PACKAGE_VERSION,
+}
+_UPSTREAM_SSM_CELLS: dict[tuple[str, str, str], frozenset[str]] = {
+ ("linux_x86_64", "11", "FALSE"): frozenset({"2.6"}),
+ ("linux_x86_64", "11", "TRUE"): frozenset({"2.6", "2.7"}),
+ ("linux_x86_64", "12", "FALSE"): frozenset({"2.6"}),
+ ("linux_x86_64", "12", "TRUE"): frozenset({"2.6", "2.7", "2.8", "2.9", "2.10"}),
+ ("linux_x86_64", "13", "TRUE"): frozenset({"2.9", "2.10"}),
+ ("linux_aarch64", "12", "TRUE"): frozenset({"2.7", "2.8", "2.9", "2.10"}),
+ ("linux_aarch64", "13", "TRUE"): frozenset({"2.9", "2.10"}),
+}
+_UPSTREAM_SSM_PYTHONS = frozenset({"cp310", "cp311", "cp312", "cp313"})
+
+
+def _upstream_ssm_cell_published(filename_prefix: str, package_version: str, env: dict) -> bool:
+ if _UPSTREAM_SSM_PINS.get(filename_prefix) != package_version:
+ return True # not a pinned release this table describes: unchanged
+ torch_mm = prebuilt_wheel_torch_mm(str(env.get("torch_mm")))
+ cell = (str(env.get("platform_tag")), str(env.get("cuda_major")), str(env.get("cxx11abi")))
+ python_tag = env.get("python_tag")
+ return (
+ torch_mm in _UPSTREAM_SSM_CELLS.get(cell, frozenset())
+ and python_tag in _UPSTREAM_SSM_PYTHONS
+ and (env.get("abi_tag") or python_tag) == python_tag
+ )
+
+
+# The newest torch the pinned causal-conv1d 1.6.1 / mamba-ssm 2.3.1 compile against: torch 2.13
+# raised its headers to C++20 and both build at C++17 (the prebuilt workflow moved to
+# causal-conv1d 1.7.0 and patches mamba-ssm for it, see .github/scripts/patch_mamba_cxx20.py).
+PINNED_SOURCE_BUILD_MAX_TORCH = (2, 12)
+
+
+def pinned_source_build_supported(env: dict[str, str] | None) -> bool:
+ """Whether a source build of the pinned causal-conv1d / mamba-ssm (or their upstream wheels)
+ can work with this torch; from torch 2.13 on only our prebuilt wheels can."""
+ try:
+ major, minor = (int(part) for part in str((env or {}).get("torch_mm") or "").split(".")[:2])
+ except ValueError:
+ return True # unknown: leave the decision to the build, as before
+ return (major, minor) <= PINNED_SOURCE_BUILD_MAX_TORCH
+
+
def direct_wheel_url(
*,
filename_prefix: str,
@@ -265,8 +315,10 @@ def direct_wheel_url(
ours = unsloth_prebuilt_wheel_url(filename_prefix = filename_prefix, env = env)
if ours is not None:
return ours
- # No upstream release has a Windows asset.
- if is_windows_wheel_env(env):
+ # No upstream release has a Windows asset, nor one for torch 2.13 or later (see above).
+ if is_windows_wheel_env(env) or not pinned_source_build_supported(env):
+ return None
+ if not _upstream_ssm_cell_published(filename_prefix, package_version, env):
return None
filename = (
@@ -478,6 +530,40 @@ def flash_attn_wheel_url(env: dict[str, str] | None) -> str | None:
)
+_UV_OFFLINE_TRUE_VALUES = frozenset({"1", "true", "t", "yes", "y", "on"})
+_HF_OFFLINE_TRUE_VALUES = frozenset({"1", "true", "yes", "on"})
+
+
+def uv_offline() -> bool:
+ """UV_OFFLINE, the operator's no-network switch for installs, in uv's boolish spellings."""
+ return os.environ.get("UV_OFFLINE", "").strip().lower() in _UV_OFFLINE_TRUE_VALUES
+
+
+# Set (to "1") while Studio itself forced HF offline only because the Hugging Face endpoint was
+# unreachable (export's _offline_window_if_unreachable), and restored when that window closes.
+# HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE then say nothing about GitHub or PyPI, so a kernel install
+# may still reach them; only UV_OFFLINE keeps it off the network. Children inherit it.
+HF_OFFLINE_AUTO_ENV = "_UNSLOTH_HF_OFFLINE_AUTO"
+
+
+def hf_offline_auto_set() -> bool:
+ """True while HF offline was set by Studio for an unreachable Hub, not by the operator."""
+ return os.environ.get(HF_OFFLINE_AUTO_ENV, "") == "1"
+
+
+def package_installs_offline_env() -> bool:
+ """The environment half of utils.utils.package_installs_offline, for callers that cannot import
+ the backend (`unsloth install-kernels` runs under --no-deps): HF_HUB_OFFLINE,
+ TRANSFORMERS_OFFLINE or UV_OFFLINE. While HF offline was only auto-set for an unreachable Hub
+ (hf_offline_auto_set), only UV_OFFLINE counts."""
+ if hf_offline_auto_set():
+ return uv_offline()
+ return uv_offline() or any(
+ os.environ.get(name, "").strip().lower() in _HF_OFFLINE_TRUE_VALUES
+ for name in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE")
+ )
+
+
def install_wheel(
wheel_url: str,
*,
@@ -510,6 +596,9 @@ def install_wheel(
attempts.append(("uv", result))
if result.returncode == 0:
return attempts
+ # uv refused the network under UV_OFFLINE; pip has no such switch and would reach it.
+ if uv_offline():
+ return attempts
pip_cmd = [python_executable, "-m", "pip", "install", "--no-deps"]
if reinstall:
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index 18bcf926963..8b615d0d7b7 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -52,7 +52,10 @@ def _nvidia_library_inventory():
from backend.utils.kernel_install import (
+ KERNELS,
+ gpu_below_flash_attn_floor,
install_prebuilt,
+ kernel_install_lock,
uninstall_command,
)
from backend.utils.wheel_utils import (
@@ -8580,6 +8583,12 @@ def _remove_stale_flash_attn() -> None:
)
+def _flash_attn_gpu_below_floor() -> bool:
+ """A visible CUDA GPU below sm80 (a T4), where Unsloth leaves FlashAttention 2 off. No visible
+ GPU is False: an image build installs ahead of the GPU it will run on."""
+ return gpu_below_flash_attn_floor()
+
+
def _ensure_flash_attn() -> None:
if _flash_attn_install_disabled():
return
@@ -8598,6 +8607,12 @@ def _ensure_flash_attn() -> None:
# a failed flash-attn probe proves staleness only when torch still imports.
stale = env is not None and _installed_distribution_version("flash-attn") is not None
wheel_url = _build_flash_attn_wheel_url(env) if env else None
+ if wheel_url and not (env or {}).get("hip_version") and _flash_attn_gpu_below_floor():
+ # Unsloth only turns FlashAttention 2 on from sm80; a T4 would download it for nothing.
+ _note("flash-attn needs an sm80 or newer GPU; skipped")
+ if stale:
+ _remove_stale_flash_attn()
+ return
wheel_available = url_exists(wheel_url) if wheel_url else False
if wheel_available:
# installing a distribution invalidates caches keyed on the action counter.
@@ -8654,6 +8669,120 @@ def _ensure_flash_attn() -> None:
_remove_stale_flash_attn()
+# The compiled extension each SSM kernel loads (kernel_install.KERNELS checks the same import).
+_SSM_KERNEL_EXTENSIONS = {"causal_conv1d": "causal_conv1d_cuda", "mamba_ssm": "selective_scan_cuda"}
+# Exit code of the probe below when torch imported and the extension raised ImportError / OSError.
+_SSM_EXTENSION_BROKEN_EXIT = 87
+_SSM_EXTENSION_PROBE = (
+ "import sys\n"
+ "import torch\n"
+ "try:\n"
+ " import {extension}\n"
+ "except (ImportError, OSError) as exc:\n"
+ " print('{extension}: %s: %s' % (type(exc).__name__, exc), file = sys.stderr)\n"
+ " sys.exit({code})\n"
+)
+
+
+def _probe_tail(stderr) -> str:
+ lines = [line for line in str(stderr or "").strip().splitlines() if line.strip()]
+ return " | ".join(lines[-3:])[-400:]
+
+
+def _run_import_probe(code: str):
+ """Run *code* in a child; None when it hung or could not start (inconclusive)."""
+ try:
+ return subprocess.run(
+ [sys.executable, "-c", code],
+ stdout = subprocess.DEVNULL,
+ stderr = subprocess.PIPE,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = _FLASH_ATTN_IMPORT_PROBE_TIMEOUT,
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return None
+
+
+def _remove_stale_ssm_kernels() -> None:
+ """Remove an installed causal-conv1d / mamba-ssm whose compiled extension no longer imports,
+ e.g. built for the torch a repair just replaced. Never installs: a model that needs them
+ installs them when it loads, and a broken native module left behind can take that load down.
+
+ Removal needs proof: torch itself imports, and the extension import then completes with an
+ ImportError / OSError. A torch that does not import, a probe killed by a signal, a hang or a
+ probe that cannot start says nothing about the kernel, and an offline machine could not get
+ it back, so each keeps it."""
+ if NO_TORCH or IS_MACOS:
+ return
+ installed = [
+ name
+ for name in _SSM_KERNEL_EXTENSIONS
+ if _installed_distribution_version(KERNELS[name][0]) is not None
+ ]
+ if not installed:
+ return
+ torch_probe = _run_import_probe("import torch")
+ if torch_probe is None or torch_probe.returncode != 0:
+ detail = "" if torch_probe is None else f": {_probe_tail(torch_probe.stderr)}"
+ _note(f"torch did not import cleanly; kept {', '.join(installed)}{detail}")
+ return
+ for name in installed:
+ distribution = KERNELS[name][0]
+ extension = _SSM_KERNEL_EXTENSIONS[name]
+ probe = _run_import_probe(
+ _SSM_EXTENSION_PROBE.format(extension = extension, code = _SSM_EXTENSION_BROKEN_EXIT)
+ )
+ if probe is None or probe.returncode == 0:
+ continue
+ tail = _probe_tail(probe.stderr)
+ if probe.returncode != _SSM_EXTENSION_BROKEN_EXIT or extension not in tail:
+ # A signal (negative code) or a crash outside the extension import: inconclusive.
+ _note(f"could not check {name} (exit {probe.returncode}); kept it: {tail}")
+ continue
+ # The verdict above was reached before this lock, which can wait up to KERNEL_LOCK_WAIT
+ # while a Studio worker or `unsloth install-kernels` (re)installs this kernel. So decide
+ # again while holding it: remove only when the extension still fails the same way, never
+ # a fresh install that loads now.
+ with kernel_install_lock(name) as held:
+ if not held:
+ _note(
+ f"{name} does not load with this torch, but its install lock is busy; kept it"
+ )
+ continue
+ recheck = _run_import_probe(
+ _SSM_EXTENSION_PROBE.format(extension = extension, code = _SSM_EXTENSION_BROKEN_EXIT)
+ )
+ if recheck is not None and recheck.returncode == 0:
+ _note(f"{name} now loads with this torch; kept it")
+ continue
+ recheck_tail = "" if recheck is None else _probe_tail(recheck.stderr)
+ if (
+ recheck is None
+ or recheck.returncode != _SSM_EXTENSION_BROKEN_EXIT
+ or extension not in recheck_tail
+ ):
+ code = "hung" if recheck is None else f"exit {recheck.returncode}"
+ _note(f"could not re-check {name} ({code}); kept it: {recheck_tail}")
+ continue
+ tail = recheck_tail
+ _count_install_action()
+ try:
+ removed = _uninstall_distribution(distribution)
+ except OSError:
+ removed = False
+ if removed:
+ _step("warning", f"{name} does not load with this torch; removed it ({tail})", _warn)
+ else:
+ _step(
+ "warning",
+ f"{name} does not load with this torch and could not be removed; "
+ f"uninstall {distribution} manually ({tail})",
+ _warn,
+ )
+
+
# -- uv bootstrap ------------------------------------------------------
USE_UV = False # Set by _bootstrap_uv() at the start of install_python_stack()
@@ -12893,6 +13022,7 @@ def install_python_stack() -> int:
if not IS_MACOS and not NO_TORCH:
_progress("flash-attn")
_ensure_flash_attn()
+ _remove_stale_ssm_kernels()
# # 6. Patch: override llama_cpp.py with fix from unsloth-zoo feature/llama-cpp-windows-support branch
# patch_package_file(
@@ -13078,6 +13208,7 @@ def install_python_stack() -> int:
_install_torchao_for_torch(_torch_after_repair)
# resync flash-attn because the repair can invalidate the wheel selected earlier.
_ensure_flash_attn()
+ _remove_stale_ssm_kernels()
# Unguarded: torch==2.10.0 accepts 2.10.0+rocm7.1, and an earlier run may have moved it.
_evict_xformers_built_for_another_torch(scope = "linux torch repair", family_only = True)
_evict_xformers_requiring_another_torch()
@@ -13094,6 +13225,7 @@ def install_python_stack() -> int:
if _torch_after_flavor and _torch_after_flavor != _torch_before_flavor:
# flash-attn was chosen for the torch the invariant just replaced, as in step 13.
_ensure_flash_attn()
+ _remove_stale_ssm_kernels()
# A direct run has no setup.ps1 postlude to swap triton back. After the invariant,
# because the swap keys off the installed +xpu label.
_ensure_xpu_triton()
diff --git a/tests/conftest.py b/tests/conftest.py
index 8b1826af955..f7a25d952cf 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -199,3 +199,14 @@ def _apply_upstream_import_fixes_for_tests() -> None:
_apply_upstream_import_fixes_for_tests()
+
+
[email protected](scope = "session")
+def _kernel_lock_dir(tmp_path_factory):
+ return str(tmp_path_factory.mktemp("kernel-locks"))
+
+
[email protected](autouse = True)
+def _isolated_kernel_install_locks(monkeypatch, _kernel_lock_dir):
+ # Tests never take (or create) the environment's real kernel install locks.
+ monkeypatch.setenv("UNSLOTH_KERNEL_LOCK_DIR", _kernel_lock_dir)
diff --git a/tests/python/test_flash_attn_install_python_stack.py b/tests/python/test_flash_attn_install_python_stack.py
index d945f8afcdf..f99768495cb 100644
--- a/tests/python/test_flash_attn_install_python_stack.py
+++ b/tests/python/test_flash_attn_install_python_stack.py
@@ -46,9 +46,16 @@ sys.path.insert(0, str(STUDIO_DIR))
sys.path.insert(0, str(STUDIO_DIR / "backend"))
import install_python_stack as ips
+import pytest
from utils import wheel_utils
[email protected](autouse = True)
+def _an_sm80_gpu(monkeypatch):
+ # The host's real GPU must not decide these tests; the capability cases below override it.
+ monkeypatch.setattr(ips, "_flash_attn_gpu_below_floor", lambda: False)
+
+
class TestPrebuiltWheelTorchMapping:
def test_torch_211_maps_to_torch210(self):
assert wheel_utils.prebuilt_wheel_torch_mm("2.11") == "2.10"
@@ -827,3 +834,64 @@ class TestWarningColor:
out = capsys.readouterr().out
assert f"\033[{code}mflash-attn install failed (exit code 1)\033[0m" in out
assert "\033[96m" not in out
+
+
+class TestFlashAttnCapabilityFloor:
+ """Unsloth only enables FlashAttention 2 from sm80: below it the wheel is never downloaded."""
+
+ def _run(self, monkeypatch, capability):
+ from utils import kernel_install
+
+ monkeypatch.setattr(
+ ips,
+ "_flash_attn_gpu_below_floor",
+ lambda: kernel_install.gpu_below_flash_attn_floor(lambda *a, **k: None),
+ )
+ monkeypatch.setattr(kernel_install, "_gpu_capability", lambda run: capability)
+ calls = {"head": 0, "install": 0}
+ notes = []
+ with (
+ mock.patch.object(ips, "NO_TORCH", False),
+ mock.patch.object(ips, "IS_WINDOWS", True),
+ mock.patch.object(ips, "IS_MACOS", False),
+ mock.patch.object(ips, "_flash_attn_importable", return_value = False),
+ mock.patch.object(ips, "_installed_distribution_version", return_value = None),
+ mock.patch.object(
+ ips,
+ "probe_torch_wheel_env",
+ return_value = {
+ "python_tag": "cp313",
+ "abi_tag": "cp313",
+ "torch_mm": "2.13",
+ "cuda_major": "13",
+ "cxx11abi": "TRUE",
+ "platform_tag": "win_amd64",
+ "hip_version": "",
+ },
+ ),
+ mock.patch.object(
+ ips,
+ "url_exists",
+ side_effect = lambda url: calls.__setitem__("head", calls["head"] + 1) or True,
+ ),
+ mock.patch.object(
+ ips,
+ "install_prebuilt",
+ side_effect = lambda *a, **k: calls.__setitem__("install", calls["install"] + 1)
+ or "installed",
+ ),
+ mock.patch.object(ips, "_note", side_effect = notes.append),
+ mock.patch.object(ips, "_count_install_action"),
+ ):
+ ips._ensure_flash_attn()
+ return calls, notes
+
+ def test_a_t4_does_not_download_flash_attn(self, monkeypatch):
+ calls, notes = self._run(monkeypatch, (7, 5))
+ assert calls == {"head": 0, "install": 0}
+ assert any("sm80" in note for note in notes)
+
+ @pytest.mark.parametrize("capability", [(8, 0), (12, 0), None])
+ def test_sm80_and_newer_or_no_visible_gpu_still_install(self, monkeypatch, capability):
+ calls, _ = self._run(monkeypatch, capability)
+ assert calls == {"head": 1, "install": 1}
diff --git a/tests/python/test_prebuilt_cuda_wheels.py b/tests/python/test_prebuilt_cuda_wheels.py
index 80b28588a60..4e39c9c7056 100644
--- a/tests/python/test_prebuilt_cuda_wheels.py
+++ b/tests/python/test_prebuilt_cuda_wheels.py
@@ -67,7 +67,8 @@ def triggers(doc: dict) -> dict:
class TestWorkflowAndResolverAgree:
def test_every_cell_the_workflow_builds_is_what_the_resolver_asks_for(self):
"""Nothing else checks the workflow's wheel name matches the installer's download; a
- mismatch is a 404."""
+ mismatch is a 404. The resolver asks only for the interpreters published (cp313: the
+ workflow's default), not every one the workflow could be dispatched to build."""
checked = 0
for package, spec in prebuilt_wheels.SPECS.items():
for torch_version in prebuilt_wheels.TORCH_VERSIONS:
@@ -80,12 +81,122 @@ class TestWorkflowAndResolverAgree:
python_tag = prebuilt_wheels.python_tag(python_version),
),
)
+ if python_version not in prebuilt_wheels.DEFAULT_PYTHON:
+ assert url is None, (package, torch_version, python_version)
+ continue
assert url is not None, (package, torch_version, python_version)
assert url.rsplit("/", 1)[1] == built
assert url.startswith(OUR_BASE)
checked += 1
- # 3 packages x 2 torch minors x 3 interpreters; guards against an emptied table.
- assert checked == 18
+ # 3 packages x 2 torch minors x cp313; guards against an emptied table.
+ assert checked == 6
+
+ def test_every_url_the_resolver_can_emit_is_a_published_cell(self):
+ """Whatever the probe reports, a URL names a cell the publish matrix builds and the release
+ carries: (platform, torch, interpreter, package). Anything else is a 404 in front of the
+ fallback."""
+ published = set()
+ for platform in prebuilt_wheels.PLATFORMS:
+ packages = prebuilt_wheels._platform(platform, "packages") or tuple(
+ prebuilt_wheels.SPECS
+ )
+ for torch_version in prebuilt_wheels._platform_torches(platform):
+ torch_mm = prebuilt_wheels.torch_minor(torch_version)
+ extra = wheel_utils._UNSLOTH_PREBUILT_EXTRA_CELLS.get((platform, torch_mm))
+ pythons = (
+ prebuilt_wheels.DEFAULT_PYTHON
+ if extra is None
+ else tuple(
+ v
+ for v in prebuilt_wheels.PYTHON_VERSIONS
+ if prebuilt_wheels.python_tag(v) in extra
+ )
+ )
+ for python_version in pythons:
+ for package in packages:
+ published.add(
+ prebuilt_wheels.wheel_name(
+ package, torch_version, python_version, platform
+ )
+ )
+ emitted = set()
+ for platform in (
+ "linux_x86_64",
+ "linux_aarch64",
+ "win_amd64",
+ "win_arm64",
+ "macosx_14_0_arm64",
+ ):
+ for torch_mm in ("2.6", "2.9", "2.10", "2.11", "2.12", "2.13", "2.14", "2.15"):
+ for python_tag in ("cp310", "cp311", "cp312", "cp313", "cp314"):
+ for cuda_version in ("12.8", "13.0", "13.4"):
+ for spec in prebuilt_wheels.SPECS.values():
+ probe = env(
+ torch_mm = torch_mm,
+ cuda_major = cuda_version.split(".")[0],
+ python_tag = python_tag,
+ platform_tag = platform,
+ )
+ probe["cuda_version"] = cuda_version
+ url = wheel_utils.unsloth_prebuilt_wheel_url(
+ filename_prefix = spec["dist"], env = probe
+ )
+ if url is not None:
+ assert url.startswith(OUR_BASE)
+ emitted.add(url.rsplit("/", 1)[1])
+ assert emitted, "the resolver emitted nothing"
+ assert emitted <= published, sorted(emitted - published)
+
+ @pytest.mark.parametrize(
+ "platform, torch_mm, python_tag, cuda_version",
+ [
+ ("linux_x86_64", "2.13", "cp312", "13.0"),
+ ("linux_x86_64", "2.14", "cp310", "13.0"),
+ ("linux_x86_64", "2.13", "cp311", "13.0"),
+ ("win_arm64", "2.14", "cp311", "13.4"),
+ ("win_arm64", "2.14", "cp312", "13.4"),
+ ],
+ )
+ def test_unpublished_interpreters_resolve_to_nothing(
+ self, platform, torch_mm, python_tag, cuda_version
+ ):
+ probe = env(torch_mm = torch_mm, python_tag = python_tag, platform_tag = platform)
+ probe["cuda_version"] = cuda_version
+ for prefix in ("causal_conv1d", "mamba_ssm"):
+ assert wheel_utils.unsloth_prebuilt_wheel_url(filename_prefix = prefix, env = probe) is None
+ # Nor an upstream URL: there is no upstream asset for torch 2.13+ either.
+ assert (
+ wheel_utils.direct_wheel_url(
+ filename_prefix = prefix,
+ package_version = "1.6.1",
+ release_tag = "v1.6.1.post4",
+ release_base_url = "https://github.com/Dao-AILab/causal-conv1d/releases/download",
+ env = probe,
+ )
+ is None
+ )
+
+ @pytest.mark.parametrize("cuda_major", ["12", "13"])
+ def test_no_upstream_url_for_torch_213_and_later(self, cuda_major):
+ probe = env(torch_mm = "2.13", cuda_major = cuda_major, python_tag = "cp312")
+ assert (
+ wheel_utils.direct_wheel_url(
+ filename_prefix = "mamba_ssm",
+ package_version = "2.3.1",
+ release_tag = "v2.3.1",
+ release_base_url = "https://github.com/state-spaces/mamba/releases/download",
+ env = probe,
+ )
+ is None
+ )
+ # torch 2.12 keeps its upstream (2.10 reuse) URL.
+ assert wheel_utils.direct_wheel_url(
+ filename_prefix = "mamba_ssm",
+ package_version = "2.3.1",
+ release_tag = "v2.3.1",
+ release_base_url = "https://github.com/state-spaces/mamba/releases/download",
+ env = env(torch_mm = "2.12", cuda_major = cuda_major, python_tag = "cp312"),
+ ).endswith("torch2.10cxx11abiTRUE-cp312-cp312-linux_x86_64.whl")
def test_the_versions_published_are_the_versions_resolved(self):
for package, spec in prebuilt_wheels.SPECS.items():
@@ -198,12 +309,17 @@ class TestOverrideScope:
wheel_utils.unsloth_prebuilt_wheel_url(filename_prefix = "flash_attn", env = None) is None
)
- @pytest.mark.parametrize("python_tag", ["cp311", "cp312", "cp313"])
- def test_every_interpreter_we_build(self, python_tag):
+ @pytest.mark.parametrize(
+ "python_tag, published", [("cp311", False), ("cp312", False), ("cp313", True)]
+ )
+ def test_only_the_interpreters_published(self, python_tag, published):
url = wheel_utils.unsloth_prebuilt_wheel_url(
filename_prefix = "flash_attn", env = env(python_tag = python_tag)
)
- assert url is not None and f"-{python_tag}-{python_tag}-" in url
+ if published:
+ assert url is not None and f"-{python_tag}-{python_tag}-" in url
+ else:
+ assert url is None
class TestBackwardsCompatible:
@@ -859,3 +975,76 @@ def test_release_lookups_do_not_pass_arg_to_gh_jq():
)
assert "--jq --arg" not in workflow
assert workflow.count("== env.TAG") == 2
+
+
+# The wheels on causal-conv1d v1.6.1.post4 and mamba v2.3.1 (identical cells for both), read from
+# the two release asset lists: (platform, CUDA major, cxx11abi, torch minor), each for cp310-cp313.
+_UPSTREAM_PUBLISHED = {
+ ("linux_x86_64", "11", "FALSE", "2.6"),
+ ("linux_x86_64", "11", "TRUE", "2.6"),
+ ("linux_x86_64", "11", "TRUE", "2.7"),
+ ("linux_x86_64", "12", "FALSE", "2.6"),
+ *(("linux_x86_64", "12", "TRUE", t) for t in ("2.6", "2.7", "2.8", "2.9", "2.10")),
+ *(("linux_x86_64", "13", "TRUE", t) for t in ("2.9", "2.10")),
+ *(("linux_aarch64", "12", "TRUE", t) for t in ("2.7", "2.8", "2.9", "2.10")),
+ *(("linux_aarch64", "13", "TRUE", t) for t in ("2.9", "2.10")),
+}
+_UPSTREAM_PINS = {
+ "causal_conv1d": (
+ "1.6.1",
+ "v1.6.1.post4",
+ "https://github.com/Dao-AILab/causal-conv1d/releases/download",
+ ),
+ "mamba_ssm": ("2.3.1", "v2.3.1", "https://github.com/state-spaces/mamba/releases/download"),
+}
+
+
+def test_every_upstream_url_the_resolver_can_emit_is_published():
+ """CUDA 13 before torch 2.9, aarch64 before 2.7, abiFALSE after 2.6, torch 2.13+ and any
+ interpreter outside cp310-cp313 have no upstream asset: no URL, not a 404."""
+ published = {
+ f"{prefix}-{version}+cu{cuda}torch{torch_mm}cxx11abi{abi}-{py}-{py}-{platform}.whl"
+ for platform, cuda, abi, torch_mm in _UPSTREAM_PUBLISHED
+ for prefix, (version, _, _) in _UPSTREAM_PINS.items()
+ for py in ("cp310", "cp311", "cp312", "cp313")
+ }
+ emitted = set()
+ for platform in ("linux_x86_64", "linux_aarch64"):
+ for cuda in ("11", "12", "13"):
+ for abi in ("TRUE", "FALSE"):
+ for torch_mm in (
+ "2.5",
+ "2.6",
+ "2.7",
+ "2.8",
+ "2.9",
+ "2.10",
+ "2.11",
+ "2.12",
+ "2.13",
+ "2.14",
+ ):
+ for py in ("cp39", "cp310", "cp311", "cp312", "cp313", "cp314"):
+ probe = env(
+ torch_mm = torch_mm,
+ cuda_major = cuda,
+ cxx11abi = abi,
+ python_tag = py,
+ platform_tag = platform,
+ )
+ probe["cuda_version"] = f"{cuda}.0"
+ for prefix, (version, tag, base) in _UPSTREAM_PINS.items():
+ url = wheel_utils.direct_wheel_url(
+ filename_prefix = prefix,
+ package_version = version,
+ release_tag = tag,
+ release_base_url = base,
+ env = probe,
+ )
+ if url is None or url.startswith(OUR_BASE):
+ continue
+ assert url.startswith(f"{base}/{tag}/"), url
+ emitted.add(url.rsplit("/", 1)[1])
+ assert emitted <= published, sorted(emitted - published)
+ # And nothing published is lost (torch 2.11 / 2.12 reuse the 2.10 wheels).
+ assert emitted == published
diff --git a/tests/python/test_ssm_kernels_install_python_stack.py b/tests/python/test_ssm_kernels_install_python_stack.py
index a25c192fb83..79cd2133b87 100644
--- a/tests/python/test_ssm_kernels_install_python_stack.py
+++ b/tests/python/test_ssm_kernels_install_python_stack.py
@@ -3,15 +3,20 @@
"""Setup installs flash-attn but not causal-conv1d or mamba-ssm: those (~550 MB) install when a
model that needs them is trained or loaded (core/training/worker.py, utils/ssm_runtime.py), and
-`unsloth install-kernels` still provisions them ahead of time for offline images."""
+`unsloth install-kernels` still provisions them ahead of time for offline images. After torch
+moves, setup still removes an installed copy whose compiled extension no longer imports."""
from __future__ import annotations
import ast
+import contextlib
import re
+import subprocess
import sys
from pathlib import Path
+import pytest
+
STUDIO_DIR = Path(__file__).resolve().parents[2] / "studio"
sys.path.insert(0, str(STUDIO_DIR))
sys.path.insert(0, str(STUDIO_DIR / "backend"))
@@ -37,16 +42,191 @@ def test_setup_has_no_ssm_kernel_step():
assert "causal-conv1d + mamba-ssm" not in SOURCE
-def test_flash_attn_is_still_installed_and_resynced_at_all_three_sites():
+def test_flash_attn_and_the_stale_kernel_check_run_at_all_three_sites():
calls = [m.start() for m in re.finditer(r"\n\s+_ensure_flash_attn\(\)\n", SOURCE)]
assert len(calls) == 3
for start in calls:
- following = SOURCE[start : start + 200]
- assert "ssm" not in following.lower(), following
+ following = SOURCE[start : start + 120]
+ assert "_remove_stale_ssm_kernels()" in following, following
assert "base_total += 1 # flash-attn, Linux and Windows" in SOURCE
+class _Kernels:
+ """Which SSM distributions are installed and how their extension import ends: True loads,
+ False raises ImportError, None hangs, an int is a raw exit code (negative = signal)."""
+
+ def __init__(
+ self,
+ installed: dict,
+ torch_imports = True,
+ ):
+ self.installed = dict(installed)
+ self.torch_imports = torch_imports
+ self.probes: list[str] = []
+ self.removed: list[str] = []
+
+ def version(self, distribution):
+ return "1.0" if distribution in self.installed else None
+
+ def run(self, cmd, *args, **kwargs):
+ code = cmd[-1]
+ self.probes.append(code)
+ if code == "import torch":
+ if self.torch_imports is None:
+ raise subprocess.TimeoutExpired(cmd, 1)
+ if self.torch_imports:
+ return subprocess.CompletedProcess(cmd, 0, stderr = "")
+ return subprocess.CompletedProcess(
+ cmd, 1, stderr = "OSError: libtorch_cuda.so: cannot open shared object file"
+ )
+ assert "import torch" in code # the extension probe proves torch imports too
+ extension = "causal_conv1d_cuda" if "causal_conv1d_cuda" in code else "selective_scan_cuda"
+ distribution = "causal-conv1d" if extension == "causal_conv1d_cuda" else "mamba-ssm"
+ state = self.installed[distribution]
+ if state is None:
+ raise subprocess.TimeoutExpired(cmd, 1)
+ if state is True:
+ return subprocess.CompletedProcess(cmd, 0, stderr = "")
+ if state is False:
+ return subprocess.CompletedProcess(
+ cmd,
+ ips._SSM_EXTENSION_BROKEN_EXIT,
+ stderr = f"{extension}: ImportError: {extension}.so: undefined symbol: _ZN3c10",
+ )
+ return subprocess.CompletedProcess(cmd, state, stderr = "Killed")
+
+ def uninstall(self, distribution):
+ self.removed.append(distribution)
+ self.installed.pop(distribution, None)
+ return True
+
+
+def _free_lock(name, **kwargs):
+ yield True
+
+
+def _run_stale_check(monkeypatch, host: _Kernels, **flags):
+ monkeypatch.setattr(ips, "NO_TORCH", flags.get("no_torch", False))
+ monkeypatch.setattr(ips, "IS_MACOS", flags.get("macos", False))
+ monkeypatch.setattr(ips, "_installed_distribution_version", host.version)
+ monkeypatch.setattr(ips, "_uninstall_distribution", host.uninstall)
+ monkeypatch.setattr(ips, "_count_install_action", lambda: None)
+ monkeypatch.setattr(ips, "kernel_install_lock", flags.get("lock", _free_lock))
+ warnings = []
+ monkeypatch.setattr(ips, "_step", lambda label, value, color_fn = None: warnings.append(value))
+ monkeypatch.setattr(ips, "_note", lambda message, color_fn = None: warnings.append(message))
+ monkeypatch.setattr(ips.subprocess, "run", host.run)
+ installs = []
+ monkeypatch.setattr(ips, "install_prebuilt", lambda *a, **k: installs.append(a) or "installed")
+ ips._remove_stale_ssm_kernels()
+ assert installs == [] # never installs anything
+ return warnings
+
+
+def test_stale_kernels_that_no_longer_import_are_removed(monkeypatch):
+ host = _Kernels({"causal-conv1d": False, "mamba-ssm": False})
+ warnings = _run_stale_check(monkeypatch, host)
+ assert host.removed == ["causal-conv1d", "mamba-ssm"]
+ assert len(warnings) == 2
+ # The probe's stderr tail says why.
+ assert "undefined symbol" in warnings[0] and "selective_scan_cuda" in warnings[1]
+
+
+def test_kernels_that_still_import_are_kept(monkeypatch):
+ host = _Kernels({"causal-conv1d": True, "mamba-ssm": False})
+ _run_stale_check(monkeypatch, host)
+ assert host.removed == ["mamba-ssm"]
+
+
+def test_absent_kernels_are_not_probed_or_installed(monkeypatch):
+ host = _Kernels({})
+ _run_stale_check(monkeypatch, host)
+ assert host.probes == [] and host.removed == []
+
+
+def test_a_hung_probe_is_inconclusive_and_keeps_the_kernel(monkeypatch):
+ host = _Kernels({"causal-conv1d": None})
+ _run_stale_check(monkeypatch, host)
+ assert host.removed == []
+
+
+def test_no_torch_or_macos_skips_the_check(monkeypatch):
+ for flags in ({"no_torch": True}, {"macos": True}):
+ host = _Kernels({"causal-conv1d": False})
+ _run_stale_check(monkeypatch, host, **flags)
+ assert host.probes == []
+
+
[email protected]("torch_imports", [False, None])
+def test_a_torch_that_does_not_import_keeps_every_kernel(monkeypatch, torch_imports):
+ # A half-finished repair or a transient failure in torch says nothing about the kernels, and an
+ # offline machine could not reinstall them.
+ host = _Kernels({"causal-conv1d": False, "mamba-ssm": False}, torch_imports = torch_imports)
+ warnings = _run_stale_check(monkeypatch, host)
+ assert host.removed == []
+ assert host.probes == ["import torch"]
+ if torch_imports is False:
+ assert "libtorch_cuda.so" in warnings[0]
+
+
[email protected]("exit_code", [-9, -11, 1, 2])
+def test_a_killed_or_unrelated_failure_is_inconclusive(monkeypatch, exit_code):
+ # A signal (OOM killer, segfault) or an exit that is not the extension's ImportError / OSError.
+ host = _Kernels({"causal-conv1d": exit_code, "mamba-ssm": exit_code})
+ warnings = _run_stale_check(monkeypatch, host)
+ assert host.removed == []
+ assert all("kept it" in message for message in warnings)
+
+
+def test_the_probed_extensions_match_the_kernel_checks():
+ for name, extension in ips._SSM_KERNEL_EXTENSIONS.items():
+ assert ips.KERNELS[name][1] == f"import torch, {extension}"
+
+
def test_the_kernel_cli_still_provisions_both_ahead_of_time():
from utils import kernel_install
assert {"causal_conv1d", "mamba_ssm"} <= set(kernel_install.KERNELS)
assert callable(kernel_install.install_kernel)
+
+
+def test_a_busy_install_lock_keeps_the_kernel(monkeypatch):
+ # A Studio load verifying a fresh install holds this lock: never remove it underneath that.
+ @contextlib.contextmanager
+ def _busy(name, **kwargs):
+ yield False
+
+ host = _Kernels({"causal-conv1d": False})
+ warnings = _run_stale_check(monkeypatch, host, lock = _busy)
+ assert host.removed == []
+ assert any("install lock is busy" in message for message in warnings)
+
+
+def test_a_kernel_reinstalled_while_waiting_on_the_lock_is_kept(monkeypatch):
+ # Broken when first probed, but a Studio load / `unsloth install-kernels` held the lock and
+ # reinstalled it: the re-probe under the lock loads, so nothing is removed.
+ host = _Kernels({"causal-conv1d": False})
+
+ @contextlib.contextmanager
+ def _lock_after_reinstall(name, **kwargs):
+ host.installed["causal-conv1d"] = True
+ yield True
+
+ warnings = _run_stale_check(monkeypatch, host, lock = _lock_after_reinstall)
+ assert host.removed == []
+ assert any("now loads" in message and "kept it" in message for message in warnings)
+ extension_probes = [code for code in host.probes if code != "import torch"]
+ assert len(extension_probes) == 2 # probed again under the lock
+
+
+def test_an_inconclusive_re_probe_under_the_lock_keeps_the_kernel(monkeypatch):
+ host = _Kernels({"causal-conv1d": False})
+
+ @contextlib.contextmanager
+ def _lock_then_hang(name, **kwargs):
+ host.installed["causal-conv1d"] = None
+ yield True
+
+ warnings = _run_stale_check(monkeypatch, host, lock = _lock_then_hang)
+ assert host.removed == []
+ assert any("kept it" in message for message in warnings)
diff --git a/tests/studio/install/test_windows_torch_flavor_invariant.py b/tests/studio/install/test_windows_torch_flavor_invariant.py
index 413c66bdc8e..7aaf5eaf50f 100644
--- a/tests/studio/install/test_windows_torch_flavor_invariant.py
+++ b/tests/studio/install/test_windows_torch_flavor_invariant.py
@@ -258,7 +258,7 @@ class TestStepThirteenWiring:
["_progress", "_torch_step_label", "_probe_installed_torch_version"]
+ repairs
+ ["_probe_installed_torch_version", "_note", "_install_torchao_for_torch"]
- + ["_ensure_flash_attn"]
+ + ["_ensure_flash_attn", "_remove_stale_ssm_kernels"]
+ ["_evict_xformers_built_for_another_torch", "_evict_xformers_requiring_another_torch"]
), step13
assert "_install_torchao_for_torch" not in _calls_in(guards[0])