Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Training subprocess entry point. Each job runs in a fresh subprocess (mp.get_context("spawn")): a
clean interpreter with no stale module state, which solves transformers version-switching.
Pattern follows core/data_recipe/jobs/worker.py."""
from __future__ import annotations
from utils.account_context import account_thread
from loggers import get_logger
import importlib
import importlib.metadata
import importlib.util
import math
import os
import shutil
import sys
import time
import traceback
import gc
import re
import types
import subprocess as _sp
from pathlib import Path
from typing import Any, Callable, TYPE_CHECKING
if TYPE_CHECKING:
from core.training.training import TrainingProgress
# WSL AMD GPU needs HSA_ENABLE_DXG_DETECTION=1 before torch init; spawned workers miss the installer env.
if sys.platform.startswith("linux") and "HSA_ENABLE_DXG_DETECTION" not in os.environ:
try:
if os.path.exists("/dev/dxg") and any(
os.path.exists(_p + "/librocdxg.so") for _p in ("/opt/rocm/lib", "/opt/rocm/lib64")
):
os.environ["HSA_ENABLE_DXG_DETECTION"] = "1"
except Exception:
pass
logger = get_logger(__name__)
from utils.child_stdio import utf8_child_env
# Fresh spawned interpreter: re-apply the process-wide network injections and CPU thread caps.
from utils.cpu_threads import install_openblas_runtime_cap
from utils.native_tls import activate_native_tls
from utils.happy_eyeballs import activate_happy_eyeballs
activate_native_tls()
activate_happy_eyeballs()
install_openblas_runtime_cap()
from utils.hardware import apply_gpu_ids
from utils.hf_dataset_options import hf_dataset_split_instruction_names
# Light module: the MLX branch runs on torch-less hosts.
from core.training.dataset_bounds import (
bound_dataset_rows,
max_train_rows_for_config,
record_row_bound,
row_bound_for_resume,
slice_dataset_rows,
world_size_from_env,
)
from core.training.resume import _checkpoint_state, session_eta_seconds
from utils.training_runs import build_default_output_dir_name
from utils.kernel_install import (
CAUSAL_CONV1D,
MAMBA_SSM,
PinnedKernel,
hipcc_gcc_install_dir,
install_prebuilt,
source_build_command,
source_build_run_kwargs,
uninstall_command,
)
from utils.ssm_runtime import (
model_is_ssm as _model_is_ssm,
model_wants_causal_conv1d as _model_wants_causal_conv1d,
)
from utils.wheel_utils import (
direct_wheel_url,
flash_attn_wheel_url,
install_wheel,
is_windows_wheel_env,
pinned_source_build_supported,
probe_torch_wheel_env,
url_exists,
)
def _output_dir_from_resume_checkpoint(resume_from_checkpoint: str | None) -> str | None:
if not resume_from_checkpoint:
return None
path = Path(resume_from_checkpoint)
return str(path.parent if path.name.startswith("checkpoint-") else path)
def _data_parallel_world_size() -> int:
"""Larger of env world size and CUDA device count, never the sum: distributed runs force n_gpu to 1."""
sizes = [world_size_from_env()]
torch_module = sys.modules.get("torch")
if torch_module is not None:
try:
distributed = getattr(torch_module, "distributed", None)
if (
distributed is not None
and distributed.is_available()
and distributed.is_initialized()
):
sizes.append(int(distributed.get_world_size()))
except Exception:
# A stubbed or half-initialised torch.distributed must not fail a run.
pass
try:
sizes.append(int(torch_module.cuda.device_count()))
except Exception:
# CPU-only builds answer 0 (dropped by the filter); a raising torch.cuda must not fail the run.
pass
return max([size for size in sizes if size > 0], default = 1)
def _model_local_files_only(config: dict) -> bool:
return bool(config.get("model_snapshot_path"))
def _dataset_local_files_only(config: dict) -> bool:
return bool(config.get("dataset_snapshot_path"))
def _adapter_trains_embeddings(model) -> bool:
"""Whether a loaded adapter already trains embed_tokens or lm_head in full (#13140)."""
for adapter in (getattr(model, "peft_config", None) or {}).values():
saved = list(getattr(adapter, "modules_to_save", None) or [])
saved += list(getattr(adapter, "modules_to_tie", None) or [])
if any(str(m).rsplit(".", 1)[-1] in ("embed_tokens", "lm_head") for m in saved):
return True
return False
def _adapter_base_changed_error(config: dict) -> str | None:
"""Refuse when the adapter now names a different base than the one checked at start (#13140)."""
# None: no adapter was admitted. "": an adapter that named no base, which must stay that way.
pinned = config.get("adapter_base_pin")
if pinned is None:
return None
from utils.models.model_config import get_base_model_from_lora
# The folder the loader reads: a cached Hub snapshot, not the repo id it came from.
if (
get_base_model_from_lora(str(_resolve_cached_model_load_name(config) or "")) or ""
) != pinned:
return "The adapter's base model changed after the run was checked. Start the run again."
return None
def _escaping_weights_error(config: dict) -> str | None:
from utils.security.file_security import escaping_weight_references
for key in ("model_name", "model_local_path", "model_snapshot_path", "adapter_base_pin"):
if config.get(key) and escaping_weight_references(config[key]):
return "This model's weight index names files outside its folder."
return None
def _untrainable_model_format_error(config: dict) -> str | None:
model_format = str(config.get("model_format") or "").strip().lower()
if model_format == "gguf":
return "GGUF models are inference-only and cannot be trained."
if model_format == "adapter":
# LoRA / CPT may continue from a saved adapter; full finetune still cannot (#13140).
# The embedding and decision runners wrap a fresh adapter, so they still need a base.
if config.get("training_type") in ("LoRA/QLoRA", "Continued Pretraining") and not (
config.get("is_embedding", False) or config.get("is_decision", False)
):
from utils.paths import is_local_path
if not is_local_path(str(config.get("model_name") or "")):
return (
"Continuing from a Hugging Face adapter repository is not supported yet. "
"Download it to a local folder and select that folder."
)
if config.get("is_dataset_image") or config.get("is_dataset_audio"):
return (
"Continuing a saved adapter on an image or audio dataset is not supported "
"yet. Start from its base model instead."
)
return None
return "Adapter models are inference-only and cannot be trained as base models."
return None
def _lora_uses_loftq(config: dict) -> bool:
"""A resume rebuilds its adapter before loading the checkpoint, so it reruns LoftQ only when the checkpoint was
trained with it: runs saved while Studio never applied LoftQ kept the original base weights."""
if not config.get("use_loftq"):
return False
resume = config.get("resume_from_checkpoint")
if not resume:
return True
import json
from core.training.resume import get_resume_checkpoint_path
checkpoint = Path(get_resume_checkpoint_path(resume) or resume)
try:
saved = json.loads((checkpoint / "adapter_config.json").read_text(encoding = "utf-8"))
except (OSError, ValueError):
return False
return isinstance(saved, dict) and saved.get("init_lora_weights") == "loftq"
def _resolve_cached_model_load_name(config: dict) -> str:
return config.get("model_snapshot_path") or config["model_name"]
def _worker_hf_token(config: dict):
"""Must stay three-valued: collapsing to None made a supplied API key count as the ambient caller."""
from hub.utils.hf_tokens import hf_token_arg
return hf_token_arg(
config.get("hf_token"), allow_ambient_token = config.get("allow_ambient", True)
)
def _effective_training_load_in_4bit(
config: dict, model_load_target: str, hf_token: str | None
) -> bool:
from .provenance import effective_training_load_in_4bit
return effective_training_load_in_4bit(config, model_load_target, hf_token)
def _training_weight_prefetch(config, trainer, hf_token, event_queue):
"""Resolve only the ordinary SFT checkpoint; uncertain routes keep the existing sequential load."""
try:
from .model_weight_prefetch import ModelWeightPrefetch, prefetch_repo
from utils.hf_cache_settings import get_hf_cache_paths
from utils.hardware import dataset_map_num_proc
repo = prefetch_repo(
config,
trainer,
load_in_4bit = lambda: _effective_training_load_in_4bit(
config, _resolve_cached_model_load_name(config), hf_token
),
capabilities = sys.modules.get("unsloth.device_type"),
offline = _model_offline_mode_enabled(),
formatting_can_fork = (
not config.get("dataset_streaming", False) and (dataset_map_num_proc() or 1) > 1
),
)
if repo is not None:
return ModelWeightPrefetch(
repo,
token = hf_token,
cache_dir = get_hf_cache_paths().hub_cache,
emit = event_queue.put,
should_stop = lambda: trainer.should_stop,
)
except Exception as error: # noqa: BLE001
logger.info("Skipping model checkpoint prefetch: %s", error)
return None
def _drop_model_pin(config: dict) -> str:
config["model_snapshot_path"] = None
return config["model_name"]
def _drop_model_pin_for_fallback(config: dict, hf_token: str | None) -> str:
from utils.transformers_version import get_transformers_activation_tier
active_target = _resolve_cached_model_load_name(config)
fallback_target = config["model_name"]
if not config.get("model_revision"):
active_tier = get_transformers_activation_tier(active_target, hf_token)
fallback_tier = get_transformers_activation_tier(fallback_target, hf_token)
if active_tier != fallback_tier:
raise RuntimeError(
"The cached model is incomplete and its Hugging Face fallback requires "
f"a different Transformers runtime ({active_tier} to {fallback_tier}). "
"Remove the incomplete cached model and retry."
)
return _drop_model_pin(config)
def _is_model_cache_artifact_error(error: BaseException | None) -> bool:
"""Scoped to the model-cache retry path, so a bare TypeError elsewhere is never retried."""
from hub.utils.dataset_cache import is_cache_artifact_error
if is_cache_artifact_error(error):
return True
markers = (
"can't load processor for",
"can't load image processor for",
"can't load feature extractor for",
"stat: path should be string, bytes, os.pathlike or integer, not nonetype",
"expected str, bytes or os.pathlike object, not nonetype",
# SentencePiece/BPE tokenizers fail on a missing vocab as a bare AttributeError; treat as cache miss.
"'nonetype' object has no attribute 'endswith'",
"'nonetype' object has no attribute 'readlines'",
"argument should be a str or an os.pathlike object",
"can't find a vocabulary file at path 'none'",
)
seen: set[int] = set()
current = error
while current is not None and id(current) not in seen:
seen.add(id(current))
if any(marker in str(current).lower() for marker in markers):
return True
current = current.__cause__ or current.__context__
return False
def _package_installs_offline() -> bool:
# UV_OFFLINE too: uv would refuse the index and install_wheel's pip fallback would not.
try:
from utils.utils import package_installs_offline
return package_installs_offline()
except Exception:
return os.environ.get("UV_OFFLINE", "").strip().lower() in {
"1",
"true",
"t",
"yes",
"y",
"on",
}
def _model_offline_mode_enabled() -> bool:
try:
from utils.utils import hf_env_offline
return hf_env_offline()
except Exception:
pass
return any(
str(os.environ.get(name, "")).strip().lower() in {"1", "true", "yes", "on"}
for name in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE")
)
def _cache_artifact_fallback_allowed(
config: dict, error: BaseException | None, resource: str
) -> bool:
require_exact = bool(
config.get("require_exact_resume_resources")
or config.get(f"require_exact_{resource}_resource")
)
if resource == "dataset":
from hub.utils.dataset_cache import dataset_cache_fallback_allowed
return dataset_cache_fallback_allowed(
error,
require_exact = require_exact,
revision = config.get("dataset_revision"),
)
if require_exact or _model_offline_mode_enabled():
return False
return _is_model_cache_artifact_error(error)
def _model_cache_fallback_error(config: dict, error: BaseException | None) -> RuntimeError | None:
"""Return an actionable error when an incomplete cache cannot be repaired."""
if not _is_model_cache_artifact_error(error):
return None
if config.get("require_exact_resume_resources") or config.get("require_exact_model_resource"):
return RuntimeError(
"The exact cached model snapshot is incomplete, so this run cannot "
"preserve its recorded model resources. Restore the missing model, "
"tokenizer, or processor files and retry."
)
if _model_offline_mode_enabled():
revision = config.get("model_revision")
revision_text = f" at revision {revision}" if revision else ""
return RuntimeError(
"Offline mode is enabled, but the cached model snapshot is incomplete. "
"Reconnect to download the missing model, tokenizer, or processor files"
f"{revision_text}, or select a complete local model."
)
return None
def _mlx_revision_fallback_error(config: dict) -> RuntimeError | None:
"""MLX remaps bnb repos to a full-precision base, where the pinned commit has no guaranteed meaning."""
model_name = str(config.get("model_name") or "")
revision = config.get("model_revision")
if (
revision
and model_name.startswith("unsloth/")
and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit"))
):
return RuntimeError(
"The cached model snapshot is incomplete, but MLX cannot safely retry "
f"'{model_name}' at revision {revision}: MLX maps its bitsandbytes "
"weights to a different base repository. Select that full-precision "
"base model directly, or restore the missing cached files."
)
return None
def _require_strict_cached_dataset(config: dict, dataset: Any, split: str) -> Any:
if (
config.get("require_exact_resume_resources") or config.get("require_exact_dataset_resource")
) and dataset is None:
raise FileNotFoundError(f"The exact cached dataset split '{split}' is no longer available.")
return dataset
def _offline_mode_enabled() -> bool:
return any(
str(os.environ.get(name, "")).strip().lower() in {"1", "true", "yes", "on"}
for name in ("HF_HUB_OFFLINE", "HF_DATASETS_OFFLINE")
)
def _verify_config_pins(config: dict, event_queue: Any) -> bool:
require_model = bool(
config.get("require_exact_resume_resources") or config.get("require_exact_model_resource")
)
require_dataset = bool(
config.get("require_exact_resume_resources") or config.get("require_exact_dataset_resource")
)
if require_model or require_dataset:
from core.training.provenance import (
ExactResumeResourcesUnavailable,
validate_exact_dataset_pin,
validate_exact_model_pin,
)
try:
model_snapshot = validate_exact_model_pin(config) if require_model else None
dataset_snapshot = validate_exact_dataset_pin(config) if require_dataset else None
except ExactResumeResourcesUnavailable as error:
event_queue.put(
{
"type": "error",
"error": str(error),
"stack": "",
"ts": time.time(),
}
)
return False
if model_snapshot is not None:
config["model_snapshot_path"] = model_snapshot
config["model_revision"] = Path(model_snapshot).name
if dataset_snapshot is not None:
config["dataset_snapshot_path"] = dataset_snapshot
for message in config.get("cache_pin_warnings") or []:
_send_status(event_queue, message)
model_path = config.get("model_snapshot_path")
require_validated_snapshot = bool(config.get("require_validated_model_snapshot"))
if require_validated_snapshot and not model_path:
event_queue.put(
{
"type": "error",
"error": (
"The cached model snapshot selected during preflight is no longer available."
),
"stack": "",
"ts": time.time(),
}
)
return False
if model_path and not require_model:
from hub.utils.hf_cache_state import (
latest_snapshot_from_cache_path,
with_load_subdirs,
)
from utils.utils import canonical_model_repo_id
pinned_repo_id = config.get("actual_model_repo_id") or canonical_model_repo_id(
config["model_name"]
)
config["model_snapshot_path"] = latest_snapshot_from_cache_path(
model_path,
"model",
pinned_repo_id,
with_load_subdirs(config["model_name"], ("config.json", "adapter_config.json")),
)
if config["model_snapshot_path"] is None:
if require_validated_snapshot:
event_queue.put(
{
"type": "error",
"error": (
"The cached model snapshot selected during preflight is no "
"longer available."
),
"stack": "",
"ts": time.time(),
}
)
return False
if not config.get("model_revision"):
config["actual_model_repo_id"] = None
else:
config["model_revision"] = Path(config["model_snapshot_path"]).name
dataset_path = config.get("dataset_snapshot_path")
if dataset_path and not require_dataset:
from hub.utils.dataset_cache import (
dataset_cache_path_from_cache_path,
dataset_snapshot_from_cache_path,
)
resolved = dataset_cache_path_from_cache_path(
dataset_path,
config.get("hf_dataset") or "",
)
snapshot = (
dataset_snapshot_from_cache_path(
str(resolved),
config.get("hf_dataset") or "",
)
if resolved is not None
else None
)
if snapshot is not None:
config["dataset_revision"] = snapshot.name
config["dataset_snapshot_path"] = str(resolved) if resolved else None
if (
config.get("dataset_revision")
and not config.get("dataset_snapshot_path")
and _offline_mode_enabled()
):
event_queue.put(
{
"type": "error",
"error": (
"The selected dataset snapshot is incomplete and its exact "
"revision cannot be downloaded while offline."
),
"stack": "",
"ts": time.time(),
}
)
return False
return True
def _mlx_diffusion_lm_error(config: dict, model_load_target: str) -> str | None:
from utils.models.diffusion_lm import mlx_refusal
return mlx_refusal(
model_load_target, _worker_hf_token(config), revision = config.get("model_revision")
)
def _validate_training_worker_config(config: dict, event_queue: Any) -> bool:
if not _verify_config_pins(config, event_queue):
return False
format_error = _untrainable_model_format_error(config) or _adapter_base_changed_error(config)
if format_error:
event_queue.put(
{
"type": "error",
"error": format_error,
"stack": "",
"ts": time.time(),
}
)
return False
return True
def _cached_dataset_row_limit(config: dict) -> int | None:
slice_end = config.get("dataset_slice_end")
slice_start = config.get("dataset_slice_start")
if isinstance(slice_end, bool) or not isinstance(slice_end, int) or slice_end < 0:
return None
if slice_start is None:
slice_start = 0
if isinstance(slice_start, bool) or not isinstance(slice_start, int):
return None
return slice_end + 1 if slice_end >= max(slice_start, 0) else None
def _load_cached_dataset_for_config(
config: dict,
split: str | None,
token: str | None = None,
*,
row_limit: int | None = None,
):
hf_dataset = config.get("hf_dataset")
local_path = config.get("dataset_snapshot_path")
if not hf_dataset or not local_path:
return None
from hub.utils.dataset_cache import load_cached_hf_dataset
kwargs = {
"subset": config.get("subset"),
"split": split or "train",
"token": token,
}
if row_limit is not None:
kwargs["row_limit"] = row_limit
return load_cached_hf_dataset(hf_dataset, local_path, **kwargs)
def _load_hf_train_and_eval_datasets(
config: dict,
token: str | None,
load_dataset: Callable,
status_callback: Callable[[str], None],
warning_callback: Callable[[str], None] | None = None,
):
from core.training.eval_dataset import (
EVAL_SPLIT_CANDIDATES,
MIN_EVAL_ROWS,
STREAMING_EVAL_MAX_SAMPLES,
evaluation_enabled,
)
hf_dataset = config["hf_dataset"]
subset = config.get("subset")
train_split = config.get("train_split", "train") or "train"
eval_split = config.get("eval_split")
revision = config.get("dataset_revision")
eval_enabled = evaluation_enabled(config.get("eval_steps"))
require_exact = bool(
config.get("require_exact_resume_resources") or config.get("require_exact_dataset_resource")
)
dataset = None
loaded_from_cache = False
config["_dataset_loaded_from_exact_snapshot"] = False
def warn(message: str) -> None:
if warning_callback is not None:
warning_callback(message)
else:
logger.warning(message)
def load_remote(split: str, streaming: bool = False):
kwargs = {"split": split, "token": token}
if subset:
kwargs["name"] = subset
if revision:
kwargs["revision"] = revision
if streaming:
kwargs["streaming"] = True
return load_dataset(hf_dataset, **kwargs)
if config.get("dataset_streaming"):
# As on CUDA: eval needs its own split, capped so each pass ends.
status_callback(f"Streaming dataset: {hf_dataset}")
dataset = load_remote(train_split, streaming = True)
eval_dataset = None
if eval_enabled and eval_split and eval_split != train_split:
eval_dataset = load_remote(eval_split, streaming = True).take(STREAMING_EVAL_MAX_SAMPLES)
return dataset, eval_dataset
if _dataset_local_files_only(config):
status_callback(f"Loading cached dataset: {hf_dataset}")
try:
row_limit = _cached_dataset_row_limit(config)
if row_limit is None:
dataset = _load_cached_dataset_for_config(config, train_split, token)
else:
dataset = _load_cached_dataset_for_config(
config,
train_split,
token,
row_limit = row_limit,
)
dataset = _require_strict_cached_dataset(config, dataset, train_split)
loaded_from_cache = dataset is not None
except Exception as error:
if not _cache_artifact_fallback_allowed(config, error, "dataset"):
raise
status_callback("Cached dataset unavailable; downloading from the Hub...")
if dataset is None:
dataset = load_remote(train_split)
eval_dataset = None
explicit_separate_eval = bool(eval_split and eval_split != train_split)
if eval_enabled and explicit_separate_eval:
if loaded_from_cache:
try:
eval_dataset = _load_cached_dataset_for_config(config, eval_split, token)
eval_dataset = _require_strict_cached_dataset(
config,
eval_dataset,
eval_split,
)
except Exception as error:
if not _cache_artifact_fallback_allowed(config, error, "dataset"):
raise
status_callback(
"Cached eval split unavailable; reloading train and eval from the Hub..."
)
remote_train = load_remote(train_split)
remote_eval = load_remote(eval_split)
dataset = remote_train
eval_dataset = remote_eval
loaded_from_cache = False
else:
eval_dataset = load_remote(eval_split)
elif eval_enabled and not eval_split:
auto_errors: list[tuple[str, Exception]] = []
try:
split_info = getattr(getattr(dataset, "info", None), "splits", None)
if loaded_from_cache:
available_splits = list(split_info or ())
else:
from datasets import get_dataset_split_names
split_kwargs = {"path": hf_dataset}
if subset:
split_kwargs["config_name"] = subset
if revision:
split_kwargs["revision"] = revision
if token:
split_kwargs["token"] = token
available_splits = get_dataset_split_names(**split_kwargs)
excluded_splits = set(hf_dataset_split_instruction_names(train_split))
for candidate in EVAL_SPLIT_CANDIDATES:
if candidate not in available_splits or candidate in excluded_splits:
continue
try:
if loaded_from_cache:
candidate_dataset = _load_cached_dataset_for_config(
config,
candidate,
token,
)
candidate_dataset = _require_strict_cached_dataset(
config,
candidate_dataset,
candidate,
)
else:
candidate_dataset = load_remote(candidate)
except Exception as error:
if require_exact:
raise
auto_errors.append((candidate, error))
continue
if len(candidate_dataset) >= MIN_EVAL_ROWS:
eval_dataset = candidate_dataset
break
except Exception as error:
if require_exact:
raise
warn(
"Automatic eval split detection failed; a held-out split will be created "
f"from the training data when enough rows are available: {error}"
)
else:
if eval_dataset is None and auto_errors:
candidate, error = auto_errors[0]
warn(
f"Automatic eval split '{candidate}' could not be loaded; a held-out "
"split will be created from the training data when enough rows are "
f"available: {error}"
)
from core.training.provenance import (
attest_loaded_dataset,
exact_dataset_snapshot_path,
)
snapshot, _ = attest_loaded_dataset(hf_dataset, dataset, eval_dataset)
if snapshot is None and loaded_from_cache:
snapshot = exact_dataset_snapshot_path(
config.get("dataset_snapshot_path"),
hf_dataset,
)
if snapshot is not None:
config["dataset_snapshot_path"] = snapshot
config["_dataset_loaded_from_exact_snapshot"] = True
return dataset, eval_dataset
def _load_embedding_hf_dataset(
config: dict, load_dataset: Callable, status_callback: Callable[[str], None]
):
hf_dataset = str(config.get("hf_dataset") or "").strip()
if not hf_dataset:
return None
subset = config.get("subset") or None
train_split = config.get("train_split", "train") or "train"
revision = config.get("dataset_revision")
token = _worker_hf_token(config)
dataset = None
config["_dataset_loaded_from_exact_snapshot"] = False
if _dataset_local_files_only(config):
status_callback(f"Loading cached dataset: {hf_dataset}")
try:
row_limit = _cached_dataset_row_limit(config)
if row_limit is None:
dataset = _load_cached_dataset_for_config(
config,
train_split,
token,
)
else:
dataset = _load_cached_dataset_for_config(
config,
train_split,
token,
row_limit = row_limit,
)
dataset = _require_strict_cached_dataset(
config,
dataset,
train_split,
)
if dataset is not None:
from core.training.provenance import exact_dataset_snapshot_path
snapshot = exact_dataset_snapshot_path(
config.get("dataset_snapshot_path"),
hf_dataset,
)
if snapshot is not None:
config["dataset_snapshot_path"] = snapshot
config["_dataset_loaded_from_exact_snapshot"] = True
except Exception as error:
if not _cache_artifact_fallback_allowed(config, error, "dataset"):
raise
status_callback("Cached dataset unavailable; downloading from the Hub...")
dataset = None
config["_dataset_loaded_from_exact_snapshot"] = False
if dataset is None:
load_kwargs = {
"split": train_split,
"token": token,
}
if revision:
load_kwargs["revision"] = revision
dataset = load_dataset(hf_dataset, subset, **load_kwargs)
from core.training.provenance import attest_loaded_dataset
snapshot, _ = attest_loaded_dataset(hf_dataset, dataset)
if snapshot is not None:
config["dataset_snapshot_path"] = snapshot
config["_dataset_loaded_from_exact_snapshot"] = True
return dataset
def _pre_detect_load_in_4bit(config: dict, model_load_target: str, hf_token: str | None) -> bool:
"""Mirrors the real load's 4-bit flip so detection reads the repo the loader fetches."""
if not bool(config.get("load_in_4bit", True)):
return False
from utils.transformers_version import latest_tier_active_for
return not latest_tier_active_for(model_load_target, hf_token)
def _pre_detect_training_model(
trainer,
config: dict,
model_name: str,
hf_token: str | None,
model_load_name: str,
local_files_only: bool,
model_revision: str | None = None,
) -> None:
trainer.pre_detect_and_load_tokenizer(
model_name = model_name,
max_seq_length = config["max_seq_length"],
hf_token = hf_token,
is_dataset_image = config.get("is_dataset_image", False),
is_dataset_audio = config.get("is_dataset_audio", False),
trust_remote_code = config.get("trust_remote_code", False),
model_load_name = model_load_name,
local_files_only = local_files_only,
model_revision = model_revision,
load_in_4bit = _pre_detect_load_in_4bit(config, model_load_name, hf_token),
)
_check_finetune_targets_after_detect(trainer, config)
_apply_diffusion_lm_overrides(trainer, config)
_LORA_DEFAULT_LR = 2e-4
# Near-duplicate head rows for <|im_end|> / <tool_call> / ... (cosine 0.97): a 2e-4 LoRA swaps them in chat.
_DIFFUSION_LM_LORA_LR = {"dream": 5e-5, "sdar": 5e-5}
def _apply_diffusion_lm_overrides(trainer, config: dict) -> None:
"""Text-diffusion runs never pack: packed rows would share one bidirectional attention window. Set before
the row bound and online tokenization read the flag, so their accounting matches what trains."""
profile = getattr(trainer, "diffusion_lm_profile", None)
if not profile:
return
config["diffusion_lm_profile"] = profile
if config.get("packing"):
config["packing"] = False
logger.info("Text-diffusion model (%s): sequence packing turned off", profile)
lowered = _DIFFUSION_LM_LORA_LR.get(profile)
try:
at_default = float(config.get("learning_rate", _LORA_DEFAULT_LR)) == _LORA_DEFAULT_LR
except (TypeError, ValueError):
at_default = False # the trainer reports the invalid value itself
if lowered is not None and at_default:
config["learning_rate"] = lowered
logger.info(
"Text-diffusion model (%s): learning rate 2e-4 lowered to %g; set any other value to override",
profile,
lowered,
)
_NOTHING_TO_TRAIN = (
"Nothing to train: select at least one layer family (finetune_language_layers or "
"finetune_vision_layers) and at least one module type (finetune_attention_modules or "
"finetune_mlp_modules)."
)
def _finetune_selectors(config: dict) -> tuple[bool, bool, bool, bool]:
"""Omitted keys take the CUDA consumer's defaults, so the guard models runs as they really train."""
return (
bool(config.get("finetune_vision_layers", True)),
bool(config.get("finetune_language_layers", True)),
bool(config.get("finetune_attention_modules", True)),
bool(config.get("finetune_mlp_modules", True)),
)
def _requests_all_linear(config: dict) -> bool:
"""all-linear forces all selectors on in PEFT, so every linear layer trains regardless of selectors."""
target_modules = config.get("target_modules")
if isinstance(target_modules, str):
return target_modules == "all-linear"
if isinstance(target_modules, (list, tuple)):
return list(target_modules) == ["all-linear"]
return False
def _check_finetune_targets_after_detect(trainer, config: dict) -> None:
"""Needs detection to settle the branch, so it runs after pre_detect and before any weights load."""
if config.get("training_type", "LoRA/QLoRA") != "LoRA/QLoRA":
return # Full Finetuning / CPT build adapters from target_modules alone
if not (getattr(trainer, "is_vlm", False) or getattr(trainer, "is_audio_vlm", False)):
return # the text branch ignores these four
if _requests_all_linear(config):
return # get_peft_model turns all five selectors on for the keyword; see below
vision, language, attention, mlp = _finetune_selectors(config)
# Mirror get_peft_regex's two guards: one layer family AND one module type.
if not (vision or language) or not (attention or mlp):
raise ValueError(_NOTHING_TO_TRAIN)
# On CPT, MLX trains these regardless of the layer-family flags.
_CPT_TARGET_NAMES = frozenset({"embed_tokens", "lm_head"})
_MLX_DEFAULT_LORA_TARGETS = (
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
)
def _trains_embedding_modules(trainable_names, full_module_keys = ()) -> bool:
"""Whether the embedding or output head trains in full, from a fresh CPT wrap or a continued adapter.
``full_module_keys`` are the weights zoo resolved for embed_tokens / lm_head, which cover heads named otherwise.
"""
for name in trainable_names:
if name in full_module_keys:
return True
parts = set(name.split("."))
if parts & _CPT_TARGET_NAMES and not parts & {"lora_a", "lora_b"}:
return True
return False
def _mlx_quantized_cpt_modules(model, names) -> set:
"""The requested embed_tokens / lm_head this load quantized (a 4-bit checkpoint or load), which zoo cannot
train in full. A tied head is the embedding, so it goes with it."""
quantized, found = set(), set()
for path, module in model.named_modules():
leaf = path.rsplit(".", 1)[-1]
if leaf in names:
found.add(leaf)
if any(hasattr(child, "scales") for _, child in module.named_modules()):
quantized.add(leaf)
if "embed_tokens" in quantized and "lm_head" in names and "lm_head" not in found:
quantized.add("lm_head")
return quantized
def _mlx_cpt_embedding_learning_rate(requested, lr_value, *, trains_embeddings):
"""The CUDA CPT rule: lr/10 unless set, and dropped when no embedding module trains."""
if not trains_embeddings:
if requested is not None:
logger.warning(
"CPT: embedding_learning_rate was provided but neither embed_tokens "
"nor lm_head is being trained; ignoring the override."
)
return None
if requested is None:
return lr_value / 10.0
return float(requested)
def _names_a_cpt_target(target_modules) -> bool:
"""Whether an explicit target list names something that trains on its own."""
if isinstance(target_modules, str):
return target_modules in _CPT_TARGET_NAMES
try:
return any(name in _CPT_TARGET_NAMES for name in target_modules)
except TypeError: # not iterable -> not a list of names, so nothing is guaranteed
return False
def _check_mlx_finetune_targets(config: dict) -> None:
"""A fused qkv survives the module filter but still gets no adapters when language layers are off."""
targets = config.get("target_modules")
if targets:
if _names_a_cpt_target(targets):
return
_, language, attention, mlp = _finetune_selectors(config)
# MLX defaults vision to False (unlike _finetune_selectors), so legacy configs are not waved through.
vision = bool(config.get("finetune_vision_layers", False))
# Vision counts: this runs before VLM detection; _check_mlx_effective_targets handles text later.
if attention or mlp or language or vision:
return
raise ValueError(_NOTHING_TO_TRAIN)
_, _, attention, mlp = _finetune_selectors(config)
if not (attention or mlp):
raise ValueError(_NOTHING_TO_TRAIN)
def _check_mlx_effective_targets(
config: dict, *, finetune_language: bool, finetune_vision: bool
) -> None:
"""Rechecks post-backfill values; with both layer families off, the run trains nothing."""
if finetune_language or finetune_vision:
return
if _names_a_cpt_target(config.get("target_modules") or ()):
return
raise ValueError(_NOTHING_TO_TRAIN)
def _fetch_spark_tts_repo(trainer, event_queue: Any, model_name: str) -> None:
"""Spark-TTS encodes its dataset with the BiCodec weights in the model repo, so the repo is fetched before the
dataset step (#13458), under the model-load lifecycle so a stalled Xet download still respawns over HTTP."""
from utils.hf_xet_fallback import start_watchdog
_send_status(event_queue, "Loading model...")
event_queue.put({"type": "model_load_started", "ts": time.time()})
# The download lands under the resolved repo, not the "Spark-TTS-0.5B/LLM" alias.
watchdog_stop = start_watchdog(
repo_ids = list(dict.fromkeys([model_name, trainer.spark_tts_hub_repo()])),
on_stall = lambda msg: event_queue.put({"type": "stall", "message": msg, "ts": time.time()}),
xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
)
try:
trainer.resolve_spark_tts_repo(
on_model_resolved = lambda repo_id: event_queue.put(
{"type": "model_load_resolved", "repo_id": repo_id, "ts": time.time()}
),
)
finally:
watchdog_stop.set()
event_queue.put({"type": "model_load_completed", "ts": time.time()})
def _reload_dataset_with_remote_model_tokenizer(
trainer,
config: dict,
model_name: str,
hf_token: str | None,
reload_dataset: Callable[[], tuple],
model_revision: str | None = None,
):
_pre_detect_training_model(
trainer,
config,
model_name,
hf_token,
model_name,
False,
model_revision,
)
return reload_dataset()
def _strip_unsloth_bnb_4bit_suffix(model_name: str) -> str:
"""unsloth.models.loader._strip_unsloth_bnb_4bit_suffix, copied rather than imported: that
module pulls in torch, and this runs before the worker is allowed to."""
stripped = model_name
for suffix in ("-unsloth-bnb-4bit", "-bnb-4bit"):
if len(stripped) >= len(suffix) and stripped.lower().endswith(suffix):
stripped = stripped[: -len(suffix)]
return stripped
def _model_load_security_error(config: dict, load_target: str, hf_token: str | None) -> dict | None:
from utils.models.model_config import get_base_model_from_lora_identifier
from utils.security import (
evaluate_file_security,
evaluate_remote_code_consent_for_targets,
load_scan_target,
security_load_subdirs,
)
requested_targets = [load_target]
try:
base_model = get_base_model_from_lora_identifier(load_target, hf_token)
if base_model:
requested_targets.append(base_model)
except Exception as error:
logger.debug("Could not resolve LoRA base for security scan: %s", error)
# Scan mirrors the loader substitutes too, else fetched bytes skip the malware/consent scan.
# Must not import torch/unsloth here; only the torch loader path uses this mapper.
try:
from core.training.training import should_use_mlx_training_backend
from utils.models.unsloth_mirror import unsloth_public_mirror
torch_loader_path = not config.get("is_embedding", False) and (
not should_use_mlx_training_backend()
)
if (
torch_loader_path
and not _model_local_files_only(config)
and not config.get("model_revision")
):
# Fallbacks only go 4-bit -> 16-bit, so a 16-bit config never scans the 4-bit mirror.
_modes = (True, False) if config.get("load_in_4bit", True) else (False,)
# Expand every seed: a remote LoRA's base has its own mirror that gets downloaded.
for _seed in list(dict.fromkeys(requested_targets)):
for _mode in _modes:
mirrored = unsloth_public_mirror(_seed, _mode)
if mirrored and mirrored != _seed:
requested_targets.append(mirrored)
# Without prequantized models (ROCm bnb < 0.49.2) the loader strips the 4-bit suffix; scan that repo too.
stripped = _strip_unsloth_bnb_4bit_suffix(mirrored)
if stripped != mirrored and stripped != _seed:
requested_targets.append(stripped)
except Exception as error: # noqa: BLE001
logger.debug("Could not resolve the mirror for the security scan: %s", error)
from utils.utils import hf_env_offline
primary_name = config["model_name"]
local_only_load = hf_env_offline()
consent_load_subdirs: dict[str, tuple] = {}
targets: list[str] = []
for requested_target in dict.fromkeys(requested_targets):
load_subdirs = security_load_subdirs(requested_target, hf_token)
if requested_target == load_target and requested_target != primary_name:
load_subdirs = tuple(
dict.fromkeys((*load_subdirs, *security_load_subdirs(primary_name, hf_token)))
)
target, load_subdirs = load_scan_target(requested_target, load_subdirs)
if target not in consent_load_subdirs:
targets.append(target)
consent_load_subdirs[target] = ()
load_subdirs = tuple(dict.fromkeys((*consent_load_subdirs[target], *load_subdirs)))
consent_load_subdirs[target] = load_subdirs
decision = evaluate_file_security(
target,
hf_token = hf_token,
load_subdirs = load_subdirs,
local_only_load = local_only_load,
)
if decision.blocked:
return {
"error": decision.reason,
"error_kind": "malware_blocked",
"security": decision.response_payload(),
}
if not config.get("trust_remote_code", False):
return None
decision = evaluate_remote_code_consent_for_targets(
targets,
hf_token = hf_token,
trust_remote_code = True,
approved_fingerprint = config.get("approved_remote_code_fingerprint"),
subject = config.get("subject"),
load_subdirs_by_target = consent_load_subdirs,
)
if not decision.blocked:
return None
return {
"error": (
f"{decision.blocked_message()}\n\n{decision.findings_summary}"
if decision.max_severity
else decision.blocked_message()
),
"error_kind": "remote_code_blocked",
"remote_code": decision.response_payload(),
}
_FLASH_ATTN_RUNTIME_MIN_SEQ_LEN = 32768
_FLASH_ATTN_SKIP_ENV = "UNSLOTH_STUDIO_SKIP_FLASHATTN_INSTALL"
_FAST_PATH_HOOKS_SKIP_ENV = "UNSLOTH_STUDIO_SKIP_FAST_PATH_HOOKS"
# Module scope so the torch.library.Library registration isn't GC'd mid-run.
_WINDOWS_ROCM_GROUPED_MM_LIB = None
def _install_grouped_mm_cpu_fallback(torch_mod, logger, label):
"""Works around a null HIP _grouped_mm kernel on RDNA4 and ROCm <= 7.12; keep the Library referenced."""
import warnings as _warnings
_gm_lib = torch_mod.library.Library("aten", "IMPL")
def _grouped_mm_safe_impl(
self,
mat2,
offs = None,
bias = None,
out_dtype = None,
):
"""Python mm/bmm fallback for _grouped_mm on gfx120X (null HIP kernel, ROCm <= 7.12)."""
_t = torch_mod
if offs is None:
# No offsets: 2-D -> mm, 3-D batched -> bmm (unconditional mm broke 3-D MoE).
if self.dim() == 3 and mat2.dim() == 3:
result = _t.bmm(self.contiguous(), mat2.contiguous())
elif self.dim() == 3 and mat2.dim() == 2:
result = _t.matmul(self.contiguous(), mat2.contiguous())
elif self.dim() == 2 and mat2.dim() == 3:
result = _t.matmul(self.contiguous(), mat2.contiguous())
else:
result = _t.mm(self.contiguous(), mat2.contiguous())
else:
# Grouped: offs[i] is the exclusive end-row of group i.
offs_list = offs.tolist()
pieces = []
prev = 0
for idx, end in enumerate(offs_list):
end = int(end)
a_part = self[prev:end].contiguous()
b_part = mat2[idx].contiguous() if mat2.dim() == 3 else mat2.contiguous()
pieces.append(_t.mm(a_part, b_part))
prev = end
# Include trailing rows not covered by offs.
if prev < self.shape[0]:
a_tail = self[prev:].contiguous()
b_tail = mat2[-1].contiguous() if mat2.dim() == 3 else mat2.contiguous()
pieces.append(_t.mm(a_tail, b_tail))
result = (
_t.cat(pieces, dim = 0)
if pieces
else _t.zeros(0, mat2.shape[-1], device = self.device, dtype = self.dtype)
)
if bias is not None:
result = result + bias
if out_dtype is not None:
result = result.to(out_dtype)
elif result.dtype != self.dtype:
result = result.to(self.dtype)
return result
with _warnings.catch_warnings():
_warnings.simplefilter("ignore")
_gm_lib.impl("_grouped_mm", _grouped_mm_safe_impl, "CUDA")
logger.info(
"%s: patched _grouped_mm CUDA dispatch (null HIP kernel on gfx120X, "
"ROCm <= 7.12 -- bypassed with Python mm fallback)",
label,
)
return _gm_lib
# Subprocesses don't inherit os.add_dll_directory; replicate main.py's ROCm DLL setup.
_ROCM_DLL_HANDLES: list = []
if sys.platform == "win32":
def _add_rocm_dll_dirs_worker() -> None:
_candidates: list[str] = []
for _var in ("HIP_PATH", "ROCM_PATH"):
_val = os.environ.get(_var)
if _val:
_candidates.append(os.path.join(_val, "bin"))
_default_root = os.path.join(
os.environ.get("ProgramFiles", r"C:\Program Files"), "AMD", "ROCm"
)
def _ver_key(name: str) -> tuple:
# Numeric tuple key so "10.0" sorts after "7.0".
parts = []
for chunk in name.split("."):
try:
parts.append((0, int(chunk)))
except ValueError:
parts.append((1, chunk))
return tuple(parts)
try:
if os.path.isdir(_default_root):
for _ver in sorted(os.listdir(_default_root), key = _ver_key, reverse = True):
_bin = os.path.join(_default_root, _ver, "bin")
if os.path.isdir(_bin):
_candidates.append(_bin)
except OSError:
pass
for _d in _candidates:
if os.path.isdir(_d):
try:
_ROCM_DLL_HANDLES.append(os.add_dll_directory(_d))
except (OSError, AttributeError):
pass
_add_rocm_dll_dirs_worker()
del _add_rocm_dll_dirs_worker
def _decision_has_llm_backbone(model_load_target: str, hf_token: str | None) -> bool:
# Clef ships joint_head_config.json next to its Qwen3.5 backbone; Laya never does.
marker = "joint_head_config.json"
try:
if (Path(model_load_target) / marker).is_file():
return True
from huggingface_hub import HfApi
siblings = HfApi(token = hf_token).model_info(model_load_target).siblings or ()
return any(getattr(s, "rfilename", None) == marker for s in siblings)
except Exception:
return False
_hipcc_gcc_install_dir = hipcc_gcc_install_dir
def _is_importable(import_name: str) -> bool:
# Invalidate finder caches so a package installed earlier in this process is seen.
importlib.invalidate_caches()
try:
__import__(import_name)
return True
except Exception as exc:
# Wrong-arch wheels raise OSError/RuntimeError, not ImportError.
logger.debug("%s is not importable (%s: %s)", import_name, type(exc).__name__, exc)
return False
# Kept in step with install_python_stack._FLASH_ATTN_IMPORT_PROBE_TIMEOUT.
_IMPORT_PROBE_TIMEOUT = 300
def _is_importable_isolated(import_name: str) -> bool:
"""Probe the import in a child: a wrong-arch wheel can abort or segfault, killing this worker."""
return _import_probe_isolated(
import_name, "import importlib, sys; importlib.import_module(sys.argv[1])"
)
def _extension_importable_isolated(extension: str) -> bool:
"""_is_importable_isolated for a bare torch extension (selective_scan_cuda), which needs
torch's libraries loaded first."""
return _import_probe_isolated(
extension, "import torch, importlib, sys; importlib.import_module(sys.argv[1])"
)
def _import_probe_isolated(import_name: str, code: str) -> bool:
try:
result = _sp.run(
[sys.executable, "-c", code, import_name],
stdout = _sp.DEVNULL,
stderr = _sp.DEVNULL,
timeout = _IMPORT_PROBE_TIMEOUT,
# No env override: the probe must see what the real in-process import will.
)
except (OSError, _sp.TimeoutExpired) as exc:
logger.debug("%s import probe did not complete (%s)", import_name, exc)
return False
if result.returncode != 0:
logger.debug("%s import probe exited %s", import_name, result.returncode)
return result.returncode == 0
def _uninstall_package(pypi_name: str, display_name: str) -> bool:
"""Remove a distribution. True iff it is gone afterwards."""
result = _sp.run(
uninstall_command(pypi_name, use_uv = bool(shutil.which("uv"))),
stdout = _sp.PIPE,
stderr = _sp.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
env = utf8_child_env(),
)
if result.returncode != 0:
logger.warning("Could not remove the rejected %s install:\n%s", display_name, result.stdout)
return False
return True
def _distribution_present(pypi_name: str) -> bool:
"""Metadata alone decides: leftover metadata makes unsloth import a native module that may abort."""
importlib.invalidate_caches()
try:
importlib.metadata.distribution(pypi_name)
return True
except Exception:
return False
def _reject_install(event_queue: Any, pypi_name: str, display_name: str, reason: str) -> None:
"""A rejected install left in place is still imported by the metadata gate, so remove it."""
if not _distribution_present(pypi_name):
return
logger.warning("%s %s", display_name, reason)
if _uninstall_package(pypi_name, display_name):
_send_status(event_queue, f"{display_name} is not usable on this GPU; removed it")
else:
_send_status(
event_queue,
f"{display_name} is not usable on this GPU and could not be removed; "
f"uninstall {pypi_name} manually before training",
)
def _install_package_wheel_first(
*,
event_queue: Any,
import_name: str,
display_name: str,
pypi_name: str,
required: bool = False,
extension: str | None = None,
**kwargs: Any,
) -> bool:
"""Install a wheel-first fast-path package; any unsuccessful exit discards what it left; offline
changes nothing; with extension, an extension-free copy is replaced, not accepted."""
from utils.ssm_runtime import _unblock_import, forget_imported_modules
_unblock_import(import_name) # an earlier in-process block; this call probes afresh
def kernel_usable() -> bool:
return _is_importable(import_name) and (extension is None or _is_importable(extension))
if kernel_usable():
logger.info("%s already installed", display_name)
return True
def drop_extension_free_copy() -> None:
# CUDA route cannot keep an extension-free mamba_ssm (transformers takes its fast path and
# fails); removal is allowed offline and runs under the same per-env lock as installs.
if extension is None or not _is_importable(import_name):
return
from utils.kernel_install import OPTIONAL_KERNEL_LOCK_WAIT, kernel_install_lock
with kernel_install_lock(import_name, wait_seconds = OPTIONAL_KERNEL_LOCK_WAIT) as held:
if not _is_importable(import_name) or _is_importable(extension):
return
if not held:
# Cannot uninstall without the lock: hide it from this process instead.
from utils.ssm_runtime import block_import, refresh_transformers_kernel_gates
logger.info(
"Hiding the extension-free %s from this run: its install lock is busy",
display_name,
)
block_import(import_name, extension)
refresh_transformers_kernel_gates(removed = True)
return
logger.info(
"Removing the extension-free %s: this run needs its CUDA kernels", display_name
)
_uninstall_package(pypi_name, display_name)
forget_imported_modules(import_name, extension)
if _is_importable(import_name) and not _is_importable(extension):
# Uninstall failed (read-only env, locked files): hide it from this process.
from utils.ssm_runtime import block_import, refresh_transformers_kernel_gates
block_import(import_name, extension)
refresh_transformers_kernel_gates(removed = True)
if _model_offline_mode_enabled() or _package_installs_offline():
logger.info("Skipping %s installation while offline", display_name)
drop_extension_free_copy()
return False
# The runtime's failed-attempt memo (same key): the eager install and the fast-path hook, or a
# chat load in the same process, never repeat a failed (possibly half-hour) build.
from utils.ssm_runtime import _FAILED_INSTALLS, _failure_key
failure_key = _failure_key(import_name, kwargs.get("wheel_url_builder"))
if failure_key in _FAILED_INSTALLS:
logger.info("%s already failed to install in this process; not retrying", display_name)
drop_extension_free_copy()
return False
from utils.kernel_install import (
KERNEL_LOCK_WAIT,
OPTIONAL_KERNEL_LOCK_WAIT,
kernel_install_lock,
)
# One install per package and env across workers: precheck, install, verify and cleanup are one
# transaction; an optional acceleration does not hold a run for a peer's build.
with kernel_install_lock(
import_name,
on_wait = lambda: _send_status(
event_queue, f"Waiting for another process installing {display_name}..."
),
wait_seconds = KERNEL_LOCK_WAIT if required else OPTIONAL_KERNEL_LOCK_WAIT,
) as held:
if not held:
_send_status(
event_queue,
f"Could not take the {display_name} install lock (another process is installing "
"it, or the lock is unavailable); continuing without it",
)
if (
extension is not None
and _is_importable(import_name)
and not _is_importable(extension)
):
# Not ours to uninstall without the lock, but this run must not use it: hidden from
# this process, transformers takes its torch path instead of selective_scan_fn.
from utils.ssm_runtime import block_import, refresh_transformers_kernel_gates
block_import(import_name, extension)
refresh_transformers_kernel_gates(removed = True)
return False
if kernel_usable():
logger.info("%s installed by another process", display_name)
if extension is not None:
forget_imported_modules(import_name, extension)
return True
# Importable but without the compiled module: the extension-free wheel, replaced here.
replacing = extension is not None and _is_importable(import_name)
if replacing:
_send_status(
event_queue, f"Replacing the extension-free {display_name} with its CUDA kernels..."
)
installed = False
try:
installed = _attempt_package_install(
event_queue = event_queue,
import_name = import_name,
display_name = display_name,
pypi_name = pypi_name,
extension = extension,
replacing = replacing,
**kwargs,
)
return installed
finally:
if replacing:
# The process imported the extension-free copy: the next import must load what is
# installed now (or nothing, after a failed replacement removes it).
forget_imported_modules(import_name, extension)
if not installed:
_FAILED_INSTALLS.add(failure_key)
_reject_install(event_queue, pypi_name, display_name, "will not import")
if extension is not None and _is_importable(import_name):
# The extension-free copy could not be removed: hide it from this process.
from utils.ssm_runtime import block_import, refresh_transformers_kernel_gates
block_import(import_name, extension)
refresh_transformers_kernel_gates(removed = True)
def _attempt_package_install(
*,
event_queue: Any,
import_name: str,
display_name: str,
pypi_name: str,
pypi_version: str | None = None,
filename_prefix: str | None = None,
release_tag: str | None = None,
release_base_url: str | None = None,
wheel_url_builder: Callable[[dict[str, str] | None], str | None] | None = None,
pypi_spec: str | None = None,
pypi_status_message: str | None = None,
allow_source_build: bool = True,
extension: str | None = None,
replacing: bool = False,
) -> bool:
"""The install itself. Call it through _install_package_wheel_first, never directly."""
def _verify() -> bool:
# Out of process: a bad wheel can take the worker down with it.
if not _is_importable_isolated(import_name):
return False
return extension is None or _extension_importable_isolated(extension)
# Set when a wheel installed but would not import; see the uninstall before the fallback.
wheel_rejected = False
env = probe_torch_wheel_env(timeout = 30, include_windows = True)
if not env or not (env.get("cuda_version") or env.get("cuda_major") or env.get("hip_version")):
# These are CUDA / ROCm extensions: CPU-only, XPU, MPS or an unreadable torch builds none.
_send_status(
event_queue, f"{display_name} needs a CUDA or ROCm build of torch; skipping it"
)
return False
# Windows is wheel-only: a PyPI fallback there is a source build needing MSVC + the CUDA toolkit.
wheel_only = sys.platform == "win32" or is_windows_wheel_env(env)
if wheel_url_builder is not None:
wheel_url = wheel_url_builder(env)
else:
wheel_url = direct_wheel_url(
filename_prefix = filename_prefix,
package_version = pypi_version,
release_tag = release_tag,
release_base_url = release_base_url,
env = env,
)
wheel_available = url_exists(wheel_url) if wheel_url else False
if wheel_url is None:
logger.info("No compatible %s wheel candidate", display_name)
elif wheel_available is not False:
# Only a 404 proves nothing is published: an unanswered HEAD (a proxy refusing HEAD but
# serving the download) still tries the wheel, and falls back only if that fails.
if wheel_available is None:
_send_status(
event_queue,
f"Could not check the {display_name} prebuilt wheel; trying to download it.",
)
_send_status(event_queue, f"Installing {display_name} for faster training...")
outcome = install_prebuilt(
wheel_url,
install = install_wheel,
verify = _verify,
on_failed = lambda installer, result: logger.warning(
"%s failed to install %s wheel:\n%s",
installer,
display_name,
result.stdout,
),
use_uv = bool(shutil.which("uv")),
run = _sp.run,
# Over the extension-free copy of the same version, which uv / pip would keep.
**({"reinstall": True} if replacing else {}),
)
if outcome == "installed":
logger.info("Installed prebuilt %s wheel successfully", display_name)
return True
if outcome == "rejected":
logger.warning(
"%s wheel installed but is not importable; %s",
display_name,
"removing it" if wheel_only or not allow_source_build else "falling back to PyPI",
)
wheel_rejected = True
else:
logger.info("No published %s wheel found: %s", display_name, wheel_url)
if wheel_only:
logger.info("%s: no usable prebuilt wheel for Windows; skipping", display_name)
return False
if not allow_source_build:
_send_status(event_queue, f"No usable prebuilt {display_name} for this torch; skipping it")
return False
if pypi_version is not None and not pinned_source_build_supported(env):
# The pinned causal-conv1d / mamba-ssm sources build at C++17 and torch 2.13+ needs
# C++20: a certain failure after minutes of compiling.
_send_status(
event_queue,
f"No prebuilt {display_name} for torch {env.get('torch_mm')}, and it cannot be built "
"from source for it; skipping it",
)
return False
is_hip = env and env.get("hip_version")
if is_hip and not shutil.which("hipcc"):
logger.error(
"%s requires hipcc for source compilation on ROCm. "
"Install the ROCm HIP SDK: https://rocm.docs.amd.com",
display_name,
)
_send_status(
event_queue,
f"{display_name}: hipcc not found (ROCm HIP SDK required)",
)
return False
if pypi_spec is None:
pypi_spec = f"{pypi_name}=={pypi_version}"
if pypi_status_message is None:
if is_hip:
pypi_status_message = (
f"Compiling {display_name} from source for ROCm (this may take several minutes)..."
)
else:
pypi_status_message = f"Installing {display_name} from PyPI for faster training..."
if wheel_rejected or replacing:
# Remove it rather than install over it (a broken wheel, or the extension-free copy being
# replaced): pip/uv would report the broken distribution as already satisfying the
# spec and do nothing. --force-reinstall is not the answer either, since both scope it to the whole resolved
# transaction, which for flash-attn means torch and the running CUDA stack. A failure here is caught by the
# _reject_install in the finally below, which is reached from every exit.
_uninstall_package(pypi_name, display_name)
_send_status(event_queue, pypi_status_message)
plain_pypi_install = pypi_version is None
if plain_pypi_install:
if shutil.which("uv"):
pypi_cmd = [
"uv",
"pip",
"install",
"--python",
sys.executable,
pypi_spec,
]
else:
pypi_cmd = [sys.executable, "-m", "pip", "install", pypi_spec]
else:
pypi_cmd = source_build_command(
pypi_spec, use_uv = bool(shutil.which("uv")), is_hip = bool(is_hip), reinstall = False
)
_run_kwargs, _gcc_dir = source_build_run_kwargs(
is_hip = bool(is_hip), gcc_install_dir = _hipcc_gcc_install_dir
)
if _gcc_dir is not None:
logger.info(
"HIP source build for %s: appended --gcc-install-dir=%s to HIPCC_COMPILE_FLAGS_APPEND",
display_name,
_gcc_dir,
)
from utils.kernel_install import run_process_tree
try:
# Bounded, and a timeout stops the compilers pip / uv started too.
result = run_process_tree(pypi_cmd, run = _sp.run, **_run_kwargs)
except _sp.TimeoutExpired:
logger.error(
"%s installation timed out after %ds",
display_name,
_run_kwargs.get("timeout"),
)
_send_status(
event_queue,
f"{display_name} installation timed out after {_run_kwargs.get('timeout')}s",
)
return False
if result.returncode != 0:
if is_hip:
error_lines = (result.stdout or "").strip().splitlines()
snippet = "\n".join(error_lines[-5:]) if error_lines else "(no output)"
logger.error(
"Failed to compile %s for ROCm:\n%s",
display_name,
result.stdout,
)
_send_status(
event_queue,
f"Failed to compile {display_name} for ROCm. "
"Check that hipcc and ROCm development headers are installed.\n"
f"{snippet}",
)
else:
logger.error(
"Failed to install %s from PyPI:\n%s",
display_name,
result.stdout,
)
return False
# rc=0 is not proof again here: pip/uv exit 0 on "Requirement already satisfied" without installing anything.
# Returning False is enough; the caller's finally discards it.
if not _verify():
logger.warning("%s installed from PyPI but will not import", display_name)
return False
if is_hip:
logger.info("Compiled and installed %s from source for ROCm", display_name)
else:
logger.info("Installed %s from PyPI", display_name)
return True
def _pinned_kernel_kwargs(kernel: PinnedKernel) -> dict[str, str]:
"""_install_package_wheel_first arguments for a pinned kernel release."""
return {
"import_name": kernel.import_name,
"display_name": kernel.display_name,
"pypi_name": kernel.pypi_name,
"pypi_version": kernel.package_version,
"filename_prefix": kernel.import_name,
"release_tag": kernel.release_tag,
"release_base_url": kernel.release_base_url,
}
def _ensure_causal_conv1d_fast_path(
event_queue: Any,
model_name: str,
*,
required: bool | None = None,
) -> None:
if required is None:
required = _model_wants_causal_conv1d(model_name)
if not required:
return
_install_package_wheel_first(event_queue = event_queue, **_pinned_kernel_kwargs(CAUSAL_CONV1D))
def _flash_linear_attention_importable() -> bool:
"""True iff the gated-delta kernels unsloth_zoo vendors and injects as `fla` import."""
try:
import fla.modules # noqa: F401
import fla.ops.gated_delta_rule # noqa: F401
return True
except Exception as exc:
logger.warning(
"flash-linear-attention is not importable; continuing on the pure-torch path: %s",
exc,
)
return False
def _ensure_mamba_ssm(
event_queue: Any,
model_name: str,
*,
required: bool | None = None,
essential: bool | None = None,
triton_only: bool = False,
) -> bool | None:
# Only Nemotron-H, Falcon-H1, Granite-4.0-H may build mamba-ssm from source, best effort: a
# missing one is left to the model load; others get prebuilt wheels only.
if required is None:
required = _model_is_ssm(model_name)
if not required:
return
if essential is None:
# Unspecified: only Nemotron-H, Falcon-H1 and Granite-4.0-H may build from source, never
# other SSMs.
from utils.ssm_runtime import model_requires_mamba_ssm
essential = model_requires_mamba_ssm(model_name)
if sys.platform == "win32" and importlib.util.find_spec("triton") is None:
# mamba_ssm imports triton at load and the wheel goes in --no-deps.
_send_status(event_queue, "mamba-ssm needs Triton on Windows (triton-windows); skipping it")
return
if triton_only:
from utils.wheel_utils import mamba_ssm_pure_wheel_url
logger.info("Mamba-2 model on Triton kernels; setting up the extension-free mamba-ssm")
# True when it imports; the caller then skips causal-conv1d and the CUDA mamba-ssm.
return _install_package_wheel_first(
event_queue = event_queue,
**_pinned_kernel_kwargs(MAMBA_SSM),
wheel_url_builder = mamba_ssm_pure_wheel_url,
allow_source_build = False,
required = essential,
)
if not essential and not _is_importable(MAMBA_SSM.import_name):
from utils.ssm_runtime import _mamba_ssm_floor_reason
reason = _mamba_ssm_floor_reason()
if reason is not None:
_send_status(event_queue, f"Skipping mamba-ssm, which {reason}")
return
logger.info("SSM model detected; setting up mamba-ssm after causal-conv1d")
from utils.ssm_runtime import MAMBA_SSM_EXTENSION
_install_package_wheel_first(
event_queue = event_queue,
**_pinned_kernel_kwargs(MAMBA_SSM),
allow_source_build = essential,
required = essential,
# Never settle for the extension-free wheel a Mamba-2 load may have left here.
extension = MAMBA_SSM_EXTENSION,
)
# Integrated-only Radeon names match as whole tokens (a substring reads RX 7600M as 600M).
_ROCM_APU_NAME_RE = re.compile(r"(?<![\w-])(?:820m|8040s|780m|760m|740m|680m|660m|610m)(?!\w)")
def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
"""Trusts props.is_integrated first (it only upgrades to unified); name match is the last resort."""
gcn_arch = ""
for _attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):
_v = (getattr(props, _attr, "") or "").split(":")[0].strip()
if _v:
gcn_arch = _v
break
# Driver's is_integrated flag can only upgrade to unified, never downgrade the known APU set.
if getattr(props, "is_integrated", 0):
return gcn_arch, True
if gcn_arch:
# gfx1152 (Krackan Point) shares system RAM like gfx1150/gfx1151.
return gcn_arch, gcn_arch.lower() in {"gfx1150", "gfx1151", "gfx1152"}
# Without arch attrs, match device name, only under _hw.IS_ROCM so NVIDIA 840M cannot hit Krackan.
dev_lower = (getattr(props, "name", "") or "").lower()
is_unified = (
"890m" in dev_lower
or "880m" in dev_lower
or "8065s" in dev_lower
or "8060s" in dev_lower
or "8050s" in dev_lower
or "860m" in dev_lower
or "840m" in dev_lower
or _ROCM_APU_NAME_RE.search(dev_lower) is not None
)
return gcn_arch, is_unified
# Fixed 16 GiB reserve: a flat 20% on 128 GiB Strix Halo is wasteful, 10% starved the OS.
_UNIFIED_OS_RESERVE_BYTES = 16 * 1024**3
_UNIFIED_MAX_RESERVE_FRACTION = 0.20
_DISCRETE_MEM_FRACTION = 0.90
# The name this guard shipped with; still wins on a ROCm host, since setups export it.
_MEM_FRACTION_ENV = "UNSLOTH_ROCM_MEM_FRACTION"
# Backend neutral, so NVIDIA has a cap too (unsloth#8178).
_GPU_MEM_FRACTION_ENV = "UNSLOTH_GPU_MEM_FRACTION"
def _parse_mem_fraction_env(env_value: str | None) -> float | None:
"""``UNSLOTH_ROCM_MEM_FRACTION`` as a float, None when unset or unusable. Shared with the OOM
guard's log line so it can say whether the override was actually honoured, rather than just
whether the variable was set."""
try:
override = float(env_value) # None -> TypeError, "" / " " -> ValueError
except (TypeError, ValueError):
return None
# Two-sided so NaN is rejected.
return override if 0.0 < override <= 1.0 else None
def _training_vram_budget_fraction(
budget_gb: float | None,
denominator_bytes: int,
current: float = 1.0,
) -> float | None:
"""The memory fraction that holds this process to ``budget_gb``, never looser than ``current``
(the OOM guard's cap); None when there is no budget or no total to divide by."""
try:
budget = float(budget_gb)
except (TypeError, ValueError):
return None
if not (budget > 0) or denominator_bytes <= 0:
return None
return min(current, budget * 1024**3 / denominator_bytes)
def _apply_training_vram_budget(
torch_mod: Any, single_gb: float | None, per_device_gb: list | None, gpu_ids: list | None
) -> dict[int, float]:
"""Cap each visible device at its own budget; returns {ordinal: fraction} for what was set."""
applied: dict[int, float] = {}
if not torch_mod.cuda.is_available():
return applied
get_fraction = getattr(torch_mod.cuda, "get_per_process_memory_fraction", None)
divides_by_props = _allocator_divides_by_props_total(getattr(torch_mod, "__version__", ""))
for index in range(torch_mod.cuda.device_count()):
budget = _device_budget_gb(single_gb, per_device_gb, index, gpu_ids)
if not budget:
continue
props = torch_mod.cuda.get_device_properties(index)
denominator = int(getattr(props, "total_memory", 0) or 0)
if not divides_by_props:
denominator = int(torch_mod.cuda.mem_get_info(index)[1])
current = get_fraction(index) if get_fraction is not None else 1.0
fraction = _training_vram_budget_fraction(budget, denominator, current)
if fraction is None:
continue
torch_mod.cuda.set_per_process_memory_fraction(fraction, index)
applied[index] = fraction
logger.info(
"Training VRAM budget: set_per_process_memory_fraction(%.4f, cuda:%d), "
"%.1f GiB of %.1f GiB",
fraction,
index,
fraction * denominator / 1024**3,
denominator / 1024**3,
)
return applied
def _device_budget_gb(
single_gb: float | None,
per_device_gb: list | None,
ordinal: int,
gpu_ids: list | None = None,
) -> float | None:
"""The budget for torch device ``ordinal``. ``per_device_gb`` is indexed by physical GPU id, so
the ordinal goes through ``gpu_ids`` (the CUDA_VISIBLE_DEVICES narrowing) first; when it is
given, a missing or null entry means no cap on that card."""
if not per_device_gb:
return single_gb
physical = gpu_ids[ordinal] if gpu_ids and ordinal < len(gpu_ids) else ordinal
try:
return per_device_gb[int(physical)]
except (IndexError, TypeError, ValueError):
return None
def _offload_plan_shape(config: dict) -> dict:
"""The per-device batch and LoRA rank an Auto offload plan sizes its training reserve for."""
return {"batch_size": config.get("batch_size"), "lora_rank": config.get("lora_r")}
def _visible_gpu_count() -> int:
"""GPUs the load spreads over when no ids were resolved (a UUID / MIG mask is inherited as is)."""
try:
import torch
return torch.cuda.device_count() if torch.cuda.is_available() else 0
except Exception:
return 0
def _with_vram_budget_hint(config: dict, message: str) -> str:
"""Point a run that does not fit at its own VRAM budget, the one cause the generic advice omits."""
budget = config.get("offload_vram_gb")
per_device = [gb for gb in config.get("offload_vram_gb_per_device") or [] if gb]
lower = (message or "").lower()
if not (budget or per_device) or not any(
k in lower for k in ("out of memory", "out of vram", "does not fit")
):
return message
cap = (
f"a {budget:g} GiB VRAM budget"
if budget and not per_device
else "per-GPU VRAM budgets (" + ", ".join(f"{gb:g}" for gb in per_device) + " GiB)"
)
return (
f"{message}\nThis run is capped at {cap}. Raise or clear the "
"VRAM budget under Training Hyperparameters > Memory."
)
def _allocator_divides_by_props_total(torch_version: str | None) -> bool:
"""Torch 2.10+ caps at props.total_memory, older torch at hipMemGetInfo; unparsable says True."""
release = str(torch_version or "").split("+", 1)[0].split(".")
try:
major, minor = int(release[0]), int(release[1])
except (IndexError, ValueError):
return True
return (major, minor) >= (2, 10)
def _rocm_memory_fraction(
total_bytes: int,
is_unified: bool,
platform: str,
env_value: str | None = None,
denominator_bytes: int | None = None,
) -> float:
"""Valid UNSLOTH_ROCM_MEM_FRACTION overrides; unified Windows gets 1.0 as WDDM excludes the OS share."""
override = _parse_mem_fraction_env(env_value)
if override is not None:
return override
if not is_unified:
return _DISCRETE_MEM_FRACTION
if platform == "win32":
return 1.0
if total_bytes <= 0:
# Missing or None total defaults to 0; with no pool size, keep the historical cap instead of solving.
return 1.0 - _UNIFIED_MAX_RESERVE_FRACTION
# Fraction space, not bytes: avoids 0.7999... ULP rounding below the historical 0.80.
reserve_fraction = min(_UNIFIED_MAX_RESERVE_FRACTION, _UNIFIED_OS_RESERVE_BYTES / total_bytes)
fraction = 1.0 - reserve_fraction
if (
reserve_fraction < _UNIFIED_MAX_RESERVE_FRACTION
and denominator_bytes
and denominator_bytes > 0
and denominator_bytes != total_bytes
):
# Floored so a larger driver total never tightens past the old 0.80; percentage arm is scale-free.
fraction = max(
fraction * total_bytes / denominator_bytes,
1.0 - _UNIFIED_MAX_RESERVE_FRACTION,
)
# Past ~160 GiB unified would get a looser cap than discrete; clamp.
return min(fraction, _DISCRETE_MEM_FRACTION)
def _mem_fraction_env_names(backend: str) -> tuple[str, ...]:
"""Most specific first, so a host already exporting the ROCm name keeps its cap."""
if backend == "rocm":
return (_MEM_FRACTION_ENV, _GPU_MEM_FRACTION_ENV)
return (_GPU_MEM_FRACTION_ENV,)
def _mem_fraction_env_value(backend: str, environ: Any = None) -> tuple[str | None, str | None]:
"""``(raw, name)``, else the first one SET so a warning names what the user exported."""
if environ is None:
environ = os.environ
first_set: tuple[str | None, str | None] = (None, None)
for name in _mem_fraction_env_names(backend):
raw = environ.get(name)
if raw is None:
continue
if _parse_mem_fraction_env(raw) is not None:
return raw, name
if first_set == (None, None):
first_set = (raw, name)
return first_set
def _gpu_memory_fraction(
total_bytes: int,
is_unified: bool,
platform: str,
backend: str,
env_value: str | None = None,
denominator_bytes: int | None = None,
) -> float:
"""An override in ``(0.0, 1.0]`` wins; else ROCm delegates unchanged and the rest
answer ``1.0``, torch's uncapped default, so setting nothing changes nothing."""
override = _parse_mem_fraction_env(env_value)
if override is not None:
return override
if backend != "rocm":
return 1.0
return _rocm_memory_fraction(total_bytes, is_unified, platform, None, denominator_bytes)
# ── Fast-path hooks ──
# Wrap is_causal_conv1d_available so the first call drives the install.
# UNSLOTH_STUDIO_SKIP_FAST_PATH_HOOKS=1 falls back to the substring path.
def _rebind_in_already_imported_modules(*, attr_name: str, old_obj: Any, new_obj: Any) -> int:
"""Rebind `attr_name -> new_obj` in every module that imported `old_obj`. `from X import Y`
creates a local binding that reassigning X.Y won't reach. Uses `__dict__.get` to skip lazy
`__getattr__` aliases."""
count = 0
missing = object()
for mod_name, mod in list(sys.modules.items()):
if mod is None:
continue
module_dict = getattr(mod, "__dict__", None)
if not isinstance(module_dict, dict):
continue
existing = module_dict.get(attr_name, missing)
if existing is old_obj:
try:
setattr(mod, attr_name, new_obj)
count += 1
except Exception as exc:
logger.debug("Could not rebind %s in %s: %s", attr_name, mod_name, exc)
return count
_TVM_FFI_BROKEN_VERSIONS = ("0.1.10", "0.1.11")
def _install_metadata_scan_cache() -> Any:
"""Share one packages_distributions() scan between transformers and TRL (unsloth's own window
opens after this worker's first transformers import). Loaded by path: the package would run
unsloth's whole init. None when the helper is missing (older unsloth) or fails."""
try:
spec = importlib.util.find_spec("unsloth")
for root in list(getattr(spec, "submodule_search_locations", None) or ()):
path = os.path.join(root, "_import_metadata_cache.py")
if not os.path.isfile(path):
continue
helper_spec = importlib.util.spec_from_file_location(
"_unsloth_studio_import_metadata_cache", path
)
helper = importlib.util.module_from_spec(helper_spec)
helper_spec.loader.exec_module(helper)
token = helper.install()
return None if token is None else (helper, token)
except Exception:
pass
return None
def _uninstall_metadata_scan_cache(installed: Any) -> None:
if installed is None:
return
helper, token = installed
try:
helper.uninstall(token)
except Exception:
pass
def _guard_fla_tilelang() -> None:
"""Default FLA_TILELANG to 0 on ROCm torch and on an apache-tvm-ffi that faults on sm_100."""
try:
import torch as _torch_for_fla
if (
getattr(_torch_for_fla.version, "hip", None)
or "rocm" in getattr(_torch_for_fla, "__version__", "").lower()
):
os.environ.setdefault("FLA_TILELANG", "0")
except Exception as exc:
logger.debug("FLA_TILELANG guard skipped: %s", exc)
# A leftover TileLang plus a broken tvm-ffi crashes the run; steer off it instead of installing.
try:
if importlib.util.find_spec("tilelang") is not None:
tvm_ffi_version = importlib.metadata.version("apache-tvm-ffi")
if tvm_ffi_version in _TVM_FFI_BROKEN_VERSIONS:
before = os.environ.get("FLA_TILELANG")
os.environ.setdefault("FLA_TILELANG", "0")
# setdefault is a no-op under an override, so only claim what actually happened.
if before is None:
logger.info(
"Disabling TileLang: apache-tvm-ffi %s faults under it; FLA_TILELANG is now %s",
tvm_ffi_version,
os.environ.get("FLA_TILELANG"),
)
else:
logger.info(
"Keeping FLA_TILELANG=%s set by the environment despite apache-tvm-ffi %s",
before,
tvm_ffi_version,
)
except Exception as exc:
logger.debug("FLA_TILELANG guard skipped: %s", exc)
def _install_fast_path_hooks(
event_queue: Any,
model_name: str,
*,
install_causal_conv1d: bool | None = None,
) -> None:
"""First gate call drives the install; UNSLOTH_STUDIO_SKIP_FAST_PATH_HOOKS=1 uses the substring gate."""
_guard_fla_tilelang()
if os.getenv(_FAST_PATH_HOOKS_SKIP_ENV) == "1":
logger.info("Fast-path hooks disabled via env; using substring fallback")
return
try:
from transformers.utils import import_utils as _iu
except Exception as exc:
logger.warning(
"transformers.utils.import_utils not importable; skipping fast-path hooks: %s",
exc,
)
return
def _make_wrapper(
original: Callable[[], bool], install_fn: Callable[[Any], bool], gate_name: str
) -> Callable[[], bool]:
state = {"installed": False}
def wrapper() -> bool:
if state["installed"]:
return original()
try:
original.cache_clear() # defensive; worker subprocess is fresh
except AttributeError:
pass
ok = original()
if not ok:
logger.info("Hook fired for %s; triggering install", gate_name)
try:
ok = bool(install_fn(event_queue))
except Exception as exc:
logger.warning("%s install raised: %s; falling back to torch", gate_name, exc)
ok = False
logger.info("%s hook done; available=%s", gate_name, ok)
state["installed"] = True
return ok
wrapper.__wrapped__ = original # type: ignore[attr-defined]
wrapper.cache_clear = getattr(original, "cache_clear", lambda: None) # type: ignore[attr-defined]
return wrapper
def _causal_conv1d_install(eq: Any) -> bool:
ok = _install_package_wheel_first(event_queue = eq, **_pinned_kernel_kwargs(CAUSAL_CONV1D))
return bool(ok)
hooks: list[tuple[str, Callable[[Any], bool]]] = []
if install_causal_conv1d is None:
install_causal_conv1d = _model_wants_causal_conv1d(model_name)
if install_causal_conv1d:
hooks.append(("is_causal_conv1d_available", _causal_conv1d_install))
for gate_name, install_fn in hooks:
original = getattr(_iu, gate_name, None)
if original is None:
logger.info(
"%s missing on transformers.utils.import_utils; skipping hook",
gate_name,
)
continue
wrapped = _make_wrapper(original, install_fn, gate_name)
setattr(_iu, gate_name, wrapped)
rebound = _rebind_in_already_imported_modules(
attr_name = gate_name, old_obj = original, new_obj = wrapped
)
logger.info("Installed fast-path hook on %s (rebound %d modules)", gate_name, rebound)
def _should_try_runtime_flash_attn_install(max_seq_length: int) -> bool:
if os.getenv(_FLASH_ATTN_SKIP_ENV) == "1":
return False
if max_seq_length < _FLASH_ATTN_RUNTIME_MIN_SEQ_LEN:
return False
return sys.platform.startswith("linux") or sys.platform == "win32"
def _flash_attn_gpu_below_floor() -> bool:
"""Whether the best visible GPU is below sm80, where Unsloth leaves FlashAttention 2 off. Asks
the torch this worker already imported when there is one (no child process); ROCm never takes
the NVIDIA floor; no visible GPU is not below it."""
torch = sys.modules.get("torch")
if torch is not None:
try:
if getattr(torch.version, "hip", None):
return False
if not torch.cuda.is_available():
return False
count = torch.cuda.device_count()
return count > 0 and max(
tuple(torch.cuda.get_device_capability(i)) for i in range(count)
) < (8, 0)
except Exception as exc:
logger.debug("In-process GPU capability check failed (%s); probing a child", exc)
from utils.kernel_install import gpu_below_flash_attn_floor
return gpu_below_flash_attn_floor()
def _ensure_flash_attn_for_long_context(event_queue: Any, max_seq_length: int) -> None:
if not _should_try_runtime_flash_attn_install(max_seq_length):
return
if _is_importable("flash_attn"):
# Already usable: nothing to download, so no capability probe either.
logger.info("flash-attn already installed")
return
if _flash_attn_gpu_below_floor():
# Unsloth only turns FlashAttention 2 on from sm80 (a T4 is sm75): nothing to download.
_send_status(event_queue, "flash-attn needs an sm80 or newer GPU; continuing without it")
return
installed = _install_package_wheel_first(
event_queue = event_queue,
import_name = "flash_attn",
display_name = "flash-attn",
pypi_name = "flash-attn",
wheel_url_builder = flash_attn_wheel_url,
# Optional acceleration: skip an unavailable wheel instead of building during training.
allow_source_build = False,
)
if not installed:
_send_status(event_queue, "Continuing without flash-attn")
def _activate_transformers_version(model_name: str, hf_token: str | None = None) -> None:
"""Activate the correct transformers version BEFORE any ML imports."""
backend_path = str(Path(__file__).resolve().parent.parent.parent)
if backend_path not in sys.path:
sys.path.insert(0, backend_path)
from utils.transformers_version import activate_transformers_for_subprocess
activate_transformers_for_subprocess(model_name, hf_token)
def _activate_transformers_version_or_warn(model_name: str, hf_token: str | None = None) -> None:
"""MLX path is non-fatal: failed transformers activation logs a warning, then falls through."""
try:
_activate_transformers_version(model_name, hf_token)
except Exception as exc:
logger.warning(
"Failed to activate transformers version for '%s' (MLX); "
"training may fail if this model requires a specific version. Error: %s",
model_name,
exc,
)
def _mlx_vocab_size(model):
"""Rows of the MLX model's input embedding table (the CUDA path reads get_input_embeddings().num_embeddings), so
a pre-tokenized id or label past it is refused before the loss indexes it. None when it cannot be found, which
skips only the range check."""
for owner in (model, getattr(model, "language_model", None)):
if owner is None:
continue
for embed in (
getattr(getattr(owner, "model", None), "embed_tokens", None),
getattr(owner, "embed_tokens", None),
):
# A LoRA-wrapped embedding keeps the table on its inner module.
for layer in (
embed,
getattr(embed, "embedding", None),
getattr(embed, "base_layer", None),
):
if layer is None:
continue
# QuantizedEmbedding packs its weight, so its row count is the stored num_embeddings.
rows = getattr(layer, "num_embeddings", None)
shape = getattr(getattr(layer, "weight", None), "shape", None)
if rows is None and shape:
rows = shape[0]
if isinstance(rows, int) and not isinstance(rows, bool) and rows > 0:
return rows
return None
def _carry_mlx_chat_template_to_processor(model, tokenizer) -> bool:
"""MLXTrainer saves the loader's processor when it has one, so a text run on a multimodal base checkpoint (Qwen3.5
Base) saved no template and Chat failed with "chat_template is not set". Give it the template the rows used."""
processor = getattr(model, "_processor", None)
if processor is None or processor is tokenizer:
return False
trained = tokenizer
if type(tokenizer).__name__ == "TokenizerWrapper":
trained = getattr(tokenizer, "_tokenizer", tokenizer)
template = getattr(trained, "chat_template", None)
if not template or getattr(processor, "chat_template", None):
return False
own = getattr(processor, "tokenizer", None)
if own is not None and own is not trained and getattr(own, "chat_template", None):
return False
processor.chat_template = template
if own is not None and own is not trained:
# Also carries get_chat_template's end-token remap.
processor.tokenizer = trained
return True
def _mlx_vlm_max_resized_size(width: int, height: int, target: int) -> tuple[int, int]:
if width <= 0 or height <= 0 or target <= 0:
return width, height
largest_side = max(width, height)
if largest_side <= target:
return width, height
# Integer formula matches unsloth_zoo's collator (round() differs by 1px).
new_w = max(1, (width * target + largest_side // 2) // largest_side)
new_h = max(1, (height * target + largest_side // 2) // largest_side)
return new_w, new_h
_MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE = {}
def _mlx_vlm_resized_image_layout(processor = None) -> str | None:
"""Return the numpy image layout expected after Unsloth-side VLM resizing."""
image_processor = getattr(processor, "image_processor", None)
if image_processor is None:
return None
cls = image_processor.__class__
key = (getattr(cls, "__module__", ""), getattr(cls, "__qualname__", cls.__name__))
if key in _MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE:
return _MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE[key]
copied_image_processor = _copy_mlx_vlm_image_processor(image_processor)
layout = (
_probe_mlx_vlm_numpy_image_layout(copied_image_processor)
if copied_image_processor is not None
else None
)
_MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE[key] = layout
return layout
def _copy_mlx_vlm_image_processor(image_processor):
import copy
try:
return copy.deepcopy(image_processor)
except Exception:
try:
return copy.copy(image_processor)
except Exception:
return None
def _probe_mlx_vlm_numpy_image_layout(image_processor) -> str | None:
try:
import numpy as np
except ImportError:
return None
def _accepts(candidate) -> bool:
try:
image_processor(images = [candidate])
return True
except TypeError:
try:
image_processor([candidate])
return True
except Exception:
return False
except Exception:
return False
# Asymmetric image so CHW-vs-HWC mistakes are visible to processors that skip 3D conversion.
hwc = np.zeros((64, 96, 3), dtype = np.uint8)
chw = np.ascontiguousarray(hwc.transpose(2, 0, 1))
if _accepts(hwc):
return None
if _accepts(chw):
return "chw"
return None
def _resize_mlx_vlm_image(
image,
resize,
image_layout = None,
):
if resize is None:
return image
try:
from PIL import Image
import numpy as np
except ImportError:
return image
if not isinstance(image, Image.Image):
return image
image = image.convert("RGB")
new_size = _mlx_vlm_max_resized_size(*image.size, int(resize))
if new_size != image.size:
resampling = getattr(Image, "Resampling", Image).LANCZOS
image = image.resize(new_size, resampling)
# Writable RGB ndarray skips mlx-vlm's PIL square-resize and HF non-writable warnings.
array = np.array(image, copy = True)
if image_layout == "chw":
return np.ascontiguousarray(array.transpose(2, 0, 1))
return array
def _resize_mlx_vlm_images(
value,
resize,
image_layout = None,
):
if isinstance(value, list):
return [_resize_mlx_vlm_image(image, resize, image_layout = image_layout) for image in value]
return _resize_mlx_vlm_image(value, resize, image_layout = image_layout)
_MLX_VIDEO_UNSUPPORTED = (
"Training on video is not supported on Apple Silicon (MLX) yet. "
"Train this dataset on an NVIDIA or AMD GPU, or drop the video column."
)
_RESOLVE_IMAGE_COLUMN = object()
def _mlx_audio_rows(
dataset,
processor,
custom_format_mapping = None,
image_col = _RESOLVE_IMAGE_COLUMN,
warn = None,
) -> tuple:
"""The CUDA transcription rows, with each clip left decoded so zoo can check its rate, and
the row's images (a mixed audio + image dataset) beside them. Returns (rows, image column);
an eval split takes the train split's image column, None for none."""
from datasets import Audio
from unsloth_zoo.mlx.utils import audio_extractor_sampling_rate
from utils.datasets.audio_decode import ensure_audio_decoding
from utils.datasets.audio_rows import (
audio_transcription_messages,
prepare_audio_vlm_image_column,
resolve_audio_columns,
resolve_audio_vlm_image_column,
)
from utils.datasets.cells import cell_text
if not ensure_audio_decoding():
raise RuntimeError(
"No audio decoder is available: install the audio runtime (librosa and soundfile)."
)
columns = resolve_audio_columns(dataset, custom_format_mapping)
audio_col, text_col = columns["audio_col"], columns["text_col"]
if not audio_col or not text_col:
raise ValueError(
f"Audio dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
train_split = image_col is _RESOLVE_IMAGE_COLUMN
if train_split:
image_col = resolve_audio_vlm_image_column(
dataset, custom_format_mapping, audio_col, text_col, columns["speaker_col"]
)
elif image_col not in dataset.column_names:
image_col = None
rate = audio_extractor_sampling_rate(processor) or 16000
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = rate))
if image_col:
dataset = prepare_audio_vlm_image_column(dataset, image_col)
rows = []
blank = 0
for row in dataset:
text = cell_text(row[text_col])
if not text.strip():
blank += 1
continue
images = row[image_col] if image_col else None
if not isinstance(images, list):
images = [] if images is None else [images]
images = [image for image in images if image is not None]
item = {"messages": audio_transcription_messages(row[audio_col], text, len(images))}
if images:
item["images"] = images
rows.append(item)
if not rows and train_split:
raise ValueError(f"Every transcript in the '{text_col}' column is blank.")
if blank:
(warn or logger.warning)(
f"Skipped {blank:,} of {blank + len(rows):,} audio rows with a blank transcript."
)
return rows, image_col
def _adapt_for_mlx_vlm(
items,
resize = None,
image_layout = None,
):
"""mlx-vlm needs images at top level to build pixel_values, not inside message content."""
adapted = []
for item in items:
images = []
messages = []
for msg in item.get("messages", []):
content = msg.get("content", "")
if isinstance(content, list):
new_content = []
for part in content:
if isinstance(part, dict) and part.get("type") == "video":
raise ValueError(_MLX_VIDEO_UNSUPPORTED)
if isinstance(part, dict) and part.get("type") == "image":
img = part.get("image")
if img is not None:
images.append(
_resize_mlx_vlm_image(
img,
resize,
image_layout = image_layout,
)
)
new_content.append({"type": "image"})
else:
new_content.append(part)
messages.append({"role": msg["role"], "content": new_content})
else:
messages.append(msg)
out = {"messages": messages}
if images:
out["image"] = images[0] if len(images) == 1 else images
elif "image" in item:
out["image"] = _resize_mlx_vlm_images(
item["image"],
resize,
image_layout = image_layout,
)
elif "images" in item:
out["images"] = _resize_mlx_vlm_images(
item["images"],
resize,
image_layout = image_layout,
)
adapted.append(out)
return adapted
_MLX_STUDIO_LR_SCHEDULERS = {"linear", "cosine", "constant"}
# Fallback mirroring unsloth_zoo._normalize_mlx_optimizer_name when mlx isn't importable.
_MLX_STUDIO_ADAMW_ALIASES = frozenset(
(
"adamw_8bit",
"paged_adamw_8bit",
"adamw_bnb_8bit",
"paged_adamw_32bit",
"adamw_torch",
"adamw_torch_fused",
"paged_adamw",
"adamw_32bit",
"adamw_hf",
"adamw_anyprecision",
"adamw_apex_fused",
)
)
_MLX_STUDIO_NATIVE_OPTIMIZERS = ("adafactor", "adamw", "adam", "sgd", "muon", "lion")
def _normalize_mlx_studio_optimizer(value):
try:
from unsloth_zoo.mlx.trainer import _normalize_mlx_optimizer_name
return _normalize_mlx_optimizer_name(value or "adamw_8bit")
except (ImportError, ValueError):
# Missing mlx, or an older zoo normalizer: map common adamw_* names locally.
opt = str(getattr(value, "value", value) or "adamw_8bit").strip().lower()
opt = opt.rsplit(".", 1)[-1].replace("-", "_")
if opt in _MLX_STUDIO_ADAMW_ALIASES:
opt = "adamw"
if opt not in _MLX_STUDIO_NATIVE_OPTIMIZERS:
supported = ", ".join(_MLX_STUDIO_NATIVE_OPTIMIZERS)
raise ValueError(
f"Unsupported optimizer for MLX training: {value!r}. "
f"Supported optimizers: {supported}."
)
return opt
def _mlx_dora_peft_kwargs(config, get_peft_model):
"""LoRA kwargs a DoRA request adds for MLX, or raise why it cannot run."""
import inspect
if not config.get("use_dora"):
return {}
try:
parameter = inspect.signature(get_peft_model).parameters.get("use_dora")
except (TypeError, ValueError):
parameter = None
# A **kwargs catch-all would silently swallow use_dora, so require a named parameter.
named = parameter is not None and parameter.kind in (
inspect.Parameter.POSITIONAL_OR_KEYWORD,
inspect.Parameter.KEYWORD_ONLY,
)
if not named:
raise NotImplementedError(
"DoRA on Apple Silicon needs an unsloth-zoo whose MLX "
"get_peft_model takes use_dora, and this install cannot be "
"confirmed to. The version that predates MLX DoRA accepts the "
"request and trains plain LoRA instead, so the run stops here "
"rather than guessing. Update unsloth-zoo, or pick a different "
"LoRA variant."
)
return {"use_dora": True}
def _normalize_mlx_studio_scheduler(value):
raw = str(value or "linear").strip().lower()
if raw not in _MLX_STUDIO_LR_SCHEDULERS:
supported = ", ".join(sorted(_MLX_STUDIO_LR_SCHEDULERS))
raise ValueError(
f"Unsupported LR scheduler for MLX training: {value!r}. Supported values: {supported}."
)
return raw
def _resolve_mlx_local_dataset_path(dataset_file) -> Path:
from utils.paths import resolve_dataset_path
dataset_path = Path(os.path.expanduser(str(dataset_file)))
if dataset_path.is_absolute():
return dataset_path
if dataset_path.exists():
return dataset_path.resolve()
return Path(str(resolve_dataset_path(str(dataset_file))))
def _resolve_mlx_local_dataset_files(file_paths: list) -> list[str]:
"""Resolve CLI paths and Unsloth local dataset uploads without importing the GPU trainer."""
from utils.paths import dataset_files_in_dir
all_files: list[str] = []
for dataset_file in file_paths or []:
file_path_obj = _resolve_mlx_local_dataset_path(dataset_file)
if file_path_obj.is_dir():
all_files.extend(str(p) for p in dataset_files_in_dir(file_path_obj))
continue
all_files.append(str(file_path_obj))
return all_files
def _mlx_local_dataset_loader_for_files(files: list[str]) -> str:
first_ext = Path(files[0]).suffix.lower()
if first_ext in (".json", ".jsonl"):
return "json"
if first_ext == ".csv":
return "csv"
if first_ext == ".parquet":
return "parquet"
raise ValueError(f"Unsupported dataset format: {files[0]}")
_MLX_WORKER_COMPLETE = "_mlx_worker_complete"
def _start_worker_stop_poller(
stop_queue,
on_stop: Callable[[bool], None],
*,
completion_type: str | None = None,
timeout: float = 1.0,
):
import queue as _queue
import threading
cancel_requested = False
def poll_stop():
nonlocal cancel_requested
while True:
try:
msg = stop_queue.get(timeout = timeout)
if not isinstance(msg, dict):
continue
message_type = msg.get("type")
if completion_type is not None and message_type == completion_type:
return
if message_type != "stop":
continue
if not bool(msg.get("save", True)):
cancel_requested = True
on_stop(not cancel_requested)
if cancel_requested:
return
except _queue.Empty:
continue
except (EOFError, OSError, ValueError):
return
stop_thread = account_thread(target = poll_stop, daemon = True)
stop_thread.start()
return stop_thread
def _start_mlx_stop_poller(stop_queue):
stop_save = [True]
stop_requested = [False]
trainer_ref = [None]
def is_stop_requested():
return stop_requested[0]
def apply_stop(save: bool) -> None:
stop_save[0] = save
stop_requested[0] = True
trainer = trainer_ref[0]
if trainer is not None:
trainer.stop_requested = True
# Early stopping set load_best_model_at_end; a manual stop keeps the current adapters instead, since the
# stop checkpoint stores this step's optimizer state next to them.
args = getattr(trainer, "args", None)
if getattr(args, "load_best_model_at_end", False):
args.load_best_model_at_end = False
stop_thread = _start_worker_stop_poller(
stop_queue,
apply_stop,
completion_type = _MLX_WORKER_COMPLETE,
timeout = 0.25,
)
return stop_save, stop_requested, trainer_ref, is_stop_requested, stop_thread
def _resolve_mlx_output_dir(config, model_name):
from utils.paths import resolve_output_dir, default_run_dir_name
output_dir = config.get("output_dir", "")
if not output_dir:
output_dir = f"{default_run_dir_name(model_name)}_{int(time.time())}"
return str(resolve_output_dir(output_dir))
if config.get("allow_external_output_dir"):
output_path = Path(output_dir).expanduser()
if not output_path.is_absolute():
output_path = Path.cwd() / output_path
return str(output_path.resolve())
return str(resolve_output_dir(output_dir))
def _resolve_mlx_training_steps(
requested_max_steps,
dataset_size,
batch_size,
gradient_accumulation_steps,
num_epochs,
world_size,
):
if requested_max_steps > 0:
return requested_max_steps
global_batch_size = batch_size * world_size
return max(
1,
math.ceil(dataset_size / global_batch_size / gradient_accumulation_steps) * num_epochs,
)
def _slice_streaming_dataset(dataset, slice_start, slice_end):
"""A stream slices lazily, as on CUDA, and has no length to bound."""
if slice_start:
dataset = dataset.skip(slice_start)
if slice_end is not None:
dataset = dataset.take(max(0, slice_end - (slice_start or 0) + 1))
return dataset
def _configure_mlx_training_schedule(
trainer,
requested_max_steps,
dataset_size,
batch_size,
gradient_accumulation_steps,
num_epochs,
warmup_ratio = None,
eval_steps_ratio = None,
):
max_steps = _resolve_mlx_training_steps(
requested_max_steps,
dataset_size,
batch_size,
gradient_accumulation_steps,
num_epochs,
trainer.distributed_world_size,
)
trainer.args.max_steps = max_steps
if warmup_ratio is not None:
trainer.args.warmup_steps = math.ceil(warmup_ratio * max_steps)
if eval_steps_ratio is not None:
trainer.args.eval_steps = max(1, math.ceil(eval_steps_ratio * max_steps))
return max_steps, int(trainer.args.eval_steps)
def _run_mlx_main_process_action(trainer, action, context):
world_size = int(trainer.distributed_world_size)
if world_size <= 1:
return action()
raise_distributed_failure = getattr(trainer, "_raise_distributed_failure", None)
if not callable(raise_distributed_failure):
raise RuntimeError(
"Unsloth MLX DDP CLI requires MLXTrainer failure coordination. "
"Upgrade unsloth-zoo to a compatible version."
)
result = None
error = None
if bool(trainer.is_main_process):
try:
result = action()
except BaseException as exc:
error = exc
raise_distributed_failure(error is not None, context, error)
return result
def _prepare_mlx_output_dir(
trainer,
output_dir,
ensure_dir,
event_queue = None,
):
def _prepare():
result = ensure_dir(Path(output_dir))
if event_queue is not None:
_emit_output_dir(event_queue, output_dir)
return result
return _run_mlx_main_process_action(
trainer,
_prepare,
"output directory setup",
)
def _setup_mlx_tracking(trainer, config, output_dir, send):
wandb_run = None
tb_writer = None
if config.get("enable_wandb", False):
def _setup_wandb():
try:
import wandb as _wandb
wandb_token = config.get("wandb_token")
if wandb_token:
os.environ["WANDB_API_KEY"] = wandb_token
# Keep the authenticated subject out of W&B run config (mirrors _sanitize_db_config).
_wandb_sensitive = {"hf_token", "wandb_token", "s3_config", "subject"}
return _wandb.init(
project = config.get("wandb_project") or "unsloth-mlx",
config = {k: v for k, v in config.items() if k not in _wandb_sensitive},
reinit = True,
)
except Exception as e:
send("status", status_message = f"wandb init failed: {e}")
return None
wandb_run = _run_mlx_main_process_action(trainer, _setup_wandb, "Weights & Biases setup")
if config.get("enable_tensorboard", False):
def _setup_tensorboard():
try:
from tensorboardX import SummaryWriter
except ImportError:
try:
from torch.utils.tensorboard import SummaryWriter
except ImportError:
SummaryWriter = None
if SummaryWriter is None:
send(
"status",
status_message = "tensorboard unavailable (install tensorboardX)",
)
return None
try:
tb_dir = config.get("tensorboard_dir") or f"{output_dir}/runs"
return SummaryWriter(log_dir = tb_dir)
except Exception as e:
send("status", status_message = f"tensorboard init failed: {e}")
return None
tb_writer = _run_mlx_main_process_action(trainer, _setup_tensorboard, "TensorBoard setup")
return wandb_run, tb_writer
def _mlx_worker_finalization_state(trainer, snapshot_stop):
trainer_stopped = bool(trainer.stop_requested)
local_stop_requested, stop_save = snapshot_stop()
stopped = trainer_stopped or bool(local_stop_requested)
cancelled = bool(local_stop_requested and not stop_save)
if int(trainer.distributed_world_size) > 1:
any_flag = getattr(trainer, "_distributed_any_flag", None)
if not callable(any_flag):
raise RuntimeError(
"Unsloth MLX DDP CLI requires MLXTrainer stop coordination. "
"Upgrade unsloth-zoo to a compatible version."
)
stopped = bool(any_flag(stopped))
cancelled = bool(any_flag(cancelled))
if stopped:
trainer.stop_requested = True
return stopped, cancelled
def _synchronize_mlx_before_final_save(trainer, synchronize):
error = None
try:
synchronize()
except BaseException as exc:
error = exc
if int(trainer.distributed_world_size) <= 1:
if error is not None:
raise error
return
raise_distributed_failure = getattr(trainer, "_raise_distributed_failure", None)
if not callable(raise_distributed_failure):
raise RuntimeError(
"Unsloth MLX DDP CLI requires MLXTrainer failure coordination. "
"Upgrade unsloth-zoo to a compatible version."
)
raise_distributed_failure(error is not None, "final synchronization", error)
def _finalize_mlx_training(
trainer, snapshot_stop, output_dir, synchronize, send, write_stop_checkpoint
):
stopped, cancelled = _mlx_worker_finalization_state(trainer, snapshot_stop)
if cancelled:
send("complete", output_dir = None, status_message = "Training cancelled")
return
if stopped:
saving_message = "Saving stopped model..."
completion_message = "Training stopped"
else:
saving_message = "Saving model..."
completion_message = "Training completed"
send("status", status_message = saving_message)
_synchronize_mlx_before_final_save(trainer, synchronize)
_run_mlx_main_process_action(
trainer,
lambda: trainer.save_model(output_dir),
"final model save",
)
stopped_after_save, cancelled_after_save = _mlx_worker_finalization_state(
trainer, snapshot_stop
)
if cancelled_after_save:
send("complete", output_dir = None, status_message = "Training cancelled")
return
if stopped_after_save:
completion_message = "Training stopped"
checkpoint_ok = _run_mlx_main_process_action(
trainer,
write_stop_checkpoint,
"stop checkpoint save",
)
if int(trainer.distributed_world_size) > 1:
checkpoint_ok = trainer._distributed_any_flag(bool(checkpoint_ok))
if not checkpoint_ok:
send(
"error",
error = (
"Failed to save a resumable checkpoint after stop. "
"Model files were saved, but this run cannot be resumed."
),
keep_error_status = True,
resume_blocked = True,
)
return
if not stopped_after_save and getattr(trainer, "_early_stopped", False):
# Zoo ends an early-stopped run without stop_requested, so it reads as a plain completion otherwise.
completion_message = "Training completed (stopped early: eval loss stopped improving)"
send(
"complete",
output_dir = output_dir if bool(trainer.is_main_process) else None,
status_message = completion_message,
)
def _resolve_mlx_max_grad_norm(value):
"""Unset stays 0.0 so MLX keeps cheap per-parameter clipping; a requested value is honored."""
if value is None:
return 0.0
try:
value = float(value)
except (TypeError, ValueError):
raise ValueError(
f"Unsloth MLX: max_grad_norm={value!r} must be a non-negative float or None."
)
# inf clears a >= 0 check but never binds, so the run would train unclipped.
if value < 0 or not math.isfinite(value):
raise ValueError(
f"Unsloth MLX: max_grad_norm={value} must be a finite value >= 0 "
"(use 0 to disable global norm clipping)."
)
return value
def _run_mlx_training(event_queue, stop_queue, config):
"""Apple Silicon MLX path via unsloth_zoo's MLXTrainer, no torch; same event_queue protocol."""
import time
import math
from pathlib import Path
def _send(event_type, **kwargs):
if event_type == "status" and "message" not in kwargs:
sm = kwargs.get("status_message")
if sm is not None:
kwargs["message"] = sm
event_queue.put({"type": event_type, "ts": time.time(), **kwargs})
_stop_save, _stop_requested, _trainer_ref, _is_stop_requested, _stop_thread = (
_start_mlx_stop_poller(stop_queue)
)
def _snapshot_stop():
return _stop_requested[0], _stop_save[0]
_send("status", status_message = "Loading MLX libraries...")
import mlx.core as mx
from mlx.utils import tree_flatten
try:
from unsloth_zoo.mlx.loader import FastMLXModel
from unsloth_zoo.mlx.trainer import (
MLXTrainer,
MLXTrainingConfig,
train_on_responses_only,
)
from unsloth_zoo.mlx.utils import iter_mlx_lora_modules
except ImportError as e:
raise ImportError(
"Unsloth: MLX training requires unsloth-zoo with the MLX modules "
"(unsloth_zoo.mlx.loader / unsloth_zoo.mlx.trainer). Reinstall via "
"install.sh on Apple Silicon."
) from e
from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset
if mx.metal.is_available():
info = mx.device_info()
rec_bytes = info.get("max_recommended_working_set_size", 0) or 0
if rec_bytes > 0:
memory_cap = int(rec_bytes * 0.85)
wired_cap = min(int(rec_bytes), memory_cap)
mx.set_memory_limit(memory_cap)
mx.set_wired_limit(wired_cap)
model_name = config["model_name"]
hf_token = _worker_hf_token(config)
if hf_token:
os.environ["HF_TOKEN"] = hf_token
model_load_name = _resolve_cached_model_load_name(config)
model_local_only = _model_local_files_only(config)
model_revision = None if model_local_only else config.get("model_revision")
if config.get("use_loftq"):
message = "LoftQ is not supported for MLX training yet."
_send("error", error = message)
raise NotImplementedError(message)
if config.get("is_embedding"):
message = "Embedding model training is not supported for MLX training yet."
_send("error", error = message)
raise NotImplementedError(message)
# Decided before the model loads, so version skew does not cost a download.
try:
mlx_dora_kwargs = _mlx_dora_peft_kwargs(config, FastMLXModel.get_peft_model)
except NotImplementedError as exc:
_send("error", error = str(exc))
raise
optim_name = _normalize_mlx_studio_optimizer(config.get("optim", "adamw_8bit"))
lr_scheduler_type = _normalize_mlx_studio_scheduler(config.get("lr_scheduler_type", "linear"))
# Force text-only for non-image datasets even on vision-capable models (e.g. Qwen3.5-VL on alpaca).
# Forward resume_from_checkpoint to MLXTrainer.train(); else Resume silently restarts from step 0.
resume_from_checkpoint = config.get("resume_from_checkpoint") or None
training_type = config.get("training_type", "LoRA/QLoRA")
is_cpt = training_type == "Continued Pretraining"
# As on CUDA, CPT trains on raw text, so an image dataset loads and formats as text.
is_dataset_image = bool(config.get("is_dataset_image", False)) and not is_cpt
# CUDA takes its raw-text path (CPT or format "raw") before the audio one.
is_dataset_audio = bool(config.get("is_dataset_audio", False)) and not (
is_cpt or config.get("format_type") == "raw"
)
use_lora = training_type in ("LoRA/QLoRA", "Continued Pretraining")
# Before the download/load below: unlike the CUDA path, this needs none of the model.
# A continued adapter keeps its own targets, and CPT builds its adapter from target_modules alone.
if use_lora and not is_cpt and config.get("model_format") != "adapter":
_check_mlx_finetune_targets(config)
# Normalize seed; explicit None must not reach the seed chain.
_raw_seed = config.get("random_seed", 3407)
random_seed = 3407 if _raw_seed is None else int(_raw_seed)
# `config.get(k, d)` only fills d when key is missing; handle explicit None too.
_model_seed = config.get("model_random_state")
model_random_state = random_seed if _model_seed is None else int(_model_seed)
_lora_seed = config.get("lora_random_state")
lora_random_state = random_seed if _lora_seed is None else int(_lora_seed)
security_error = _model_load_security_error(config, model_load_name, hf_token)
if security_error:
_send("error", **security_error)
return
# Fetch the dataset before the model so a dataset error does not wait on the model download.
_send("status", status_message = "Loading dataset...")
if config.get("is_dataset_audio"):
# Raw-text runs still read rows carrying the Audio column, so install the decoder for them too.
from utils.datasets.audio_decode import ensure_audio_decoding
ensure_audio_decoding()
hf_dataset = config.get("hf_dataset", "")
dataset_streaming = bool(hf_dataset and config.get("dataset_streaming"))
config["_dataset_loaded_from_exact_snapshot"] = False
def _load_local(file_paths):
from utils.datasets.saved import load_saved_dataset
if len(file_paths) == 1:
saved = load_saved_dataset(_resolve_mlx_local_dataset_path(file_paths[0]))
if saved is not None:
return saved
all_files = _resolve_mlx_local_dataset_files(file_paths)
if not all_files:
raise ValueError("No local dataset files found")
from utils.datasets.cells import csv_as_text_kwargs
loader = _mlx_local_dataset_loader_for_files(all_files)
return load_dataset(
loader,
data_files = all_files,
split = "train",
**csv_as_text_kwargs(all_files),
)
eval_dataset = None
if hf_dataset:
dataset, eval_dataset = _load_hf_train_and_eval_datasets(
config,
hf_token,
load_dataset,
lambda message: _send("status", status_message = message),
lambda message: _send("warning", message = message),
)
elif config.get("local_datasets"):
dataset = _load_local(config["local_datasets"])
elif config.get("s3_config"):
from core.training.s3_dataset import (
S3DownloadCancelled,
prepare_s3_dataset_download,
)
_send("status", status_message = "Downloading dataset from S3...")
try:
s3_download = prepare_s3_dataset_download(
config["s3_config"],
cancel_callback = _is_stop_requested,
)
try:
dataset = _load_local(s3_download.files)
finally:
s3_download.cleanup()
except S3DownloadCancelled:
_send("complete", output_dir = None, status_message = "Training cancelled")
return
else:
raise ValueError("No dataset specified")
from core.training.eval_dataset import evaluation_enabled
eval_enabled = evaluation_enabled(config.get("eval_steps"))
if eval_enabled and not hf_dataset and config.get("local_eval_datasets"):
eval_dataset = _load_local(config["local_eval_datasets"])
if _is_stop_requested():
_send("complete", output_dir = None, status_message = "Training cancelled")
return
_send("status", status_message = f"Loading {model_name}...")
try:
model, tokenizer = FastMLXModel.from_pretrained(
model_load_name,
load_in_4bit = config.get("load_in_4bit", True),
full_finetuning = not use_lora,
text_only = None if is_dataset_image or is_dataset_audio else True,
token = hf_token,
trust_remote_code = bool(config.get("trust_remote_code", False)),
random_state = model_random_state,
revision = model_revision,
)
except Exception as error:
if not model_local_only:
raise
fallback_error = _model_cache_fallback_error(config, error)
if fallback_error is not None:
raise fallback_error from error
if not _cache_artifact_fallback_allowed(config, error, "model"):
raise
revision_error = _mlx_revision_fallback_error(config)
if revision_error is not None:
raise revision_error from error
_send(
"status",
status_message = (
f"Cached files for {model_name} are incomplete; retrying from Hugging Face..."
),
)
model_load_name = _drop_model_pin_for_fallback(config, hf_token)
# Scan the Hub target we fall back to, not the cached pin already scanned above.
security_error = _model_load_security_error(config, model_load_name, hf_token)
if security_error:
_send("error", **security_error)
return
model_local_only = False
model_revision = config.get("model_revision")
model, tokenizer = FastMLXModel.from_pretrained(
model_load_name,
load_in_4bit = config.get("load_in_4bit", True),
full_finetuning = not use_lora,
text_only = None if is_dataset_image or is_dataset_audio else True,
token = hf_token,
trust_remote_code = bool(config.get("trust_remote_code", False)),
random_state = model_random_state,
revision = model_revision,
)
from utils.models.model_identity import restore_hf_cache_repo_identity
restored_repo_id = restore_hf_cache_repo_identity(
model,
model_load_name,
expected_repo_id = config.get("actual_model_repo_id") or model_name,
)
if restored_repo_id:
logger.info(
"Restored Hub model identity for saved MLX adapter metadata: %s",
restored_repo_id,
)
loaded_model_for_provenance = model
is_vlm = bool((is_dataset_image or is_dataset_audio) and getattr(model, "_is_vlm_model", False))
model._is_vlm_model = is_vlm
if is_dataset_audio and not is_vlm:
# zoo trains audio only as input to an audio-capable vision-language model.
raise ValueError(
"Audio dataset training on Apple Silicon needs an audio-input model such as "
"Gemma 3n or Gemma 4; text-to-speech and speech-recognition checkpoints are not "
"supported on MLX yet."
)
vision_image_size = config.get("vision_image_size")
# DeepSeek OCR uses a coupled preset tuple; skip resize like the Torch path.
_model_name_lower = str(config.get("model_name", "")).lower()
_is_deepseek_ocr = "deepseek" in _model_name_lower and "ocr" in _model_name_lower
if is_vlm and vision_image_size is not None and _is_deepseek_ocr:
_send(
"status",
status_message = (
"MLX vision image resize ignored for DeepSeek OCR (uses fixed Gundam preset)."
),
)
vision_image_size = None
elif is_vlm and vision_image_size is not None:
vision_image_size = int(vision_image_size)
_send(
"status",
status_message = f"MLX vision image resize: {vision_image_size} (max dimension)",
)
# gradient_checkpointing stays a string; get_peft_model and MLXTrainer both accept strings.
gc_setting = config.get("gradient_checkpointing", "mlx")
if isinstance(gc_setting, str):
use_grad_checkpoint = (
gc_setting if gc_setting.lower() not in ("false", "none", "") else False
)
else:
use_grad_checkpoint = gc_setting
# A reloaded adapter stays trainable; never stack a second one on it (#13140).
continues_adapter = use_lora and next(iter_mlx_lora_modules(model), None) is not None
if continues_adapter:
_send(
"status",
status_message = "Continuing from existing LoRA adapters (kept the adapter's settings)",
)
elif is_cpt:
_send("status", status_message = "Configuring LoRA for continued pretraining...")
# As on CUDA, embed_tokens / lm_head train as full modules rather than through LoRA.
user_modules = config.get("target_modules") or []
leaf = lambda module: str(module).rsplit(".", 1)[-1] # noqa: E731
full_modules = {leaf(m) for m in user_modules} & _CPT_TARGET_NAMES
quantized = _mlx_quantized_cpt_modules(model, full_modules)
if quantized:
_send(
"warning",
message = (
f"This 4-bit load quantized {' and '.join(sorted(quantized))}, which cannot "
"train in full, so continued pretraining leaves them frozen. Load the 16-bit "
"model to train them."
),
)
full_modules -= quantized
peft_kwargs = dict(
r = config.get("lora_r", 128),
lora_alpha = config.get("lora_alpha", 32),
lora_dropout = config.get("lora_dropout", 0.0),
use_rslora = config.get("use_rslora", False),
random_state = lora_random_state,
target_modules = [m for m in user_modules if leaf(m) not in _CPT_TARGET_NAMES]
or list(_MLX_DEFAULT_LORA_TARGETS),
modules_to_save = sorted(full_modules) or None,
use_gradient_checkpointing = use_grad_checkpoint,
)
elif use_lora:
_send("status", status_message = "Configuring LoRA adapters...")
peft_kwargs = dict(
r = config.get("lora_r", 16),
lora_alpha = config.get("lora_alpha", 16),
lora_dropout = config.get("lora_dropout", 0.0),
use_rslora = config.get("use_rslora", False),
init_lora_weights = config.get("init_lora_weights", True),
random_state = lora_random_state,
target_modules = config.get("target_modules") or list(_MLX_DEFAULT_LORA_TARGETS),
use_gradient_checkpointing = use_grad_checkpoint,
)
finetune_language = config.get("finetune_language_layers", True)
finetune_attention = config.get("finetune_attention_modules", True)
finetune_mlp = config.get("finetune_mlp_modules", True)
# Audio-only rows give vision LoRA no gradient, and zoo refuses Gemma 3n's linear-free vision tower.
finetune_vision = (
config.get("finetune_vision_layers", False) if is_vlm and is_dataset_image else False
)
if (finetune_attention or finetune_mlp) and not finetune_language and not finetune_vision:
finetune_language = True
# is_vlm and the back-fill's outcome are known now; the preflight could only guess.
_check_mlx_effective_targets(
config,
finetune_language = finetune_language,
finetune_vision = finetune_vision,
)
peft_kwargs["finetune_language_layers"] = finetune_language
peft_kwargs["finetune_attention_modules"] = finetune_attention
peft_kwargs["finetune_mlp_modules"] = finetune_mlp
if is_vlm:
peft_kwargs["finetune_vision_layers"] = finetune_vision
if use_lora and not continues_adapter:
peft_kwargs.update(mlx_dora_kwargs)
model = FastMLXModel.get_peft_model(model, **peft_kwargs)
slice_start = config.get("dataset_slice_start")
slice_end = config.get("dataset_slice_end")
# max_steps runs cannot reach the whole dataset; bound rows before mapping.
mlx_raw_text_mode = (
training_type == "Continued Pretraining" or config.get("format_type") == "raw"
)
# mlx.launch shards the batch like DDP and advertises the process count in the env this reads.
mlx_max_train_rows = max_train_rows_for_config(
config,
branch_never_packs = is_vlm and not mlx_raw_text_mode,
world_size = _data_parallel_world_size(),
)
# Resume replays a batch cursor, so the bound must match the checkpoint's marker.
mlx_max_train_rows, mlx_max_train_rows_seed = row_bound_for_resume(
resume_from_checkpoint, mlx_max_train_rows, random_seed
)
# A bracketed split names rows the same way the numeric fields do.
mlx_split_names_rows = "[" in (config.get("train_split") or "")
# The bound keeps a uniform sample of the rows, so a pass over it is this share of a dataset pass.
mlx_kept_row_fraction = [1.0]
def _on_bound(kept, total):
mlx_kept_row_fraction[0] = kept / total
_send("status", status_message = f"Using {kept} of {total} rows (max_steps run)")
def _slice(ds):
if dataset_streaming:
return _slice_streaming_dataset(ds, slice_start, slice_end)
if slice_start is not None or slice_end is not None:
# The user named these rows; the bound below defers to that.
return slice_dataset_rows(ds, slice_start, slice_end)[0]
if mlx_split_names_rows:
return ds
return bound_dataset_rows(
ds,
mlx_max_train_rows,
mlx_max_train_rows_seed,
on_bound = _on_bound,
)
dataset = _slice(dataset)
if not _emit_resource_provenance(
event_queue,
config,
loaded_model_for_provenance,
model_load_target = model_load_name,
model_load_in_4bit = bool(config.get("load_in_4bit")),
dataset_loaded_from_exact_snapshot = bool(config.get("_dataset_loaded_from_exact_snapshot")),
):
return
# Reuse the GPU format pipeline for VLM (OCR/caption/llava/sharegpt+images) and text.
format_type = "raw" if is_cpt else config.get("format_type", "")
custom_format_mapping = config.get("custom_format_mapping")
dataset_final_format = ""
eval_final_format = ""
try:
from utils.datasets import format_and_template_dataset
def _fmt_progress(status_message = "", **_kw):
_send("status", status_message = status_message)
if is_dataset_audio:
_send("status", status_message = "Formatting audio dataset...")
_warn = lambda message: _send("warning", message = message)
dataset, audio_image_col = _mlx_audio_rows(
dataset, tokenizer, custom_format_mapping, warn = _warn
)
if eval_dataset is not None:
eval_dataset, _ = _mlx_audio_rows(
eval_dataset, tokenizer, custom_format_mapping, audio_image_col, warn = _warn
)
if not eval_dataset:
_send(
"warning",
message = "The eval dataset is empty after preprocessing, so this run has no evaluation.",
)
eval_dataset = None
eval_enabled = False
if audio_image_col:
image_layout = (
_mlx_vlm_resized_image_layout(tokenizer)
if vision_image_size is not None
else None
)
dataset = _adapt_for_mlx_vlm(
dataset, resize = vision_image_size, image_layout = image_layout
)
if eval_dataset is not None:
eval_dataset = _adapt_for_mlx_vlm(
eval_dataset, resize = vision_image_size, image_layout = image_layout
)
elif is_vlm:
from utils.datasets.format_detection import detect_video_columns, video_sample
# Before formatting: conversion would fetch every Hub clip only to be refused.
first_row = next(iter(dataset), None)
if isinstance(first_row, dict) and detect_video_columns(
video_sample(dataset, first_row)
):
raise ValueError(_MLX_VIDEO_UNSUPPORTED)
_send("status", status_message = "Formatting VLM dataset...")
vlm_info = format_and_template_dataset(
dataset,
model_name = model_name,
tokenizer = tokenizer,
is_vlm = True,
dataset_name = hf_dataset or "local",
dataset_revision = config.get("dataset_revision"),
custom_format_mapping = custom_format_mapping,
progress_callback = _fmt_progress,
)
if vlm_info.get("success"):
vision_image_layout = (
_mlx_vlm_resized_image_layout(tokenizer)
if vision_image_size is not None
else None
)
dataset = _adapt_for_mlx_vlm(
vlm_info["dataset"],
resize = vision_image_size,
image_layout = vision_image_layout,
)
else:
errors = vlm_info.get("errors", [])
raise ValueError(f"VLM dataset format conversion failed: {'; '.join(errors)}")
if eval_dataset is not None:
ev_info = format_and_template_dataset(
eval_dataset,
model_name = model_name,
tokenizer = tokenizer,
is_vlm = True,
dataset_name = hf_dataset or "local",
dataset_revision = config.get("dataset_revision"),
custom_format_mapping = custom_format_mapping,
)
if ev_info.get("success"):
vision_image_layout = (
_mlx_vlm_resized_image_layout(tokenizer)
if vision_image_size is not None
else None
)
eval_dataset = _adapt_for_mlx_vlm(
ev_info["dataset"],
resize = vision_image_size,
image_layout = vision_image_layout,
)
elif format_type:
_send("status", status_message = f"Formatting dataset ({format_type})...")
info = format_and_template_dataset(
dataset,
model_name = model_name,
tokenizer = tokenizer,
is_vlm = False,
format_type = format_type,
dataset_name = hf_dataset or "local",
custom_format_mapping = custom_format_mapping,
progress_callback = _fmt_progress,
)
if info.get("success", True):
dataset = info.get("dataset", dataset)
else:
errors = info.get("errors", [])
raise ValueError(f"Dataset format conversion failed: {'; '.join(errors)}")
if info.get("dropped_rows_warning"):
_send("warning", message = info["dropped_rows_warning"])
for message in info.get("run_warnings", []):
_send("warning", message = message)
dataset_final_format = str(info.get("final_format", "") or "").lower()
eval_final_format = dataset_final_format
if eval_dataset is not None:
ev = format_and_template_dataset(
eval_dataset,
model_name = model_name,
tokenizer = tokenizer,
is_vlm = False,
format_type = format_type,
dataset_name = hf_dataset or "local",
custom_format_mapping = custom_format_mapping,
split_name = "eval",
raw_text_column = info.get("raw_text_column"),
)
if ev.get("success", True):
eval_dataset = ev.get("dataset", eval_dataset)
# A separately supplied eval split is detected on its own and may not be pre-tokenized.
eval_final_format = str(ev.get("final_format", "") or "").lower()
if hasattr(eval_dataset, "__len__") and len(eval_dataset) == 0:
_send(
"warning",
message = "The eval dataset is empty after preprocessing, so this run has no evaluation.",
)
eval_dataset = None
# A user-supplied split that filters to nothing must not carve one out of train.
eval_enabled = False
else:
eval_errors = ev.get("errors", [])
raise ValueError(
f"Eval dataset format conversion failed: {'; '.join(eval_errors)}"
)
if ev.get("dropped_rows_warning"):
_send("warning", message = f"Eval dataset: {ev['dropped_rows_warning']}")
for message in ev.get("run_warnings", []):
_send("warning", message = message)
except ImportError:
_send("status", status_message = "Format helper unavailable, using raw dataset")
if not is_vlm and dataset_final_format:
# get_chat_template returns a new tokenizer unless it is an mlx-lm wrapper, so the rows' template stayed in
# the formatter; mirror the CUDA trainer and re-apply it here (no-op for models with a template).
from utils.datasets.chat_templates import get_training_chat_template
tokenizer = get_training_chat_template(tokenizer, model_name, dataset_final_format)
if _carry_mlx_chat_template_to_processor(model, tokenizer):
logger.info("Saving the training chat template with the processor")
if eval_enabled and eval_dataset is None:
from core.training.eval_dataset import (
MIN_TOTAL_ROWS_FOR_EVAL,
split_dataset_for_evaluation,
)
split_result = split_dataset_for_evaluation(dataset)
if split_result is None:
_send(
"warning",
message = (
f"Evaluation is enabled, but the training dataset has only {len(dataset)} "
f"rows; at least {MIN_TOTAL_ROWS_FOR_EVAL} are required to create a "
"held-out eval split. Training will continue without evaluation."
),
)
else:
dataset, eval_dataset = split_result
eval_final_format = dataset_final_format
# zoo's MLX batcher reads input_ids / labels and ignores attention_mask, so masked padding is trimmed here as on
# CUDA. MLX truncates to max_seq_length itself. Each split goes by its own format: a separately supplied eval
# split can be pre-tokenized while the training split is text, and the reverse.
if dataset_final_format == "pretokenized":
from utils.datasets.pretokenized import prepare_pretokenized_dataset
dataset = prepare_pretokenized_dataset(
dataset, max_seq_length = None, vocab_size = _mlx_vocab_size(model), mlx = True
)
if eval_dataset is not None and eval_final_format == "pretokenized":
from utils.datasets.pretokenized import prepare_pretokenized_dataset
eval_dataset = prepare_pretokenized_dataset(
eval_dataset, max_seq_length = None, vocab_size = _mlx_vocab_size(model), mlx = True
)
requested_max_steps = config.get("max_steps", 0) or 0
num_epochs = config.get("num_epochs", 3)
max_seq_length = config.get("max_seq_length", 2048)
batch_size = config.get("batch_size", 4)
grad_accum = config.get("gradient_accumulation_steps", 4)
lr_value = float(config.get("learning_rate", "5e-5" if is_cpt else "2e-4"))
embedding_lr_value = (
_mlx_cpt_embedding_learning_rate(
config.get("embedding_learning_rate"),
lr_value,
trains_embeddings = _trains_embedding_modules(
(name for name, _ in tree_flatten(model.trainable_parameters())),
getattr(model, "_unsloth_cpt_full_module_weight_keys", None) or (),
),
)
if is_cpt
else None
)
# Warmup: prefer warmup_steps; fall back to warmup_ratio
warmup_steps = config.get("warmup_steps")
warmup_ratio = config.get("warmup_ratio")
warmup_uses_ratio = warmup_steps is None and warmup_ratio is not None
if warmup_steps is None:
warmup_steps = 0 if warmup_uses_ratio else 5
# Resolve to ~/.unsloth/studio/outputs/ so the export page finds it
from utils.paths import ensure_dir
# Resume must land in the original run dir even when config lacks output_dir.
resume_dir = config.get("output_dir", "") or _output_dir_from_resume_checkpoint(
resume_from_checkpoint
)
output_dir = _resolve_mlx_output_dir(
{**config, "output_dir": resume_dir} if resume_dir else config, model_name
)
raw_eval_steps = config.get("eval_steps", 0)
if evaluation_enabled(raw_eval_steps):
eval_steps_value = float(raw_eval_steps)
else:
eval_steps_value = 0.0
eval_steps_ratio = eval_steps_value if 0 < eval_steps_value < 1 else None
eval_steps_val = 0 if eval_steps_ratio is not None else int(eval_steps_value)
# Re-validate for direct worker callers; training.py normalizes the main path.
max_grad_norm = _resolve_mlx_max_grad_norm(config.get("max_grad_norm"))
max_grad_value = config.get("max_grad_value")
if max_grad_value is not None:
max_grad_value = float(max_grad_value)
if max_grad_value < 0 or not math.isfinite(max_grad_value):
raise ValueError(
f"Unsloth MLX: max_grad_value={max_grad_value} must be finite and >= 0 "
"(0 or None disables elementwise clipping)."
)
max_grad_leaf_norm = config.get("max_grad_leaf_norm")
if max_grad_leaf_norm is not None:
max_grad_leaf_norm = float(max_grad_leaf_norm)
if max_grad_leaf_norm < 0 or not math.isfinite(max_grad_leaf_norm):
raise ValueError(
f"Unsloth MLX: max_grad_leaf_norm={max_grad_leaf_norm} must be finite and >= 0 "
"(0 or None disables proportional leaf-norm clipping)."
)
weight_decay = config.get("weight_decay", 0.001)
weight_decay = 0.001 if weight_decay is None else float(weight_decay)
# A sized dataset stays eager: without a pass length zoo cannot end an epoch on an optimizer step.
mlx_config_kwargs = dict(
per_device_train_batch_size = batch_size,
gradient_accumulation_steps = grad_accum,
max_steps = requested_max_steps,
learning_rate = lr_value,
warmup_steps = warmup_steps,
lr_scheduler_type = lr_scheduler_type,
optim = optim_name,
weight_decay = weight_decay,
max_grad_norm = max_grad_norm,
max_grad_value = max_grad_value,
logging_steps = 1,
max_seq_length = max_seq_length,
seed = random_seed,
use_cce = True,
compile = True,
gradient_checkpointing = use_grad_checkpoint,
packing = bool(config.get("packing", False)),
streaming = dataset_streaming,
output_dir = output_dir,
save_steps = int(config.get("save_steps", 0) or 0),
eval_steps = eval_steps_val,
)
# Also gates the masking skip below, so defined outside the feature-detect block.
raw_text_mode = training_type == "Continued Pretraining" or format_type == "raw"
# Feature-detect optional fields so this PR works without the paired zoo bump.
_supported_fields = getattr(MLXTrainingConfig, "__dataclass_fields__", {})
if "cast_norm_output_to_input_dtype" in _supported_fields:
# Explicit None falls back to True (default).
_raw_cast = config.get("cast_norm_output_to_input_dtype", True)
mlx_config_kwargs["cast_norm_output_to_input_dtype"] = (
True if _raw_cast is None else bool(_raw_cast)
)
if "dataset_order" in _supported_fields:
# CUDA's trainer reads a stream in source order; a sized dataset gets its RandomSampler order.
mlx_config_kwargs["dataset_order"] = "sequential" if dataset_streaming else "torch_randperm"
if "max_grad_leaf_norm" in _supported_fields:
mlx_config_kwargs["max_grad_leaf_norm"] = max_grad_leaf_norm
if "report_grad_norm" in _supported_fields:
# Grad norm is free under global-norm clipping; switching clip modes would change loss.
mlx_config_kwargs["report_grad_norm"] = True
if (
"early_stopping_patience" in _supported_fields
and "load_best_model_at_end" in _supported_fields
):
from core.training.early_stopping import early_stopping_patience
# Zoo counts patience in evaluations and restores the best adapters itself, the CUDA path's contract.
patience = early_stopping_patience(config.get("early_stopping_patience"))
if patience and eval_dataset is not None and eval_steps_value > 0:
mlx_config_kwargs["early_stopping_patience"] = patience
mlx_config_kwargs["load_best_model_at_end"] = True
if embedding_lr_value is not None:
mlx_config_kwargs["embedding_learning_rate"] = embedding_lr_value
if "append_eos" in _supported_fields:
# Unsloth SFT formatting owns rendered examples; raw/CPT text still needs MLX to append EOS.
mlx_config_kwargs["append_eos"] = bool(raw_text_mode)
trainer = MLXTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset,
eval_dataset = eval_dataset,
args = MLXTrainingConfig(**mlx_config_kwargs),
)
max_steps, eval_steps_val = _configure_mlx_training_schedule(
trainer,
requested_max_steps,
None if dataset_streaming else len(dataset),
batch_size,
grad_accum,
num_epochs,
warmup_ratio = warmup_ratio if warmup_uses_ratio else None,
eval_steps_ratio = eval_steps_ratio,
)
_prepare_mlx_output_dir(trainer, output_dir, ensure_dir, event_queue)
def _record_mlx_row_bound():
# Pin the subset before any checkpoint lands here; a resume reads it back.
if not record_row_bound(output_dir, mlx_max_train_rows, mlx_max_train_rows_seed) and (
mlx_max_train_rows
):
_send(
"warning",
message = (
f"Could not record the max_steps row bound in {output_dir}: "
"resuming this run later will read it as unbounded"
),
)
_run_mlx_main_process_action(trainer, _record_mlx_row_bound, "max_steps row bound setup")
_trainer_ref[0] = trainer
if _is_stop_requested():
trainer.stop_requested = True
# Tell the parent eval is configured so the frontend shows the eval chart
if eval_dataset is not None and eval_steps_val > 0:
_send("eval_configured")
# Auto-detect markers from the chat template first, manual table as fallback. Mirror the CUDA skips: raw/CPT text
# has no chat turns. Check the resolved format too, since format_type="auto" can land on alpaca or raw.
from utils.datasets.pretokenized import has_custom_labels
if (
config.get("train_on_completions", False)
and dataset_final_format == "pretokenized"
and not has_custom_labels(dataset)
):
# zoo's MLX masker re-tokenizes text, so token-id rows have nothing it can mask (CUDA masks input_ids).
_send(
"warning",
message = (
"'Train on completions' is not available for pre-tokenized datasets on Mac: "
"training will run on full sequences. Add a labels column to choose the trained tokens."
),
)
if (
config.get("train_on_completions", False)
and not raw_text_mode
and dataset_final_format not in ("raw_text", "pretokenized")
):
_send("status", status_message = "Configuring response-only training...")
# No catch: an exception here is a real masking failure and must fail the run.
from utils.datasets.completion_masking import apply_completion_masking
trainer, masking_applied = apply_completion_masking(
trainer,
model_name,
train_on_responses_only,
notify = lambda level, message: _send("status", status_message = message),
dataset_template = "alpaca" if dataset_final_format == "alpaca" else None,
)
if not masking_applied:
# Masking miss changes the run's objective, so warn in the sticky warning list, not a status line.
_send(
"warning",
message = (
f"'Train on completions' could not be applied for {model_name}: no "
f"instruction/response markers were found. Training will run on full "
f"sequences (prompts included)."
),
)
wandb_run, tb_writer = _setup_mlx_tracking(trainer, config, output_dir, _send)
_send("status", status_message = f"Training {model_name}...")
start_step = 0
if resume_from_checkpoint:
start_step = _checkpoint_state(Path(resume_from_checkpoint)) or 0
def _on_step(
step,
total,
loss,
lr,
tok_s,
peak_gb,
elapsed,
num_tokens,
grad_norm = None,
):
eta = session_eta_seconds(elapsed, step, start_step, total) or 0
_send(
"progress",
step = step,
epoch = (
round(trainer.state.epoch * mlx_kept_row_fraction[0], 2)
if trainer.state.epoch
else 0
),
loss = loss,
learning_rate = lr,
total_steps = total,
elapsed_seconds = elapsed,
eta_seconds = eta,
session_start_step = start_step,
grad_norm = grad_norm,
num_tokens = num_tokens,
eval_loss = None,
status_message = None,
peak_memory_gb = peak_gb,
)
if wandb_run is not None:
try:
wandb_run.log(
{
"train/loss": loss,
"train/learning_rate": lr,
"train/tokens_per_sec": tok_s,
"train/peak_gb": peak_gb,
"train/num_tokens": num_tokens,
**({"train/grad_norm": grad_norm} if grad_norm is not None else {}),
},
step = step,
)
except Exception:
pass
if tb_writer is not None:
try:
tb_writer.add_scalar("train/loss", loss, step)
tb_writer.add_scalar("train/learning_rate", lr, step)
tb_writer.add_scalar("train/tokens_per_sec", tok_s, step)
tb_writer.add_scalar("train/peak_gb", peak_gb, step)
if grad_norm is not None:
tb_writer.add_scalar("train/grad_norm", grad_norm, step)
except Exception:
pass
trainer.add_step_callback(_on_step)
def _on_eval(step, eval_loss, perplexity):
_send("progress", step = step, eval_loss = eval_loss)
if wandb_run is not None:
try:
wandb_run.log({"eval/loss": eval_loss, "eval/perplexity": perplexity}, step = step)
except Exception:
pass
if tb_writer is not None:
try:
tb_writer.add_scalar("eval/loss", eval_loss, step)
tb_writer.add_scalar("eval/perplexity", perplexity, step)
except Exception:
pass
trainer.add_eval_callback(_on_eval)
_opt_ref = [None]
_orig_build_optimizer = getattr(trainer, "_build_optimizer", None)
if callable(_orig_build_optimizer):
def _capture_optimizer(total_steps):
_opt_ref[0] = _orig_build_optimizer(total_steps)
return _opt_ref[0]
trainer._build_optimizer = _capture_optimizer
gc.collect()
mx.synchronize()
_save_model = trainer.save_model
def _skip_internal_final_save(*args, **kwargs):
raise ValueError("worker owns final save")
trainer.save_model = _skip_internal_final_save
try:
trainer.train(resume_from_checkpoint = resume_from_checkpoint)
finally:
trainer.save_model = _save_model
def _finish_tracking() -> None:
# Runs on every save/finalize exit so TB/W&B never leak on early return.
if tb_writer is not None:
try:
tb_writer.close()
except Exception:
pass
if wandb_run is not None:
try:
wandb_run.finish()
except Exception:
pass
try:
_finalize_mlx_training(
trainer,
_snapshot_stop,
output_dir,
mx.synchronize,
_send,
lambda: _write_mlx_stop_checkpoint(trainer, _opt_ref[0], output_dir),
)
finally:
_finish_tracking()
def _is_current_process_apple_silicon() -> bool:
import platform
return platform.system() == "Darwin" and platform.machine() == "arm64"
def run_mlx_training_process(
*,
event_queue: Any,
stop_queue: Any,
config: dict,
transformers_activated: bool = False,
config_prevalidated: bool = False,
) -> None:
"""MLX worker entrypoint shared by Unsloth subprocesses and the CLI adapter."""
backend_path = str(Path(__file__).resolve().parent.parent.parent)
if backend_path not in sys.path:
sys.path.insert(0, backend_path)
from utils.hf_xet_fallback import child_should_disable_xet
if child_should_disable_xet(config):
os.environ["HF_HUB_DISABLE_XET"] = "1"
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"
if not config_prevalidated and not _validate_training_worker_config(config, event_queue):
return
model_load_target = _resolve_cached_model_load_name(config)
if not transformers_activated:
# Must precede detect_hardware(): its MLX stack check imports mlx_lm, hence transformers.
_activate_transformers_version_or_warn(
model_load_target,
_worker_hf_token(config),
)
from utils.hardware import hardware as _hw
_hw.detect_hardware()
if _hw.DEVICE != _hw.DeviceType.MLX:
event_queue.put(
{
"type": "error",
"error": "MLX training requires Apple Silicon with the MLX backend available.",
"stack": "",
"ts": time.time(),
}
)
return
try:
try:
if config.get("is_decision"):
# Its own pipeline, as on the torch path, behind the same security gate.
security_error = _model_load_security_error(
config, model_load_target, _worker_hf_token(config)
)
if security_error:
event_queue.put({"type": "error", **security_error, "ts": time.time()})
else:
from core.training.decision_trainer import run_decision_training
run_decision_training(event_queue, stop_queue, config)
else:
_run_mlx_training(event_queue, stop_queue, config)
finally:
try:
stop_queue.put({"type": _MLX_WORKER_COMPLETE})
except (EOFError, OSError, ValueError):
pass
except BaseException as exc:
event_queue.put(
{
"type": "error",
"error": str(exc) or type(exc).__name__,
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
_end_abandoned_decision_download(event_queue)
def _end_abandoned_decision_download(event_queue: Any) -> None:
trainer = sys.modules.get("core.training.decision_trainer")
if trainer is not None:
trainer.end_abandoned_download(event_queue)
def _training_job_is_local(config) -> bool:
"""True when neither model nor dataset needs the Hub; unresolvable means remote (fail closed)."""
try:
from utils.paths import is_local_path
except Exception:
return False
if config.get("hf_dataset"):
return False
model = config.get("model_name")
try:
if not (model and is_local_path(model)):
return False
# A local checkpoint can name a remote base to fetch later, so decide from disk, not network.
base, needs_hub = _recorded_local_base(model)
if needs_hub:
return False
return not base or is_local_path(base)
except Exception:
return False
def _recorded_local_base(model_name) -> "tuple[str | None, bool]":
"""``(base, needs_hub)`` for the base this checkpoint records on disk. Delegates to the
resolver's own disk reads so the gate cannot drift from what activation later resolves. Fail
closed on an unavailable reader."""
try:
from utils.transformers_version import recorded_local_base
return recorded_local_base(model_name)
except Exception:
return None, True
def _download_decision_checkpoint(event_queue: Any, config: dict) -> None:
from core.systemone import laya_runtime
from core.systemone.catalog import Checkpoint
from utils.hf_xet_fallback import start_watchdog
from utils.paths import is_local_path
model_name = config["model_name"]
# An LLM that gets a new decision head downloads through FastModel's own loader in the trainer.
if is_local_path(model_name) or config.get("decision_layout") == "llm":
return
hf_token = _worker_hf_token(config)
if hf_token:
os.environ["HF_TOKEN"] = hf_token
_send_status(event_queue, "Loading decision model...")
# Under the stall watchdog, so the parent can retry a stalled Xet download over HTTP.
event_queue.put({"type": "model_load_started", "ts": time.time()})
watchdog_stop = start_watchdog(
repo_ids = [model_name],
on_stall = lambda msg: event_queue.put({"type": "stall", "message": msg, "ts": time.time()}),
xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
)
try:
laya_runtime._checkpoint_dir(
Checkpoint(
"base",
model_name,
config.get("model_subfolder") or None,
"",
layout = config.get("decision_layout") or "laya",
)
)
except Exception as exc:
# The trainer's own load reports it.
logger.info("Could not download %s ahead of the trainer: %s", model_name, exc)
finally:
watchdog_stop.set()
event_queue.put({"type": "model_load_completed", "ts": time.time()})
def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> None:
"""Fresh subprocess entrypoint: events out on event_queue, stop commands in on stop_queue."""
# Off on Linux: forked map() workers deadlock.
os.environ["TOKENIZERS_PARALLELISM"] = (
"true" if sys.platform in ("win32", "darwin") else "false"
)
os.environ["PYTHONWARNINGS"] = "ignore" # before imports
# Until unsloth is imported below; transformers is first imported well before that.
metadata_scan_cache = _install_metadata_scan_cache()
from utils.security.hub_data_files import install as guard_hub_data_files
from utils.security.dataset_media import allow_dataset_repos, install as guard_dataset_media
guard_hub_data_files()
guard_dataset_media()
allow_dataset_repos(config.get("hf_dataset"))
if not config.get("allow_ambient", True):
from hub.utils.hf_tokens import apply_token_to_child_env, hf_token_arg, is_anonymous
hf_token = hf_token_arg(config.get("hf_token"), allow_ambient_token = False)
apply_token_to_child_env(os.environ, hf_token)
if is_anonymous(hf_token):
os.environ["HF_TOKEN_PATH"] = os.devnull
# HTTP-fallback respawn: disable Xet before any huggingface_hub import (read at import time).
from utils.hf_xet_fallback import child_should_disable_xet
if child_should_disable_xet(config):
os.environ["HF_HUB_DISABLE_XET"] = "1"
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"
print(
"Xet transport disabled for this training worker (HF_HUB_DISABLE_XET=1).",
file = sys.stderr,
flush = True,
)
# Offline auto-detect: skip ~25s of HF retries per call when the hub is unreachable. Skipped for a filesystem-only
# job: a local checkpoint with a local dataset never reaches the Hub, and probing unconditionally would add
# seconds to every such startup.
if "HF_HUB_OFFLINE" not in os.environ and not _training_job_is_local(config):
_offline = False
_network_offline = False
try:
from utils.utils import hf_dns_dead, hf_env_offline, hf_probe_disabled
# Hub ignores TRANSFORMERS_OFFLINE, so translate it before probing.
_offline = hf_env_offline()
# hf_dns_dead follows HF_ENDPOINT and stands down behind a proxy, so a mirror still counts.
if not _offline:
_offline = _network_offline = hf_dns_dead()
if not _offline and not hf_probe_disabled():
# DNS can answer without egress; only a connection failure counts, not a 502/503.
from utils.transformers_version import hf_endpoint_unreachable
_offline = _network_offline = hf_endpoint_unreachable(
gateway_errors_offline = False,
proxy_timeouts_offline = False,
)
except Exception:
_offline = _network_offline = False
if _offline:
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
# TRANSFORMERS_OFFLINE alone must not force cache-only datasets.
if _network_offline:
os.environ.setdefault("HF_DATASETS_OFFLINE", "1")
# logger isn't configured yet; print to stderr instead.
print(
"Hugging Face endpoint unreachable; HF_HUB_OFFLINE=1 set for this worker.",
file = sys.stderr,
flush = True,
)
import warnings
from loggers.config import LogConfig, keep_progress_bars_countable
if os.getenv("ENVIRONMENT_TYPE", "production") == "production":
warnings.filterwarnings("ignore")
# The monitor polls tqdm._instances, so redirect bars instead of disabling them.
keep_progress_bars_countable()
LogConfig.setup_logging(
service_name = "unsloth-studio-training-worker",
env = os.getenv("ENVIRONMENT_TYPE", "production"),
)
# As the inference worker: a recovered traceback reaching fd 2 must not read as the crash.
from utils.worker_stderr import mark_log_record_continuations
mark_log_record_continuations()
gpu_ids = config.get("resolved_gpu_ids")
if config.get("is_decision") and gpu_ids and len(gpu_ids) > 1:
gpu_ids = gpu_ids[:1]
event_queue.put(
{
"type": "warning",
"message": f"Decision models train on one GPU; using GPU {gpu_ids[0]}.",
"ts": time.time(),
}
)
apply_gpu_ids(gpu_ids, backend = config.get("device_backend"))
if not _validate_training_worker_config(config, event_queue):
return
# Before any backend dispatch: the MLX, decision and embedding runners load these weights too.
weights_error = _escaping_weights_error(config)
if weights_error:
event_queue.put({"type": "error", "error": weights_error, "stack": "", "ts": time.time()})
return
model_name = config["model_name"]
model_load_target = _resolve_cached_model_load_name(config)
# MLX fast path, before any torch/transformers import.
backend_path = str(Path(__file__).resolve().parent.parent.parent)
if backend_path not in sys.path:
sys.path.insert(0, backend_path)
from .training import is_apple_silicon_training_platform, should_use_mlx_training_backend
mlx_backend_requested = is_apple_silicon_training_platform()
mlx_transformers_activated = False
if mlx_backend_requested and _is_current_process_apple_silicon():
# Must precede detect_hardware(): its MLX stack check imports mlx_lm, hence transformers.
_activate_transformers_version_or_warn(
model_load_target,
_worker_hf_token(config),
)
mlx_transformers_activated = True
from utils.hardware import hardware as _hw
_hw.detect_hardware()
if _hw.trains_on_cpu(_hw.CHAT_ONLY_REASON):
# Studio's verdict wins over the inherited UNSLOTH_DISABLE_CPU_FALLBACK=1 and zoo's own CPU refusals.
os.environ["UNSLOTH_FORCE_CPU"] = "1"
if mlx_backend_requested or should_use_mlx_training_backend(device = _hw.DEVICE):
diffusion_error = _mlx_diffusion_lm_error(config, model_load_target)
if diffusion_error:
event_queue.put(
{"type": "error", "error": diffusion_error, "stack": "", "ts": time.time()}
)
return
run_mlx_training_process(
event_queue = event_queue,
stop_queue = stop_queue,
config = config,
transformers_activated = mlx_transformers_activated,
config_prevalidated = True,
)
return
try:
_activate_transformers_version(
model_load_target,
_worker_hf_token(config),
)
except Exception as exc:
event_queue.put(
{
"type": "error",
"error": f"Failed to activate transformers version: {exc}",
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
# NemotronH needs trust_remote_code=True for config parsing bugs (not Llama-Nemotron).
from utils.security.trusted_org import is_trusted_org_repo
_NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano")
_lowered = model_name.lower()
if (
any(sub in _lowered for sub in _NEMOTRON_TRUST_SUBSTRINGS)
and (_lowered.startswith("unsloth/") or _lowered.startswith("nvidia/"))
# Confirm a genuine first-party Hub repo (not a spoofed "unsloth/" name); authenticated.
and is_trusted_org_repo(model_name, hf_token = _worker_hf_token(config))
and not config.get("trust_remote_code", False)
):
config["trust_remote_code"] = True
logger.info(
"Auto-enabled trust_remote_code for Nemotron model: %s",
model_name,
)
security_error = _model_load_security_error(
config,
_resolve_cached_model_load_name(config),
_worker_hf_token(config),
)
if security_error:
event_queue.put({"type": "error", **security_error, "ts": time.time()})
return
# ── 1b. Install fast-path kernel libraries for the chosen model.
# 1) causal-conv1d runs eagerly for matching architectures: some SSM modeling files
# lazy_load it without calling is_causal_conv1d_available.
# 2) mamba-ssm + flash-attn keep their substring / size gates.
# 3) FLA gated-delta kernels: vendored by unsloth_zoo, nothing to install.
# Laya decision models are ModernBERT encoders: none of these apply.
# Clef decision models are Qwen3.5 backbones and need the same gated-delta / conv kernels.
if (
not config.get("is_decision")
or config.get("decision_layout") in ("clef", "llm")
or _decision_has_llm_backbone(model_load_target, _worker_hf_token(config))
):
try:
from utils.ssm_runtime import resolve_ssm_kernels
# A continued local adapter loads from its folder, but its base decides the kernels:
# the authorized pin when the run was checked, else the adapter's recorded base.
kernels = resolve_ssm_kernels(
model_name,
model_load_target,
_worker_hf_token(config),
adapter_base = config.get("adapter_base_pin") or None,
# Whether the load runs a checkpoint's remote code, not just whether it has some.
trust_remote_code = bool(config.get("trust_remote_code", False)),
)
is_ssm = kernels.mamba_ssm
wants_causal_conv1d = kernels.causal_conv1d
from utils.ssm_runtime import mamba2_runs_on_triton
# Mamba-2 on Triton drop-ins: no causal-conv1d, extension-free mamba_ssm wheel.
triton_only = mamba2_runs_on_triton(kernels)
if triton_only:
# The extension-free wheel first. Not published, refused or not importable: this
# run takes the CUDA-wheel route (causal-conv1d, then mamba-ssm) instead.
triton_only = bool(
_ensure_mamba_ssm(
event_queue,
model_name,
required = is_ssm,
essential = kernels.mamba_ssm_required,
triton_only = True,
)
)
if triton_only:
wants_causal_conv1d = False
else:
_send_status(
event_queue,
"The extension-free mamba-ssm wheel is not available; using the CUDA "
"kernels instead",
)
_ensure_causal_conv1d_fast_path(
event_queue,
model_name,
required = wants_causal_conv1d,
)
_install_fast_path_hooks(
event_queue,
model_name,
install_causal_conv1d = wants_causal_conv1d,
)
if not triton_only:
_ensure_mamba_ssm(
event_queue,
model_name,
required = is_ssm,
essential = kernels.mamba_ssm_required,
)
_ensure_flash_attn_for_long_context(
event_queue,
int(config.get("max_seq_length", 2048)),
)
except Exception as exc:
event_queue.put(
{
"type": "error",
"error": (
f"Please choose another model to train, since "
f"a fast-path kernel library "
f"(causal-conv1d / mamba-ssm) failed to install "
f"with error: {exc}"
),
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
# No start-method override: Dataset.map() uses `multiprocess`, not stdlib.
# Windows: importable Triton isn't enough on AMD; clang-cl JIT needs MSVC CRT headers.
if sys.platform == "win32":
from core._msvc_env import gate_torch_compile_on_windows
gate_torch_compile_on_windows(logger)
# Stub torchao on Windows ROCm; see core/_torchao_stub.py (no RCCL there). Run before transformers/unsloth_zoo.
from core._torchao_stub import install_torchao_windows_rocm_stub
install_torchao_windows_rocm_stub()
# Single-GPU never inits the process group, but transformers/trl import these torch.distributed attrs anyway.
_td_stubs = {
"is_initialized": lambda: False,
"is_available": lambda: False,
"is_torchelastic_launched": lambda: False,
"get_rank": lambda: 0,
"get_world_size": lambda: 1,
"barrier": lambda: None,
}
try:
import torch.distributed as _td
for _name, _stub in _td_stubs.items():
if not hasattr(_td, _name):
setattr(_td, _name, _stub)
except Exception:
_td_mock = types.ModuleType("torch.distributed")
for _name, _stub in _td_stubs.items():
setattr(_td_mock, _name, _stub)
sys.modules["torch.distributed"] = _td_mock
try:
import torch as _torch
_torch.distributed = _td_mock
except Exception:
pass
# Windows ROCm <= 7.12: _grouped_mm null HIP kernel on gfx1200 crashes; JitDecomp bypasses
# TORCHDYNAMO_DISABLE, so override the CUDA dispatch key. Fixed in rocm7.13.
global _WINDOWS_ROCM_GROUPED_MM_LIB
if sys.platform == "win32":
_torch_for_rocm = sys.modules.get("torch")
# AMD SDK / Radeon wheels don't always set torch.version.hip.
_build_version_for_rocm = (
getattr(_torch_for_rocm, "__version__", "").lower()
if _torch_for_rocm is not None
else ""
)
_is_win_rocm_torch = bool(
_torch_for_rocm is not None
and (
getattr(getattr(_torch_for_rocm, "version", None), "hip", None)
or "rocm" in _build_version_for_rocm
)
)
if _is_win_rocm_torch:
# Belt-and-suspenders; the JitDecomp patch is the real fix, but this covers other paths.
if "TORCHDYNAMO_DISABLE" not in os.environ:
os.environ["TORCHDYNAMO_DISABLE"] = "1"
logger.info("Windows ROCm: torch.compile (dynamo) disabled")
# bitsandbytes probes hipinfo.exe from PATH; the venv Scripts dir is only on PATH when activated.
_scripts_dir = os.path.dirname(sys.executable)
if os.path.isfile(os.path.join(_scripts_dir, "hipInfo.exe")):
import shutil as _shutil
if not _shutil.which("hipinfo.exe"):
os.environ["PATH"] = _scripts_dir + os.pathsep + os.environ.get("PATH", "")
# AMD's Windows BNB DLL suffix may not match torch.version.hip; detect the real one. Caller-set BNB_ROCM_VERSION stays authoritative.
if (
"BNB_ROCM_VERSION" not in os.environ
or os.environ.get("UNSLOTH_BNB_ROCM_VERSION_SOURCE") == "sitecustomize"
):
_bnb_rocm_ver = None
_found_rocm_bnb = False
try:
import glob as _glob
import importlib.util as _ilu
import re as _re
_bnb_spec = _ilu.find_spec("bitsandbytes")
if _bnb_spec and _bnb_spec.submodule_search_locations:
_all_vers: list[str] = []
for _pkg_dir in _bnb_spec.submodule_search_locations:
for _dll in _glob.glob(
os.path.join(_pkg_dir, "libbitsandbytes_rocm*.dll")
):
_found_rocm_bnb = True
_m = _re.search(
r"libbitsandbytes_rocm(\d+)\.dll",
os.path.basename(_dll),
)
if _m:
_all_vers.append(_m.group(1))
# Highest numeric suffix wins (glob order isn't sorted).
if _all_vers:
_bnb_rocm_ver = max(_all_vers, key = lambda v: int(v))
except Exception:
pass
# Set BNB_ROCM_VERSION only if a ROCm bnb DLL exists, so later fixes can still redetect or opt out.
if _found_rocm_bnb:
_bnb_rocm_ver = _bnb_rocm_ver or os.environ.get("BNB_ROCM_VERSION") or "72"
os.environ["BNB_ROCM_VERSION"] = _bnb_rocm_ver
os.environ["UNSLOTH_BNB_ROCM_VERSION_SOURCE"] = "detected"
logger.info(
"Windows ROCm: set BNB_ROCM_VERSION=%s "
"(detected from installed BNB wheel; "
"overrides torch.version.hip auto-detection)",
_bnb_rocm_ver,
)
# Drop only bnb's benign BNB_ROCM_VERSION override notice.
if os.environ.get("BNB_ROCM_VERSION"):
import logging as _logging
_logging.getLogger("bitsandbytes.cextension").addFilter(
lambda _r: "environment variable detected" not in _r.getMessage()
)
# Kernel-fix gate reads HIP version, else torch.__version__ (AMD SDK wheels leave version.hip unset).
def _hip_ver_at_least(major: int, minor: int) -> bool:
_hip_str = getattr(getattr(_torch_for_rocm, "version", None), "hip", None)
if not _hip_str:
# Try the standard "+rocmX.Y.Z" embedded version first.
_ver_match = re.search(r"rocm(\d+)\.(\d+)", _build_version_for_rocm)
if _ver_match:
return (
int(_ver_match.group(1)),
int(_ver_match.group(2)),
) >= (major, minor)
# "+rocmsdk<date>" wheels postdate the gfx120X null-kernel fix (ROCm 7.13); treat as >= 7.13.
if "rocmsdk" in _build_version_for_rocm:
logger.debug(
"Windows ROCm: AMD SDK wheel detected (%r); "
"assuming HIP >= %d.%d (rocmsdk wheels post-date "
"the gfx120X null-kernel fix)",
_build_version_for_rocm,
major,
minor,
)
return True
return False
try:
_parts = [int(x) for x in str(_hip_str).split(".")[:2]]
if len(_parts) < 2:
logger.warning(
"Windows ROCm: torch.version.hip %r has fewer than "
"two components; cannot compare against %d.%d",
_hip_str,
major,
minor,
)
return False
return (_parts[0], _parts[1]) >= (major, minor)
except ValueError:
logger.warning(
"Windows ROCm: could not parse torch.version.hip %r as "
"a version number; assuming HIP < %d.%d",
_hip_str,
major,
minor,
)
return False
# Only on affected versions (ROCm <= 7.12); 7.13+ uses the real GPU kernel.
if not _hip_ver_at_least(7, 13):
try:
_WINDOWS_ROCM_GROUPED_MM_LIB = _install_grouped_mm_cpu_fallback(
_torch_for_rocm, logger, "Windows ROCm"
)
except Exception as _patch_exc:
logger.warning(
"Windows ROCm: could not patch _grouped_mm — "
"training may crash with 0xC0000005: %s",
_patch_exc,
)
else:
logger.info(
"Windows ROCm: HIP >= 7.13 — _grouped_mm kernel is functional, "
"skipping Python fallback (AMD fixed gfx1200 null kernel in ROCm 7.13)"
)
# The win32 guard above misses Linux: RDNA4 (gfx1200/gfx1201) hits the same null HIP _grouped_mm kernel at ROCm <=
# 7.12 (fixed 7.13, ROCm/TheRock #5284). Gate on arch + HIP.
# ── 1f-linux. Linux ROCm RDNA4 _grouped_mm null kernel ──
if sys.platform.startswith("linux") and _hw.IS_ROCM:
try:
_torch_lin = sys.modules.get("torch")
if _torch_lin is not None and _torch_lin.cuda.is_available():
# Unknown version on gfx120X: assume affected unless rocmsdk.
_hip_str = str(getattr(getattr(_torch_lin, "version", None), "hip", "") or "")
_ver = getattr(_torch_lin, "__version__", "").lower()
_m = re.match(r"(\d+)\.(\d+)", _hip_str) or re.search(r"rocm(\d+)\.(\d+)", _ver)
if _m:
_hip_lt_713 = (int(_m.group(1)), int(_m.group(2))) < (7, 13)
else:
_hip_lt_713 = "rocmsdk" not in _ver
# Scan every GPU: device_map="balanced" can place layers on a later RDNA4 card.
_rdna4 = False
for _i in range(_torch_lin.cuda.device_count()):
_props = _torch_lin.cuda.get_device_properties(_i)
_lin_arch, _ = _rocm_classify_unified_memory(_props)
_lin_name = (getattr(_props, "name", "") or "").lower()
if _lin_arch.lower() in ("gfx1200", "gfx1201") or (
not _lin_arch and re.search(r"rx\s*90[0-9]0|r9700", _lin_name)
):
_rdna4 = True
break
if _rdna4 and _hip_lt_713:
_WINDOWS_ROCM_GROUPED_MM_LIB = _install_grouped_mm_cpu_fallback(
_torch_lin, logger, "Linux ROCm gfx120X"
)
except Exception as _gm_lin_exc:
logger.warning("Linux ROCm gfx120X: could not patch _grouped_mm: %s", _gm_lin_exc)
# ROCm OOM guard: exhausting VRAM can hang the HIP driver instead of raising, so set_per_process_memory_fraction
# caps the allocator and PyTorch raises OutOfMemoryError first. Unified hosts share GPU+system RAM and need OS
# headroom, so the cap depends on the classification and the pool size (see _rocm_memory_fraction and
# _rocm_classify_unified_memory). Skipped if no torch.
# ── 1g. ROCm OOM guard ──
if _hw.IS_ROCM:
try:
import torch as _torch_mem
if _torch_mem.cuda.is_available():
# torch keeps the fraction PER DEVICE: a sharded run left the rest uncapped.
_unified_seen = False
for _mem_index in range(_torch_mem.cuda.device_count()):
# Classify unified vs discrete (see _rocm_classify_unified_memory's docstring).
_props = _torch_mem.cuda.get_device_properties(_mem_index)
_dev_name = _props.name
_gcn_arch, _is_unified = _rocm_classify_unified_memory(_props)
if _is_unified and not _gcn_arch:
logger.debug(
"ROCm OOM guard: gcnArchName absent -- inferred "
"unified memory from device name %r; applying unified cap",
_dev_name,
)
# Windows mem_get_info total is WDDM budget (OS share excluded); Linux needs headroom; torch 2.10+ scales props.total_memory, older hipMemGetInfo.
_total_bytes = int(getattr(_props, "total_memory", 0) or 0)
_driver_total = 0
if not _allocator_divides_by_props_total(
getattr(_torch_mem, "__version__", "")
):
try:
_driver_total = int(_torch_mem.cuda.mem_get_info(_mem_index)[1])
except Exception:
_driver_total = 0
# ROCm name first, so an existing export keeps the cap it had.
_env_raw, _env_name = _mem_fraction_env_value("rocm")
_env_fraction = _parse_mem_fraction_env(_env_raw)
if _env_raw and _env_fraction is None:
logger.warning(
"ROCm OOM guard: ignoring %s=%r (needs a float in (0.0, 1.0]); "
"using the computed cap instead",
_env_name,
_env_raw,
)
_mem_fraction = _gpu_memory_fraction(
_total_bytes,
_is_unified,
sys.platform,
"rocm",
_env_raw,
_driver_total or None,
)
_allowed = (
f"{_total_bytes * _mem_fraction / 1024**3:.1f} of "
f"{_total_bytes / 1024**3:.1f} GiB allowed"
if _total_bytes > 0
else "device total unreported by this wheel"
)
_torch_mem.cuda.set_per_process_memory_fraction(_mem_fraction, _mem_index)
logger.info(
"ROCm OOM guard: set_per_process_memory_fraction(%.4f, cuda:%d) — "
"%s memory host (%s, %s), %s, %s",
_mem_fraction,
_mem_index,
"unified" if _is_unified else "discrete",
_dev_name,
_gcn_arch or "unknown arch",
_allowed,
f"from {_env_name}"
if _env_fraction is not None
else f"computed; override with {_MEM_FRACTION_ENV} or {_GPU_MEM_FRACTION_ENV}",
)
# Differing totals: the budget printed above is not the one enforced.
if (
_is_unified
and sys.platform != "win32"
and _env_fraction is None
and _total_bytes > 0
and _driver_total > 0
and abs(_driver_total - _total_bytes) > _total_bytes // 100
):
logger.info(
"ROCm OOM guard: props.total_memory is %.1f GiB but this torch caps "
"against the driver's %.1f GiB, so the fraction is solved for that "
"total and %.1f GiB stays free against the intended %.1f GiB. Adjust "
"with %s.",
_total_bytes / 1024**3,
_driver_total / 1024**3,
(_total_bytes - _mem_fraction * _driver_total) / 1024**3,
_UNIFIED_OS_RESERVE_BYTES / 1024**3,
_MEM_FRACTION_ENV,
)
_unified_seen = _unified_seen or _is_unified
# Windows APU WDDM budget is user-raisable but nothing says so; tell the user.
if _unified_seen and sys.platform == "win32":
try:
import psutil as _psutil
_phys = _psutil.virtual_memory().total
_granted = _torch_mem.cuda.mem_get_info(0)[1]
if _granted < 0.75 * _phys:
logger.info(
"Windows grants the GPU %.1f GiB of %.1f GiB "
"system RAM (driver/WDDM budget). To raise it: "
"increase the BIOS UMA frame buffer size, or "
"AMD Software > Performance > Tuning > "
"Variable Graphics Memory.",
_granted / 1024**3,
_phys / 1024**3,
)
except Exception:
pass
except Exception as _oom_guard_err:
logger.debug("Could not set GPU memory fraction: %s", _oom_guard_err)
# A user preference, unlike the driver-forced ROCm arm; XPU and MPS are not wired yet.
# ── 1h. Explicit GPU memory cap ──
if not _hw.IS_ROCM:
_cap_raw, _cap_name = _mem_fraction_env_value("cuda")
_cap_fraction = _parse_mem_fraction_env(_cap_raw)
if _cap_raw and _cap_fraction is None:
logger.warning(
"Ignoring %s=%r (needs a float in (0.0, 1.0]); leaving GPU memory uncapped",
_cap_name,
_cap_raw,
)
elif _cap_fraction is not None:
try:
import torch as _torch_cap
if _torch_cap.cuda.is_available():
_cap = _gpu_memory_fraction(0, False, sys.platform, "cuda", _cap_raw)
# Fraction is per device, so cap every visible GPU.
for _cap_index in range(_torch_cap.cuda.device_count()):
_torch_cap.cuda.set_per_process_memory_fraction(_cap, _cap_index)
_cap_props = _torch_cap.cuda.get_device_properties(_cap_index)
_cap_total = int(getattr(_cap_props, "total_memory", 0) or 0)
logger.info(
"GPU memory cap: set_per_process_memory_fraction(%.4f, cuda:%d) from %s — %s, %s",
_cap,
_cap_index,
_cap_name,
getattr(_cap_props, "name", "unknown device"),
f"{_cap_total * _cap / 1024**3:.1f} of {_cap_total / 1024**3:.1f} GiB allowed"
if _cap_total > 0
else "device total unreported by this wheel",
)
else:
logger.debug(
"%s is set but no torch CUDA device is available; nothing to cap",
_cap_name,
)
except Exception as _cap_err:
logger.debug("Could not set GPU memory fraction: %s", _cap_err)
# ── 2. Now import ML libraries (fresh in this clean process) ──
try:
_send_status(event_queue, "Importing Unsloth...")
backend_path = str(Path(__file__).resolve().parent.parent.parent)
if backend_path not in sys.path:
sys.path.insert(0, backend_path)
from core.training.trainer import UnslothTrainer
from utils.paths import (
ensure_dir,
resolve_output_dir,
resolve_tensorboard_dir,
datasets_root,
default_run_dir_name,
)
import transformers
logger.info("Subprocess loaded transformers %s", transformers.__version__)
# unsloth_zoo injects its vendored fla when the trainer imports unsloth above.
logger.info(
"flash-linear-attention fast path importable: %s",
_flash_linear_attention_importable(),
)
except Exception as exc:
event_queue.put(
{
"type": "error",
"error": f"Failed to import ML libraries: {exc}",
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
finally:
_uninstall_metadata_scan_cache(metadata_scan_cache)
if config.get("offload_layers"):
from utils.models.diffusion_lm import diffusion_lm_profile_for_model
# The diffusion load has no block swapping; off before the budget below sizes itself to offloading.
_diffusion = diffusion_lm_profile_for_model(
model_load_target, _worker_hf_token(config), revision = config.get("model_revision")
)
if _diffusion:
config["offload_layers"] = 0
event_queue.put(
{
"type": "warning",
"message": f"Layer offloading is not available for text-diffusion models ({_diffusion}); "
"training without it.",
"ts": time.time(),
}
)
# Offload layers auto-size to allocator budget, so two runs on one card each take a share.
# ── 2b. Training VRAM budget ──
# Only "auto" sizes to a budget, and only LoRA runs outside decision / embedding offload.
_wants_budget = bool(
(config.get("offload_vram_gb") or config.get("offload_vram_gb_per_device"))
and config.get("offload_layers") == "auto"
and config.get("training_type", "LoRA/QLoRA") in ("LoRA/QLoRA", "Continued Pretraining")
and not config.get("is_decision")
and not config.get("is_embedding")
)
# Vision / audio loads on several GPUs cannot offload at load, and LoRA setup allocates adapters before it swaps:
# a cap would OOM either step, so those runs offload Auto uncapped.
_budget_unsupported = (
_wants_budget
and (len(gpu_ids) if gpu_ids else _visible_gpu_count()) > 1
and bool(config.get("is_dataset_image") or config.get("is_dataset_audio"))
)
if _budget_unsupported:
logger.info(
"Training VRAM budget not applied: multi-GPU vision / audio runs offload at LoRA setup"
)
# An out-of-memory error then must not blame a cap that was never set.
config["offload_vram_gb"] = None
config["offload_vram_gb_per_device"] = None
elif _wants_budget:
try:
import torch as _torch_budget
_apply_training_vram_budget(
_torch_budget,
config.get("offload_vram_gb"),
config.get("offload_vram_gb_per_device"),
gpu_ids,
)
except Exception as _budget_err:
logger.warning("Could not apply the training VRAM budget: %s", _budget_err)
if config.get("is_decision", False):
try:
from core.training.decision_trainer import run_decision_training
run_decision_training(event_queue, stop_queue, config)
except Exception as exc:
event_queue.put(
{
"type": "error",
"error": str(exc),
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
_end_abandoned_decision_download(event_queue)
return
if config.get("batch_members"):
try:
from core.training.multi_lora_trainer import run_multi_lora_training
return run_multi_lora_training(event_queue, stop_queue, config)
except Exception as exc:
event_queue.put(
{
"type": "error",
"error": str(exc),
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
# Embedding models branch early: FastSentenceTransformer plus MultipleNegativesRankingLoss.
if config.get("is_embedding", False):
try:
_run_embedding_training(event_queue, stop_queue, config)
except Exception as exc:
event_queue.put(
{
"type": "error",
"error": str(exc),
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
trainer = UnslothTrainer()
trainer.add_progress_callback(_create_trainer_progress_callback(event_queue))
def _apply_stop(save: bool) -> None:
trainer.should_stop = True
trainer.save_on_stop = save
logger.info("Stop signal received (save=%s)", save)
_start_worker_stop_poller(stop_queue, _apply_stop)
# Pipeline order: detect -> dataset -> model -> prepare -> train, so both never hold VRAM at once.
try:
hf_token = _worker_hf_token(config)
model_load_name = _resolve_cached_model_load_name(config)
model_local_only = _model_local_files_only(config)
model_revision = None if model_local_only else config.get("model_revision")
dataset_local_only = _dataset_local_files_only(config)
eval_steps = config.get("eval_steps", 0.00)
hf_dataset = config.get("hf_dataset", "")
training_type = config.get("training_type", "LoRA/QLoRA")
is_cpt_for_dataset = training_type == "Continued Pretraining"
# Filled in below, after the model probe; the closure runs after both.
max_train_rows = None
max_train_rows_seed = config.get("random_seed", 3407)
def _load_training_dataset():
result = trainer.load_and_format_dataset(
dataset_source = hf_dataset if hf_dataset and hf_dataset.strip() else None,
format_type = config.get("format_type", ""),
local_datasets = config.get("local_datasets") or None,
local_eval_datasets = config.get("local_eval_datasets") or None,
custom_format_mapping = config.get("custom_format_mapping"),
subset = config.get("subset"),
train_split = config.get("train_split", "train"),
eval_split = config.get("eval_split"),
dataset_streaming = config.get("dataset_streaming", False),
eval_steps = eval_steps,
dataset_slice_start = config.get("dataset_slice_start"),
dataset_slice_end = config.get("dataset_slice_end"),
is_cpt = is_cpt_for_dataset,
s3_config = config.get("s3_config"),
dataset_local_files_only = dataset_local_only,
dataset_local_path = config.get("dataset_snapshot_path"),
dataset_revision = config.get("dataset_revision"),
require_exact_resume_resources = bool(
config.get("require_exact_resume_resources")
or config.get("require_exact_dataset_resource")
),
hf_token = hf_token,
max_train_rows = max_train_rows,
max_train_rows_seed = max_train_rows_seed,
)
if isinstance(result, tuple):
loaded_dataset, loaded_eval_dataset = result
else:
loaded_dataset = result
loaded_eval_dataset = None
if eval_steps is not None and float(eval_steps) <= 0:
loaded_eval_dataset = None
snapshot = getattr(trainer, "dataset_snapshot_path", None)
if snapshot:
config["dataset_snapshot_path"] = snapshot
return loaded_dataset, loaded_eval_dataset
_send_status(event_queue, "Detecting model type...")
try:
_pre_detect_training_model(
trainer,
config,
model_name,
hf_token,
model_load_name,
model_local_only,
model_revision,
)
except Exception as error:
if not model_local_only:
raise
fallback_error = _model_cache_fallback_error(config, error)
if fallback_error is not None:
raise fallback_error from error
if not _cache_artifact_fallback_allowed(config, error, "model"):
raise
_send_status(
event_queue,
f"Cached files for {model_name} are incomplete; retrying from Hugging Face...",
)
model_load_name = _drop_model_pin_for_fallback(config, hf_token)
# Scan the Hub target we fall back to, not the cached pin already scanned above.
security_error = _model_load_security_error(config, model_load_name, hf_token)
if security_error:
event_queue.put({"type": "error", **security_error, "ts": time.time()})
return
model_local_only = False
model_revision = config.get("model_revision")
_pre_detect_training_model(
trainer,
config,
model_name,
hf_token,
model_load_name,
model_local_only,
model_revision,
)
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
return
# Packing opt-out reads the probed branch, not client flags; streaming and explicit ranges opt out.
raw_text_mode = is_cpt_for_dataset or config.get("format_type") == "raw"
branch_never_packs = bool(getattr(trainer, "_audio_type", None)) or (
bool(getattr(trainer, "is_vlm", False) or getattr(trainer, "is_audio_vlm", False))
and not raw_text_mode
)
# Every replica draws its own batch, so size the subset for this launch, after model probing settles torch and the GPU mask.
max_train_rows = max_train_rows_for_config(
config,
branch_never_packs = branch_never_packs,
world_size = _data_parallel_world_size(),
)
# Resume uses the recorded subset; with no marker it trained unbounded, so bounding now would continue on unrelated rows.
resumed_rows, max_train_rows_seed = row_bound_for_resume(
config.get("resume_from_checkpoint"), max_train_rows, max_train_rows_seed
)
if resumed_rows != max_train_rows:
logger.info(
"Resuming with the row bound recorded at the original start "
f"({resumed_rows} rows) instead of {max_train_rows}\n"
)
if resumed_rows and max_train_rows and resumed_rows < max_train_rows:
# Fewer-replica subset kept (re-deriving gives unrelated rows); extra ranks may re-read rows.
logger.info(
"That subset was sized for a smaller data-parallel world than this "
"one, so the added replicas may re-read rows; start a new run "
"instead of resuming to size it for this machine\n"
)
max_train_rows = resumed_rows
if getattr(trainer, "_audio_type", None) == "bicodec":
_fetch_spark_tts_repo(trainer, event_queue, model_name)
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
return
_send_status(event_queue, "Loading and formatting dataset...")
from .model_weight_prefetch import prepare_dataset_with_prefetch
dataset, eval_dataset = prepare_dataset_with_prefetch(
_load_training_dataset,
_training_weight_prefetch(config, trainer, hf_token, event_queue),
)
if dataset is None or trainer.should_stop:
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
else:
event_queue.put(
{
"type": "error",
"error": trainer.training_progress.error or "Failed to load dataset",
"stack": "",
"ts": time.time(),
}
)
return
# Start the tqdm monitor early to capture download + tokenization bars.
import threading as _th
_tqdm_stop = _th.Event()
def _monitor_tqdm():
from tqdm.auto import tqdm as _tqdm_cls
while not _tqdm_stop.is_set():
for bar in list(getattr(_tqdm_cls, "_instances", set())):
try:
n, total = bar.n or 0, bar.total or 0
desc = getattr(bar, "desc", "") or ""
if total > 0 and n > 0 and desc:
pct = min(int(n * 100 / total), 100)
_send_status(event_queue, f"{desc.strip()} {pct}% ({n:,}/{total:,})")
except (AttributeError, ReferenceError):
pass
_tqdm_stop.wait(3)
_tqdm_thread = account_thread(target = _monitor_tqdm, daemon = True)
_tqdm_thread.start()
training_type = config.get("training_type", "LoRA/QLoRA")
is_cpt = training_type == "Continued Pretraining"
use_lora = training_type in ("LoRA/QLoRA", "Continued Pretraining")
cpt_trains_embeddings = False
# Model load uses VRAM; watchdog lets the parent recover a stalled Xet download via respawn.
_send_status(event_queue, "Loading model...")
from utils.hf_xet_fallback import start_watchdog
# Recheck right before the load: the adapter folder may have been edited since start (#13140).
base_error = _adapter_base_changed_error(config) or _escaping_weights_error(config)
if base_error:
event_queue.put({"type": "error", "error": base_error, "stack": "", "ts": time.time()})
return
event_queue.put({"type": "model_load_started", "ts": time.time()})
_load_watchdog_stop = start_watchdog(
repo_ids = [model_name],
on_stall = lambda msg: event_queue.put(
{"type": "stall", "message": msg, "ts": time.time()}
),
xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
)
def _report_model_repo(repo_id):
# Local cache loads have no active Hub download to track.
if os.path.isdir(os.path.expanduser(repo_id)):
return
event_queue.put({"type": "model_load_resolved", "repo_id": repo_id, "ts": time.time()})
# Latest-sidecar models load 16-bit: bnb 4-bit feeds quantized experts into unvalidated paths.
try:
_train_load_in_4bit = _effective_training_load_in_4bit(
config,
model_load_name,
hf_token,
)
if config["load_in_4bit"] and not _train_load_in_4bit:
logger.info(
"Latest-transformers sidecar active for %s - forcing a 16-bit "
"training load (4-bit is disabled for brand-new architectures)",
model_load_name,
)
success = trainer.load_model(
on_model_resolved = _report_model_repo,
model_name = model_name,
max_seq_length = config["max_seq_length"],
load_in_4bit = _train_load_in_4bit,
full_finetuning = not use_lora,
hf_token = hf_token,
is_dataset_image = config.get("is_dataset_image", False),
is_dataset_audio = config.get("is_dataset_audio", False),
trust_remote_code = config.get("trust_remote_code", False),
gpu_ids = config.get("resolved_gpu_ids"),
model_load_name = model_load_name,
local_files_only = model_local_only,
actual_model_repo_id = config.get("actual_model_repo_id"),
model_revision = model_revision,
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
offload_layers = config.get("offload_layers", 0),
prefetch_depth = config.get("prefetch_depth", 2),
offload_plan_shape = _offload_plan_shape(config),
)
fallback_error = (
_model_cache_fallback_error(config, trainer.model_load_error)
if not success and model_local_only and not trainer.should_stop
else None
)
if fallback_error is not None:
trainer.training_progress.error = str(fallback_error)
if (
not success
and model_local_only
and not trainer.should_stop
and fallback_error is None
and _cache_artifact_fallback_allowed(config, trainer.model_load_error, "model")
):
_send_status(
event_queue,
f"Cached files for {model_name} are incomplete; retrying from Hugging Face...",
)
model_load_name = _drop_model_pin_for_fallback(config, hf_token)
# Scan the Hub target we fall back to, not the cached pin already scanned above.
security_error = _model_load_security_error(config, model_load_name, hf_token)
if security_error:
event_queue.put({"type": "error", **security_error, "ts": time.time()})
return
model_local_only = False
model_revision = config.get("model_revision")
trainer.model = None
trainer.tokenizer = None
dataset = None
eval_dataset = None
gc.collect()
from utils.hardware import clear_gpu_cache
clear_gpu_cache()
_send_status(
event_queue,
"Reloading and formatting the dataset with the Hub tokenizer...",
)
dataset, eval_dataset = _reload_dataset_with_remote_model_tokenizer(
trainer,
config,
model_name,
hf_token,
_load_training_dataset,
model_revision,
)
if dataset is None or trainer.should_stop:
success = False
else:
success = trainer.load_model(
on_model_resolved = _report_model_repo,
model_name = model_name,
max_seq_length = config["max_seq_length"],
load_in_4bit = _train_load_in_4bit,
full_finetuning = not use_lora,
hf_token = hf_token,
is_dataset_image = config.get("is_dataset_image", False),
is_dataset_audio = config.get("is_dataset_audio", False),
trust_remote_code = config.get("trust_remote_code", False),
gpu_ids = config.get("resolved_gpu_ids"),
model_load_name = model_load_name,
local_files_only = model_local_only,
actual_model_repo_id = config.get("actual_model_repo_id"),
model_revision = model_revision,
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
offload_layers = config.get("offload_layers", 0),
prefetch_depth = config.get("prefetch_depth", 2),
offload_plan_shape = _offload_plan_shape(config),
)
finally:
_load_watchdog_stop.set()
event_queue.put({"type": "model_load_completed", "ts": time.time()})
if not success or trainer.should_stop:
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
else:
error_msg = _with_vram_budget_hint(
config, trainer.training_progress.error or "Failed to load model"
)
event_queue.put(
{
"type": "error",
"error": error_msg,
"stack": "",
"ts": time.time(),
}
)
return
if not _emit_resource_provenance(
event_queue,
config,
trainer.model,
model_load_target = model_load_name,
model_load_in_4bit = _train_load_in_4bit,
dataset_loaded_from_exact_snapshot = bool(
getattr(trainer, "dataset_loaded_from_exact_snapshot", False)
),
):
return
if eval_dataset is not None:
event_queue.put(
{
"type": "eval_configured",
"ts": time.time(),
}
)
if is_cpt:
_send_status(event_queue, "Configuring LoRA for continued pretraining...")
# LoRA on embed_tokens/lm_head never trains, so use modules_to_save.
_embedding_modules = ("embed_tokens", "lm_head")
_user_modules = config.get("target_modules") or []
_leaf = lambda m: str(m).rsplit(".", 1)[-1] # noqa: E731
_wants = [m for m in _user_modules if _leaf(m) in _embedding_modules]
# Either module in modules_to_save fills the embedding_learning_rate group.
cpt_trains_embeddings = bool(_wants)
cpt_target_modules = [m for m in _user_modules if _leaf(m) not in _embedding_modules]
if not cpt_target_modules:
cpt_target_modules = [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
]
success = trainer.prepare_model_for_training(
use_lora = True,
target_modules = cpt_target_modules,
modules_to_save = _wants or None,
lora_r = config.get("lora_r", 128),
lora_alpha = config.get("lora_alpha", 32),
lora_dropout = config.get("lora_dropout", 0.0),
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
use_rslora = config.get("use_rslora", False),
use_loftq = _lora_uses_loftq(config),
use_dora = config.get("use_dora", False),
random_state = config.get("random_seed", 3407),
)
elif use_lora:
_send_status(event_queue, "Configuring LoRA adapters...")
success = trainer.prepare_model_for_training(
use_lora = True,
finetune_vision_layers = config.get("finetune_vision_layers", True),
finetune_language_layers = config.get("finetune_language_layers", True),
finetune_attention_modules = config.get("finetune_attention_modules", True),
finetune_mlp_modules = config.get("finetune_mlp_modules", True),
target_modules = config.get("target_modules"),
lora_r = config.get("lora_r", 16),
lora_alpha = config.get("lora_alpha", 16),
lora_dropout = config.get("lora_dropout", 0.0),
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
use_rslora = config.get("use_rslora", False),
use_loftq = _lora_uses_loftq(config),
use_dora = config.get("use_dora", False),
random_state = config.get("random_seed", 3407),
)
else:
_send_status(event_queue, "Preparing model for full finetuning...")
success = trainer.prepare_model_for_training(
use_lora = False,
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
)
if not success or trainer.should_stop:
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
else:
event_queue.put(
{
"type": "error",
"error": _with_vram_budget_hint(
config, trainer.training_progress.error or "Failed to prepare model"
),
"stack": "",
"ts": time.time(),
}
)
return
if is_cpt and not cpt_trains_embeddings:
# A continued adapter keeps its own modules_to_save, so it may train them anyway.
cpt_trains_embeddings = _adapter_trains_embeddings(getattr(trainer, "model", None))
lr_default = "5e-5" if is_cpt else "2e-4"
try:
lr_value = float(config.get("learning_rate", lr_default))
except ValueError:
event_queue.put(
{
"type": "error",
"error": f"Invalid learning rate: {config.get('learning_rate')}",
"stack": "",
"ts": time.time(),
}
)
return
# Pydantic already validated embedding_learning_rate (Optional[float], gt=0, lt=1.0).
embedding_lr_value = config.get("embedding_learning_rate")
if is_cpt:
if cpt_trains_embeddings:
if embedding_lr_value is None:
# Default embedding_learning_rate = lr/10 (Unsloth CPT notebook).
embedding_lr_value = lr_value / 10.0
logger.info(
f"CPT: using default embedding_learning_rate={embedding_lr_value:.1e} "
f"(lr/10). Set explicitly to override.\n"
)
elif embedding_lr_value is not None:
logger.warning(
"CPT: embedding_learning_rate was provided but neither embed_tokens "
"nor lm_head is being trained; ignoring the override.\n"
)
embedding_lr_value = None
resume_from_checkpoint = config.get("resume_from_checkpoint")
output_dir = config.get("output_dir") or _output_dir_from_resume_checkpoint(
resume_from_checkpoint
)
if not output_dir:
output_dir = build_default_output_dir_name(
model_name,
config.get("project_name"),
)
output_dir = str(resolve_output_dir(output_dir))
ensure_dir(Path(output_dir))
_emit_output_dir(event_queue, output_dir)
# Pinned before any checkpoint, so resume reads it back rather than re-deriving it.
if not record_row_bound(output_dir, max_train_rows, max_train_rows_seed) and max_train_rows:
# Not fatal, no fallback: the dataset is already bounded; a resume will read this run as unbounded.
logger.warning(
f"Could not record the max_steps row bound in {output_dir}: "
"resuming this run later will read it as unbounded\n"
)
tensorboard_dir = config.get("tensorboard_dir")
if config.get("enable_tensorboard", False):
tensorboard_dir = str(resolve_tensorboard_dir(tensorboard_dir))
ensure_dir(Path(tensorboard_dir))
# Start training directly - no inner thread, we ARE the subprocess.
dataset_display = config.get("hf_dataset", "") or config.get("uploaded_file", "") or ""
_send_status(
event_queue,
f'Training "{model_name}"'
+ (f"\nDataset = {dataset_display}" if dataset_display else ""),
)
max_steps = config.get("max_steps", 0)
save_steps = config.get("save_steps", 0)
trainer._train_worker(
dataset,
output_dir = output_dir,
num_epochs = config.get("num_epochs", 3),
learning_rate = lr_value,
embedding_learning_rate = embedding_lr_value,
batch_size = config.get("batch_size", 2),
gradient_accumulation_steps = config.get("gradient_accumulation_steps", 4),
warmup_steps = config.get("warmup_steps"),
warmup_ratio = config.get("warmup_ratio"),
max_steps = max_steps if max_steps and max_steps > 0 else 0,
save_steps = save_steps if save_steps and save_steps > 0 else 0,
weight_decay = config.get("weight_decay", 0.001),
random_seed = config.get("random_seed", 3407),
packing = config.get("packing", False),
train_on_completions = False if is_cpt else config.get("train_on_completions", False),
enable_wandb = config.get("enable_wandb", False),
wandb_project = config.get("wandb_project", "unsloth-training"),
wandb_token = config.get("wandb_token"),
enable_tensorboard = config.get("enable_tensorboard", False),
tensorboard_dir = tensorboard_dir,
eval_dataset = eval_dataset,
eval_steps = eval_steps,
early_stopping_patience = config.get("early_stopping_patience", 0),
max_seq_length = config.get("max_seq_length", 2048),
vision_image_size = config.get("vision_image_size"),
optim = config.get("optim", "adamw_8bit"),
lr_scheduler_type = config.get("lr_scheduler_type", "linear"),
is_cpt = is_cpt,
resume_from_checkpoint = resume_from_checkpoint,
)
_tqdm_stop.set()
progress = trainer.get_training_progress()
drafter_note = ""
# A Stop during the drafter phase drops only the drafter; the fine-tune is already saved.
stopped_in_training = trainer.should_stop
if config.get("drafter") and not progress.error and not trainer.should_stop:
from core.training.drafter import run_drafter_phase
manifest = run_drafter_phase(trainer, config, output_dir, event_queue, _send_status)
if manifest is not None:
label = "MTP head" if manifest["strategy"] == "mtp" else "EAGLE-3 drafter"
drafter_note = f"\nTrained {label} saved with the model."
progress = trainer.get_training_progress()
elif output_dir:
from core.training.drafter import discard_drafter_artifacts
discard_drafter_artifacts(output_dir)
if progress.error:
event_queue.put(
{
"type": "error",
"error": progress.error,
"stack": "",
"ts": time.time(),
}
)
else:
saved_output_dir = (
None if stopped_in_training and not trainer.save_on_stop else output_dir
)
event_queue.put(
{
"type": "complete",
"output_dir": saved_output_dir,
"status_message": (progress.status_message or "Training completed")
+ drafter_note,
"ts": time.time(),
}
)
except Exception as exc:
_exc_str = str(exc).lower()
_is_oom = (
"out of memory" in _exc_str
or "hip out of memory" in _exc_str
or "cuda out of memory" in _exc_str
or type(exc).__name__ == "OutOfMemoryError"
)
if _is_oom:
_oom_msg = (
"GPU ran out of VRAM during training.\n"
"To fix: reduce max_seq_length (e.g. 2048–4096), enable "
"gradient_checkpointing=True, lower per_device_train_batch_size, "
"or use a smaller model / higher quantization."
)
logger.error("Training stopped: GPU OOM — %s", exc)
event_queue.put(
{
"type": "error",
"error": _with_vram_budget_hint(config, _oom_msg),
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
else:
event_queue.put(
{
"type": "error",
"error": str(exc),
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
def _send_status(event_queue: Any, message: str) -> None:
"""Send a status update to the parent process."""
event_queue.put(
{
"type": "status",
"message": message,
"ts": time.time(),
}
)
def _emit_output_dir(event_queue: Any, output_dir: str) -> None:
try:
event_queue.put({"type": "output_dir", "output_dir": output_dir, "ts": time.time()})
except Exception:
pass
def _emit_resource_provenance(
event_queue: Any,
config: dict,
model: Any,
*,
model_load_target: str,
model_load_in_4bit: bool,
dataset_loaded_from_exact_snapshot: bool,
) -> bool:
"""False, after reporting why, when a resumed run loaded a different base than it trained on."""
from core.training.provenance import (
build_worker_provenance_event,
incomplete_worker_provenance_event,
resumed_model_mismatch,
)
try:
event = build_worker_provenance_event(
config,
model,
model_load_target = model_load_target,
model_load_in_4bit = model_load_in_4bit,
dataset_loaded_from_exact_snapshot = dataset_loaded_from_exact_snapshot,
)
except Exception:
logger.warning("Could not attest training resource provenance", exc_info = True)
event = incomplete_worker_provenance_event("provenance_attestation_failed")
event["ts"] = time.time()
expected = config.get("resume_expected_model")
mismatch = resumed_model_mismatch(expected, event) if expected else None
if mismatch:
# Record only the dataset, so the next resume still checks this run's expected base.
event_queue.put({**event, "model": incomplete_worker_provenance_event()["model"]})
event_queue.put({"type": "error", "error": mismatch, "stack": "", "ts": time.time()})
return False
event_queue.put(event)
return True
def _mlx_has_checkpoint_at_step(output_dir, step: int) -> bool:
if step <= 0:
return False
from core.training.resume import is_resume_checkpoint_valid
return is_resume_checkpoint_valid(
Path(output_dir) / f"checkpoint-{step}", expected_step = step, backend = "mlx"
)
def _write_mlx_stop_checkpoint(trainer, optimizer, output_dir) -> bool:
"""Writes a resume checkpoint for a stopped MLX run; True if the current step is covered."""
step = int(getattr(trainer, "_global_step", 0) or 0)
# A periodic save or a resumed run may already cover the current step.
if _mlx_has_checkpoint_at_step(output_dir, step):
return True
if step <= 0 or optimizer is None:
return False
ckpt_dir = Path(output_dir) / f"checkpoint-{step}"
if ckpt_dir.is_symlink():
# Refuse a symlinked dir: it could redirect writes outside output_dir.
logger.error("Refusing to write MLX stop checkpoint through symlink: %s", ckpt_dir)
return False
try:
ckpt_dir.mkdir(parents = True, exist_ok = True)
from unsloth_zoo.mlx.utils import (
save_optimizer_state,
save_trainable_adapters,
save_trainer_state,
)
save_trainable_adapters(trainer.model, str(ckpt_dir))
save_optimizer_state(optimizer, str(ckpt_dir))
save_trainer_state(
{
"global_step": step,
"train_loss_history": list(getattr(trainer, "_train_loss_history", [])),
},
str(ckpt_dir),
)
logger.info("Saved stop checkpoint to %s", ckpt_dir)
except Exception:
logger.exception("Failed to write stop checkpoint under %s", output_dir)
return _mlx_has_checkpoint_at_step(output_dir, step)
def _create_trainer_progress_callback(event_queue: Any) -> Callable[[TrainingProgress], None]:
"""Metrics go out only when changed, since the parent appends every progress event to its histories; status only when non-empty."""
sent_warnings: set[str] = set()
last_metrics: list = [None]
def _on_progress(progress: TrainingProgress) -> None:
has_train_loss = progress.step > 0 and progress.loss is not None
has_eval_loss = progress.eval_loss is not None
# The run summary flag marks the terminal record (early-stopped runs never reach total_steps).
is_terminal = bool(getattr(progress, "is_run_summary", False)) or (
progress.total_steps > 0 and progress.step >= progress.total_steps
)
# Wall-clock fields excluded so status replays don't look like new measurements.
metrics = (
progress.step,
progress.loss,
progress.learning_rate,
progress.grad_norm,
progress.num_tokens,
progress.epoch,
progress.eval_loss,
)
is_repeat = metrics == last_metrics[0]
if (
(progress.step == 0 and progress.total_steps > 0)
or has_train_loss
or has_eval_loss
or is_terminal
) and not is_repeat:
last_metrics[0] = metrics
event_queue.put(
{
"type": "progress",
"step": progress.step,
"epoch": progress.epoch,
"loss": progress.loss,
"learning_rate": progress.learning_rate,
"total_steps": progress.total_steps,
"elapsed_seconds": progress.elapsed_seconds,
"eta_seconds": progress.eta_seconds,
"session_start_step": progress.session_start_step,
"grad_norm": progress.grad_norm,
"num_tokens": progress.num_tokens,
"eval_loss": progress.eval_loss,
"status_message": progress.status_message,
"offload": getattr(progress, "offload", None),
"ts": time.time(),
}
)
if progress.status_message:
_send_status(event_queue, progress.status_message)
for message in progress.warnings:
if message not in sent_warnings:
sent_warnings.add(message)
event_queue.put({"type": "warning", "message": message, "ts": time.time()})
return _on_progress
def _create_embedding_progress_callback(
event_queue: Any,
*,
total_steps: int,
training_start_time: float,
should_stop: Callable[[], bool],
):
"""TrainerCallback that reports embedding training progress to the parent. ``should_stop`` is
polled in on_train_begin and on_step_end, so a stop signal arriving mid-run is seen."""
from transformers import TrainerCallback
from core.training.startup_progress import TrainingStartupProgress
startup = TrainingStartupProgress()
class _EmbeddingProgressCallback(TrainerCallback):
_start_step = 0
_training_start_time = training_start_time
def on_train_begin(self, args, state, control, **kwargs):
self._start_step = state.global_step
if state.global_step > 0:
self._training_start_time = time.time()
if should_stop():
return
_send_status(event_queue, startup.begin(args, state))
def on_step_begin(self, args, state, control, **kwargs):
if not should_stop() and (message := startup.step_begin()):
_send_status(event_queue, message)
def on_substep_end(self, args, state, control, **kwargs):
if not should_stop() and (message := startup.substep_end()):
_send_status(event_queue, message)
def on_log(
self,
args,
state,
control,
logs = None,
**kwargs,
):
if not logs:
return
# Trainer end-of-run summary (train_runtime, per-second rates, total_flos) is published nowhere else.
from core.training.trainer import _RESERVED_LOG_KEYS, _TRAINER_SUMMARY_KEYS
if any(k in logs for k in _TRAINER_SUMMARY_KEYS):
logger.info(
"trainer summary",
**{
k: v
for k, v in logs.items()
if isinstance(k, str) and k not in _RESERVED_LOG_KEYS
},
)
# train_loss in HF's summary is the run mean, not a step loss, so it must never become the final step.
loss_value = logs.get("loss")
if loss_value is None and logs.get("train_loss") is not None:
print(
f"Training finished: mean train_loss={logs.get('train_loss')} "
f"over {state.global_step} steps",
flush = True,
)
current_step = state.global_step
elapsed = time.time() - self._training_start_time
eta = session_eta_seconds(elapsed, current_step, self._start_step, total_steps)
event_queue.put(
{
"type": "progress",
"step": current_step,
"epoch": round(state.epoch, 2) if state.epoch else 0,
"loss": loss_value,
"learning_rate": logs.get("learning_rate", None),
"total_steps": total_steps,
"elapsed_seconds": elapsed,
"eta_seconds": eta,
"session_start_step": self._start_step,
"grad_norm": logs.get("grad_norm"),
"num_tokens": getattr(state, "num_input_tokens_seen", None),
"eval_loss": logs.get("eval_loss"),
"status_message": "" if should_stop() else startup.log(logs) or "",
"ts": time.time(),
}
)
def on_step_end(self, args, state, control, **kwargs):
if should_stop():
logger.info("Embedding training: stop at step %d", state.global_step)
control.should_training_stop = True
return control
if message := startup.step_end(state):
_send_status(event_queue, message)
return _EmbeddingProgressCallback()
def _csv_number(cell):
from utils.datasets.cells import _NA_CELLS
text = (cell or "").strip()
if text.lower() in ("true", "false"):
return float(text.lower() == "true")
return None if text in _NA_CELLS else float(text)
def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> None:
"""Separate from UnslothTrainer's LLM/VLM/audio paths: embeddings use SentenceTransformerTrainer."""
import math
model_name = config["model_name"]
model_load_name = _resolve_cached_model_load_name(config)
model_local_only = _model_local_files_only(config)
model_revision = None if model_local_only else config.get("model_revision")
training_start_time = time.time()
_send_status(event_queue, "Importing embedding libraries...")
try:
# Recover from a namespace-package shadow (embedding imports unsloth directly).
from core.import_guards import ensure_real_packages
ensure_real_packages("unsloth_zoo", "unsloth")
from unsloth import FastSentenceTransformer, is_bfloat16_supported
from sentence_transformers import (
SentenceTransformerTrainer,
SentenceTransformerTrainingArguments,
)
from sentence_transformers.losses import MultipleNegativesRankingLoss
from sentence_transformers.training_args import BatchSamplers
from datasets import Dataset
from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset
from utils.paths import datasets_root, resolve_output_dir, default_run_dir_name
except ImportError as e:
event_queue.put(
{
"type": "error",
"error": f"Failed to import embedding libraries: {e}. "
"Ensure 'sentence_transformers' and 'unsloth' are installed.",
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
# datasets loads only here, so quiet its Generating/Map bars now or they land in the structured log.
try:
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
except Exception: # noqa: BLE001 - never let log tidying stop a run
pass
_should_stop = False
_save_on_stop = True
def _apply_stop(save: bool) -> None:
nonlocal _should_stop, _save_on_stop
_save_on_stop = save
_should_stop = True
logger.info(
"Embedding training: stop signal received (save=%s)",
_save_on_stop,
)
_start_worker_stop_poller(stop_queue, _apply_stop)
_send_status(event_queue, "Loading embedding model...")
try:
hf_token = _worker_hf_token(config)
max_seq_length = config.get("max_seq_length", 512)
training_type = config.get("training_type", "LoRA/QLoRA")
use_lora = training_type == "LoRA/QLoRA"
security_error = _model_load_security_error(config, model_load_name, hf_token)
if security_error:
event_queue.put({"type": "error", **security_error, "ts": time.time()})
return
try:
model = FastSentenceTransformer.from_pretrained(
model_name = model_load_name,
max_seq_length = max_seq_length,
full_finetuning = not use_lora,
token = hf_token,
revision = model_revision,
use_exact_model_name = model_revision is not None,
# Not config[trust_remote_code]: consent scan checks only auto_map, so forwarding consent would run unscanned modules.json code.
trust_remote_code = False,
)
except Exception as error:
if not model_local_only:
raise
fallback_error = _model_cache_fallback_error(config, error)
if fallback_error is not None:
raise fallback_error from error
if not _cache_artifact_fallback_allowed(config, error, "model"):
raise
_send_status(
event_queue,
f"Cached files for {model_name} are incomplete; retrying from Hugging Face...",
)
model_load_name = _drop_model_pin_for_fallback(config, hf_token)
# Scan the Hub target we fall back to, not the cached pin already scanned above.
security_error = _model_load_security_error(config, model_load_name, hf_token)
if security_error:
event_queue.put({"type": "error", **security_error, "ts": time.time()})
return
model_local_only = False
model_revision = config.get("model_revision")
model = FastSentenceTransformer.from_pretrained(
model_name = model_load_name,
max_seq_length = max_seq_length,
full_finetuning = not use_lora,
token = hf_token,
revision = model_revision,
use_exact_model_name = model_revision is not None,
# Deliberately False, as above.
trust_remote_code = False,
)
except Exception as e:
event_queue.put(
{
"type": "error",
"error": f"Failed to load embedding model '{model_name}': {e}",
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
loaded_model_for_provenance = model
if _should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
return
if use_lora:
_send_status(event_queue, "Configuring LoRA adapters (FEATURE_EXTRACTION)...")
try:
gradient_checkpointing = config.get("gradient_checkpointing", False)
# Normalize "none"/empty to False.
if gradient_checkpointing in ("none", "", None):
gradient_checkpointing = False
from core.training.trainer import (
LOFTQ_SKIPPED_ON_QUANTIZED,
_has_quantized_weights,
_loftq_peft_kwargs,
)
use_loftq = _lora_uses_loftq(config)
# The fast encoder prepares an intrinsically quantized repo for k-bit training, and LoftQ fails on it.
if use_loftq and _has_quantized_weights(model):
_send_status(event_queue, LOFTQ_SKIPPED_ON_QUANTIZED)
use_loftq = False
model = FastSentenceTransformer.get_peft_model(
model,
r = config.get("lora_r", 32),
target_modules = config.get("target_modules")
or ["q_proj", "k_proj", "v_proj", "o_proj"],
lora_alpha = config.get("lora_alpha", 64),
lora_dropout = config.get("lora_dropout", 0.0),
bias = "none",
use_gradient_checkpointing = gradient_checkpointing,
random_state = config.get("random_seed", 3407),
use_rslora = config.get("use_rslora", False),
use_dora = config.get("use_dora", False),
**_loftq_peft_kwargs(use_loftq),
task_type = "FEATURE_EXTRACTION",
)
except Exception as e:
event_queue.put(
{
"type": "error",
"error": f"Failed to configure LoRA for embedding model: {e}",
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
if _should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
return
_send_status(event_queue, "Loading dataset...")
try:
config["_dataset_loaded_from_exact_snapshot"] = False
hf_dataset = str(config.get("hf_dataset") or "").strip()
local_datasets = config.get("local_datasets") or []
csv_text_columns: list[str] = []
def _load_local_embedding_dataset(dataset_paths: list[str]):
from utils.paths import dataset_files_in_dir
all_files: list[str] = []
for dataset_file in dataset_paths:
file_path = (
dataset_file
if os.path.isabs(dataset_file)
else os.path.join(
str(datasets_root()),
dataset_file,
)
)
if os.path.isdir(file_path):
file_path_obj = Path(file_path)
all_files.extend(str(p) for p in dataset_files_in_dir(file_path_obj))
else:
all_files.append(file_path)
if not all_files:
raise ValueError("No local dataset files found")
first_ext = Path(all_files[0]).suffix.lower()
if first_ext in (".json", ".jsonl"):
loader = "json"
elif first_ext == ".csv":
loader = "csv"
elif first_ext == ".parquet":
loader = "parquet"
else:
raise ValueError(f"Unsupported local dataset format: {all_files[0]}")
from datasets import Features, Value
from utils.datasets.cells import csv_as_text_kwargs, typed_csv_columns
dataset = load_dataset(
loader,
data_files = all_files,
split = "train",
**csv_as_text_kwargs(all_files),
)
if loader != "csv":
return dataset
labels = typed_csv_columns(dataset) & {"label", "labels", "score", "scores"}
for column in labels:
dataset = dataset.map(
lambda cells: {column: [_csv_number(cell) for cell in cells]},
input_columns = column,
batched = True,
features = Features({**dataset.features, column: Value("float64")}),
)
csv_text_columns[:] = [
column for column in dataset.column_names if column not in labels
]
return dataset
if hf_dataset:
dataset = _load_embedding_hf_dataset(
config,
load_dataset,
lambda message: _send_status(event_queue, message),
)
elif local_datasets:
dataset = _load_local_embedding_dataset(local_datasets)
elif config.get("s3_config"):
from core.training.s3_dataset import (
S3DownloadCancelled,
prepare_s3_dataset_download,
)
_send_status(event_queue, "Downloading dataset from S3...")
s3_download = None
try:
s3_download = prepare_s3_dataset_download(
config["s3_config"],
cancel_callback = lambda: _should_stop,
)
dataset = _load_local_embedding_dataset(s3_download.files)
except S3DownloadCancelled:
event_queue.put(
{
"type": "complete",
"output_dir": None,
"status_message": "Training cancelled",
"ts": time.time(),
}
)
return
finally:
if s3_download is not None:
s3_download.cleanup()
else:
event_queue.put(
{
"type": "error",
"error": "No dataset specified for embedding training.",
"stack": "",
"ts": time.time(),
}
)
return
slice_start = config.get("dataset_slice_start")
slice_end = config.get("dataset_slice_end")
if slice_start is not None or slice_end is not None:
dataset = slice_dataset_rows(dataset, slice_start, slice_end)[0]
if csv_text_columns:
kept = dataset.filter(
lambda *columns: [None not in cells for cells in zip(*columns)],
input_columns = csv_text_columns,
batched = True,
)
if len(dataset) and not len(kept):
raise ValueError(f"All {len(dataset):,} rows have an empty text cell.")
if len(kept) < len(dataset):
event_queue.put(
{
"type": "warning",
"message": f"Dropped {len(dataset) - len(kept):,} of {len(dataset):,} "
"rows because a text cell was empty.",
"ts": time.time(),
}
)
dataset = kept
logger.info(f"Embedding dataset loaded: {len(dataset)} samples")
except Exception as e:
event_queue.put(
{
"type": "error",
"error": f"Failed to load dataset: {e}",
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
if _should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
return
if not _emit_resource_provenance(
event_queue,
config,
loaded_model_for_provenance,
model_load_target = model_load_name,
model_load_in_4bit = False,
dataset_loaded_from_exact_snapshot = bool(config.get("_dataset_loaded_from_exact_snapshot")),
):
return
loss = MultipleNegativesRankingLoss(model)
_send_status(event_queue, "Configuring training...")
try:
lr_value = float(config.get("learning_rate", "2e-4"))
except ValueError:
event_queue.put(
{
"type": "error",
"error": f"Invalid learning rate: {config.get('learning_rate')}",
"stack": "",
"ts": time.time(),
}
)
return
resume_from_checkpoint = config.get("resume_from_checkpoint")
output_dir = config.get("output_dir") or _output_dir_from_resume_checkpoint(
resume_from_checkpoint
)
if not output_dir:
output_dir = build_default_output_dir_name(
model_name,
config.get("project_name"),
)
output_dir = str(resolve_output_dir(output_dir))
_emit_output_dir(event_queue, output_dir)
num_epochs = config.get("num_epochs", 2)
batch_size = config.get("batch_size", 256)
gradient_accumulation_steps = config.get("gradient_accumulation_steps", 1)
max_steps_val = config.get("max_steps", 0)
save_steps_val = config.get("save_steps", 0)
warmup_ratio = config.get("warmup_ratio", 0.03)
warmup_steps_val = config.get("warmup_steps")
log_frequency = config.get("log_frequency", 50)
from core.training.trainer import _drop_hf_stdout_callbacks, _hf_stdout_progress_disabled
from core.training.training import apply_save_strategy
training_args_kwargs = {
"output_dir": output_dir,
"per_device_train_batch_size": batch_size,
"gradient_accumulation_steps": gradient_accumulation_steps,
"learning_rate": lr_value,
"fp16": not is_bfloat16_supported(),
"bf16": is_bfloat16_supported(),
"logging_steps": 1,
"report_to": ["wandb"] if config.get("enable_wandb") else "none",
"lr_scheduler_type": config.get("lr_scheduler_type", "linear"),
"batch_sampler": BatchSamplers.NO_DUPLICATES,
"optim": config.get("optim", "adamw_8bit"),
"weight_decay": config.get("weight_decay", 0.001),
"seed": config.get("random_seed", 3407),
# No terminal (stdout goes to the server log); embedding progress callback already publishes the bar.
"disable_tqdm": _hf_stdout_progress_disabled(),
}
if max_steps_val and max_steps_val > 0:
training_args_kwargs["max_steps"] = max_steps_val
else:
training_args_kwargs["num_train_epochs"] = num_epochs if num_epochs > 0 else 2
# warmup: prefer warmup_ratio (standard for embedding scripts), else steps
if warmup_ratio is not None and warmup_ratio > 0:
training_args_kwargs["warmup_ratio"] = warmup_ratio
elif warmup_steps_val is not None and warmup_steps_val > 0:
training_args_kwargs["warmup_steps"] = warmup_steps_val
apply_save_strategy(training_args_kwargs, save_steps_val)
args = SentenceTransformerTrainingArguments(**training_args_kwargs)
if max_steps_val and max_steps_val > 0:
total_steps = max_steps_val
else:
effective_epochs = num_epochs if num_epochs > 0 else 2
len_dataloader = math.ceil(len(dataset) / batch_size)
steps_per_epoch = max(len_dataloader // gradient_accumulation_steps, 1)
total_steps = steps_per_epoch * effective_epochs
progress_callback = _create_embedding_progress_callback(
event_queue,
total_steps = total_steps,
training_start_time = training_start_time,
should_stop = lambda: _should_stop,
)
_send_status(event_queue, "Starting embedding training...")
try:
trainer = SentenceTransformerTrainer(
model = model,
train_dataset = dataset,
loss = loss,
args = args,
callbacks = [progress_callback],
)
# disable_tqdm only swaps ProgressCallback for PrinterCallback, both to stdout.
_drop_hf_stdout_callbacks(trainer)
trainer.train(resume_from_checkpoint = resume_from_checkpoint)
except Exception as e:
event_queue.put(
{
"type": "error",
"error": f"Embedding training failed: {e}",
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
if _should_stop and not _save_on_stop:
event_queue.put(
{
"type": "complete",
"output_dir": None,
"status_message": "Training cancelled",
"ts": time.time(),
}
)
return
_send_status(event_queue, "Saving model...")
try:
if _should_stop and _save_on_stop:
trainer._save_checkpoint(trainer.model, trial = None)
model.save_pretrained(output_dir)
model.tokenizer.save_pretrained(output_dir)
logger.info("Embedding model saved to %s", output_dir)
except Exception as e:
logger.error("Failed to save embedding model: %s", e)
event_queue.put(
{
"type": "error",
"error": f"Training completed but failed to save: {e}",
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
)
return
event_queue.put(
{
"type": "complete",
"output_dir": output_dir,
"status_message": "Embedding training completed",
"ts": time.time(),
}
)