Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Unsloth training backend: integrates Unsloth training with the FastAPI backend."""
from utils.account_context import account_thread
import gc
import os
import sys
import types
# Off on Linux so forked map() workers cannot deadlock.
os.environ["TOKENIZERS_PARALLELISM"] = "true" if sys.platform in ("win32", "darwin") else "false"
# Do NOT import unsloth_zoo.compiler here: it pulls in heavy torch/triton imports.
if sys.platform in ("win32", "darwin"):
_compile_cache = os.environ.get("UNSLOTH_COMPILE_LOCATION", "unsloth_compiled_cache")
if not os.path.isabs(_compile_cache):
_compile_cache = os.path.abspath(_compile_cache)
os.environ["UNSLOTH_COMPILE_LOCATION"] = _compile_cache
_pp = os.environ.get("PYTHONPATH", "")
if _compile_cache not in _pp.split(os.pathsep):
os.environ["PYTHONPATH"] = _compile_cache + (os.pathsep + _pp if _pp else "")
if _compile_cache not in sys.path:
sys.path.insert(0, _compile_cache)
import torch
from utils.hardware import (
clear_gpu_cache,
dataset_map_num_proc,
get_device_map,
raise_if_offloaded,
get_visible_gpu_count,
)
# getattr: torch._dynamo is lazy and a bare access can return a half-built module.
_dynamo_config = getattr(getattr(torch, "_dynamo", None), "config", None)
if _dynamo_config is not None and hasattr(_dynamo_config, "recompile_limit"):
_dynamo_config.recompile_limit = 64
# Drop any unsloth/unsloth_zoo namespace-package shadow before importing them.
from core.import_guards import ensure_real_packages as _ensure_real_packages
_ensure_real_packages("unsloth_zoo", "unsloth")
from unsloth import FastLanguageModel, FastVisionModel, is_bfloat16_supported
import json
import threading
import math
from loggers import get_logger
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Callable, Union
from datasets import Dataset
from core.training.eval_dataset import STREAMING_EVAL_MAX_SAMPLES, evaluation_enabled
from utils.datasets.audio_decode import ensure_audio_decoding
from utils.datasets.audio_rows import (
audio_transcription_messages,
image_slot_count,
prepare_audio_vlm_image_column,
resolve_audio_columns,
resolve_audio_vlm_image_column,
)
from utils.datasets.tokenizer_fingerprint import install_tokenizer_fingerprint
from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset
from utils.hf_dataset_options import hf_dataset_split_instruction_names
from utils.third_party_source import (
ensure_dac_speech_weights,
ensure_outetts_source,
ensure_spark_tts_source,
import_outetts_module,
import_sparktts_module,
)
from utils.models import is_vision_model, audio_vlm_multimodal_lm_checked, detect_audio_type_checked
from utils.models.diffusion_lm import (
diffusion_lm_profile_for_model,
loaded_diffusion_profile,
require_diffusion_api,
)
from utils.models.model_identity import restore_hf_cache_repo_identity
from utils.models.unsloth_mirror import unsloth_public_mirror
from utils.models.model_config import _env_offline
from utils.datasets import format_and_template_dataset
from utils.datasets.cells import cell_text, csv_as_text_kwargs
from utils.datasets.chat_templates import get_training_chat_template
from utils.datasets.completion_masking import (
apply_completion_masking,
count_truncated_drops,
truncated_drop_cause,
)
from utils.datasets.iterable import is_streaming_dataset as detect_streaming_dataset
from utils.datasets.iterable import take_rows
from utils.datasets.pretokenized import (
PRETOKENIZED_FORMAT,
has_custom_labels,
prepare_pretokenized_dataset,
resolve_completion_only_loss,
)
from utils.datasets.raw_text import prepare_raw_text_dataset, resolve_column_names
from utils.paths import (
dataset_files_in_dir,
ensure_dir,
resolve_dataset_path,
resolve_output_dir,
resolve_tensorboard_dir,
)
from trl import SFTTrainer, SFTConfig
from .resume import session_eta_seconds
from .training import (
TrainingProgress,
apply_save_strategy,
create_mlx_trainer_adapter,
should_use_mlx_training_backend,
)
from .dataset_bounds import (
bound_dataset_rows,
slice_dataset_rows,
world_size_env_report,
world_size_from_env,
)
logger = get_logger(__name__)
_STUDIO_DEFAULT_TARGETS = (
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
)
_CPT_EMBEDDING_TARGETS = ("embed_tokens", "lm_head")
def _build_report_targets(training_args) -> list[str] | str:
report_to: list[str] = []
if training_args.get("enable_wandb", False):
report_to.append("wandb")
if training_args.get("enable_tensorboard", False):
report_to.append("tensorboard")
return report_to or "none"
def _verbose_logging_requested() -> bool:
"""Whether `unsloth studio --verbose` is in effect. --verbose zeroes both access-log windows
(unsloth_cli/commands/studio.py), and the env is inherited by the training subprocess, so the
same pair is the signal here."""
def _zero(name: str) -> bool:
raw = (os.environ.get(name) or "").strip()
try:
return raw != "" and int(raw) <= 0
except ValueError:
return False
return _zero("UNSLOTH_STUDIO_ACCESS_LOG_DEDUP_MS") and _zero(
"UNSLOTH_STUDIO_ACCESS_LOG_POLL_DEDUP_MS"
)
def _hf_stdout_progress_disabled() -> bool:
"""The subprocess stdout is teed to the log, where tqdm fragments can break structlog JSON."""
return not _verbose_logging_requested()
_EMBEDDING_GETTERS = {"embed_tokens": "get_input_embeddings", "lm_head": "get_output_embeddings"}
def _embedding_module_path(model, name: str):
"""Full module path of the model's input embedding / output head for Studio's generic CPT names."""
getter = getattr(model, _EMBEDDING_GETTERS.get(name, ""), None)
if not callable(getter):
return None
try:
module = getter()
except Exception:
return None
if module is None:
return None
return next((path for path, m in model.named_modules() if m is module and path), None)
def _tokenizer_blocks_map_workers(tokenizer) -> bool:
"""True when `dataset_num_proc` workers cannot receive this tokenizer. dill pickles a remote-code
tokenizer class by value (transformers 5.5 does not set its module on the parent package), dragging
in module globals up to huggingface_hub's live HTTP client: "cannot pickle 'SSLContext' object"
(Dream's DreamTokenizer). Stock tokenizer classes pickle by reference, so only those are probed."""
inner = getattr(tokenizer, "tokenizer", tokenizer)
if not str(getattr(type(inner), "__module__", "")).startswith("transformers_modules"):
return False
try:
import dill
dill.dumps(tokenizer)
except Exception:
return True
return False
# Keys that only ever appear on Trainer's end-of-run / end-of-eval summary record.
_TRAINER_SUMMARY_KEYS = (
"train_runtime",
"train_samples_per_second",
"train_steps_per_second",
"total_flos",
"eval_runtime",
"eval_samples_per_second",
"eval_steps_per_second",
)
# structlog fills these itself; a metric of the same name would collide.
_RESERVED_LOG_KEYS = frozenset({"event", "level", "timestamp", "logger"})
def _drop_hf_stdout_callbacks(trainer) -> None:
"""disable_tqdm only swaps in PrinterCallback, which prints a raw dict per step; remove both."""
if _verbose_logging_requested():
return
try:
from transformers.trainer_callback import PrinterCallback, ProgressCallback
except Exception: # noqa: BLE001 - never let log tidying break a training run
return
for callback_cls in (PrinterCallback, ProgressCallback):
try:
trainer.remove_callback(callback_cls)
except Exception: # noqa: BLE001 - not attached, or an incompatible trainer
pass
# These load_model branches force load_in_4bit=False: the base is 16-bit or wider (bicodec fp32).
_FORCED_16BIT_AUDIO_TYPES = frozenset({"csm", "whisper", "bicodec", "dac"})
def _audio_vlm_auto_model_kwargs(multimodal: bool) -> dict:
"""No vision tower (LFM3-Audio): the default load builds a backbone with no LM head, so use Chat's class."""
if not multimodal:
return {}
import transformers
auto_model = getattr(transformers, "AutoModelForMultimodalLM", None)
return {"auto_model": auto_model} if auto_model is not None else {}
def _bitsandbytes_allows_4bit() -> bool:
"""from_pretrained drops 4-bit when bitsandbytes is unusable, so the metadata read must check it."""
try:
from unsloth.device_type import ALLOW_BITSANDBYTES
return bool(ALLOW_BITSANDBYTES)
except Exception: # noqa: BLE001 -- unreadable means "cannot narrow", so keep the request
return True
def _metadata_lookup_name(
model_name: str,
lookup_name: str,
local_files_only: bool,
model_revision: Optional[str],
load_in_4bit: bool,
) -> str:
"""Return the repo whose config and tokenizer the loader will read."""
if local_files_only or model_revision is not None or lookup_name != model_name:
return lookup_name
# MLX loads the name as given; there is no mirror mapping on Apple Silicon.
from core.training.training import should_use_mlx_training_backend
if should_use_mlx_training_backend():
return lookup_name
mirror = unsloth_public_mirror(model_name, load_in_4bit and _bitsandbytes_allows_4bit())
if mirror is None:
return lookup_name
logger.info(
"Reading %s config and tokenizer from %s, the repo Unsloth loads", model_name, mirror
)
return mirror
def _spark_tts_tokenizer_kwargs(audio_type: Optional[str], lookup_name: str) -> dict:
"""``subfolder`` for a Spark-TTS repo root, empty for anything else. Only the canonical repo
layout needs it: a local checkpoint or an alias that already names LLM/ points at the
tokenizer directly."""
if audio_type != "bicodec":
return {}
name = str(lookup_name).replace("\\", "/").rstrip("/")
if name.endswith("/LLM"):
return {}
# A snapshot root holding an LLM/ child is not the tokenizer dir; point at LLM/.
if os.path.isdir(os.path.join(lookup_name, "LLM")):
return {"subfolder": "LLM"}
# A local checkpoint that carries its own tokenizer is already the right directory.
if os.path.isfile(os.path.join(lookup_name, "tokenizer_config.json")):
return {}
return {"subfolder": "LLM"}
def _spark_tts_hub_repo(model_name: str) -> str:
"""The Hub repo behind a Spark-TTS name: the "Spark-TTS-0.5B/LLM" registry alias is a subfolder, not a repo."""
if model_name.endswith("/LLM"):
return f"unsloth/{model_name.rsplit('/', 1)[0]}"
return model_name
def _spark_tts_repo_dir(
model_name: str,
lookup_name: str,
local_files_only: bool,
model_revision: Optional[str],
on_model_resolved: Optional[Callable[[str], None]] = None,
) -> str:
"""Spark-TTS repo root: LLM/ for training, BiCodec/ and wav2vec2 beside it for the codec."""
local = os.path.abspath(os.path.expanduser(lookup_name))
# Local folders are never cache-pinned, so local_files_only is False for them.
if local_files_only or os.path.isdir(local):
parent = os.path.dirname(local)
if os.path.basename(local) == "LLM" and os.path.isdir(os.path.join(parent, "BiCodec")):
return parent
return local
from huggingface_hub import snapshot_download
hf_repo = _spark_tts_hub_repo(model_name)
if on_model_resolved is not None:
on_model_resolved(hf_repo)
return os.path.abspath(snapshot_download(hf_repo, revision = model_revision))
# Enough rows to see past a leading null or malformed value, few enough to stay cheap on a streamed dataset.
_AUDIO_SNIFF_ROWS = 16
# Keep in step with detect_multimodal_dataset, so the veto answers False only for the columns that set the audio flag.
_AUDIO_COLUMN_KEYWORDS = ("audio", "speech", "wav", "waveform", "sound")
def _dataset_has_audio_column(dataset) -> Optional[bool]:
"""Checks the Audio feature or row values, not column names; None when undecidable, and the caller keeps refusing unless False."""
try:
from datasets.features.audio import Audio
except Exception: # noqa: BLE001 - datasets missing or too old to have the feature
return None
features = getattr(dataset, "features", None)
if not features:
return None
try:
if any(isinstance(f, Audio) for f in features.values()):
return True
except Exception: # noqa: BLE001 - a mapping that is not a features dict
return None
# Several rows, not one: JSON/CSV audio can start with a null or malformed value. Only an audio-NAMED column can answer False; no evidence stays None.
try:
from utils.datasets.format_detection import (
_AUDIO_EXTENSIONS,
_is_audio_value,
_keyword_in_column,
)
def looks_like_audio(value) -> bool:
if _is_audio_value(value):
return True
return isinstance(value, str) and value.lower().endswith(_AUDIO_EXTENSIONS)
saw_a_usable_value = False
for index, row in enumerate(dataset):
if index >= _AUDIO_SNIFF_ROWS:
break
for name, value in row.items():
if looks_like_audio(value):
return True
if not any(_keyword_in_column(k, str(name)) for k in _AUDIO_COLUMN_KEYWORDS):
continue
if isinstance(value, str):
saw_a_usable_value = saw_a_usable_value or bool(value.strip())
elif value is not None:
saw_a_usable_value = True
except Exception: # noqa: BLE001 - unreadable row, empty dataset, odd row type
return None
return False if saw_a_usable_value else None
def _raise_if_empty_train_split(dataset, stage: str) -> None:
# Format detection and SFTTrainer die with a bare StopIteration on an empty split.
if hasattr(dataset, "__len__") and len(dataset) == 0:
where = f" {stage}" if stage else ""
raise ValueError(
f"The training dataset has no rows{where}. "
"Add at least one example before starting training."
)
# Marks an omitted mode, which keeps the loaded one; a literal default would overwrite it.
_UNSET = object()
def normalize_gradient_checkpointing(value) -> Union[str, bool]:
if isinstance(value, str):
text = value.strip().lower()
if text in ("", "unsloth"):
return "unsloth"
if text in ("true", "1", "yes"):
return True
if text in ("false", "0", "no", "none", "off"):
return False
elif value in (True, False, "unsloth"):
return value
logger.warning(f"Invalid gradient_checkpointing value: {value}, defaulting to 'unsloth'")
return "unsloth"
def _ignored_adapter_settings(
peft_config,
lora_r,
lora_alpha,
modules_to_save,
target_modules = None,
lora_dropout = None,
use_dora = False,
use_rslora = False,
use_loftq = False,
) -> list[str]:
"""Requested LoRA settings a continued adapter keeps from its own config instead."""
adapter = next(iter(peft_config.values()), None) if isinstance(peft_config, dict) else None
if adapter is None:
return []
def leaf(module):
return str(module).rsplit(".", 1)[-1]
ignored = []
saved_r = getattr(adapter, "r", None)
if saved_r is not None and saved_r != lora_r:
ignored.append(f"rank {lora_r} (adapter has {saved_r})")
saved_alpha = getattr(adapter, "lora_alpha", None)
if saved_alpha is not None and saved_alpha != lora_alpha:
ignored.append(f"alpha {lora_alpha} (adapter has {saved_alpha})")
saved_dropout = getattr(adapter, "lora_dropout", None)
if lora_dropout is not None and saved_dropout is not None and saved_dropout != lora_dropout:
ignored.append(f"dropout {lora_dropout} (adapter has {saved_dropout})")
if bool(use_dora) != bool(getattr(adapter, "use_dora", False)):
ignored.append("DoRA on" if use_dora else "DoRA off")
if bool(use_rslora) != bool(getattr(adapter, "use_rslora", False)):
ignored.append("rsLoRA on" if use_rslora else "rsLoRA off")
if use_loftq:
# LoftQ only initializes a new adapter; an existing one keeps its weights.
ignored.append("LoftQ initialization")
# A string target is a PEFT regex or "all-linear"; only explicit lists compare cleanly.
saved_targets = getattr(adapter, "target_modules", None)
requested_list = target_modules if isinstance(target_modules, (list, tuple)) else []
wants_all_linear = target_modules == "all-linear" or "all-linear" in requested_list
requested = [m for m in requested_list if m != "all-linear"]
if isinstance(saved_targets, str) and requested:
# Saved as "all-linear" or a PEFT regex: an explicit selection cannot narrow it.
ignored.append(
f"target modules {', '.join(sorted({leaf(m) for m in requested}))} "
f"(adapter has {saved_targets})"
)
elif isinstance(saved_targets, (list, set, tuple)) and (requested or wants_all_linear):
saved_names = ", ".join(sorted({leaf(m) for m in saved_targets}))
if wants_all_linear:
ignored.append(f"all-linear targets (adapter has {saved_names})")
elif {leaf(m) for m in requested} != {leaf(m) for m in saved_targets}:
ignored.append(
f"target modules {', '.join(sorted({leaf(m) for m in requested}))} "
f"(adapter has {saved_names})"
)
saved_modules = {leaf(m) for m in (getattr(adapter, "modules_to_save", None) or [])}
missing = [m for m in (modules_to_save or []) if leaf(m) not in saved_modules]
if missing:
ignored.append(f"training {', '.join(missing)} (adapter does not train them)")
return ignored
def _tokenizer_is_empty(tokenizer) -> bool:
# Not a length check: a snapshot keeping tokenizer_config.json but no vocab still lists its special tokens.
inner = getattr(tokenizer, "tokenizer", tokenizer)
try:
return not inner("hello world", add_special_tokens = False)["input_ids"]
except Exception:
return False
LOFTQ_SKIPPED_ON_QUANTIZED = (
"LoftQ needs 16-bit weights and this model is loaded quantized, so it trains without LoftQ. "
"Choose LoRA (16-bit) to use LoftQ."
)
def _has_quantized_weights(model) -> bool:
try:
from unsloth.models._utils import _has_quantized_linears
except ImportError:
return False
# routed_ok mirrors Unsloth's own LoftQ check: its loader densifies routed NVFP4 / FP8 weights.
return _has_quantized_linears(model, routed_ok = True)
def _loftq_peft_kwargs(use_loftq) -> dict:
# PEFT ignores loftq_config unless init_lora_weights is "loftq", so the toggle alone trained plain LoRA.
if not use_loftq:
return {"loftq_config": None}
return {"init_lora_weights": "loftq", "loftq_config": {"loftq_bits": 4, "loftq_iter": 1}}
def _load_adapter_aware_tokenizer(lookup_name, hf_token, **kwargs):
"""AutoTokenizer for *lookup_name*. A PEFT adapter saved without tokenizer files falls back
to the base it names, as Unsloth's own adapter loader does (#13140)."""
from transformers import AutoTokenizer
try:
return AutoTokenizer.from_pretrained(lookup_name, token = hf_token, **kwargs)
except Exception:
from utils.models.model_config import get_base_model_from_lora_identifier
base = get_base_model_from_lora_identifier(lookup_name, hf_token)
if not base:
raise
# The adapter's revision and subfolder do not apply to its base repo.
kwargs.pop("revision", None)
kwargs.pop("subfolder", None)
return AutoTokenizer.from_pretrained(base, token = hf_token, **kwargs)
class UnslothTrainer:
def __new__(cls, *args, **kwargs):
if cls is UnslothTrainer and should_use_mlx_training_backend():
return create_mlx_trainer_adapter(*args, **kwargs)
return super().__new__(cls)
def __init__(self):
self.model = None
self.tokenizer = None
self.trainer = None
self.training_thread = None
self.training_progress = TrainingProgress()
self.progress_callbacks = []
self.is_training = False
self.should_stop = False
self.save_on_stop = True
self._early_stopping = None
self.load_in_4bit = True
self.is_cpt = False
self.is_vlm = False
self.is_audio = False
self.is_audio_vlm = False
self._audio_type = None
self.diffusion_lm_profile = None
self._use_gradient_checkpointing = "unsloth"
self._offload_layers = 0
self._prefetch_depth = 2
self._gpu_ids = None
self._offload_layer_devices = None
self._offload_plan_shape = {}
# True until a probe says otherwise, so a path that never probes cannot trip the inconclusive-detection guard.
self._audio_type_known = True
self._is_dataset_audio = False
self._cuda_audio_used = False
self._spark_tts_repo_dir = None
self._spark_tts_repo_source = None
self._spark_tts_code_dir = None
self._outetts_code_dir = None
self.model_name = None
self.model_load_error = None
self.dataset_loaded_from_exact_snapshot = False
self.dataset_snapshot_path = None
# A max_steps bound keeps a uniform sample of the rows, so a pass over it is this share of a dataset pass.
self._kept_row_fraction = 1.0
self.training_start_time: Optional[float] = None
self.session_start_step: int = 0
self.batch_size: Optional[int] = None
self.max_seq_length: Optional[int] = None
self.gradient_accumulation_steps: Optional[int] = None
self._lock = threading.Lock()
def pre_detect_and_load_tokenizer(
self,
model_name: str,
max_seq_length: int = 2048,
hf_token: Optional[str] = None,
is_dataset_image: bool = False,
is_dataset_audio: bool = False,
trust_remote_code: bool = False,
model_load_name: Optional[str] = None,
local_files_only: bool = False,
model_revision: Optional[str] = None,
load_in_4bit: bool = True,
) -> None:
"""Loads no weights, so it can run before load_and_format_dataset() without VRAM contention."""
self.model_name = model_name
self.max_seq_length = max_seq_length
self.trust_remote_code = trust_remote_code
lookup_name = _metadata_lookup_name(
model_name,
model_load_name or model_name,
local_files_only,
model_revision,
load_in_4bit,
)
if hf_token:
os.environ["HF_TOKEN"] = hf_token
# Checked: an unreadable tokenizer_config would misroute a TTS run to the text path.
self._audio_type, self._audio_type_known = detect_audio_type_checked(
lookup_name,
hf_token,
local_files_only = local_files_only,
revision = model_revision,
)
self._is_dataset_audio = is_dataset_audio
if self._audio_type == "audio_vlm":
self.is_audio = False
self.is_audio_vlm = is_dataset_audio
self._audio_type = None
else:
self.is_audio = self._audio_type is not None
self.is_audio_vlm = False
if not self.is_audio and not self.is_audio_vlm:
self._cuda_audio_used = False
vision = (
is_vision_model(
lookup_name,
hf_token = hf_token,
local_files_only = local_files_only,
revision = model_revision,
)
if not self.is_audio
else False
)
self.is_vlm = not self.is_audio_vlm and vision and is_dataset_image
self._detect_diffusion_lm(lookup_name, hf_token, local_files_only, model_revision)
logger.info(
"pre_detect: audio_type=%s, is_audio=%s, is_audio_vlm=%s, is_vlm=%s, diffusion_lm=%s",
self._audio_type,
self.is_audio,
self.is_audio_vlm,
self.is_vlm,
self.diffusion_lm_profile,
)
# CPU-only load: Whisper needs AutoProcessor, others AutoTokenizer (CSM loads its own inline).
def _load_tokenizer_for_detected_type():
if self._audio_type == "whisper":
from transformers import AutoProcessor
self.tokenizer = AutoProcessor.from_pretrained(
lookup_name,
# Labels are encoded before load_model: match FastModel's English/transcribe prefix.
language = "English",
task = "transcribe",
trust_remote_code = trust_remote_code,
token = hf_token,
local_files_only = local_files_only,
revision = model_revision,
)
else:
self.tokenizer = _load_adapter_aware_tokenizer(
lookup_name,
hf_token,
trust_remote_code = trust_remote_code,
local_files_only = local_files_only,
revision = model_revision,
# Spark-TTS tokenizer lives under LLM/; the root has no vocab and misleadingly blames sentencepiece.
**_spark_tts_tokenizer_kwargs(self._audio_type, lookup_name),
)
# Transformers 5 builds a one-token tokenizer from a snapshot without vocab files instead of raising, which
# skipped the incomplete-cache Hub retry and trained to NaN loss (#6958).
if local_files_only and _tokenizer_is_empty(self.tokenizer):
raise FileNotFoundError(
f"The cached snapshot of {model_name} has no tokenizer files."
)
if model_revision and _tokenizer_is_empty(self.tokenizer):
raise FileNotFoundError(
f"{model_name} has no tokenizer files at revision {model_revision}, the revision of "
"its cached copy. Delete the cached model and retry to use the latest revision."
)
# Probe and tokenizer load are separate reads; a transient 5xx may affect only one.
def _retry_audio_detection() -> bool:
"""Re-probe an inconclusive type. True if the answer changed."""
if self._audio_type_known:
return False
previous_type = self._audio_type
retried_type, retried_known = detect_audio_type_checked(
lookup_name,
hf_token,
local_files_only = local_files_only,
revision = model_revision,
)
if not retried_known:
return False
self._audio_type, self._audio_type_known = retried_type, True
# audio_vlm is detected as an audio_type now; handle separately
if self._audio_type == "audio_vlm":
self.is_audio = False
self.is_audio_vlm = is_dataset_audio
self._audio_type = None
else:
self.is_audio = self._audio_type is not None
self.is_audio_vlm = False
logger.info(
"pre_detect retry: audio_type=%s, is_audio=%s",
self._audio_type,
self.is_audio,
)
return self._audio_type != previous_type
# An inconclusive probe leaves Spark-TTS at its tokenizer-less root; re-probe.
try:
_load_tokenizer_for_detected_type()
except Exception:
if not _retry_audio_detection():
raise
logger.info(
"Retrying the tokenizer load for %s as %s after the first read failed",
model_name,
self._audio_type,
)
_load_tokenizer_for_detected_type()
logger.info("Pre-loaded tokenizer for %s", model_name)
# A changed answer needs a different tokenizer object (whisper processor, Spark LLM/).
if _retry_audio_detection():
_load_tokenizer_for_detected_type()
logger.info(
"Reloaded tokenizer for %s after detection changed to %s",
model_name,
self._audio_type,
)
# The worker formats the dataset before load_model, so BiCodec preprocessing resolves the repo itself (#13458).
self._spark_tts_repo_source = (
model_name,
model_load_name or model_name,
local_files_only,
model_revision,
)
def resolve_spark_tts_repo(
self, on_model_resolved: Optional[Callable[[str], None]] = None
) -> None:
"""Set the Spark-TTS repo root from what pre_detect recorded, unless load_model already set it."""
if self._spark_tts_repo_dir is None:
self._spark_tts_repo_dir = _spark_tts_repo_dir(
*self._spark_tts_repo_source, on_model_resolved
)
def spark_tts_hub_repo(self) -> str:
return _spark_tts_hub_repo(self._spark_tts_repo_source[0])
def _detect_diffusion_lm(self, lookup_name, hf_token, local_files_only, model_revision):
"""Text-diffusion LMs train with their own denoising objective on the text path, never the VLM one
(DiffusionGemma ships a vision tower the diffusion objective does not use)."""
self.diffusion_lm_profile = (
None
if self.is_audio or self.is_audio_vlm
else diffusion_lm_profile_for_model(
lookup_name,
hf_token = hf_token,
local_files_only = local_files_only,
revision = model_revision,
)
)
if self.diffusion_lm_profile and self.is_vlm:
self.is_vlm = False
self._record_warning(
"Text-diffusion models train on text only here; the dataset's images are ignored."
)
def add_progress_callback(self, callback: Callable[[TrainingProgress], None]):
self.progress_callbacks.append(callback)
def _update_progress(self, **kwargs):
"""Update training progress and notify callbacks"""
with self._lock:
for key, value in kwargs.items():
if hasattr(self.training_progress, key):
setattr(self.training_progress, key, value)
for callback in self.progress_callbacks:
try:
callback(self.training_progress)
except Exception as e:
logger.error(f"Error in progress callback: {e}")
def _record_warning(self, message: str) -> None:
message = str(message).strip()
if not message:
return
with self._lock:
if message in self.training_progress.warnings:
return
self.training_progress.warnings.append(message)
logger.warning(message)
for callback in self.progress_callbacks:
try:
callback(self.training_progress)
except Exception as e:
logger.error(f"Error in progress callback: {e}")
def _offload_load_kwargs(self, device_map = None) -> dict:
# Loading the offloaded layers straight into host RAM is what lets a model larger than the card load at all.
if not self._offload_layers:
return {}
# A fixed count loads to host only onto one card (core raises on a multi-GPU map); get_peft_model swaps it there.
if self._offload_layers != "auto" and device_map in ("unsloth_balanced", "balanced"):
return {}
return {
"offload_layers": self._offload_layers,
# Auto plans at load; without the run's batch and rank it sizes for batch 1, rank 16.
"device_map_planner_kwargs": {
"prefetch_depth": self._prefetch_depth,
**self._offload_plan_shape,
},
}
def _offload_peft_kwargs(self) -> dict:
if not self._offload_layers:
return {}
return {"offload_layers": self._offload_layers, "prefetch_depth": self._prefetch_depth}
def _offload_snapshot(self) -> Optional[dict]:
"""Where each decoder layer is and what the last steps' copies cost, for the live panel."""
swapper = (
getattr(self.model, "_unsloth_block_swap", None) if self.model is not None else None
)
stats = getattr(swapper, "stats", None)
if stats is None:
return None
try:
snap = stats()
# JSON object keys must be strings.
snap["state"] = {str(k): v for k, v in snap["state"].items()}
if torch.cuda.is_available():
device = getattr(swapper, "device", None)
snap["vram_allocated_bytes"] = torch.cuda.memory_allocated(device)
snap["vram_peak_bytes"] = torch.cuda.max_memory_allocated(device)
snap["vram_total_bytes"] = torch.cuda.get_device_properties(device).total_memory
get_fraction = getattr(torch.cuda, "get_per_process_memory_fraction", None)
if get_fraction is not None:
snap["vram_fraction"] = get_fraction(device)
snap["vram_devices"] = self._offload_vram_devices()
snap["layer_device"] = self._offload_layer_device_map(swapper)
return snap
except Exception as exc:
logger.debug("offload stats unavailable: %s", exc)
return None
def _offload_vram_devices(self) -> list:
gpu_ids = getattr(self, "_gpu_ids", None)
get_fraction = getattr(torch.cuda, "get_per_process_memory_fraction", None)
devices = []
for i in range(torch.cuda.device_count()):
entry = {
"index": i,
"gpu_id": gpu_ids[i] if gpu_ids and i < len(gpu_ids) else i,
"name": torch.cuda.get_device_properties(i).name,
"allocated_bytes": torch.cuda.memory_allocated(i),
"peak_bytes": torch.cuda.max_memory_allocated(i),
"total_bytes": torch.cuda.get_device_properties(i).total_memory,
}
if get_fraction is not None:
entry["fraction"] = get_fraction(i)
devices.append(entry)
return devices
def _offload_layer_device_map(self, swapper) -> dict:
"""Torch ordinal of every decoder layer: an offloaded one's home card, a resident one's weights."""
key = (id(swapper), tuple(getattr(swapper, "indices", ())))
cached = getattr(self, "_offload_layer_devices", None)
if cached is not None and cached[0] == key:
return cached[1]
devices = {}
for li, block in zip(getattr(swapper, "indices", ()), getattr(swapper, "blocks", ())):
home = getattr(block, "home", None)
if home is not None and home.type == "cuda" and home.index is not None:
devices[str(li)] = home.index
try:
from unsloth_zoo.block_swap import find_decoder_layers
for li, layer in enumerate(find_decoder_layers(self.model)):
if str(li) in devices:
continue
p = next(layer.parameters(), None)
if p is not None and p.device.type == "cuda" and p.device.index is not None:
devices[str(li)] = p.device.index
except Exception as exc:
logger.debug("offload layer devices unavailable: %s", exc)
self._offload_layer_devices = (key, devices)
return devices
def _create_progress_callback(self):
"""Create a TrainerCallback for progress tracking. Reused by all training branches."""
from transformers import TrainerCallback
from core.training.startup_progress import TrainingStartupProgress
trainer_ref = self
startup = TrainingStartupProgress()
class _ProgressCallback(TrainerCallback):
# Throttled republish: without HF's tqdm bar a long eval looks stalled after the last step log.
_eval_seen = 0
_eval_last_report = 0.0
def on_train_begin(self, args, state, control, **kwargs):
trainer_ref.session_start_step = state.global_step
if state.global_step > 0:
trainer_ref.training_start_time = time.time()
if trainer_ref.should_stop:
return
trainer_ref._update_progress(status_message = startup.begin(args, state))
def on_step_begin(self, args, state, control, **kwargs):
if not trainer_ref.should_stop and (message := startup.step_begin()):
trainer_ref._update_progress(status_message = message)
def on_substep_end(self, args, state, control, **kwargs):
if not trainer_ref.should_stop and (message := startup.substep_end()):
trainer_ref._update_progress(status_message = message)
def on_evaluate(self, args, state, control, **kwargs):
cls = type(self)
had_status = cls._eval_last_report > 0.0
cls._eval_seen = 0
cls._eval_last_report = 0.0
# An empty status is ignored downstream, so the UI would stay on "Evaluating...".
if had_status and not trainer_ref.should_stop:
trainer_ref._update_progress(status_message = "Training in progress...")
def on_prediction_step(
self,
args,
state,
control,
eval_dataloader = None,
**kwargs,
):
cls = type(self)
cls._eval_seen += 1
total = None
try:
total = len(eval_dataloader) if eval_dataloader is not None else None
except TypeError:
total = None
now = time.time()
if cls._eval_last_report and (now - cls._eval_last_report) < 15.0:
return
cls._eval_last_report = now
where = f"{cls._eval_seen:,}" + (f"/{total:,}" if total else "")
trainer_ref._update_progress(status_message = f"Evaluating... {where} batches")
logger.info("evaluating", batches = cls._eval_seen, total_batches = total)
def on_log(
self,
args,
state,
control,
logs = None,
**kwargs,
):
if not logs:
return
# Trainer summaries carry numbers found nowhere else, so they are logged, not left to raw stdout.
if any(k in logs for k in _TRAINER_SUMMARY_KEYS):
logger.info(
"trainer summary",
**{
k: v
for k, v in logs.items()
if isinstance(k, str) and k not in _RESERVED_LOG_KEYS
},
)
# Not train_loss: that is the run MEAN.
loss_value = logs.get("loss")
is_run_summary = loss_value is None and (
logs.get("train_loss") is not None or logs.get("train_runtime") is not None
)
if is_run_summary:
logger.info(
"training finished: mean train_loss=%s over %s steps",
logs.get("train_loss"),
state.global_step,
)
current_step = state.global_step
grad_norm = logs.get("grad_norm", None)
elapsed_seconds = None
if trainer_ref.training_start_time is not None:
elapsed_seconds = time.time() - trainer_ref.training_start_time
eta_seconds = session_eta_seconds(
elapsed_seconds,
current_step,
trainer_ref.session_start_step,
trainer_ref.training_progress.total_steps,
)
num_tokens = getattr(state, "num_input_tokens_seen", None)
trainer_ref._update_progress(
step = current_step,
epoch = round(state.epoch * trainer_ref._kept_row_fraction, 2)
if state.epoch
else 0,
loss = loss_value,
learning_rate = logs.get("learning_rate", None),
elapsed_seconds = elapsed_seconds,
eta_seconds = eta_seconds,
session_start_step = trainer_ref.session_start_step,
grad_norm = grad_norm,
num_tokens = num_tokens,
eval_loss = logs.get("eval_loss", None),
is_run_summary = is_run_summary,
offload = trainer_ref._offload_snapshot(),
status_message = "" if trainer_ref.should_stop else startup.log(logs) or "",
)
def on_epoch_end(self, args, state, control, **kwargs):
trainer_ref._update_progress(
epoch = state.epoch * trainer_ref._kept_row_fraction, step = state.global_step
)
def on_step_end(self, args, state, control, **kwargs):
if trainer_ref.should_stop:
logger.info(f"Stop detected at step {state.global_step}\n")
control.should_training_stop = True
return control
if message := startup.step_end(state):
trainer_ref._update_progress(status_message = message)
return _ProgressCallback()
def _attach_early_stopping(self, training_args):
from core.training.early_stopping import attach_early_stopping
self._early_stopping = attach_early_stopping(
self.trainer,
training_args.get("early_stopping_patience"),
user_stopped = lambda: self.should_stop,
)
def _early_stopping_summary(self) -> str:
es = self._early_stopping
if es is None:
return ""
summary = f"Stopped early at step {es.stopped_at}. " if es.stopped_at is not None else ""
if es.restored:
summary += f"Kept the weights from step {es.best_step} (eval loss {es.best_loss:.4f}). "
return summary
def _calculate_total_steps(self, num_samples, batch_size, grad_accum, num_epochs, max_steps):
"""Calculate total training steps from dataset size and training params."""
if max_steps and max_steps > 0:
return max_steps
len_dataloader = math.ceil(num_samples / batch_size)
steps_per_epoch = max(
len_dataloader // grad_accum + int(len_dataloader % grad_accum > 0), 1
)
return steps_per_epoch * num_epochs
def _build_audio_training_args(
self,
training_args,
output_dir,
*,
extra_args = None,
):
"""Build the training args dict for audio branches: common config (batch size, lr, warmup,
fp16/bf16, etc.) with per-branch overrides via extra_args."""
batch_size = training_args.get("batch_size", 2)
gradient_accumulation_steps = training_args.get("gradient_accumulation_steps", 4)
warmup_steps_val = training_args.get("warmup_steps", 5)
max_steps_val = training_args.get("max_steps", 0)
learning_rate = training_args.get("learning_rate", 2e-4)
weight_decay = training_args.get("weight_decay", 0.001)
lr_scheduler_type = training_args.get("lr_scheduler_type", "linear")
random_seed = training_args.get("random_seed", 3407)
optim_value = training_args.get("optim", "adamw_8bit")
config = {
"per_device_train_batch_size": batch_size,
"gradient_accumulation_steps": gradient_accumulation_steps,
"warmup_steps": warmup_steps_val if warmup_steps_val is not None else 5,
"learning_rate": learning_rate,
"fp16": not is_bfloat16_supported(),
"bf16": is_bfloat16_supported(),
"logging_steps": 1,
"optim": optim_value,
"weight_decay": weight_decay,
"lr_scheduler_type": lr_scheduler_type,
"seed": random_seed,
"output_dir": output_dir,
"report_to": _build_report_targets(training_args),
# The subprocess stdout is teed into the server log, so HF's bar is noise there; --verbose restores it.
"disable_tqdm": _hf_stdout_progress_disabled(),
}
if training_args.get("enable_tensorboard", False):
config["logging_dir"] = str(
resolve_tensorboard_dir(training_args.get("tensorboard_dir"))
)
if max_steps_val and max_steps_val > 0:
config["max_steps"] = max_steps_val
else:
config["num_train_epochs"] = training_args.get("num_epochs", 3)
apply_save_strategy(config, training_args.get("save_steps", 0))
if extra_args:
config.update(extra_args)
return config
def _finalize_training(
self,
output_dir,
label = "",
):
"""Save model after training and update progress. Used by all training branches."""
if self.should_stop and self.save_on_stop:
self.trainer._save_checkpoint(self.trainer.model, trial = None)
self.trainer.save_model()
self.tokenizer.save_pretrained(output_dir)
self._patch_adapter_config(output_dir)
msg = f"{label} training stopped" if label else "Training stopped"
logger.info(f"\n{msg}. Model saved to {output_dir}\n")
self._update_progress(
is_training = False,
status_message = (
f"Training stopped. {self._early_stopping_summary()}Model saved to {output_dir}"
),
)
elif self.should_stop:
msg = f"{label} training cancelled" if label else "Training cancelled"
logger.info(f"\n{msg}.\n")
self._update_progress(is_training = False, status_message = "Training cancelled.")
else:
self.trainer.save_model()
self.tokenizer.save_pretrained(output_dir)
self._patch_adapter_config(output_dir)
msg = f"{label} training completed" if label else "Training completed"
logger.info(f"\n{msg}! Model saved to {output_dir}\n")
self._update_progress(
is_training = False,
is_completed = True,
status_message = (
f"Training completed! {self._early_stopping_summary()}Model saved to {output_dir}"
),
)
def _cleanup_audio_artifacts(self):
"""Leftover audio modules would make the next dataset.map(num_proc=N) fork children deadlock."""
audio_paths = []
if self._spark_tts_code_dir:
audio_paths.append(str(self._spark_tts_code_dir))
if self._outetts_code_dir:
audio_paths.append(str(self._outetts_code_dir))
removed_paths = []
for path in audio_paths:
if path in sys.path:
sys.path.remove(path)
removed_paths.append(path)
prefixes = ("snac", "whisper", "sparktts", "outetts")
removed_modules = [key for key in sys.modules if key.startswith(prefixes)]
for key in removed_modules:
del sys.modules[key]
self._spark_tts_code_dir = None
self._outetts_code_dir = None
if removed_paths or removed_modules:
logger.info(
f"Cleaned up audio artifacts: {len(removed_paths)} paths, "
f"{len(removed_modules)} modules\n"
)
def load_model(
self,
model_name: str,
max_seq_length: int = 2048,
load_in_4bit: bool = True,
hf_token: Optional[str] = None,
is_dataset_image: bool = False,
is_dataset_audio: bool = False,
trust_remote_code: bool = False,
full_finetuning: bool = False,
gpu_ids: Optional[list[int]] = None,
model_load_name: Optional[str] = None,
local_files_only: bool = False,
actual_model_repo_id: Optional[str] = None,
model_revision: Optional[str] = None,
use_gradient_checkpointing: Union[str, bool] = "unsloth",
on_model_resolved: Optional[Callable[[str], None]] = None,
offload_layers: Union[int, str] = 0,
prefetch_depth: Union[int, str] = 2,
offload_plan_shape: Optional[dict] = None,
) -> bool:
"""Load model for training (supports both text and vision models)"""
# Offloading streams frozen base weights, so it has nothing to do in a full finetune.
self._offload_layers = 0 if full_finetuning else (offload_layers or 0)
self._prefetch_depth = prefetch_depth or 2
# Physical ids behind each torch ordinal, so the panel names cards as the settings do.
self._gpu_ids = list(gpu_ids) if gpu_ids else None
self._offload_plan_shape = {k: v for k, v in (offload_plan_shape or {}).items() if v}
self.load_in_4bit = load_in_4bit
self.trust_remote_code = trust_remote_code
# The loader installs the checkpointing implementation; a full finetune never reinstalls it.
use_gradient_checkpointing = normalize_gradient_checkpointing(use_gradient_checkpointing)
self._use_gradient_checkpointing = use_gradient_checkpointing
lookup_name = model_load_name or model_name
metadata_name = _metadata_lookup_name(
model_name, lookup_name, local_files_only, model_revision, load_in_4bit
)
self.model_load_error = None
try:
if self.model is not None:
del self.model
if self.tokenizer is not None:
del self.tokenizer
if self.trainer is not None:
del self.trainer
logger.info("\nClearing GPU memory before training...")
clear_gpu_cache()
# Clear audio sys.path/sys.modules state; stale entries deadlock forked map() workers
self._cleanup_audio_artifacts()
# Reload patched modules first: __UNSLOTH_PATCHED__ blocks re-compilation.
import importlib
for _key, _mod in list(sys.modules.items()):
if "transformers.models." in _key and ".modeling_" in _key:
if hasattr(_mod, "__UNSLOTH_PATCHED__"):
try:
importlib.reload(_mod)
except Exception:
pass
from utils.cache_cleanup import clear_unsloth_compiled_cache
_preserve = ["Unsloth*Trainer.py"] if sys.platform in ("win32", "darwin") else None
clear_unsloth_compiled_cache(preserve_patterns = _preserve)
# Use the checked variant: it reassigns _audio_type, else the flag describes the previous probe.
self._audio_type, self._audio_type_known = detect_audio_type_checked(
metadata_name,
hf_token,
local_files_only = local_files_only,
revision = model_revision,
)
self._is_dataset_audio = is_dataset_audio
if self._audio_type == "audio_vlm":
self.is_audio = False
self.is_audio_vlm = is_dataset_audio
self._audio_type = None
else:
self.is_audio = self._audio_type is not None
self.is_audio_vlm = False
if not self.is_audio and not self.is_audio_vlm:
self._cuda_audio_used = False
vision = (
is_vision_model(
metadata_name,
hf_token = hf_token,
local_files_only = local_files_only,
revision = model_revision,
)
if not self.is_audio
else False
)
self.is_vlm = not self.is_audio_vlm and vision and is_dataset_image
# Only a definitive no (not an unreadable config) takes the multimodal class.
audio_vlm_multimodal = False
if self.is_audio_vlm and not vision:
takes, known = audio_vlm_multimodal_lm_checked(
metadata_name,
hf_token = hf_token,
local_files_only = local_files_only,
revision = model_revision,
)
audio_vlm_multimodal = known and takes
self._detect_diffusion_lm(metadata_name, hf_token, local_files_only, model_revision)
self.model_name = model_name
self.max_seq_length = max_seq_length
logger.info(
f"Audio type: {self._audio_type}, is_audio: {self.is_audio}, is_audio_vlm: {self.is_audio_vlm}"
)
logger.info(f"Dataset has images: {is_dataset_image}, audio: {is_dataset_audio}")
logger.info(f"Using VLM path: {self.is_vlm}")
self._update_progress(
is_training = True,
is_completed = False,
error = None,
step = 0,
loss = 0.0,
epoch = 0,
)
model_display = model_name.split("/")[-1] if "/" in model_name else model_name
model_type_label = (
"audio"
if self.is_audio
else (
"vision"
if self.is_vlm
else ("text-diffusion" if self.diffusion_lm_profile else "text")
)
)
self._update_progress(
status_message = f"Loading {model_type_label} model... {model_display}"
)
logger.info(f"\nLoading {model_type_label} model: {model_name}")
if hf_token:
os.environ["HF_TOKEN"] = hf_token
# Gate-check the repo Unsloth will fetch, which may be a public mirror of it.
if "/" in model_name and not local_files_only and not _env_offline():
try:
from huggingface_hub import model_info as hf_model_info
model_info_kwargs = {"token": hf_token or None}
if model_revision:
model_info_kwargs["revision"] = model_revision
info = hf_model_info(metadata_name, **model_info_kwargs)
# model_info works on gated repos (metadata is public); info.gated flags acceptance.
if info.gated and not hf_token:
friendly = (
f"Access denied for '{model_name}'. This model is gated. "
f"Please add a Hugging Face token with access and try again."
)
logger.error(
f"Model '{model_name}' is gated (gated={info.gated}) and no HF token provided"
)
self._update_progress(error = friendly, is_training = False)
return False
except Exception as gate_err:
from huggingface_hub.utils import (
GatedRepoError,
RepositoryNotFoundError,
)
if isinstance(gate_err, (GatedRepoError, RepositoryNotFoundError)):
from hub.utils.hf_errors import modelscope_missing
friendly = modelscope_missing(gate_err) or (
f"Access denied for '{model_name}'. This model is gated or private. "
f"Please add a Hugging Face token with access and try again."
)
logger.error(f"Gated model check failed: {gate_err}")
self._update_progress(error = friendly, is_training = False)
return False
device_map = get_device_map(gpu_ids)
logger.info(
f"Using device_map='{device_map}' ({get_visible_gpu_count()} GPU(s) visible)"
)
# ROCm without native bf16 crashes on bf16 kernels, so force float16 there; NVIDIA keeps None so T4/V100 are not coerced.
_is_rocm = (
bool(getattr(torch.version, "hip", None)) or "rocm" in torch.__version__.lower()
)
_auto_dtype = torch.float16 if (_is_rocm and not is_bfloat16_supported()) else None
# Record the precision that actually loaded; Chat reloads the base at it.
if self._audio_type in _FORCED_16BIT_AUDIO_TYPES:
self.load_in_4bit = False
if self._audio_type == "csm":
# Whisper: FastModel, auto_model=WhisperForConditionalGeneration, load_in_4bit=False
from unsloth import FastModel
from transformers import CsmForConditionalGeneration
self.model, self.tokenizer = FastModel.from_pretrained(
model_name = lookup_name,
max_seq_length = max_seq_length,
dtype = _auto_dtype,
auto_model = CsmForConditionalGeneration,
load_in_4bit = False,
device_map = device_map,
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
)
logger.info("Loaded CSM audio model")
elif self._audio_type == "whisper":
from unsloth import FastModel
from transformers import WhisperForConditionalGeneration
self.model, self.tokenizer = FastModel.from_pretrained(
model_name = lookup_name,
dtype = _auto_dtype,
load_in_4bit = False,
device_map = device_map,
full_finetuning = full_finetuning,
auto_model = WhisperForConditionalGeneration,
whisper_language = "English",
whisper_task = "transcribe",
token = hf_token,
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
)
self.model.generation_config.language = "<|en|>"
self.model.generation_config.task = "transcribe"
self.model.config.suppress_tokens = []
self.model.generation_config.forced_decoder_ids = None
logger.info("Loaded Whisper audio model (FastModel)")
elif self._audio_type == "snac":
# Orpheus: language model with audio codec tokens
self.model, self.tokenizer = FastLanguageModel.from_pretrained(
model_name = lookup_name,
max_seq_length = max_seq_length,
dtype = _auto_dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
**self._offload_load_kwargs(device_map),
)
logger.info(f"Loaded {self._audio_type} audio model (FastLanguageModel)")
elif self._audio_type == "bicodec":
# Spark-TTS loads only the LLM subfolder of the full repo.
from unsloth import FastModel
self._spark_tts_repo_dir = _spark_tts_repo_dir(
model_name,
lookup_name,
local_files_only,
model_revision,
on_model_resolved,
)
llm_path = os.path.join(self._spark_tts_repo_dir, "LLM")
self.model, self.tokenizer = FastModel.from_pretrained(
model_name = llm_path,
max_seq_length = max_seq_length,
dtype = torch.float32, # Spark-TTS requires float32
attn_implementation = "sdpa", # Flash Attention cannot run float32
load_in_4bit = False,
device_map = device_map,
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
)
logger.info("Loaded Spark-TTS (bicodec) model")
elif self._audio_type == "dac":
from unsloth import FastModel
self.model, self.tokenizer = FastModel.from_pretrained(
lookup_name,
max_seq_length = max_seq_length,
load_in_4bit = False,
device_map = device_map,
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
)
logger.info("Loaded OuteTTS (dac) model (FastModel)")
elif self.is_audio_vlm:
from unsloth import FastModel
audio_vlm_kwargs = _audio_vlm_auto_model_kwargs(audio_vlm_multimodal)
self.model, self.tokenizer = FastModel.from_pretrained(
**audio_vlm_kwargs,
model_name = lookup_name,
max_seq_length = max_seq_length,
dtype = _auto_dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
**self._offload_load_kwargs(device_map),
)
logger.info(
f"Loaded audio VLM model (FastModel{', AutoModelForMultimodalLM' if audio_vlm_kwargs else ''})"
)
elif self.diffusion_lm_profile:
# FastModel routes every registered text-diffusion family to its diffusion loader.
from unsloth import FastModel
self.model, self.tokenizer = FastModel.from_pretrained(
model_name = lookup_name,
max_seq_length = max_seq_length,
dtype = _auto_dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
)
loaded = loaded_diffusion_profile(self.model)
if loaded is None:
raise RuntimeError(
f"Unsloth: {model_name} looks like a text-diffusion model "
f"({self.diffusion_lm_profile}) but loaded as an autoregressive one, so it "
"would train on the wrong objective. Update Unsloth (`unsloth studio update`)."
)
self.diffusion_lm_profile = loaded
logger.info(f"Loaded text-diffusion model (profile: {loaded})")
elif self.is_vlm:
self.model, self.tokenizer = FastVisionModel.from_pretrained(
model_name = lookup_name,
max_seq_length = max_seq_length,
dtype = _auto_dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
**self._offload_load_kwargs(device_map),
)
logger.info("Loaded vision model")
from transformers import ProcessorMixin
tok = self.tokenizer
has_image_proc = isinstance(tok, ProcessorMixin) or hasattr(tok, "image_processor")
logger.info(f"\n[VLM Diagnostic] FastVisionModel returned: {type(tok).__name__}")
logger.info(
f"[VLM Diagnostic] Is ProcessorMixin: {isinstance(tok, ProcessorMixin)}"
)
logger.info(
f"[VLM Diagnostic] Has image_processor: {hasattr(tok, 'image_processor')}"
)
logger.info(f"[VLM Diagnostic] Usable as vision processor: {has_image_proc}\n")
else:
self.model, self.tokenizer = FastLanguageModel.from_pretrained(
model_name = lookup_name,
max_seq_length = max_seq_length,
dtype = _auto_dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
**self._offload_load_kwargs(device_map),
)
logger.info("Loaded text model")
# The config probe misses a Hub LoRA (no config.json), but core loads a diffusion base through
# its diffusion path: train it with its own objective, never next-token SFT.
loaded = loaded_diffusion_profile(self.model)
if loaded:
self.diffusion_lm_profile = loaded
logger.info(f"Loaded text-diffusion model (profile: {loaded})")
raise_if_offloaded(self.model, device_map, "Unsloth training")
restored_repo_id = restore_hf_cache_repo_identity(
self.model,
lookup_name,
expected_repo_id = actual_model_repo_id or model_name,
)
if restored_repo_id:
logger.info(
f"Restored Hub model identity for saved adapter metadata: {restored_repo_id}"
)
if self.should_stop:
return False
if full_finetuning and self.diffusion_lm_profile:
from unsloth import FastModel
FastModel.for_training(
self.model, use_gradient_checkpointing = bool(use_gradient_checkpointing)
)
elif full_finetuning:
self.model.for_training(
use_gradient_checkpointing = use_gradient_checkpointing,
)
self._update_progress(status_message = "Model loaded successfully")
logger.info("Model loaded successfully")
return True
except OSError as e:
self.model_load_error = e
if "could not get source code" in str(e) and not getattr(
self, "_source_code_retried", False
):
# Stale Unsloth patch state can break inspect.getsource() when switching families; retry once.
self._source_code_retried = True
logger.info(f"\n'could not get source code' — retrying once...\n")
return self.load_model(
model_name = model_name,
max_seq_length = max_seq_length,
load_in_4bit = load_in_4bit,
hf_token = hf_token,
is_dataset_image = is_dataset_image,
is_dataset_audio = is_dataset_audio,
trust_remote_code = trust_remote_code,
full_finetuning = full_finetuning,
gpu_ids = gpu_ids,
model_load_name = model_load_name,
local_files_only = local_files_only,
actual_model_repo_id = actual_model_repo_id,
model_revision = model_revision,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
offload_layers = offload_layers,
prefetch_depth = prefetch_depth,
offload_plan_shape = offload_plan_shape,
)
error_msg = str(e)
error_lower = error_msg.lower()
if any(
k in error_lower
for k in (
"gated repo",
"access to it at",
"401",
"403",
"unauthorized",
"forbidden",
)
):
error_msg = (
f"Access denied for '{model_name}'. This model is gated or private. "
f"Please add a Hugging Face token with access and try again."
)
# The traceback names the failing call; str(e) alone (e.g. "[WinError 6]") does not (#15583).
logger.error(f"Error loading model: {e}", exc_info = True)
self._update_progress(error = error_msg, is_training = False)
return False
except Exception as e:
self.model_load_error = e
error_msg = str(e)
error_lower = error_msg.lower()
if any(
k in error_lower
for k in (
"gated repo",
"access to it at",
"401",
"403",
"unauthorized",
"forbidden",
)
):
error_msg = (
f"Access denied for '{model_name}'. This model is gated or private. "
f"Please add a Hugging Face token with access and try again."
)
logger.error(f"Error loading model: {e}", exc_info = True)
self._update_progress(error = error_msg, is_training = False)
return False
finally:
self._source_code_retried = False
def _prepare_diffusion_lora(
self,
target_modules,
modules_to_save,
lora_r,
lora_alpha,
lora_dropout,
use_gradient_checkpointing,
use_rslora,
use_loftq,
use_dora,
random_state = 3407,
) -> bool:
"""LoRA for a text-diffusion model through its profile. Studio's default target list names
autoregressive projections (LLaDA has none of o/gate/down_proj, and the MoE families must keep their
experts and router frozen), so the default list means "the profile's own targets"."""
from unsloth import FastModel
unsupported = [name for name, on in (("DoRA", use_dora), ("LoftQ", use_loftq)) if on]
if unsupported:
raise ValueError(
f"{' and '.join(unsupported)} {'is' if len(unsupported) == 1 else 'are'} not supported "
"for text-diffusion models yet. Turn it off and train with LoRA."
)
requested = (
[target_modules] if isinstance(target_modules, str) else list(target_modules or [])
)
if not requested or set(requested) - set(_CPT_EMBEDDING_TARGETS) == set(
_STUDIO_DEFAULT_TARGETS
):
target_modules = None
elif requested == ["all-linear"]:
target_modules = "all-linear"
else:
target_modules = [m for m in requested if m != "all-linear"] or "all-linear"
leaf_names = {name.rsplit(".", 1)[-1] for name, _ in self.model.named_modules()}
if modules_to_save:
# LLaDA names them transformer.wte / transformer.ff_out; the full path keeps each block's ff_out out.
resolved = {
m: m if m in leaf_names else _embedding_module_path(self.model, m)
for m in modules_to_save
}
kept = list(dict.fromkeys(path for path in resolved.values() if path))
dropped = sorted(m for m, path in resolved.items() if not path)
if dropped:
logger.info(
f"Text-diffusion model has no {dropped}; training them is skipped for this model.\n"
)
modules_to_save = kept or None
logger.info(
f"Text-diffusion LoRA ({self.diffusion_lm_profile}): target modules = "
f"{'profile default' if target_modules is None else target_modules}\n"
)
peft_kwargs = dict(
r = lora_r,
target_modules = target_modules,
lora_alpha = lora_alpha,
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = bool(use_gradient_checkpointing),
random_state = random_state,
)
if modules_to_save:
peft_kwargs["modules_to_save"] = modules_to_save
if use_rslora:
peft_kwargs["use_rslora"] = True
self.model = FastModel.get_peft_model(self.model, **peft_kwargs)
if self.should_stop:
logger.info("Stopped during LoRA configuration\n")
return False
self._update_progress(status_message = "LoRA adapters configured")
logger.info("LoRA adapters configured successfully\n")
return True
def prepare_model_for_training(
self,
use_lora: bool = True,
finetune_vision_layers: bool = True,
finetune_language_layers: bool = True,
finetune_attention_modules: bool = True,
finetune_mlp_modules: bool = True,
target_modules: list = None,
lora_r: int = 16,
lora_alpha: int = 16,
lora_dropout: float = 0.0,
use_gradient_checkpointing: Union[str, bool] = _UNSET,
use_rslora: bool = False,
use_loftq: bool = False,
use_dora: bool = False,
modules_to_save: list = None,
random_state: int = 3407,
) -> bool:
try:
if self.model is None:
raise ValueError("Model not loaded. Call load_model() first.")
if use_gradient_checkpointing is _UNSET:
use_gradient_checkpointing = self._use_gradient_checkpointing
else:
use_gradient_checkpointing = normalize_gradient_checkpointing(
use_gradient_checkpointing
)
if not use_lora:
if use_gradient_checkpointing != self._use_gradient_checkpointing:
logger.warning(
f"Gradient checkpointing {use_gradient_checkpointing} was requested after the "
f"model was loaded with {self._use_gradient_checkpointing}. A full finetune "
f"keeps the loaded mode, so pass it to load_model() instead."
)
self._update_progress(status_message = "Full finetuning mode - no LoRA adapters")
logger.info("Full finetuning mode - training all parameters\n")
return True
self._use_gradient_checkpointing = use_gradient_checkpointing
# Saved LoRA dirs load as PeftModel via from_pretrained; do not stack a second adapter (#13140).
if getattr(self.model, "peft_config", None):
ignored = _ignored_adapter_settings(
self.model.peft_config,
lora_r,
lora_alpha,
modules_to_save,
target_modules = target_modules,
lora_dropout = lora_dropout,
use_dora = use_dora,
use_rslora = use_rslora,
use_loftq = use_loftq,
)
status = "Continuing from existing LoRA adapters"
if ignored:
# The adapter's own shape wins; say so instead of dropping the request silently.
status += f" (kept the adapter's settings, ignoring {'; '.join(ignored)})"
logger.warning(f"{status}\n")
self._update_progress(status_message = status)
logger.info(
"Model already has LoRA adapters from the checkpoint; "
"continuing training without creating a new adapter\n"
)
return True
if use_loftq and _has_quantized_weights(self.model):
# Unsloth refuses LoftQ on quantized weights; train as before rather than fail a QLoRA run or its resume.
logger.warning(LOFTQ_SKIPPED_ON_QUANTIZED)
self._update_progress(status_message = LOFTQ_SKIPPED_ON_QUANTIZED)
use_loftq = False
if getattr(self, "diffusion_lm_profile", None):
return self._prepare_diffusion_lora(
target_modules = target_modules,
modules_to_save = modules_to_save,
lora_r = lora_r,
lora_alpha = lora_alpha,
lora_dropout = lora_dropout,
use_gradient_checkpointing = use_gradient_checkpointing,
use_rslora = use_rslora,
use_loftq = use_loftq,
use_dora = use_dora,
random_state = random_state,
)
# LoRA/QLoRA. "all-linear" is a PEFT keyword targeting every linear layer.
if isinstance(target_modules, list) and "all-linear" in target_modules:
if len(target_modules) == 1:
target_modules = "all-linear"
else:
target_modules = [m for m in target_modules if m != "all-linear"]
elif target_modules is None or (
isinstance(target_modules, list) and len(target_modules) == 0
):
target_modules = [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
]
if self.model is None:
error_msg = "Model is None - model was not loaded properly"
logger.error(error_msg)
self._update_progress(error = error_msg)
return False
if not hasattr(self.model, "config"):
error_msg = (
"Model does not have config attribute - model may not be loaded correctly"
)
logger.error(error_msg)
self._update_progress(error = error_msg)
return False
logger.info(f"Configuring LoRA adapters (r={lora_r}, alpha={lora_alpha})...\n")
logger.info(
f"Gradient checkpointing: {use_gradient_checkpointing} (type: {type(use_gradient_checkpointing).__name__})\n"
)
if self._audio_type in ("csm", "bicodec", "dac") or self.is_audio_vlm:
from unsloth import FastModel
label = self._audio_type or "audio_vlm"
logger.info(f"{label} LoRA configuration:")
logger.info(f" - Target modules: {target_modules}")
# Audio VLM models support VLM-style layer selection
if self.is_audio_vlm:
logger.info(f" - Finetune vision layers: {finetune_vision_layers}")
logger.info(f" - Finetune language layers: {finetune_language_layers}")
logger.info(f" - Finetune attention modules: {finetune_attention_modules}")
logger.info(f" - Finetune MLP modules: {finetune_mlp_modules}")
peft_kwargs = dict(
r = lora_r,
target_modules = target_modules,
modules_to_save = modules_to_save,
lora_alpha = lora_alpha,
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
**self._offload_peft_kwargs(),
)
if self.is_audio_vlm:
peft_kwargs.update(
finetune_vision_layers = finetune_vision_layers,
finetune_language_layers = finetune_language_layers,
finetune_attention_modules = finetune_attention_modules,
finetune_mlp_modules = finetune_mlp_modules,
)
self.model = FastModel.get_peft_model(self.model, **peft_kwargs)
elif self._audio_type == "whisper":
# Whisper: FastModel.get_peft_model with task_type=None
from unsloth import FastModel
logger.info(f"Audio model (whisper) LoRA configuration:")
logger.info(f" - Target modules: {target_modules}\n")
self.model = FastModel.get_peft_model(
self.model,
r = lora_r,
target_modules = target_modules,
modules_to_save = modules_to_save,
lora_alpha = lora_alpha,
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
task_type = None,
)
elif self._audio_type == "snac":
logger.info(f"Audio model ({self._audio_type}) LoRA configuration:")
logger.info(f" - Target modules: {target_modules}\n")
self.model = FastLanguageModel.get_peft_model(
self.model,
r = lora_r,
target_modules = target_modules,
modules_to_save = modules_to_save,
lora_alpha = lora_alpha,
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
)
elif self.is_vlm:
logger.info(f"Vision model LoRA configuration:")
logger.info(f" - Finetune vision layers: {finetune_vision_layers}")
logger.info(f" - Finetune language layers: {finetune_language_layers}")
logger.info(f" - Finetune attention modules: {finetune_attention_modules}")
logger.info(f" - Finetune MLP modules: {finetune_mlp_modules}\n")
self.model = FastVisionModel.get_peft_model(
self.model,
finetune_vision_layers = finetune_vision_layers,
finetune_language_layers = finetune_language_layers,
finetune_attention_modules = finetune_attention_modules,
finetune_mlp_modules = finetune_mlp_modules,
r = lora_r,
target_modules = target_modules,
lora_alpha = lora_alpha,
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
modules_to_save = modules_to_save,
**self._offload_peft_kwargs(),
)
else:
logger.info(f"Text model LoRA configuration:")
logger.info(f" - Target modules: {target_modules}\n")
if modules_to_save:
logger.info(f" - Modules to save: {modules_to_save}\n")
self.model = FastLanguageModel.get_peft_model(
self.model,
r = lora_r,
target_modules = target_modules,
lora_alpha = lora_alpha,
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
modules_to_save = modules_to_save,
**self._offload_peft_kwargs(),
)
if self.should_stop:
logger.info("Stopped during LoRA configuration\n")
return False
self._update_progress(status_message = "LoRA adapters configured")
logger.info("LoRA adapters configured successfully\n")
return True
except Exception as e:
import traceback
import sys
error_details = (
f"{type(e).__name__}: {str(e)}" if str(e) else f"{type(e).__name__} (no message)"
)
full_traceback = traceback.format_exc()
logger.error(f"Error preparing model: {error_details}")
logger.error(f"Full traceback:\n{full_traceback}")
logger.info(f"\n[ERROR] Error preparing model: {error_details}")
logger.info(f"[ERROR] Full traceback:\n{full_traceback}")
self._update_progress(error = error_details)
return False
def _apply_csm_forward_fix(self):
"""The stock forward leaks Trainer kwargs into the depth decoder, causing a None loss and a
crash."""
import torch
import torch.nn as nn
from transformers.models.csm.modeling_csm import (
CsmForConditionalGeneration,
CsmOutputWithPast,
)
base_csm = self.model.base_model.model
# Original forward (@can_return_tuple wrapped version)
_original_forward = CsmForConditionalGeneration.forward
# Keys the depth decoder and its sub-layers understand
_TRANSFORMERS_KWARGS = {
"num_items_in_batch",
"output_hidden_states",
"output_attentions",
"output_router_logits",
"cu_seq_lens_q",
"cu_seq_lens_k",
"max_length_q",
"max_length_k",
}
def _fixed_csm_forward(
self,
input_ids = None,
input_values = None,
attention_mask = None,
input_values_cutoffs = None,
position_ids = None,
past_key_values = None,
inputs_embeds = None,
labels = None,
use_cache = None,
cache_position = None,
logits_to_keep = 0,
**kwargs,
):
# Strip non-standard kwargs from Unsloth/PEFT.
output_attentions = kwargs.pop("output_attentions", None)
output_hidden_states = kwargs.pop("output_hidden_states", None)
kwargs.pop("return_dict", None)
kwargs.pop("causal_mask", None)
kwargs.pop("num_logits_to_keep", None)
kwargs.pop("task_ids", None)
clean_kwargs = {k: v for k, v in kwargs.items() if k in _TRANSFORMERS_KWARGS}
if input_ids is not None and input_ids.ndim == 2:
merged = self._merge_input_ids_with_input_values(
input_ids, input_values, input_values_cutoffs, labels
)
inputs_embeds = merged["inputs_embeds"]
labels = merged["labels"]
input_ids = None
backbone_outputs = self.backbone_model(
input_ids = input_ids,
attention_mask = attention_mask,
position_ids = position_ids,
past_key_values = past_key_values,
inputs_embeds = inputs_embeds,
use_cache = use_cache,
cache_position = cache_position,
output_attentions = output_attentions,
output_hidden_states = output_hidden_states,
**clean_kwargs,
)
backbone_hidden_states = backbone_outputs[0]
slice_indices = (
slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
)
backbone_logits = self.lm_head(backbone_hidden_states[:, slice_indices, :])
loss = None
backbone_loss = None
depth_decoder_loss = None
depth_decoder_outputs = None
if labels is not None:
backbone_labels = labels[:, :, 0]
backbone_loss = self.loss_function(
logits = backbone_logits,
labels = backbone_labels,
vocab_size = self.config.vocab_size,
**clean_kwargs,
)
train_mask = ~(labels[:, :, 1:] == -100).all(dim = -1)
depth_decoder_input_ids = labels[train_mask][..., : self.config.num_codebooks - 1]
depth_decoder_input_ids = nn.functional.pad(
depth_decoder_input_ids, (1, 0), value = 0
)
train_idxs = train_mask.nonzero(as_tuple = True)
backbone_last_hidden_states = backbone_hidden_states[
train_idxs[0], train_idxs[1] - 1, :
]
depth_decoder_labels = labels[train_mask]
# Scale num_items_in_batch for the depth decoder's 31 codebooks.
dd_kwargs = clean_kwargs.copy()
if "num_items_in_batch" in dd_kwargs:
dd_kwargs["num_items_in_batch"] = dd_kwargs["num_items_in_batch"] * (
self.config.num_codebooks - 1
)
depth_decoder_outputs = self.depth_decoder(
input_ids = depth_decoder_input_ids,
backbone_last_hidden_state = backbone_last_hidden_states,
use_cache = False,
return_dict = True,
labels = depth_decoder_labels,
output_attentions = output_attentions,
output_hidden_states = output_hidden_states,
**dd_kwargs,
)
depth_decoder_loss = depth_decoder_outputs.loss
if depth_decoder_loss is None:
logger.warning(
"CSM depth_decoder_loss is None! "
f"labels shape={depth_decoder_labels.shape}, "
f"train_mask sum={train_mask.sum().item()}"
)
# Fallback: use only backbone loss to avoid crashing
loss = backbone_loss
else:
loss = backbone_loss + depth_decoder_loss
return CsmOutputWithPast(
loss = loss,
backbone_loss = backbone_loss,
depth_decoder_loss = depth_decoder_loss,
logits = backbone_logits,
past_key_values = backbone_outputs.past_key_values,
hidden_states = backbone_outputs.hidden_states,
attentions = backbone_outputs.attentions,
depth_decoder_logits = (
depth_decoder_outputs.logits if depth_decoder_outputs else None
),
depth_decoder_past_key_values = (
depth_decoder_outputs.past_key_values if depth_decoder_outputs else None
),
depth_decoder_hidden_states = (
depth_decoder_outputs.hidden_states if depth_decoder_outputs else None
),
depth_decoder_attentions = (
depth_decoder_outputs.attentions if depth_decoder_outputs else None
),
)
# Instance level: catches BaseTuner.forward -> self.model.forward().
base_csm.forward = types.MethodType(_fixed_csm_forward, base_csm)
# Class level: catches paths resolving through the class dict.
CsmForConditionalGeneration.forward = _fixed_csm_forward
logger.info("Applied CSM forward fix (class + instance level)\n")
def _preprocess_csm_dataset(
self,
dataset,
custom_format_mapping = None,
):
"""Preprocess dataset for CSM TTS training (exact notebook copy)."""
from transformers import AutoProcessor
from datasets import Audio
import torch
processor = AutoProcessor.from_pretrained(
self.model_name,
trust_remote_code = getattr(self, "trust_remote_code", False),
)
# A leaked pad_to_multiple_of is rejected by EncodecFeatureExtractor.
processor.tokenizer.init_kwargs.pop("pad_to_multiple_of", None)
resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
speaker_key = resolved["speaker_col"]
if audio_col is None:
raise ValueError(f"No audio column found in dataset. Columns: {dataset.column_names}")
if text_col is None:
raise ValueError(f"No text column found in dataset. Columns: {dataset.column_names}")
if speaker_key is None:
logger.info("No speaker found, adding default 'source' of 0 for all examples\n")
dataset = dataset.add_column("source", ["0"] * len(dataset))
speaker_key = "source"
logger.info(
f"CSM preprocessing: audio_col='{audio_col}', text_col='{text_col}', speaker_key='{speaker_key}'\n"
)
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 24000))
required_keys = [
"input_ids",
"attention_mask",
"labels",
"input_values",
"input_values_cutoffs",
]
self._update_progress(status_message = "Preprocessing CSM dataset...")
processed_examples = []
skipped = 0
for idx in range(len(dataset)):
if self.should_stop:
logger.info("Stopped during CSM preprocessing\n")
break
example = dataset[idx]
try:
conversation = [
{
"role": str(example[speaker_key]),
"content": [
{"type": "text", "text": example.get(text_col, "")},
{"type": "audio", "path": example[audio_col]["array"]},
],
}
]
# pad_to_multiple_of omitted: _merge_kwargs leaks it to EncodecFeatureExtractor.
model_inputs = processor.apply_chat_template(
conversation,
tokenize = True,
return_dict = True,
output_labels = True,
text_kwargs = {
"padding": "max_length",
"max_length": 256,
"padding_side": "right",
},
audio_kwargs = {
"sampling_rate": 24_000,
"max_length": 240001,
"padding": "max_length",
},
common_kwargs = {"return_tensors": "pt"},
)
out = {}
for k in required_keys:
if k not in model_inputs:
raise KeyError(f"Missing required key '{k}' in model outputs")
out[k] = model_inputs[k][0]
if not all(isinstance(out[k], torch.Tensor) for k in out):
skipped += 1
continue
processed_examples.append(out)
except Exception as e:
logger.warning(f"Error processing CSM example {idx}: {e}")
skipped += 1
continue
if (idx + 1) % 100 == 0:
self._update_progress(
status_message = f"Preprocessing CSM... {idx + 1}/{len(dataset)}"
)
if not processed_examples:
raise ValueError(f"No valid examples after CSM preprocessing (skipped {skipped})")
result_dataset = Dataset.from_list(processed_examples)
logger.info(
f"CSM preprocessing complete: {len(result_dataset)} examples ({skipped} skipped)\n"
)
return result_dataset
_resolve_audio_vlm_image_column = staticmethod(resolve_audio_vlm_image_column)
def _format_audio_vlm_dataset(
self,
dataset,
custom_format_mapping = None,
):
from datasets import Audio
resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
if not audio_col or not text_col:
raise ValueError(
f"Audio VLM dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
self._audio_vlm_audio_col = audio_col
image_col = self._resolve_audio_vlm_image_column(
dataset, custom_format_mapping, audio_col, text_col, resolved["speaker_col"]
)
self._audio_vlm_image_col = image_col
total = len(dataset)
dataset = dataset.filter(
lambda text: bool(cell_text(text).strip()), input_columns = [text_col]
)
if not len(dataset):
raise ValueError(f"Every transcript in the '{text_col}' column is blank.")
if len(dataset) < total:
self._record_warning(
f"Skipped {total - len(dataset):,} of {total:,} audio rows with a blank transcript."
)
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 16000))
if image_col:
dataset = prepare_audio_vlm_image_column(dataset, image_col)
logger.info(f"Audio VLM dataset also has images in '{image_col}'; training on both\n")
def format_messages(samples):
formatted = {"messages": []}
for idx in range(len(samples[audio_col])):
row_images = samples[image_col][idx] if image_col else None
formatted["messages"].append(
audio_transcription_messages(
samples[audio_col][idx]["array"],
cell_text(samples[text_col][idx]),
image_slot_count(row_images),
)
)
return formatted
self._update_progress(status_message = "Formatting audio VLM dataset...")
dataset = dataset.map(
format_messages,
batched = True,
batch_size = 4,
num_proc = dataset_map_num_proc(4),
)
logger.info(f"Audio VLM dataset formatted: {len(dataset)} examples\n")
return dataset
def _preprocess_snac_dataset(
self,
dataset,
custom_format_mapping = None,
):
"""Interleaves 7 SNAC codes per frame and trains on the full sequence, with no label masking."""
import torch
import torchaudio.transforms as T
SNAC_MODEL_NAME = "hubertsiuzdak/snac_24khz"
SNAC_SAMPLE_RATE = 24000
# SNAC unvalidated on Intel XPU; keep the pre-PR CPU fallback for non-CUDA hosts.
device = "cuda" if torch.cuda.is_available() else "cpu"
max_length = self.max_seq_length or 2048
tokenizer = self.tokenizer
START_OF_HUMAN = 128259
END_OF_HUMAN = 128260
START_OF_AI = 128261
END_OF_AI = 128262
START_OF_SPEECH = 128257
END_OF_SPEECH = 128258
END_OF_TEXT = 128009
AUDIO_OFFSET = 128266
resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
speaker_col = resolved["speaker_col"]
has_source = speaker_col is not None
if not audio_col or not text_col:
raise ValueError(
f"SNAC dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
# Cast audio so datasets 4.x AudioDecoder objects decode to dicts
from datasets import Audio
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = SNAC_SAMPLE_RATE))
# Sample rate from first example (after cast, always SNAC_SAMPLE_RATE)
first_audio = dataset[0][audio_col]
ds_sample_rate = (
first_audio.get("sampling_rate", SNAC_SAMPLE_RATE)
if isinstance(first_audio, dict)
else SNAC_SAMPLE_RATE
)
self._update_progress(status_message = "Loading SNAC codec model...")
logger.info("Loading SNAC codec model...\n")
from snac import SNAC
snac_model = SNAC.from_pretrained(SNAC_MODEL_NAME)
snac_model = snac_model.to(device).eval()
resample_transform = (
T.Resample(orig_freq = ds_sample_rate, new_freq = SNAC_SAMPLE_RATE)
if ds_sample_rate != SNAC_SAMPLE_RATE
else None
)
self._update_progress(status_message = "Encoding audio with SNAC...")
logger.info(
f"SNAC preprocessing: audio_col='{audio_col}', text_col='{text_col}', "
f"has_source={has_source}, ds_sample_rate={ds_sample_rate}\n"
)
processed_examples = []
skipped = 0
for idx in range(len(dataset)):
if self.should_stop:
logger.info("Stopped during SNAC preprocessing\n")
break
example = dataset[idx]
try:
text = example.get(text_col)
if not text:
skipped += 1
continue
audio_data = example.get(audio_col)
if audio_data is None or audio_data.get("array") is None:
skipped += 1
continue
waveform = (
torch.from_numpy(audio_data["array"]).unsqueeze(0).to(dtype = torch.float32)
)
if resample_transform is not None:
waveform = resample_transform(waveform)
waveform = waveform.unsqueeze(0).to(device)
with torch.inference_mode():
codes = snac_model.encode(waveform)
all_codes = []
# One host copy per level: .item() is a device sync per scalar on CUDA.
c0, c1, c2 = (level[0].tolist() for level in codes[:3])
for i in range(codes[0].shape[1]):
all_codes.append(c0[i] + AUDIO_OFFSET)
all_codes.append(c1[2 * i] + AUDIO_OFFSET + 4096)
all_codes.append(c2[4 * i] + AUDIO_OFFSET + (2 * 4096))
all_codes.append(c2[(4 * i) + 1] + AUDIO_OFFSET + (3 * 4096))
all_codes.append(c1[(2 * i) + 1] + AUDIO_OFFSET + (4 * 4096))
all_codes.append(c2[(4 * i) + 2] + AUDIO_OFFSET + (5 * 4096))
all_codes.append(c2[(4 * i) + 3] + AUDIO_OFFSET + (6 * 4096))
if len(all_codes) == 0:
skipped += 1
continue
deduped = all_codes[:7]
for i in range(7, len(all_codes), 7):
if all_codes[i] != deduped[-7]:
deduped.extend(all_codes[i : i + 7])
all_codes = deduped
text_prompt = (
f"{example[speaker_col]}: {text}"
if has_source and example.get(speaker_col)
else text
)
text_ids = tokenizer.encode(text_prompt, add_special_tokens = True)
text_ids.append(END_OF_TEXT)
input_ids = (
[START_OF_HUMAN]
+ text_ids
+ [END_OF_HUMAN]
+ [START_OF_AI]
+ [START_OF_SPEECH]
+ all_codes
+ [END_OF_SPEECH]
+ [END_OF_AI]
)
input_ids = input_ids[:max_length]
# Labels = input_ids (no masking; Orpheus trains full sequence)
labels = list(input_ids)
attention_mask = [1] * len(input_ids)
processed_examples.append(
{
"input_ids": input_ids,
"labels": labels,
"attention_mask": attention_mask,
}
)
except Exception as e:
logger.warning(f"Error processing SNAC example {idx}: {e}")
skipped += 1
continue
if (idx + 1) % 100 == 0:
self._update_progress(status_message = f"Encoding audio... {idx + 1}/{len(dataset)}")
logger.info("Freeing SNAC codec model from GPU...\n")
snac_model.to("cpu")
del snac_model
gc.collect()
clear_gpu_cache()
self._cuda_audio_used = True
if not processed_examples:
raise ValueError(f"No valid examples after SNAC preprocessing (skipped {skipped})")
result_dataset = Dataset.from_list(processed_examples)
logger.info(
f"SNAC preprocessing complete: {len(result_dataset)} examples ({skipped} skipped)\n"
)
return result_dataset
def _preprocess_bicodec_dataset(
self,
dataset,
custom_format_mapping = None,
):
"""Preprocess dataset for Spark-TTS training with BiCodec tokenizer. Mirrors
Spark_TTS_(0_5B).ipynb: encode audio with BiCodec (semantic + global tokens), format as
special-token text strings for SFTTrainer with dataset_text_field="text"."""
import torch
import numpy as np
import torchaudio.transforms as T
# Spark-TTS BiCodec unvalidated on Intel XPU; keep the pre-PR CPU fallback.
device = "cuda" if torch.cuda.is_available() else "cpu"
self._update_progress(status_message = "Preparing Spark-TTS codec source...")
self.resolve_spark_tts_repo()
spark_code_dir = ensure_spark_tts_source(self._spark_tts_repo_dir)
self._spark_tts_code_dir = spark_code_dir
BiCodecTokenizer = import_sparktts_module(
"sparktts.models.audio_tokenizer",
spark_code_dir,
).BiCodecTokenizer
audio_volume_normalize = import_sparktts_module(
"sparktts.utils.audio",
spark_code_dir,
).audio_volume_normalize
resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
speaker_col = resolved["speaker_col"]
has_source = speaker_col is not None
if not audio_col or not text_col:
raise ValueError(
f"BiCodec dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
# Cast so datasets 4.x AudioDecoder yields dicts; skip resample since BiCodec target_sr may differ.
from datasets import Audio
dataset = dataset.cast_column(audio_col, Audio())
self._update_progress(status_message = "Loading BiCodec tokenizer...")
logger.info("Loading BiCodec tokenizer...\n")
audio_tokenizer = BiCodecTokenizer(self._spark_tts_repo_dir, device)
target_sr = audio_tokenizer.config["sample_rate"]
self._update_progress(status_message = "Encoding audio with BiCodec...")
logger.info(
f"BiCodec preprocessing: audio_col='{audio_col}', text_col='{text_col}', "
f"has_source={has_source}, target_sr={target_sr}\n"
)
def extract_wav2vec2_features(wavs: torch.Tensor) -> torch.Tensor:
"""Extract wav2vec2 features (average of layers 11, 14, 16)."""
if wavs.shape[0] != 1:
raise ValueError(f"Expected batch size 1, but got shape {wavs.shape}")
wav_np = wavs.squeeze(0).cpu().numpy()
processed = audio_tokenizer.processor(
wav_np,
sampling_rate = 16000,
return_tensors = "pt",
padding = True,
)
input_values = processed.input_values.to(audio_tokenizer.feature_extractor.device)
model_output = audio_tokenizer.feature_extractor(input_values)
if model_output.hidden_states is None:
raise ValueError("Wav2Vec2Model did not return hidden states.")
feats_mix = (
model_output.hidden_states[11]
+ model_output.hidden_states[14]
+ model_output.hidden_states[16]
) / 3
return feats_mix
processed_examples = []
skipped = 0
for idx in range(len(dataset)):
if self.should_stop:
logger.info("Stopped during BiCodec preprocessing\n")
break
example = dataset[idx]
try:
text = example.get(text_col)
if not text:
skipped += 1
continue
audio_data = example.get(audio_col)
if audio_data is None or audio_data.get("array") is None:
skipped += 1
continue
audio_array = audio_data["array"]
sampling_rate = audio_data.get("sampling_rate", target_sr)
if sampling_rate != target_sr:
resampler = T.Resample(orig_freq = sampling_rate, new_freq = target_sr)
audio_tensor_temp = torch.from_numpy(audio_array).float()
audio_array = resampler(audio_tensor_temp).numpy()
if audio_tokenizer.config.get("volume_normalize", False):
audio_array = audio_volume_normalize(audio_array)
ref_wav_np = audio_tokenizer.get_ref_clip(audio_array)
audio_tensor = torch.from_numpy(audio_array).unsqueeze(0).float().to(device)
ref_wav_tensor = torch.from_numpy(ref_wav_np).unsqueeze(0).float().to(device)
feat = extract_wav2vec2_features(audio_tensor)
batch = {
"wav": audio_tensor,
"ref_wav": ref_wav_tensor,
"feat": feat.to(device),
}
semantic_token_ids, global_token_ids = audio_tokenizer.model.tokenize(batch)
global_tokens = "".join(
[f"<|bicodec_global_{i}|>" for i in global_token_ids.squeeze().cpu().numpy()]
)
semantic_tokens = "".join(
[
f"<|bicodec_semantic_{i}|>"
for i in semantic_token_ids.squeeze().cpu().numpy()
]
)
text_content = (
f"{example[speaker_col]}: {text}"
if has_source and example.get(speaker_col)
else text
)
formatted = "".join(
[
"<|task_tts|>",
"<|start_content|>",
text_content,
"<|end_content|>",
"<|start_global_token|>",
global_tokens,
"<|end_global_token|>",
"<|start_semantic_token|>",
semantic_tokens,
"<|end_semantic_token|>",
"<|im_end|>",
]
)
processed_examples.append({"text": formatted})
except Exception as e:
logger.warning(f"Error processing BiCodec example {idx}: {e}")
skipped += 1
continue
if (idx + 1) % 100 == 0:
self._update_progress(
status_message = f"Encoding audio with BiCodec... {idx + 1}/{len(dataset)}"
)
logger.info("Freeing BiCodec tokenizer from GPU...\n")
audio_tokenizer.model.cpu()
audio_tokenizer.feature_extractor.cpu()
del audio_tokenizer
gc.collect()
clear_gpu_cache()
self._cuda_audio_used = True
if not processed_examples:
raise ValueError(f"No valid examples after BiCodec preprocessing (skipped {skipped})")
result_dataset = Dataset.from_list(processed_examples)
logger.info(
f"BiCodec preprocessing complete: {len(result_dataset)} examples ({skipped} skipped)\n"
)
sample = result_dataset[0]["text"]
logger.info(f"Sample text (first 200 chars): {sample[:200]}...\n")
logger.info(f"Sample text length: {len(sample)} chars\n")
return result_dataset
def _preprocess_dac_dataset(
self,
dataset,
custom_format_mapping = None,
):
"""Outputs text strings consumed by SFTTrainer via dataset_text_field='text'."""
import io
import tempfile
import torch
import numpy as np
import soundfile as sf
from datasets import Dataset as HFDataset
from utils.paths import ensure_dir, tmp_root
# OuteTTS DAC/Whisper preprocess unvalidated on Intel XPU; CPU fallback kept.
device = "cuda" if torch.cuda.is_available() else "cpu"
self._update_progress(status_message = "Preparing OuteTTS codec source...")
outetts_code_dir = ensure_outetts_source()
self._outetts_code_dir = outetts_code_dir
AudioProcessor = import_outetts_module(
"outetts.version.v3.audio_processor",
outetts_code_dir,
).AudioProcessor
PromptProcessor = import_outetts_module(
"outetts.version.v3.prompt_processor",
outetts_code_dir,
).PromptProcessor
OuteTTSModelConfig = import_outetts_module(
"outetts.models.config",
outetts_code_dir,
).ModelConfig
text_normalizations = import_outetts_module(
"outetts.utils.preprocessing",
outetts_code_dir,
).text_normalizations
resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
if not audio_col or not text_col:
raise ValueError(
f"DAC dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
from datasets import Audio
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 24000))
logger.info("Cast audio column to 24kHz\n")
self._update_progress(status_message = "Loading Whisper model for word timings...")
logger.info("Loading Whisper model for word timings...\n")
import whisper
whisper_model = whisper.load_model("turbo", device = device)
self._update_progress(status_message = "Loading OuteTTS AudioProcessor...")
logger.info("Loading OuteTTS AudioProcessor...\n")
audio_codec_path = ensure_dac_speech_weights()
dummy_config = OuteTTSModelConfig(
tokenizer_path = None,
device = device,
audio_codec_path = str(audio_codec_path),
)
audio_processor = AudioProcessor(config = dummy_config)
if self.tokenizer is None:
raise RuntimeError("OuteTTS tokenizer is not loaded")
prompt_processor = PromptProcessor(None)
prompt_processor.tokenizer = self.tokenizer
prompt_processor.get_audio_token_map()
self._update_progress(status_message = "Preprocessing audio with OuteTTS...")
logger.info(f"DAC preprocessing: audio_col='{audio_col}', text_col='{text_col}'\n")
processed_examples = []
skipped = 0
for idx in range(len(dataset)):
if self.should_stop:
logger.info("Stopped during DAC preprocessing\n")
break
example = dataset[idx]
try:
text = example.get(text_col)
if not text or not isinstance(text, str):
skipped += 1
continue
audio_data = example.get(audio_col)
if audio_data is None or audio_data.get("array") is None:
skipped += 1
continue
audio_array = np.array(audio_data["array"], dtype = np.float32)
sampling_rate = audio_data.get("sampling_rate", 24000)
buf = io.BytesIO()
sf.write(buf, audio_array, sampling_rate, format = "WAV", subtype = "FLOAT")
buf.seek(0)
audio_bytes = buf.getvalue()
with tempfile.NamedTemporaryFile(
suffix = ".wav",
delete = False,
dir = str(ensure_dir(tmp_root())),
) as tmp:
tmp.write(audio_bytes)
tmp.flush()
tmp_path = tmp.name
try:
whisper_result = whisper_model.transcribe(tmp_path, word_timestamps = True)
finally:
Path(tmp_path).unlink(missing_ok = True)
normalized_transcript = text_normalizations(text)
words_with_timings = []
if whisper_result and "segments" in whisper_result:
for segment in whisper_result["segments"]:
for word_info in segment.get("words", []):
cleaned = word_info["word"].strip()
if cleaned:
words_with_timings.append(
{
"word": cleaned,
"start": float(word_info["start"]),
"end": float(word_info["end"]),
}
)
if not words_with_timings:
skipped += 1
continue
speaker_data_dict = {
"audio": {"bytes": audio_bytes},
"text": normalized_transcript,
"words": words_with_timings,
}
speaker = audio_processor.create_speaker_from_dict(speaker_data_dict)
if speaker is None:
skipped += 1
continue
prompt = prompt_processor.get_training_prompt(speaker)
if prompt:
processed_examples.append({"text": prompt})
except Exception as e:
logger.warning(f"Error processing DAC example {idx}: {e}")
skipped += 1
continue
if (idx + 1) % 100 == 0:
self._update_progress(
status_message = f"Preprocessing audio with OuteTTS... {idx + 1}/{len(dataset)}"
)
logger.info("Moving Whisper model to CPU...\n")
whisper_model.to("cpu")
del whisper_model
del audio_processor
del prompt_processor
gc.collect()
clear_gpu_cache()
self._cuda_audio_used = True
if not processed_examples:
raise ValueError(f"No valid examples after DAC preprocessing (skipped {skipped})")
result_dataset = HFDataset.from_list(processed_examples)
logger.info(
f"DAC preprocessing complete: {len(result_dataset)} examples ({skipped} skipped)\n"
)
sample = result_dataset[0]["text"]
logger.info(f"Sample text (first 200 chars): {sample[:200]}...\n")
return result_dataset
def _preprocess_audio_eval_split(self, eval_dataset, preprocess, custom_format_mapping):
if eval_dataset is None:
return None
if self.should_stop:
# A stopped train pass still returns partial rows, so eval would reload a codec model.
logger.info("Stopped before eval preprocessing\n")
return None
try:
return preprocess(eval_dataset, custom_format_mapping)
except Exception as e:
if self.should_stop:
# A stop reads as "no valid examples": the cancel, not a bad eval file.
logger.info("Stopped during eval preprocessing\n")
return None
self._record_warning(
"The eval dataset could not be prepared for this audio model, so this run has "
f"no evaluation: {e}"
)
return None
def _format_audio_vlm_eval_split(self, eval_dataset, custom_format_mapping):
if eval_dataset is None:
return None
train_audio_col = getattr(self, "_audio_vlm_audio_col", None)
train_image_col = getattr(self, "_audio_vlm_image_col", None)
eval_mapping = custom_format_mapping
image_feature = eval_dataset.features.get(train_image_col)
while getattr(image_feature, "feature", None) is not None:
image_feature = image_feature.feature
if train_image_col and getattr(image_feature, "dtype", None) == "null":
eval_mapping = {train_image_col: "image", **(custom_format_mapping or {})}
formatted = self._preprocess_audio_eval_split(
eval_dataset, self._format_audio_vlm_dataset, eval_mapping
)
eval_audio_col = getattr(self, "_audio_vlm_audio_col", None)
eval_image_col = getattr(self, "_audio_vlm_image_col", None)
if formatted is not None and eval_audio_col != train_audio_col:
self._record_warning(
f"The eval dataset stores audio in '{eval_audio_col}' but the training dataset "
f"uses '{train_audio_col}', so this run has no evaluation. Give both splits the "
"same audio column name."
)
formatted = None
elif formatted is not None and eval_image_col != train_image_col:
self._record_warning(
f"The eval dataset's image column ('{eval_image_col}') does not match the training "
f"dataset's ('{train_image_col}'), so this run has no evaluation. Give both splits "
"the same image column."
)
formatted = None
self._audio_vlm_audio_col = train_audio_col
self._audio_vlm_image_col = train_image_col
return formatted
def _audio_eval_config(self, training_args):
eval_dataset = training_args.get("eval_dataset", None)
eval_steps = training_args.get("eval_steps", 0.00)
if eval_dataset is None:
return {}, None
# evaluation_enabled also rejects bools, inf and NaN, unlike `eval_steps <= 0`.
if not evaluation_enabled(eval_steps):
logger.info(f"⚠️ Eval dataset provided but eval_steps={eval_steps} (disabled)\n")
return {}, None
rows = len(eval_dataset) if hasattr(eval_dataset, "__len__") else "?"
if rows == 0:
# An empty dataloader yields no eval_loss, so the run would report none.
self._record_warning(
"The eval dataset is empty after preprocessing, so this run has no evaluation."
)
return {}, None
logger.info(f"✅ Evaluation enabled: eval_steps={eval_steps}, eval rows={rows}\n")
return {
"eval_strategy": "steps",
# float(): a numeric string passes the gate but TrainingArguments needs a number.
"eval_steps": float(eval_steps),
# Avoid HF's default of 8, which can OOM audio runs.
"per_device_eval_batch_size": training_args.get("batch_size") or 2,
}, eval_dataset
def _preprocess_whisper_dataset(
self,
dataset,
eval_split = None,
custom_format_mapping = None,
eval_dataset = None,
):
"""The 6% eval split is a fallback used only when eval_dataset is absent."""
from datasets import Audio
WHISPER_SAMPLE_RATE = 16000
resolved = resolve_audio_columns(dataset, custom_format_mapping)
audio_col = resolved["audio_col"]
text_col = resolved["text_col"]
if not audio_col or not text_col:
raise ValueError(
f"Whisper dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = WHISPER_SAMPLE_RATE))
eval_dataset_raw = None
if eval_dataset is not None:
# cast_column silently adds an all-null column for a missing Audio name.
eval_columns = list(getattr(eval_dataset, "column_names", None) or [])
missing = [c for c in (audio_col, text_col) if c not in eval_columns]
if missing:
self._record_warning(
f"The eval dataset has no {' or '.join(missing)} column, so this run has no "
f"evaluation. Its columns are: {eval_columns}"
)
else:
try:
eval_dataset_raw = eval_dataset.cast_column(
audio_col, Audio(sampling_rate = WHISPER_SAMPLE_RATE)
)
logger.info(
"Whisper eval: using the separate eval split "
f"({len(eval_dataset_raw)} rows)\n"
)
except Exception as e:
self._record_warning(
"The eval dataset could not be prepared for this audio model, so this "
f"run has no evaluation: {e}"
)
eval_dataset_raw = None
elif eval_split:
splits = dataset.train_test_split(test_size = 0.06, seed = 42)
dataset = splits["train"]
eval_dataset_raw = splits["test"]
self._update_progress(status_message = "Processing audio for Whisper...")
logger.info(
f"Whisper preprocessing: audio_col='{audio_col}', text_col='{text_col}', "
f"samples={len(dataset)}\n"
)
def process_split(ds, split_name = "train"):
processed = []
skipped = 0
for idx in range(len(ds)):
if self.should_stop:
logger.info(f"Stopped during Whisper {split_name} preprocessing\n")
break
example = ds[idx]
try:
audio_data = example.get(audio_col)
text = example.get(text_col)
if audio_data is None or audio_data.get("array") is None or not text:
skipped += 1
continue
features = self.tokenizer.feature_extractor(
audio_data["array"], sampling_rate = audio_data["sampling_rate"]
)
tokenized_text = self.tokenizer.tokenizer(text)
processed.append(
{
"input_features": features.input_features[0],
"labels": tokenized_text.input_ids,
}
)
except Exception as e:
logger.warning(f"Error processing Whisper {split_name} example {idx}: {e}")
skipped += 1
continue
if (idx + 1) % 100 == 0:
self._update_progress(
status_message = f"Processing {split_name} audio... {idx + 1}/{len(ds)}"
)
logger.info(
f"Whisper {split_name} preprocessing: {len(processed)} examples ({skipped} skipped)\n"
)
return processed
train_data = process_split(dataset, "train")
eval_data = process_split(eval_dataset_raw, "eval") if eval_dataset_raw else None
if not train_data:
raise ValueError("No valid examples after Whisper preprocessing")
if eval_dataset_raw and not eval_data and not self.should_stop:
# Every row was skipped; the trainer branch would silently read [] as "no eval".
self._record_warning(
"No usable rows were left in the eval dataset after preprocessing, so this run "
"has no evaluation."
)
return (train_data, eval_data)
@staticmethod
def _resolve_local_path(dataset_file: str) -> Path:
dataset_file = os.path.expanduser(dataset_file)
if os.path.isabs(dataset_file):
return Path(dataset_file)
if os.path.exists(dataset_file):
return Path(os.path.abspath(dataset_file))
return Path(str(resolve_dataset_path(dataset_file)))
@staticmethod
def _load_saved_dataset(file_paths: list):
if len(file_paths) != 1:
return None
from utils.datasets.saved import load_saved_dataset
return load_saved_dataset(UnslothTrainer._resolve_local_path(file_paths[0]))
@staticmethod
def _resolve_local_files(file_paths: list) -> list[str]:
"""Resolve a list of local dataset paths to concrete file paths."""
all_files: list[str] = []
for dataset_file in file_paths:
file_path_obj = UnslothTrainer._resolve_local_path(dataset_file)
if file_path_obj.is_dir():
all_files.extend(str(p) for p in dataset_files_in_dir(file_path_obj))
else:
all_files.append(str(file_path_obj))
return all_files
@staticmethod
def _loader_for_files(files: list[str]) -> str:
"""Determine the HF datasets loader type from file extensions."""
first_ext = Path(files[0]).suffix.lower()
if first_ext in (".json", ".jsonl"):
return "json"
elif first_ext == ".csv":
return "csv"
elif first_ext == ".parquet":
return "parquet"
raise ValueError(f"Unsupported dataset format: {files[0]}")
def load_and_format_dataset(
self,
dataset_source: Optional[str],
format_type: str = "auto",
local_datasets: Optional[List[str]] = None,
local_eval_datasets: Optional[List[str]] = None,
custom_format_mapping: Optional[Dict[str, Any]] = None,
subset: Optional[str] = None,
train_split: str = "train",
eval_split: Optional[str] = None,
dataset_streaming: bool = False,
eval_steps: float = 0.00,
dataset_slice_start: Optional[int] = None,
dataset_slice_end: Optional[int] = None,
is_cpt: bool = False,
s3_config: dict = None,
dataset_local_files_only: bool = False,
dataset_local_path: Optional[str] = None,
dataset_revision: Optional[str] = None,
require_exact_resume_resources: bool = False,
hf_token: Optional[str] = None,
max_train_rows: Optional[int] = None,
max_train_rows_seed: int = 3407,
) -> Optional[tuple]:
"""hf_token must reach every load_dataset call; otherwise gated datasets read the ambient
HF_TOKEN."""
from core.training.s3_dataset import S3DownloadCancelled
# datasets has no env var; silence its bars via the class before any load.
try:
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
except Exception: # noqa: BLE001 - never let log tidying stop a run
pass
# This worker lacks the API process's torchcodec shim; install it before Audio decode.
try:
ensure_audio_decoding()
except Exception: # noqa: BLE001 - never let a broken librosa stop a text run
pass
# Every map below and TRL's / Unsloth's tokenization hash the tokenizer to name cache files.
install_tokenizer_fingerprint()
s3_download = None
try:
self.dataset_loaded_from_exact_snapshot = False
self.dataset_snapshot_path = None
self._kept_row_fraction = 1.0
dataset = None
eval_dataset = None
dataset_attestation_source = None
has_separate_eval_source = False
# Not `eval_steps > 0`: inf and NaN pass that and codec-encode a split later discarded.
eval_enabled = evaluation_enabled(eval_steps)
raw_text_mode = is_cpt or format_type == "raw"
dataset_loaded_from_cache = False
def _load_selected_cached_dataset(
split_name: Optional[str], *, row_limit: Optional[int] = None
):
if not dataset_source or not dataset_local_path:
return None
from hub.utils.dataset_cache import load_cached_hf_dataset
kwargs = {
"subset": subset,
"split": split_name or "train",
}
if row_limit is not None:
kwargs["row_limit"] = row_limit
return load_cached_hf_dataset(dataset_source, dataset_local_path, **kwargs)
def _raw_mode_label() -> str:
return "CPT" if is_cpt else "raw text"
raw_text_column = {}
def _apply_raw_text_prep(ds: Dataset, split_name: str) -> Dataset:
try:
result = prepare_raw_text_dataset(
ds,
mode_label = _raw_mode_label(),
split_name = split_name,
eos_token = getattr(self.tokenizer, "eos_token", None),
append_eos = True,
text_column = raw_text_column.get("train"),
)
except ValueError as exc:
error_msg = str(exc)
logger.error(error_msg)
self._update_progress(error = error_msg)
raise
for notice in result.notices:
if notice.level == "warning":
if notice.update_status:
self._record_warning(notice.message)
else:
logger.warning(notice.message)
else:
logger.info(f"{notice.message}\n")
raw_text_column.setdefault(split_name, result.source_column)
return result.dataset
# S3 datasets download to a local temp dir, then use the local-file path below.
if s3_config and not local_datasets:
from core.training.s3_dataset import prepare_s3_dataset_download
self._update_progress(status_message = "Downloading dataset from S3...")
s3_download = prepare_s3_dataset_download(
s3_config,
cancel_callback = lambda: self.should_stop,
)
local_datasets = s3_download.files
if self.should_stop:
logger.info("Stopped during S3 download\n")
return None
logger.info(f"Downloaded {len(local_datasets)} file(s) from S3\n")
if local_datasets:
# load_dataset() is Arrow-backed; an in-memory Dataset.from_list() has no cache and forces num_proc=1
# during tokenization.
saved = self._load_saved_dataset(local_datasets)
all_files = [] if saved is not None else self._resolve_local_files(local_datasets)
if saved is not None:
dataset = saved
logger.info(f"Loaded {len(dataset)} samples from a saved dataset folder\n")
if all_files:
loader = self._loader_for_files(all_files)
dataset = load_dataset(
loader,
data_files = all_files,
split = "train",
**csv_as_text_kwargs(all_files),
)
if self.should_stop:
logger.info("Stopped during dataset loading\n")
return None
self._update_progress(
status_message = f"Loaded {len(dataset)} samples from local files"
)
logger.info(f"Loaded {len(dataset)} samples from local files\n")
logger.info(f"[DEBUG] Dataset cache_files: {dataset.cache_files}\n")
if local_eval_datasets and eval_enabled:
eval_saved = self._load_saved_dataset(local_eval_datasets)
eval_all_files = (
[]
if eval_saved is not None
else self._resolve_local_files(local_eval_datasets)
)
if eval_saved is not None:
eval_dataset = eval_saved
has_separate_eval_source = True
logger.info(
f"Loaded {len(eval_dataset)} eval samples from a saved dataset folder\n"
)
if eval_all_files:
eval_loader = self._loader_for_files(eval_all_files)
eval_dataset = load_dataset(
eval_loader,
data_files = eval_all_files,
split = "train",
**csv_as_text_kwargs(eval_all_files),
)
has_separate_eval_source = True
logger.info(
f"Loaded {len(eval_dataset)} eval samples from local eval files\n"
)
elif dataset_source:
split_name = train_split or "train"
load_kwargs = {"path": dataset_source, "split": split_name}
if subset:
load_kwargs["name"] = subset
if dataset_revision:
load_kwargs["revision"] = dataset_revision
if hf_token:
load_kwargs["token"] = hf_token
if dataset_streaming:
self._update_progress(status_message = f"Streaming dataset: {dataset_source}...")
dataset = load_dataset(**load_kwargs, streaming = True)
if dataset_slice_start is not None and dataset_slice_start > 0:
dataset = dataset.skip(dataset_slice_start)
if dataset_slice_end is not None:
slice_start = dataset_slice_start or 0
take_count = dataset_slice_end - slice_start + 1
if take_count <= 0:
raise ValueError(
"Train Split End must be greater than or equal to Train Split Start."
)
dataset = dataset.take(take_count)
# take(N) yields at most N -- a shorter source silently gives fewer rows.
logger.warning(
f"Streaming slice requested up to {take_count} rows "
f"[{slice_start}, {dataset_slice_end}]; actual yield "
f"may be smaller if the dataset has fewer rows."
)
if take_count == 1:
# start == end is valid but yields a single training row, almost always user error.
logger.warning(
"Dataset slice resolves to a single row "
f"(start == end == {slice_start}); training on 1 "
"sample is likely unintended."
)
logger.info(
f"Loaded Hugging Face dataset in streaming mode: {dataset_source}\n"
)
self._update_progress(status_message = f"Streaming {dataset_source}")
else:
# With a slice end, stream just the needed rows and materialize them, not download everything.
_slice_start = dataset_slice_start or 0
# streaming=True rejects slice syntax, so fall back to a normal download.
_split_has_slice = (train_split or "").find("[") != -1
dataset = None
if dataset_local_files_only:
self._update_progress(
status_message = f"Loading cached dataset: {dataset_source}..."
)
try:
cached_row_limit = None
if (
not _split_has_slice
and dataset_slice_end is not None
and dataset_slice_end >= 0
and dataset_slice_end >= _slice_start
):
cached_row_limit = dataset_slice_end + 1
dataset = _load_selected_cached_dataset(
split_name,
row_limit = cached_row_limit,
)
if require_exact_resume_resources and dataset is None:
raise FileNotFoundError(
f"The exact cached dataset split '{split_name}' "
"is no longer available."
)
dataset_loaded_from_cache = True
from core.training.provenance import (
exact_dataset_snapshot_path,
)
self.dataset_snapshot_path = exact_dataset_snapshot_path(
dataset_local_path,
dataset_source,
)
self.dataset_loaded_from_exact_snapshot = bool(
self.dataset_snapshot_path
)
logger.info(
f"Loaded cached dataset for {dataset_source}: {len(dataset)} rows\n"
)
except Exception as error:
from hub.utils.dataset_cache import (
dataset_cache_fallback_allowed,
)
if not dataset_cache_fallback_allowed(
error,
require_exact = require_exact_resume_resources,
revision = dataset_revision,
):
raise
self._update_progress(
status_message = (
f"Cached dataset unavailable; downloading {dataset_source}..."
)
)
dataset = None
if (
dataset is None
and not _split_has_slice
and dataset_slice_end is not None
and dataset_slice_end >= 0
and dataset_slice_end >= _slice_start
):
rows_to_stream = dataset_slice_end + 1
logger.info(
f"[dataset-slice] Manual slice specified "
f"(start={dataset_slice_start}, end={dataset_slice_end}), "
f"streaming {rows_to_stream} rows\n"
)
stream = load_dataset(**load_kwargs, streaming = True)
dataset_attestation_source = stream
dataset = Dataset.from_list(take_rows(stream, rows_to_stream))
logger.info(
f"[dataset-slice] Downloaded {len(dataset)} rows "
f"(requested {rows_to_stream})\n"
)
elif dataset is None:
self._update_progress(
status_message = f"Downloading dataset: {dataset_source}..."
)
dataset = load_dataset(**load_kwargs)
n_rows = len(dataset) if hasattr(dataset, "__len__") else 0
self._update_progress(
status_message = f"Downloaded {dataset_source} ({n_rows:,} rows)"
)
logger.info(
f"Loaded dataset from Hugging Face: {dataset_source} ({n_rows:,} rows)\n"
)
if self.should_stop:
logger.info("Stopped during dataset loading\n")
return None
if eval_enabled:
effective_train = train_split or "train"
if eval_split and eval_split != effective_train:
logger.info(f"Loading explicit eval split: '{eval_split}'\n")
eval_load_kwargs = {"path": dataset_source, "split": eval_split}
if subset:
eval_load_kwargs["name"] = subset
if dataset_revision:
eval_load_kwargs["revision"] = dataset_revision
if hf_token:
eval_load_kwargs["token"] = hf_token
if dataset_streaming:
# Streaming does not validate the split name; check it now.
from datasets import get_dataset_split_names
probe_kwargs = {"path": dataset_source}
if subset:
probe_kwargs["config_name"] = subset
if dataset_revision:
probe_kwargs["revision"] = dataset_revision
if hf_token:
probe_kwargs["token"] = hf_token
try:
available_splits = get_dataset_split_names(**probe_kwargs)
except Exception as probe_err:
raise ValueError(
f"Could not list splits for '{dataset_source}' "
f"to validate eval_split='{eval_split}': {probe_err}"
)
# Streaming rejects HF slice syntax and the request validator blocks bracketed streaming
# splits, so eval_split is always a bare split name here.
if eval_split not in available_splits:
raise ValueError(
f"Requested eval split '{eval_split}' not found in "
f"dataset '{dataset_source}'. Available splits: "
f"{available_splits}"
)
eval_dataset = load_dataset(**eval_load_kwargs, streaming = True)
# Streaming eval has no __len__; .take() stays lazy and survives the later format and
# raw-text map() passes.
if not hasattr(eval_dataset, "__len__"):
eval_dataset = eval_dataset.take(STREAMING_EVAL_MAX_SAMPLES)
logger.info(
f"Streaming eval split capped to "
f"{STREAMING_EVAL_MAX_SAMPLES} samples\n"
)
else:
if dataset_loaded_from_cache:
try:
eval_dataset = _load_selected_cached_dataset(eval_split)
if require_exact_resume_resources and eval_dataset is None:
raise FileNotFoundError(
f"The exact cached dataset split '{eval_split}' "
"is no longer available."
)
except Exception as error:
from hub.utils.dataset_cache import (
dataset_cache_fallback_allowed,
)
if not dataset_cache_fallback_allowed(
error,
require_exact = require_exact_resume_resources,
revision = dataset_revision,
):
raise
self._update_progress(
status_message = (
"Cached eval split unavailable; reloading train "
"and eval from the Hub..."
)
)
remote_train = load_dataset(**load_kwargs)
remote_eval = load_dataset(**eval_load_kwargs)
dataset = remote_train
dataset_attestation_source = None
eval_dataset = remote_eval
dataset_loaded_from_cache = False
self.dataset_loaded_from_exact_snapshot = False
self.dataset_snapshot_path = None
else:
eval_dataset = load_dataset(**eval_load_kwargs)
has_separate_eval_source = True
if hasattr(eval_dataset, "__len__"):
logger.info(
f"Loaded eval split '{eval_split}' with {len(eval_dataset)} rows\n"
)
else:
logger.info(f"Loaded eval split '{eval_split}' in streaming mode\n")
elif eval_split and eval_split == effective_train:
if dataset_streaming:
raise ValueError(
"Streaming mode does not support using the same split for both train and eval. "
"Please provide a separate eval split or set eval_steps to 0."
)
logger.info(
f"Eval split '{eval_split}' is the same as train split — will split 80/20\n"
)
else:
if dataset_streaming:
raise ValueError(
"Streaming mode currently requires an explicit eval split when evaluation is enabled."
)
if dataset_loaded_from_cache:
split_info = getattr(getattr(dataset, "info", None), "splits", None)
available_splits = list(split_info or ())
split_loader = _load_selected_cached_dataset
else:
available_splits = None
split_loader = None
eval_dataset = self._auto_detect_eval_split_from_hf(
dataset_source = dataset_source,
subset = subset,
available_splits = available_splits,
split_loader = split_loader,
excluded_split = hf_dataset_split_instruction_names(
train_split or "train"
),
revision = dataset_revision,
token = hf_token,
strict_split_loading = (
require_exact_resume_resources and dataset_loaded_from_cache
),
)
if eval_dataset is not None:
has_separate_eval_source = True
else:
logger.info("Eval disabled (eval_steps <= 0), skipping eval split detection\n")
if not dataset_streaming:
from core.training.provenance import attest_loaded_dataset
train_attestation_source = (
dataset_attestation_source
if dataset_attestation_source is not None
else dataset
)
snapshot, _ = attest_loaded_dataset(
dataset_source,
train_attestation_source,
eval_dataset if has_separate_eval_source else None,
)
if snapshot is not None:
self.dataset_snapshot_path = snapshot
self.dataset_loaded_from_exact_snapshot = True
if dataset is None:
raise ValueError("No dataset provided")
# Streaming slices lazily via skip()/take(); non-streaming fetches end+1 rows, trimmed here.
if (not dataset_streaming) and (
dataset_slice_start is not None or dataset_slice_end is not None
):
total_rows = len(dataset)
dataset, start, end = slice_dataset_rows(
dataset, dataset_slice_start, dataset_slice_end
)
logger.info(
f"Sliced dataset to rows [{start}, {end}]: {len(dataset)} of {total_rows} rows\n"
)
self._update_progress(
status_message = f"Sliced dataset to {len(dataset)} rows (indices {start}-{end})"
)
# Skipped when streaming or for an explicit user range.
if (
(not dataset_streaming)
and dataset_slice_start is None
and dataset_slice_end is None
and "[" not in (train_split or "")
):
def _log_bound(kept, total):
self._kept_row_fraction = kept / total
logger.info(
f"Bounded dataset to {kept} of {total} rows for a "
f"max_steps run (seed {max_train_rows_seed})\n"
)
self._update_progress(
status_message = f"Using {kept} of {total} rows (max_steps run)"
)
dataset = bound_dataset_rows(
dataset,
max_train_rows,
max_train_rows_seed,
on_bound = _log_bound,
)
if self.should_stop:
logger.info("Stopped before applying chat template\n")
# No separate HF eval split - caller handles programmatic splitting
return None
# An inconclusive probe must not read as "not an audio model": falling through lands on the text path,
# which fails later with a column-mapping complaint. _dataset_has_audio_column is the tiebreaker because
# _is_dataset_audio is true on a column-NAME match alone; raw/CPT is exempt.
# ========== AUDIO MODELS: custom preprocessing ==========
if (
not self._audio_type
and not getattr(self, "_audio_type_known", True)
and getattr(self, "_is_dataset_audio", False)
and not raw_text_mode
and _dataset_has_audio_column(dataset) is not False
):
raise RuntimeError(
"Could not determine whether this model is an audio model: its "
"tokenizer_config.json could not be read (gated repo, offline, or a "
"cache holding the weights but not the tokenizer files). The dataset "
"has an audio column, so training cannot fall back to the text path. "
"Re-download the model or provide a token, then retry."
)
# Every audio branch below, and the VLM path, casts an Audio column and reads its array.
if self._audio_type or self.is_audio_vlm:
if not ensure_audio_decoding():
raise RuntimeError(
"This host cannot decode audio datasets: torchcodec cannot load its "
"FFmpeg libraries, and the soundfile fallback needs soundfile and "
"librosa. Install an FFmpeg full-shared build, or install both."
)
if self._audio_type == "csm":
processed = self._preprocess_csm_dataset(dataset, custom_format_mapping)
return (
processed,
self._preprocess_audio_eval_split(
eval_dataset, self._preprocess_csm_dataset, custom_format_mapping
),
)
elif self._audio_type == "whisper":
train_data, eval_data = self._preprocess_whisper_dataset(
dataset,
# Whisper's 6% carve-out keys off eval_split, not the cadence, so gate it here.
eval_split = eval_split if eval_enabled else None,
custom_format_mapping = custom_format_mapping,
eval_dataset = eval_dataset,
)
return (train_data, eval_data)
elif self._audio_type == "snac":
processed = self._preprocess_snac_dataset(dataset, custom_format_mapping)
return (
processed,
self._preprocess_audio_eval_split(
eval_dataset, self._preprocess_snac_dataset, custom_format_mapping
),
)
elif self._audio_type == "bicodec":
processed = self._preprocess_bicodec_dataset(dataset, custom_format_mapping)
return (
{"dataset": processed, "final_format": "audio_bicodec"},
self._preprocess_audio_eval_split(
eval_dataset, self._preprocess_bicodec_dataset, custom_format_mapping
),
)
elif self._audio_type == "dac":
processed = self._preprocess_dac_dataset(dataset, custom_format_mapping)
return (
{"dataset": processed, "final_format": "audio_dac"},
self._preprocess_audio_eval_split(
eval_dataset, self._preprocess_dac_dataset, custom_format_mapping
),
)
if raw_text_mode:
# Name the env var: a stale world size from mpirun/HPC images is otherwise invisible.
logger.info(
f"{_raw_mode_label().capitalize()} mode: bypassing chat template, "
"using raw text\n"
)
dataset = _apply_raw_text_prep(dataset, "train")
if has_separate_eval_source and eval_dataset is not None:
eval_dataset = _apply_raw_text_prep(eval_dataset, "eval")
dataset_info = {
"dataset": dataset,
"detected_format": "raw_text",
"final_format": "raw_text",
"success": True,
}
if has_separate_eval_source and eval_dataset is not None:
eval_rows = (
f"{len(eval_dataset):,} rows"
if hasattr(eval_dataset, "__len__")
else "streaming"
)
logger.info(
f"{_raw_mode_label().capitalize()}: eval dataset "
f"({eval_rows}) kept as raw text\n"
)
elif eval_enabled and not has_separate_eval_source and not dataset_streaming:
# _resolve_eval_split_from_dataset does a train_test_split, which needs len and random access.
split_result = self._resolve_eval_split_from_dataset(dataset)
if split_result is not None:
train_portion, eval_dataset = split_result
dataset_info["dataset"] = train_portion
train_dataset = dataset_info["dataset"]
n = len(train_dataset) if hasattr(train_dataset, "__len__") else None
n_display = f"{n:,}" if isinstance(n, int) else "streaming"
self._update_progress(
status_message = f"Dataset ready ({n_display} samples, raw text)"
)
logger.info(f"Raw-text dataset ready ({n_display} samples)\n")
# Streaming column_names can be None, breaking the in check; use resolve_column_names.
train_columns = resolve_column_names(train_dataset)
if "text" not in train_columns:
raise ValueError(f"Raw-text dataset missing 'text' column: {train_columns}")
return (dataset_info, eval_dataset)
elif self.is_audio_vlm:
formatted = self._format_audio_vlm_dataset(dataset, custom_format_mapping)
return (
formatted,
self._format_audio_vlm_eval_split(eval_dataset, custom_format_mapping),
)
_raise_if_empty_train_split(dataset, "")
# ========== FORMAT FIRST ==========
logger.info(f"Formatting dataset with format_type='{format_type}'...\n")
dataset_info = format_and_template_dataset(
dataset,
model_name = self.model_name,
tokenizer = self.tokenizer,
is_vlm = self.is_vlm,
format_type = format_type,
dataset_name = dataset_source,
dataset_revision = dataset_revision,
custom_format_mapping = custom_format_mapping,
progress_callback = self._update_progress,
)
if self.should_stop:
logger.info("Stopped during dataset formatting\n")
return None
if not dataset_info.get("success", True):
errors = dataset_info.get("errors", [])
error_msg = "; ".join(errors) if errors else "Dataset formatting failed"
logger.error(f"Dataset conversion failed: {error_msg}")
self._update_progress(error = error_msg)
return None
if dataset_info.get("dropped_rows_warning"):
self._record_warning(dataset_info["dropped_rows_warning"])
detected = dataset_info.get("detected_format", "unknown")
final_ds = dataset_info.get("dataset")
final_n = len(final_ds) if hasattr(final_ds, "__len__") else "?"
self._update_progress(
status_message = f"Dataset ready ({final_n:,} samples, {detected} format)"
if isinstance(final_n, int)
else f"Dataset ready ({final_n} samples, {detected} format)"
)
logger.info(f"Dataset formatted successfully ({final_n} samples, {detected})\n")
if has_separate_eval_source and eval_dataset is not None:
eval_n = len(eval_dataset) if hasattr(eval_dataset, "__len__") else "?"
logger.info(f"Formatting eval dataset ({eval_n} rows)...\n")
eval_info = format_and_template_dataset(
eval_dataset,
model_name = self.model_name,
tokenizer = self.tokenizer,
is_vlm = self.is_vlm,
format_type = format_type,
dataset_name = dataset_source,
dataset_revision = dataset_revision,
custom_format_mapping = custom_format_mapping,
)
if not eval_info.get("success", True):
eval_errors = eval_info.get("errors", [])
error_msg = "; ".join(eval_errors) or "Eval dataset formatting failed"
logger.error(f"Eval dataset conversion failed: {error_msg}")
self._update_progress(error = error_msg)
return None
if eval_info.get("dropped_rows_warning"):
self._record_warning(f"Eval dataset: {eval_info['dropped_rows_warning']}")
eval_dataset = eval_info["dataset"]
# Formatted on its own, so it can resolve to another format than the training split (a split carved
# out of train has no entry and shares the training format).
dataset_info["eval_final_format"] = str(
eval_info.get("final_format", "") or ""
).lower()
logger.info("Eval dataset formatted successfully\n")
elif eval_enabled and not has_separate_eval_source and not dataset_streaming:
formatted_dataset = dataset_info["dataset"]
split_result = self._resolve_eval_split_from_dataset(formatted_dataset)
if split_result is not None:
train_portion, eval_dataset = split_result
dataset_info["dataset"] = train_portion
_raise_if_empty_train_split(dataset_info["dataset"], "after formatting")
return (dataset_info, eval_dataset)
except S3DownloadCancelled:
logger.info("Stopped during S3 download\n")
return None
except Exception as e:
logger.error(f"Error loading dataset: {e}")
self._update_progress(error = str(e))
return None
finally:
if s3_download is not None:
s3_download.cleanup()
def _auto_detect_eval_split_from_hf(
self,
dataset_source: str,
subset: Optional[str],
*,
available_splits: Optional[list[str]] = None,
split_loader: Optional[Callable[[str], Dataset]] = None,
excluded_split: Any = None,
revision: Optional[str] = None,
token: Optional[str] = None,
strict_split_loading: bool = False,
) -> Optional[Dataset]:
"""Auto-detect an eval split from an HF dataset (named split only)."""
try:
if available_splits is None and split_loader is None:
from datasets import get_dataset_split_names
load_kwargs = {"path": dataset_source}
if subset:
load_kwargs["config_name"] = subset
if revision:
load_kwargs["revision"] = revision
if token:
load_kwargs["token"] = token
available_splits = get_dataset_split_names(**load_kwargs)
elif available_splits is None or split_loader is None:
raise ValueError("Cached split names and loader must be provided together")
logger.info(f"Available splits: {available_splits}\n")
from core.training.eval_dataset import EVAL_SPLIT_CANDIDATES, MIN_EVAL_ROWS
excluded_splits = (
{excluded_split} if isinstance(excluded_split, str) else set(excluded_split or ())
)
for candidate in EVAL_SPLIT_CANDIDATES:
if candidate in available_splits and candidate not in excluded_splits:
if split_loader is not None:
candidate_ds = split_loader(candidate)
else:
eval_load_kwargs = {"path": dataset_source, "split": candidate}
if subset:
eval_load_kwargs["name"] = subset
if revision:
eval_load_kwargs["revision"] = revision
if token:
eval_load_kwargs["token"] = token
candidate_ds = load_dataset(**eval_load_kwargs)
if len(candidate_ds) >= MIN_EVAL_ROWS:
logger.info(
f"Auto-detected eval split '{candidate}' with {len(candidate_ds)} rows\n"
)
return candidate_ds
else:
logger.info(
f"Found eval split '{candidate}' but only {len(candidate_ds)} rows "
f"(< {MIN_EVAL_ROWS}), skipping\n"
)
except Exception as e:
if strict_split_loading and split_loader is not None:
raise
self._record_warning(
"Automatic eval split detection failed; a held-out split will be created "
f"from the training data when enough rows are available: {e}"
)
return None
def _resolve_eval_split_from_dataset(self, dataset) -> Optional[tuple]:
"""Split a dataset into train and eval portions. Returns (train_dataset, eval_dataset), or
None if too small."""
from core.training.eval_dataset import (
MIN_TOTAL_ROWS_FOR_EVAL,
split_dataset_for_evaluation,
)
n = len(dataset)
split_result = split_dataset_for_evaluation(dataset)
if split_result is None:
self._record_warning(
f"Evaluation is enabled, but the training dataset has only {n} rows; "
f"at least {MIN_TOTAL_ROWS_FOR_EVAL} are required to create a held-out "
"eval split. Training will continue without evaluation."
)
return None
train_dataset, eval_dataset = split_result
eval_size = len(eval_dataset)
logger.info(
f"Auto-splitting: {eval_size} rows for eval from {n} total "
f"(minimum total: {MIN_TOTAL_ROWS_FOR_EVAL})\n"
)
logger.info(f"Split complete: {len(train_dataset)} train, {len(eval_dataset)} eval\n")
return train_dataset, eval_dataset
def start_training(
self,
dataset: Dataset,
eval_dataset: Dataset = None,
eval_steps: float = 0.00,
output_dir: str | None = None,
num_epochs: int = 3,
learning_rate: float = 2e-4,
embedding_learning_rate: float | None = None,
batch_size: int = 2,
gradient_accumulation_steps: int = 4,
warmup_steps: int = None,
warmup_ratio: float = None,
max_steps: int = 0,
save_steps: int = 0,
weight_decay: float = 0.001,
random_seed: int = 3407,
packing: bool = True,
train_on_completions: bool = True,
enable_wandb: bool = False,
wandb_project: str = "unsloth-training",
wandb_token: str = None,
enable_tensorboard: bool = False,
tensorboard_dir: str | None = None,
**kwargs,
) -> bool:
if self.is_training:
logger.warning("Training already in progress")
return False
if self.model is None or self.tokenizer is None:
self._update_progress(error = "Model not loaded")
return False
# Pre-import on the main thread: Unsloth's patched_import is not thread-safe.
import transformers # noqa: F401 - ensures submodules are cached
from transformers import ( # noqa: F401
Trainer as _HFTrainer,
TrainingArguments as _TrainingArguments,
TrainerCallback as _TrainerCallback,
)
if self._audio_type == "whisper":
from transformers import ( # noqa: F401
Seq2SeqTrainer as _Seq2SeqTrainer,
Seq2SeqTrainingArguments as _Seq2SeqTrainingArguments,
)
self.training_thread = account_thread(
target = self._train_worker,
args = (dataset,),
kwargs = {
"output_dir": output_dir,
"num_epochs": num_epochs,
"learning_rate": learning_rate,
"embedding_learning_rate": embedding_learning_rate,
"batch_size": batch_size,
"gradient_accumulation_steps": gradient_accumulation_steps,
"warmup_steps": warmup_steps,
"warmup_ratio": warmup_ratio,
"max_steps": max_steps,
"save_steps": save_steps,
"weight_decay": weight_decay,
"random_seed": random_seed,
"packing": packing,
"train_on_completions": train_on_completions,
"enable_wandb": enable_wandb,
"wandb_project": wandb_project,
"wandb_token": wandb_token,
"enable_tensorboard": enable_tensorboard,
"tensorboard_dir": tensorboard_dir,
"eval_dataset": eval_dataset,
"eval_steps": eval_steps,
**kwargs,
},
)
self.should_stop = False
self.is_training = True
try:
self.training_thread.start()
return True
except Exception as e:
self.is_training = False
logger.error(f"Failed to start training thread: {e}")
return False
def _chat_template_renders_empty(self) -> bool:
"""True when the chat template renders a sample to empty text (base-model signature)."""
try:
ds = getattr(self.trainer, "train_dataset", None)
if ds is None or len(ds) == 0:
return False
row = ds[0]
messages = row.get("messages") if isinstance(row, dict) else None
if not messages:
return False
tok = self.tokenizer
if not hasattr(tok, "apply_chat_template"):
return False
rendered = tok.apply_chat_template(
messages, tokenize = False, add_generation_prompt = False
)
return not (isinstance(rendered, str) and rendered.strip())
except Exception:
return False
def _configure_online_tokenization(
self,
*,
config_args: dict,
dataset,
eval_dataset,
training_args: dict,
data_collator,
raw_text_mode: bool,
is_deepseek_ocr: bool,
):
"""Lazy worker-side tokenization for plain text; never raises, any failure degrades to eager."""
from utils.datasets.online_tokenization import (
OnlineTokenizationDecision,
attach_online_tokenization,
decide_online_tokenization,
first_sample_text,
online_config_args,
quiet_tokenizer_fork_warning,
resolve_add_special_tokens,
)
self._online_eval_dataset = eval_dataset
train_dataset = dataset["dataset"] if isinstance(dataset, dict) else dataset
# A step-capped run says nothing about passes, and world size scales the rows consumed per step.
resolved_epochs = None
max_steps = int(config_args.get("max_steps") or 0)
if max_steps > 0:
try:
rows = len(train_dataset)
# Env-only on purpose: multi-GPU load is model-parallel, so do not count devices.
world_size = world_size_from_env()
if world_size > 1:
# Name the env var: a stale mpirun size otherwise shows only as this veto.
logger.info(
f"Launcher environment reports {world_size} data-parallel "
f"processes ({world_size_env_report()}); a step-capped run "
f"consumes that many times the rows per step\n"
)
per_step = (
int(config_args.get("per_device_train_batch_size", 1))
* int(config_args.get("gradient_accumulation_steps", 1))
* world_size
)
resolved_epochs = (max_steps * per_step) / rows if rows else None
except Exception: # noqa: BLE001 - unresolvable stays unresolved
resolved_epochs = None
try:
decision = decide_online_tokenization(
dataset = train_dataset,
eval_dataset = eval_dataset,
processing_class = self.tokenizer,
model = self.model,
text_field = config_args.get("dataset_text_field", "text") or "text",
packing = bool(config_args.get("packing", False)),
is_vlm = bool(self.is_vlm),
is_audio = bool(self.is_audio or self._cuda_audio_used),
is_audio_vlm = bool(self.is_audio_vlm),
is_deepseek_ocr = bool(is_deepseek_ocr),
is_cpt = bool(training_args.get("is_cpt", False)),
raw_text_mode = bool(raw_text_mode),
has_custom_collator = data_collator is not None,
train_on_completions = bool(training_args.get("train_on_completions", False)),
dataset_streaming = bool(
training_args.get("dataset_streaming", False)
or detect_streaming_dataset(train_dataset)
),
num_train_epochs = config_args.get("num_train_epochs"),
max_steps = config_args.get("max_steps"),
grad_accum = config_args.get("gradient_accumulation_steps", 1),
resolved_max_steps_epochs = resolved_epochs,
)
except Exception as exc: # noqa: BLE001 - a gating failure must not fail a run
logger.warning(f"Online tokenization gating failed, using eager path: {exc}")
return OnlineTokenizationDecision(enabled = False, reason = f"gating error: {exc}")
logger.info(f"{decision.as_log_line()}\n")
if not decision.enabled:
return decision
try:
text_field = config_args.get("dataset_text_field", "text") or "text"
max_length = int(config_args.get("max_seq_length") or 2048)
# Apply the model's max_seq_length cap here; the generated __init__ runs later.
model_cap = getattr(self.model, "max_seq_length", None)
try:
if model_cap is not None and 0 < int(model_cap) < max_length:
logger.info(
f"Online tokenization: max_length {max_length} exceeds the "
f"model's {int(model_cap)}, using the model's\n"
)
max_length = int(model_cap)
except (TypeError, ValueError):
pass
add_special_tokens = resolve_add_special_tokens(
self.tokenizer, first_sample_text(train_dataset, text_field)
)
quiet_tokenizer_fork_warning()
view = attach_online_tokenization(
train_dataset,
tokenizer = self.tokenizer,
text_field = text_field,
max_length = max_length,
add_special_tokens = add_special_tokens,
)
if isinstance(dataset, dict):
dataset["dataset"] = view
if eval_dataset is not None:
# Probe eval's own first row: splits may disagree about a leading BOS.
eval_add_special_tokens = resolve_add_special_tokens(
self.tokenizer, first_sample_text(eval_dataset, text_field)
)
self._online_eval_dataset = attach_online_tokenization(
eval_dataset,
tokenizer = self.tokenizer,
text_field = text_field,
max_length = max_length,
add_special_tokens = eval_add_special_tokens,
)
config_args.update(online_config_args(decision))
self._online_prewarm_batches = decision.prewarm_batches
logger.info(
f"Online tokenization attached: max_length={max_length}, "
f"add_special_tokens={add_special_tokens}\n"
)
return decision
except Exception as exc: # noqa: BLE001
logger.warning(f"Online tokenization setup failed, using eager path: {exc}")
if isinstance(dataset, dict):
dataset["dataset"] = train_dataset
self._online_eval_dataset = eval_dataset
self._online_prewarm_batches = 0
return OnlineTokenizationDecision(enabled = False, reason = f"setup error: {exc}")
def _release_online_dataloader(self) -> None:
"""Persistent workers would otherwise stay resident through merge and GGUF export."""
if not getattr(self, "_online_prewarm_batches", 0):
return
trainer = getattr(self, "trainer", None)
if trainer is None:
return
try:
from utils.datasets.online_tokenization import release_train_dataloader
released = release_train_dataloader(trainer)
except Exception as exc: # noqa: BLE001 - cleanup must never fail a finished run
logger.warning(f"Online tokenization worker shutdown failed: {exc}")
return
# _online_prewarm_batches records how the run was configured and the A/B harness reads it back; a second call
# is a no-op.
if released:
logger.info(f"Online tokenization: shut down {released} DataLoader workers\n")
def _preflight_first_batch(self) -> Optional[str]:
"""Catches empty chat templates that would crash step 1; batches are re-tokenized on the
online path."""
prewarm = int(getattr(self, "_online_prewarm_batches", 0) or 0)
if prewarm:
from utils.datasets.online_tokenization import memoize_train_dataloader
# Memoize, or train() forks fresh workers and this prewarm is wasted.
memoize_train_dataloader(self.trainer)
try:
loader = self.trainer.get_train_dataloader()
iterator = iter(loader)
batch = next(iterator)
for _ in range(max(0, prewarm - 1)):
try:
next(iterator)
except StopIteration:
break
# Drop the local names: on the eager path that tears the loader down, while on the online path the memo
# still holds it so the workers survive.
del iterator, loader
except StopIteration:
return (
"Cannot start training: the dataset produced no training rows. "
"This usually means a split/slice or streaming filter removed every "
"row. Check your train split, slice range, and dataset filters."
)
except Exception as e:
model = self.model_name or "this model"
return (
f"Cannot start training: failed to build the first training batch "
f"for '{model}': {e}"
)
try:
input_ids = batch["input_ids"] if "input_ids" in batch else None
except Exception:
input_ids = getattr(batch, "input_ids", None)
if input_ids is None:
return None
seq_len = input_ids.shape[-1] if input_ids.ndim > 0 else 0
if not (input_ids.is_floating_point() or input_ids.numel() == 0 or seq_len == 0):
return None
model = self.model_name or "this model"
if self._chat_template_renders_empty():
low = model.lower()
suffix = (
f" such as '{model}-Instruct'"
if not any(t in low for t in ("instruct", "chat", "-it", "_it"))
else ""
)
return (
f"Cannot start training: the chat template for '{model}' produced "
f"no text for your dataset, so the first batch had empty token IDs. "
f"'{model}' looks like a base (pretrained) model without a chat "
f"template suited to conversational fine-tuning. Use the "
f"instruction-tuned variant{suffix} or provide a chat template."
)
return (
f"Cannot start training: the first batch produced invalid token IDs "
f"(dtype={input_ids.dtype}, length={seq_len}). Check that your dataset "
f"columns are mapped correctly for '{model}'."
)
def _train_worker(self, dataset: Dataset | dict, **training_args):
"""Streaming datasets always arrive wrapped in the dict, never as a bare IterableDataset."""
try:
# On spawn, compiled-cache dirs go on sys.path/PYTHONPATH before dataset.map() so workers can import.
if sys.platform in ("win32", "darwin"):
from utils.cache_cleanup import register_compiled_cache_on_path
register_compiled_cache_on_path()
self.batch_size = training_args.get("batch_size", 2)
self.max_seq_length = training_args.get("max_seq_length", 2048)
self.gradient_accumulation_steps = training_args.get("gradient_accumulation_steps", 4)
self.training_start_time = time.time()
self._update_progress(is_training = True, error = None)
if training_args.get("enable_wandb", False) and training_args.get("wandb_token"):
os.environ["WANDB_API_KEY"] = training_args["wandb_token"]
import wandb
wandb.init(project = training_args.get("wandb_project", "unsloth-training"))
output_dir = str(resolve_output_dir(training_args.get("output_dir")))
ensure_dir(Path(output_dir))
if self._audio_type == "csm":
# CSM uses plain HF Trainer with remove_unused_columns=False for the depth decoder.
from transformers import Trainer as HFTrainer, TrainingArguments
self._apply_csm_forward_fix()
eval_args, eval_dataset = self._audio_eval_config(training_args)
config = self._build_audio_training_args(
training_args,
output_dir,
extra_args = {
"remove_unused_columns": False,
**eval_args,
},
)
trainer_kwargs = {
"model": self.model,
"train_dataset": dataset,
"args": TrainingArguments(**config),
}
if eval_dataset is not None:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = HFTrainer(**trainer_kwargs)
self.trainer.add_callback(self._create_progress_callback())
self._attach_early_stopping(training_args)
# Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that has
# no terminal; --verbose keeps them.
_drop_hf_stdout_callbacks(self.trainer)
batch_size = training_args.get("batch_size", 2)
total = self._calculate_total_steps(
len(dataset),
batch_size,
training_args.get("gradient_accumulation_steps", 4),
training_args.get("num_epochs", 3),
training_args.get("max_steps", 0),
)
self._update_progress(total_steps = total, status_message = "Starting CSM training...")
logger.info(f"CSM training config: {config}\n")
self.trainer.train(
resume_from_checkpoint = training_args.get("resume_from_checkpoint")
)
self._finalize_training(output_dir, "CSM")
return
elif self._audio_type == "snac":
# DataCollatorForSeq2Seq pads variable-length sequences per batch and pads labels with -100.
from transformers import (
Trainer as HFTrainer,
TrainingArguments,
DataCollatorForSeq2Seq,
)
eval_args, eval_dataset = self._audio_eval_config(training_args)
config = self._build_audio_training_args(
training_args, output_dir, extra_args = eval_args
)
trainer_kwargs = {
"model": self.model,
"train_dataset": dataset,
"args": TrainingArguments(**config),
"data_collator": DataCollatorForSeq2Seq(
tokenizer = self.tokenizer,
padding = True,
pad_to_multiple_of = 8,
),
}
if eval_dataset is not None:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = HFTrainer(**trainer_kwargs)
self.trainer.add_callback(self._create_progress_callback())
self._attach_early_stopping(training_args)
_drop_hf_stdout_callbacks(self.trainer)
batch_size = training_args.get("batch_size", 2)
total = self._calculate_total_steps(
len(dataset),
batch_size,
training_args.get("gradient_accumulation_steps", 4),
training_args.get("num_epochs", 3),
training_args.get("max_steps", 0),
)
self._update_progress(total_steps = total, status_message = "Starting SNAC training...")
logger.info(f"SNAC training config: {config}\n")
self.trainer.train(
resume_from_checkpoint = training_args.get("resume_from_checkpoint")
)
self._finalize_training(output_dir, "SNAC")
return
elif self._audio_type == "whisper":
from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments
from utils.datasets import DataCollatorSpeechSeq2SeqWithPadding
eval_dataset = training_args.get("eval_dataset", None)
eval_steps_val = training_args.get("eval_steps", 5)
extra = {"remove_unused_columns": False, "label_names": ["labels"]}
if eval_dataset and not evaluation_enabled(eval_steps_val):
# The carve-out keys off eval_split, not the cadence: only this gate stops inf.
logger.info(
f"⚠️ Eval dataset provided but eval_steps={eval_steps_val} (disabled)\n"
)
elif eval_dataset:
extra["eval_strategy"] = "steps"
extra["eval_steps"] = float(eval_steps_val)
# HF's default of 8 can OOM audio runs, as the codec branches already note.
extra["per_device_eval_batch_size"] = training_args.get("batch_size") or 2
config = self._build_audio_training_args(
training_args, output_dir, extra_args = extra
)
trainer_kwargs = {
"model": self.model,
"train_dataset": dataset,
"data_collator": DataCollatorSpeechSeq2SeqWithPadding(
processor = self.tokenizer,
decoder_start_token_id = getattr(
self.model.config, "decoder_start_token_id", None
),
),
"processing_class": self.tokenizer.feature_extractor,
"args": Seq2SeqTrainingArguments(**config),
}
if eval_dataset:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = Seq2SeqTrainer(**trainer_kwargs)
self.trainer.add_callback(self._create_progress_callback())
self._attach_early_stopping(training_args)
_drop_hf_stdout_callbacks(self.trainer)
batch_size = training_args.get("batch_size", 2)
total = self._calculate_total_steps(
len(dataset),
batch_size,
training_args.get("gradient_accumulation_steps", 4),
training_args.get("num_epochs", 3),
training_args.get("max_steps", 0),
)
self._update_progress(
total_steps = total, status_message = "Starting Whisper training..."
)
logger.info(f"Whisper training config: {config}\n")
self.trainer.train(
resume_from_checkpoint = training_args.get("resume_from_checkpoint")
)
self._finalize_training(output_dir, "Whisper")
return
elif self._audio_type is not None and self._audio_type not in (
"bicodec",
"dac",
):
# bicodec/dac use the standard SFTTrainer text path below
raise NotImplementedError(
f"Audio training for '{self._audio_type}' not yet implemented"
)
model_name_lower = self.model_name.lower()
is_deepseek_ocr = "deepseek" in model_name_lower and "ocr" in model_name_lower
logger.info("Configuring data collator...\n")
dataset_final_format = (
str(dataset.get("final_format", "")).lower() if isinstance(dataset, dict) else ""
)
raw_text_mode = dataset_final_format == "raw_text"
pretokenized_mode = dataset_final_format == PRETOKENIZED_FORMAT
eval_final_format = (
str(dataset.get("eval_final_format", dataset_final_format) or "").lower()
if isinstance(dataset, dict)
else ""
)
# Without a labels column, "Train on responses only" can still mask prompts from input_ids.
user_labels = pretokenized_mode and has_custom_labels(dataset["dataset"])
self.tokenizer = get_training_chat_template(
self.tokenizer, self.model_name, dataset_final_format
)
data_collator = None
audio_response_masker = None
if is_deepseek_ocr:
logger.info("Detected DeepSeek OCR model\n")
if not _ensure_deepseek_ocr_installed():
# No snapshot_download hint: a local deepseek_ocr dir would shadow the pinned source.
error_msg = (
"Could not prepare the DeepSeek OCR module. "
"Check network access to huggingface.co and try again."
)
logger.error(error_msg)
self._update_progress(error = error_msg, is_training = False)
return
try:
from backend.data_utils import DeepSeekOCRDataCollator
logger.info("Configuring DeepSeek OCR data collator...\n")
FastVisionModel.for_training(
self.model,
use_gradient_checkpointing = self._use_gradient_checkpointing,
)
# (image_size, base_size, crop_mode) is a coupled preset.
if training_args.get("vision_image_size") is not None:
logger.info(
"Vision image resize ignored for DeepSeek OCR "
"(uses fixed Gundam preset).\n"
)
data_collator = DeepSeekOCRDataCollator(
tokenizer = self.tokenizer,
model = self.model,
image_size = 640,
base_size = 1024,
crop_mode = True,
train_on_responses_only = training_args.get("train_on_completions", False),
)
logger.info("DeepSeek OCR data collator configured successfully\n")
except Exception as e:
logger.error(f"Failed to configure DeepSeek OCR collator: {e}")
error_msg = f"Error configuring DeepSeek OCR: {str(e)}"
self._update_progress(error = error_msg, is_training = False)
return
elif self.is_audio_vlm and not raw_text_mode:
# Audio VLM collator (e.g. Gemma 3N), mirrors the Gemma3N_(4B)-Audio notebook.
logger.info("Configuring audio VLM data collator...\n")
from unsloth import FastModel
# The module flags decide whether a layer checkpoints, as in the image VLM branch.
FastModel.for_training(
self.model,
use_gradient_checkpointing = self._use_gradient_checkpointing,
)
processor = self.tokenizer
audio_col_name = getattr(self, "_audio_vlm_audio_col", "audio")
image_col_name = getattr(self, "_audio_vlm_image_col", None)
# Placeholder ids stay out of the loss; LFM3-Audio keeps them on the processor, Gemma 3n on its tokenizer.
placeholder_ids = set()
for holder in (processor, getattr(processor, "tokenizer", None)):
for attr in (
"audio_token_id",
"audio_out_token_id",
"image_token_id",
"boi_token_id",
"eoi_token_id",
):
token_id = getattr(holder, attr, None)
if type(token_id) is int:
placeholder_ids.add(token_id)
if training_args.get("train_on_completions", False) and not training_args.get(
"is_cpt", False
):
from unsloth.chat_templates import train_on_responses_only
def _build_response_masker(holder, **kwargs):
holder.masker = train_on_responses_only(
None, tokenizer = processor, return_function = True, **kwargs
)
return holder
def _notify_masking(level, message):
if level == "warning":
logger.warning(message)
else:
logger.info(f"{message}\n")
# Same marker policy as the text path: auto-detect, template table, else full sequences.
holder, audio_masking_applied = apply_completion_masking(
types.SimpleNamespace(processing_class = processor, masker = None),
self.model_name,
_build_response_masker,
notify = _notify_masking,
)
if audio_masking_applied:
audio_response_masker = holder.masker
def audio_vlm_collate_fn(examples):
texts = []
audios = []
images = []
for example in examples:
text = processor.apply_chat_template(
example["messages"],
tokenize = False,
add_generation_prompt = False,
).strip()
texts.append(text)
audios.append(example[audio_col_name]["array"])
if image_col_name:
row = example[image_col_name]
row = row if isinstance(row, list) else [row]
images.append(
[
i.convert("RGB") if hasattr(i, "convert") else i
for i in row
if i is not None
]
)
image_kwargs = {"images": images} if any(images) else {}
batch = processor(
text = texts,
audio = audios,
return_tensors = "pt",
padding = True,
**image_kwargs,
)
labels = batch["input_ids"].clone()
labels[labels == processor.tokenizer.pad_token_id] = -100
for token_id in placeholder_ids:
labels[labels == token_id] = -100
batch["labels"] = labels
if audio_response_masker is not None:
batch["labels"] = audio_response_masker(batch)["labels"]
return batch
data_collator = audio_vlm_collate_fn
logger.info("Audio VLM data collator configured\n")
elif self.is_vlm and not raw_text_mode:
logger.info("Using UnslothVisionDataCollator for vision model\n")
from unsloth.trainer import UnslothVisionDataCollator
FastVisionModel.for_training(
self.model,
use_gradient_checkpointing = self._use_gradient_checkpointing,
)
vision_image_size = training_args.get("vision_image_size")
if vision_image_size is None:
data_collator = UnslothVisionDataCollator(self.model, self.tokenizer)
else:
logger.info(f"Vision image resize: {vision_image_size} (max dimension)\n")
data_collator = UnslothVisionDataCollator(
self.model,
self.tokenizer,
resize = vision_image_size,
resize_dimension = "max",
)
logger.info("Vision data collator configured\n")
warmup_steps_val = training_args.get("warmup_steps", None)
warmup_ratio_val = training_args.get("warmup_ratio", None)
lr_value = training_args.get("learning_rate", 2e-4)
logger.info(
f"[DEBUG] learning_rate from training_args: {lr_value} (type: {type(lr_value).__name__})\n"
)
config_args = {
"per_device_train_batch_size": training_args.get("batch_size", 2),
"gradient_accumulation_steps": training_args.get("gradient_accumulation_steps", 4),
"num_train_epochs": training_args.get("num_epochs", 3),
"learning_rate": lr_value,
"fp16": not is_bfloat16_supported(),
"bf16": is_bfloat16_supported(),
"logging_steps": 1,
"weight_decay": training_args.get("weight_decay", 0.001),
"seed": training_args.get("random_seed", 3407),
"output_dir": output_dir,
"report_to": _build_report_targets(training_args),
"disable_tqdm": _hf_stdout_progress_disabled(),
"include_num_input_tokens_seen": True,
# Config boundary: None lets the shared policy size workers from affinity/cgroups.
"dataset_num_proc": dataset_map_num_proc(
1
if (
self.is_audio
or self.is_audio_vlm
or self._cuda_audio_used
or _tokenizer_blocks_map_workers(self.tokenizer)
)
else None,
serial_as_none = False,
),
"max_seq_length": training_args.get("max_seq_length", 2048),
# TRL defaults this on, and train() re-enables checkpointing from it.
"gradient_checkpointing": bool(self._use_gradient_checkpointing),
}
if training_args.get("enable_tensorboard", False):
config_args["logging_dir"] = str(
resolve_tensorboard_dir(training_args.get("tensorboard_dir"))
)
logger.info(
f"[DEBUG] dataset_num_proc={config_args['dataset_num_proc']} (is_audio={self.is_audio}, is_audio_vlm={self.is_audio_vlm}, _cuda_audio_used={self._cuda_audio_used})"
)
# Disable DataLoader multiprocessing: a modified sys.path breaks spawned workers.
if sys.platform in ("win32", "darwin"):
import transformers as _tf
if _tf.__version__.startswith("5."):
config_args["dataloader_num_workers"] = 0
if warmup_ratio_val is not None:
config_args["warmup_ratio"] = warmup_ratio_val
logger.info(f"Using warmup_ratio: {warmup_ratio_val}\n")
elif warmup_steps_val is not None:
config_args["warmup_steps"] = warmup_steps_val
logger.info(f"Using warmup_steps: {warmup_steps_val}\n")
else:
config_args["warmup_steps"] = 5
logger.info("Using default warmup_steps: 5\n")
apply_save_strategy(config_args, training_args.get("save_steps", 0))
max_steps_val = training_args.get("max_steps", 0)
if max_steps_val and max_steps_val > 0:
del config_args["num_train_epochs"]
config_args["max_steps"] = max_steps_val
logger.info(f"Training for {max_steps_val} steps\n")
else:
logger.info(f"Training for {config_args['num_train_epochs']} epochs\n")
eval_dataset = training_args.get("eval_dataset", None)
eval_steps_val = training_args.get("eval_steps", 0.00)
if eval_dataset is not None:
eval_rows = len(eval_dataset) if hasattr(eval_dataset, "__len__") else None
# Same gate as _audio_eval_config: BiCodec and DAC land here, not an audio branch.
if not evaluation_enabled(eval_steps_val):
logger.info(
f"⚠️ Eval dataset provided but eval_steps={eval_steps_val} (disabled)\n"
)
logger.info("To enable evaluation, set eval_steps > 0.0\n")
elif eval_rows == 0:
self._record_warning(
"The eval dataset is empty after preprocessing, so this run has no "
"evaluation."
)
else:
config_args["eval_strategy"] = "steps"
config_args["eval_steps"] = float(eval_steps_val)
config_args["per_device_eval_batch_size"] = config_args[
"per_device_train_batch_size"
]
logger.info(
f"✅ Evaluation enabled: eval_steps={eval_steps_val} (fraction of total steps)\n"
)
if eval_rows is None:
logger.info("Eval dataset is streaming / length unknown\n")
else:
logger.info(f"Eval dataset: {eval_rows} rows\n")
else:
logger.info("No eval dataset — evaluation disabled\n")
optim_value = training_args.get("optim", "adamw_8bit")
lr_scheduler_type_value = training_args.get("lr_scheduler_type", "linear")
if (self.is_vlm or self.is_audio_vlm) and not raw_text_mode:
# Vision/audio VLMs need skip_prepare_dataset and remove_unused_columns; raw-text takes text path.
label = "audio VLM" if self.is_audio_vlm else "vision"
logger.info(f"Configuring {label} model training parameters\n")
optim_value = training_args.get("optim", "adamw_torch_fused")
lr_scheduler_type_value = training_args.get("lr_scheduler_type", "cosine")
config_args.update(
{
"optim": optim_value,
"lr_scheduler_type": lr_scheduler_type_value,
"max_grad_norm": 0.3,
"remove_unused_columns": False,
"dataset_text_field": "",
"dataset_kwargs": {"skip_prepare_dataset": True},
"max_length": training_args.get("max_seq_length", 2048),
}
)
if config_args["gradient_checkpointing"]:
config_args["gradient_checkpointing_kwargs"] = {"use_reentrant": False}
else:
is_cpt = training_args.get("is_cpt", False)
self.is_cpt = is_cpt
if is_cpt:
logger.info("Configuring Continued Pretraining (CPT) parameters\n")
elif raw_text_mode:
logger.info("Configuring raw-text training parameters\n")
else:
logger.info("Configuring text model training parameters\n")
config_args.update(
{
"optim": optim_value,
"lr_scheduler_type": lr_scheduler_type_value,
"dataset_text_field": "text",
}
)
# Packing for text models only (DeepSeek OCR is VLM)
if not is_deepseek_ocr:
packing_enabled = training_args.get("packing", False)
if packing_enabled and training_args.get("dataset_streaming", False):
logger.warning(
"Sequence packing is enabled with dataset streaming: "
"max_steps governs training length and packed-sample "
"counts are approximate since the stream length is unknown.\n"
)
config_args["packing"] = packing_enabled
logger.info(
f"Sequence packing: {'enabled' if packing_enabled else 'disabled'}\n"
)
# Audio codec overrides - BiCodec/DAC use the text SFTTrainer path
if self._audio_type == "bicodec":
config_args["packing"] = False
logger.info("Applied BiCodec overrides: packing=False\n")
elif self._audio_type == "dac":
config_args["packing"] = False
logger.info("Applied DAC overrides: packing=False\n")
if getattr(self, "diffusion_lm_profile", None) and config_args.get("packing"):
# Packed rows share one bidirectional attention window, so documents would see each other.
config_args["packing"] = False
logger.info("Text-diffusion model: sequence packing turned off\n")
eval_pretokenized = (
eval_dataset is not None and eval_final_format == PRETOKENIZED_FORMAT
)
if pretokenized_mode or eval_pretokenized:
try:
vocab_size = int(self.model.get_input_embeddings().num_embeddings)
except Exception:
vocab_size = None
max_seq = training_args.get("max_seq_length", 2048)
# completion_mask is applied (so checked) only under completion_only_loss, which TRL resolves
# once from the train split when unset and uses for eval too.
completion_only = resolve_completion_only_loss(
dataset["dataset"], config_args.get("completion_only_loss")
)
# Each split goes by its own format: a separately supplied eval split can be pre-tokenized while the
# training split is text, and the reverse.
if pretokenized_mode:
# Packing on TRL < 1.7 keeps only input_ids, silently replacing the user's labels.
if config_args.get("packing"):
logger.info("Pre-tokenized dataset: sequence packing turned off\n")
config_args["packing"] = False
dataset["dataset"] = prepare_pretokenized_dataset(
dataset["dataset"],
max_seq_length = max_seq,
vocab_size = vocab_size,
completion_only_loss = completion_only,
)
if eval_pretokenized:
# TRL packs eval by `packing` unless eval_packing says otherwise, and packing keeps only input_ids,
# so a pre-tokenized eval split beside a packed text train split would lose its labels.
if config_args.get("packing"):
logger.info("Pre-tokenized eval dataset: eval sequence packing turned off\n")
config_args["eval_packing"] = False
eval_dataset = prepare_pretokenized_dataset(
eval_dataset,
max_seq_length = max_seq,
vocab_size = vocab_size,
completion_only_loss = completion_only,
)
# Plain-text single-pass runs tokenize in DataLoader workers; other runs keep eager .map().
self._online_prewarm_batches = 0
online_decision = self._configure_online_tokenization(
config_args = config_args,
dataset = dataset,
eval_dataset = eval_dataset,
training_args = training_args,
data_collator = data_collator,
raw_text_mode = raw_text_mode,
is_deepseek_ocr = is_deepseek_ocr,
)
if online_decision.enabled:
eval_dataset = self._online_eval_dataset
logger.info(f"The configuration is: {config_args}")
logger.info("Training configuration prepared\n")
if self.is_audio_vlm and not raw_text_mode:
# Audio VLM: raw Dataset, processing_class=processor.tokenizer; image VLM: dict wrapper.
train_dataset = dataset["dataset"] if isinstance(dataset, dict) else dataset
processing_class = (
self.tokenizer.tokenizer
if hasattr(self.tokenizer, "tokenizer")
else self.tokenizer
)
trainer_kwargs = {
"model": self.model,
"train_dataset": train_dataset,
"processing_class": processing_class,
"data_collator": data_collator,
"args": SFTConfig(**config_args),
}
if eval_dataset is not None:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = SFTTrainer(**trainer_kwargs)
elif self.is_vlm and not raw_text_mode:
train_dataset = dataset["dataset"] if isinstance(dataset, dict) else dataset
trainer_kwargs = {
"model": self.model,
"train_dataset": train_dataset,
"processing_class": self.tokenizer,
"data_collator": data_collator,
"args": SFTConfig(**config_args),
}
if eval_dataset is not None:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = SFTTrainer(**trainer_kwargs)
else:
# Unwrap Processors for text-only, else SFTTrainer skips _prepare_dataset.
from transformers import ProcessorMixin
sft_tokenizer = self.tokenizer
if isinstance(self.tokenizer, ProcessorMixin) and hasattr(
self.tokenizer, "tokenizer"
):
logger.info("Unwrapping Processor → raw tokenizer for text-only SFTTrainer")
sft_tokenizer = self.tokenizer.tokenizer
if getattr(self, "diffusion_lm_profile", None):
# Same SFT data path; the loss is the family's denoising objective. CPT = every token
# supervised, which is what the objective does when no label is -100.
DiffusionTrainer = require_diffusion_api("DiffusionTrainer")
DiffusionConfig = require_diffusion_api("DiffusionConfig")
fields = getattr(DiffusionConfig, "__dataclass_fields__", {})
diffusion_args = {k: v for k, v in config_args.items() if k in fields}
if (
"max_seq_length" in config_args
and "max_seq_length" not in fields
and "max_length" in fields
):
# Unpatched TRL (CPU runs) only knows max_length: keep the chosen context length.
diffusion_args.setdefault("max_length", config_args["max_seq_length"])
embedding_lr = training_args.get("embedding_learning_rate") if is_cpt else None
if embedding_lr is not None:
if "embedding_learning_rate" not in fields:
raise RuntimeError(
"Continued pretraining of a text-diffusion model with an embedding "
"learning rate needs a newer Unsloth. Upgrade with: "
"`pip install -U unsloth unsloth_zoo`."
)
diffusion_args["embedding_learning_rate"] = embedding_lr
skipped = sorted(set(config_args) - set(diffusion_args))
if skipped:
logger.info(f"DiffusionConfig has no {skipped}; not passed\n")
trainer_kwargs = {
"model": self.model,
"tokenizer": sft_tokenizer,
"train_dataset": dataset["dataset"],
"data_collator": data_collator,
"args": DiffusionConfig(**diffusion_args),
}
if eval_dataset is not None:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = DiffusionTrainer(**trainer_kwargs)
logger.info(
f"Text-diffusion trainer: {type(self.trainer).__name__} "
f"(profile {self.diffusion_lm_profile})\n"
)
elif is_cpt:
try:
from unsloth import (
UnslothTrainer as _UnslothCPTTrainer,
UnslothTrainingArguments as _UnslothTrainingArguments,
)
except ImportError as exc:
raise RuntimeError(
"CPT requires a newer Unsloth install that exports "
"`UnslothTrainer` and `UnslothTrainingArguments` "
"(for embedding_learning_rate support). "
"Upgrade with: `pip install -U unsloth unsloth_zoo`."
) from exc
embedding_lr = training_args.get("embedding_learning_rate")
logger.info(
f"CPT: using UnslothTrainer with embedding_learning_rate={embedding_lr}\n"
)
cpt_args = _UnslothTrainingArguments(
embedding_learning_rate = embedding_lr,
**config_args,
)
if config_args.get("packing", False):
cpt_args.packing_strategy = "wrapped"
logger.info("CPT packing strategy: wrapped\n")
trainer_kwargs = {
"model": self.model,
"tokenizer": sft_tokenizer,
"train_dataset": dataset["dataset"],
"data_collator": data_collator,
"args": cpt_args,
}
if eval_dataset is not None:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = _UnslothCPTTrainer(**trainer_kwargs)
else:
trainer_kwargs = {
"model": self.model,
"tokenizer": sft_tokenizer,
"train_dataset": dataset["dataset"],
"data_collator": data_collator,
"args": SFTConfig(**config_args),
}
if eval_dataset is not None:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = SFTTrainer(**trainer_kwargs)
# Restore the full processor so checkpoints include preprocessor_config.json (GGUF export).
if sft_tokenizer is not self.tokenizer:
self.trainer.processing_class = self.tokenizer
logger.info("Trainer initialized\n")
# Raw-text datasets always train on all tokens.
is_cpt = training_args.get("is_cpt", False)
train_on_responses_enabled = (
False
if (is_cpt or raw_text_mode or user_labels)
else training_args.get("train_on_completions", False)
)
if is_cpt:
logger.info("CPT mode: skipping train_on_responses_only — training on all tokens\n")
elif raw_text_mode:
logger.info(
"Raw-text mode: skipping train_on_responses_only — training on all tokens\n"
)
elif user_labels:
logger.info(
"Pre-tokenized dataset: its labels define the loss, no response masking\n"
)
# Markers auto-detect from chat template, falling back to manual table; gpt-oss stays manual.
if (
train_on_responses_enabled
and not self.is_audio_vlm
and not self.is_audio
and not is_deepseek_ocr
):
from unsloth.chat_templates import train_on_responses_only
logger.info("Configuring train on responses only...\n")
def _notify(level, message):
if level == "warning":
logger.warning(message)
else:
logger.info(f"{message}\n")
# Rows as the trainer holds them (packed when packing is on), so the drop check counts masking only.
pre_mask_dataset = self.trainer.train_dataset
pre_mask_len = (
None
if pre_mask_dataset is None or detect_streaming_dataset(pre_mask_dataset)
else len(pre_mask_dataset)
)
# No try/except: an exception here is a real masking failure.
self.trainer, masking_applied = apply_completion_masking(
self.trainer,
self.model_name,
train_on_responses_only,
num_proc = config_args["dataset_num_proc"],
notify = _notify,
dataset_template = ("alpaca" if dataset_final_format == "alpaca" else None),
)
if not masking_applied:
train_on_responses_enabled = False
if masking_applied:
try:
# Rows lacking the response template become all -100; len()-based, so not for streaming.
if detect_streaming_dataset(self.trainer.train_dataset):
logger.info("Skipping post-filter length check for streaming dataset\n")
else:
filtered_len = len(self.trainer.train_dataset)
original_len = pre_mask_len
if original_len is None:
original_dataset_obj = (
dataset["dataset"] if isinstance(dataset, dict) else dataset
)
original_len = len(original_dataset_obj)
dropped = original_len - filtered_len
drop_pct = (
round(100 * dropped / original_len, 1) if original_len > 0 else 0
)
if filtered_len == 0 or drop_pct > 30:
max_seq = training_args.get("max_seq_length", 2048)
cause = None
# Packed rows all reach the cap, so a truncated prompt cannot be told apart there.
if not config_args.get("packing"):
try:
cap = (
getattr(self.trainer.args, "max_length", None)
or max_seq
)
cut = count_truncated_drops(
pre_mask_dataset["input_ids"],
self.trainer.train_dataset["input_ids"],
cap,
)
cause = truncated_drop_cause(cut, dropped, cap)
except Exception as e: # noqa: BLE001 - keep the generic message
logger.info(f"Could not count truncated rows: {e}\n")
error_msg = (
f"{dropped}/{original_len} samples ({drop_pct}%) were "
f"dropped after applying 'Train on completions': after "
f"masking, those rows had no trainable response tokens "
f"left. "
) + (
cause
or (
f"The usual cause is that this model's response "
f"template was not found in the formatted samples, so "
f"every token was masked out. That typically means the "
f"dataset is already formatted, or its structure does "
f"not match the model's chat template, so 'Train on "
f"completions' should be turned off for this dataset. "
f"Less commonly, a max_seq_length ({max_seq}) shorter "
f"than the prompt can truncate the response away; only "
f"raise it if your samples are actually longer than that."
)
)
logger.error(error_msg)
self._update_progress(error = error_msg, is_training = False)
return
if dropped > 0:
logger.info(
f"⚠️ {dropped}/{original_len} samples "
f"({drop_pct}%) were dropped (all labels "
f"masked). {filtered_len} samples remain.\n"
)
logger.info(f"Post-filter dataset size: {filtered_len} samples\n")
except Exception as e:
logger.warning(f"Post-masking dataset size check failed: {e}")
else:
if train_on_responses_enabled and is_deepseek_ocr:
logger.info("Train on responses handled by DeepSeek OCR collator\n")
elif train_on_responses_enabled and audio_response_masker is not None:
logger.info("Train on responses handled by audio VLM collator\n")
else:
logger.info("Training on full sequences (including prompts)\n")
self.trainer.add_callback(self._create_progress_callback())
self._attach_early_stopping(training_args)
# Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that has no
# terminal; --verbose keeps them.
_drop_hf_stdout_callbacks(self.trainer)
train_dataset_obj = dataset["dataset"] if isinstance(dataset, dict) else dataset
is_streaming_dataset = detect_streaming_dataset(train_dataset_obj)
max_steps_value = training_args.get("max_steps")
max_steps = 0 if max_steps_value is None else int(max_steps_value)
if is_streaming_dataset and max_steps <= 0:
raise ValueError(
"Streaming mode requires max_steps > 0 because the training dataset has no length."
)
if is_streaming_dataset:
total_steps = max_steps
else:
# Prefer the trainer's processed length (post-filtering); fall back to the raw dataset.
num_samples = None
if getattr(self.trainer, "train_dataset", None) is not None:
try:
num_samples = len(self.trainer.train_dataset)
except TypeError:
num_samples = None
if num_samples is None:
num_samples = len(train_dataset_obj)
batch_size = training_args.get("batch_size", 2)
total_steps = self._calculate_total_steps(
num_samples,
batch_size,
training_args.get("gradient_accumulation_steps", 4),
training_args.get("num_epochs", 3),
max_steps,
)
self._update_progress(total_steps = total_steps)
# Fail fast on an invalid first batch (empty/float input_ids) vs a step-1 crash.
preflight_error = self._preflight_first_batch()
if preflight_error:
logger.error(preflight_error)
self._update_progress(error = preflight_error, is_training = False)
return
self._update_progress(total_steps = total_steps, status_message = "Starting training...")
logger.info("Starting training...\n")
try:
self.trainer.train(
resume_from_checkpoint = training_args.get("resume_from_checkpoint")
)
finally:
# Release before finalize: workers hold a fork during merge/export.
self._release_online_dataloader()
self._finalize_training(output_dir)
except Exception as e:
import traceback
logger.error(f"Training error: {e}")
logger.error(f"Full traceback:\n{traceback.format_exc()}")
self._update_progress(is_training = False, error = str(e))
finally:
# Backstop for returns that skip train(), e.g. a preflight error after workers already forked.
self._release_online_dataloader()
self.is_training = False
def _patch_adapter_config(self, output_dir: str) -> None:
"""Both keys use the effective 4-bit flag, so a run that fell back to 16-bit is recorded as lora."""
config_path = os.path.join(output_dir, "adapter_config.json")
if not os.path.exists(config_path):
logger.info("No adapter_config.json found — skipping training method patch")
return
try:
with open(config_path, "r", encoding = "utf-8") as f:
config = json.load(f)
trained_in_4bit = bool(self.load_in_4bit) and _bitsandbytes_allows_4bit()
if self.is_cpt:
method = "CPT"
elif trained_in_4bit:
method = "qlora"
else:
method = "lora"
config["unsloth_training_method"] = method
config["unsloth_load_in_4bit"] = trained_in_4bit
logger.info(f"Patching adapter_config.json with unsloth_training_method='{method}'")
with open(config_path, "w", encoding = "utf-8") as f:
json.dump(config, f, indent = 2)
except Exception as e:
logger.warning(f"Failed to patch adapter_config.json: {e}")
def stop_training(self, save: bool = True):
logger.info(f"\nStopping training (save={save})...")
self.should_stop = True
self.save_on_stop = save
stop_msg = (
"Stopping training and saving checkpoint..." if save else "Cancelling training..."
)
self._update_progress(status_message = stop_msg)
if self.trainer:
try:
# The callback catches should_stop and stops the loop
logger.info("Training will stop at next step...\n")
except Exception as e:
logger.error(f"Error stopping trainer: {e}")
def get_training_progress(self) -> TrainingProgress:
with self._lock:
return self.training_progress
def cleanup(self):
if self.trainer:
self.trainer = None
if self.model:
self.model = None
if self.tokenizer:
self.tokenizer = None
clear_gpu_cache()
def _ensure_deepseek_ocr_installed():
"""Installs from a pinned revision and verifies the import came from it, not any sys.path copy."""
try:
logger.info("Preparing the DeepSeek OCR module...")
from utils.third_party_source import (
ensure_deepseek_ocr_source,
import_deepseek_ocr_module,
)
source = ensure_deepseek_ocr_source()
import_deepseek_ocr_module("deepseek_ocr.modeling_deepseekocr", source)
logger.info("DeepSeek OCR module ready")
return True
except Exception as e:
logger.error(f"Failed to install DeepSeek OCR module: {e}")
logger.info(f"\n❌ Failed to install DeepSeek OCR module: {e}\n")
return False
_trainer_instance = None
def get_trainer() -> UnslothTrainer:
global _trainer_instance
if _trainer_instance is None:
_trainer_instance = UnslothTrainer()
return _trainer_instance