Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

257 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Training subprocess entry point. Each job runs in a fresh subprocess (mp.get_context("spawn")): a
clean interpreter with no stale module state, which solves transformers version-switching.
Pattern follows core/data_recipe/jobs/worker.py."""

from __future__ import annotations

from utils.account_context import account_thread
from loggers import get_logger
import importlib
import importlib.metadata
import importlib.util
import math
import os
import shutil
import sys
import time
import traceback
import gc
import re
import types
import subprocess as _sp
from pathlib import Path
from typing import Any, Callable, TYPE_CHECKING

if TYPE_CHECKING:
    from core.training.training import TrainingProgress

# WSL AMD GPU needs HSA_ENABLE_DXG_DETECTION=1 before torch init; spawned workers miss the installer env.
if sys.platform.startswith("linux") and "HSA_ENABLE_DXG_DETECTION" not in os.environ:
    try:
        if os.path.exists("/dev/dxg") and any(
            os.path.exists(_p + "/librocdxg.so") for _p in ("/opt/rocm/lib", "/opt/rocm/lib64")
        ):
            os.environ["HSA_ENABLE_DXG_DETECTION"] = "1"
    except Exception:
        pass

logger = get_logger(__name__)
from utils.child_stdio import utf8_child_env

# Fresh spawned interpreter: re-apply the process-wide network injections and CPU thread caps.
from utils.cpu_threads import install_openblas_runtime_cap
from utils.native_tls import activate_native_tls
from utils.happy_eyeballs import activate_happy_eyeballs

activate_native_tls()
activate_happy_eyeballs()
install_openblas_runtime_cap()

from utils.hardware import apply_gpu_ids
from utils.hf_dataset_options import hf_dataset_split_instruction_names

# Light module: the MLX branch runs on torch-less hosts.
from core.training.dataset_bounds import (
    bound_dataset_rows,
    max_train_rows_for_config,
    record_row_bound,
    row_bound_for_resume,
    slice_dataset_rows,
    world_size_from_env,
)
from core.training.resume import _checkpoint_state, session_eta_seconds
from utils.training_runs import build_default_output_dir_name
from utils.kernel_install import (
    CAUSAL_CONV1D,
    MAMBA_SSM,
    PinnedKernel,
    hipcc_gcc_install_dir,
    install_prebuilt,
    source_build_command,
    source_build_run_kwargs,
    uninstall_command,
)
from utils.ssm_runtime import (
    model_is_ssm as _model_is_ssm,
    model_wants_causal_conv1d as _model_wants_causal_conv1d,
)
from utils.wheel_utils import (
    direct_wheel_url,
    flash_attn_wheel_url,
    install_wheel,
    is_windows_wheel_env,
    pinned_source_build_supported,
    probe_torch_wheel_env,
    url_exists,
)


def _output_dir_from_resume_checkpoint(resume_from_checkpoint: str | None) -> str | None:
    if not resume_from_checkpoint:
        return None
    path = Path(resume_from_checkpoint)
    return str(path.parent if path.name.startswith("checkpoint-") else path)


def _data_parallel_world_size() -> int:
    """Larger of env world size and CUDA device count, never the sum: distributed runs force n_gpu to 1."""
    sizes = [world_size_from_env()]
    torch_module = sys.modules.get("torch")
    if torch_module is not None:
        try:
            distributed = getattr(torch_module, "distributed", None)
            if (
                distributed is not None
                and distributed.is_available()
                and distributed.is_initialized()
            ):
                sizes.append(int(distributed.get_world_size()))
        except Exception:
            # A stubbed or half-initialised torch.distributed must not fail a run.
            pass
        try:
            sizes.append(int(torch_module.cuda.device_count()))
        except Exception:
            # CPU-only builds answer 0 (dropped by the filter); a raising torch.cuda must not fail the run.
            pass
    return max([size for size in sizes if size > 0], default = 1)


def _model_local_files_only(config: dict) -> bool:
    return bool(config.get("model_snapshot_path"))


def _dataset_local_files_only(config: dict) -> bool:
    return bool(config.get("dataset_snapshot_path"))


def _adapter_trains_embeddings(model) -> bool:
    """Whether a loaded adapter already trains embed_tokens or lm_head in full (#13140)."""
    for adapter in (getattr(model, "peft_config", None) or {}).values():
        saved = list(getattr(adapter, "modules_to_save", None) or [])
        saved += list(getattr(adapter, "modules_to_tie", None) or [])
        if any(str(m).rsplit(".", 1)[-1] in ("embed_tokens", "lm_head") for m in saved):
            return True
    return False


def _adapter_base_changed_error(config: dict) -> str | None:
    """Refuse when the adapter now names a different base than the one checked at start (#13140)."""
    # None: no adapter was admitted. "": an adapter that named no base, which must stay that way.
    pinned = config.get("adapter_base_pin")
    if pinned is None:
        return None
    from utils.models.model_config import get_base_model_from_lora

    # The folder the loader reads: a cached Hub snapshot, not the repo id it came from.
    if (
        get_base_model_from_lora(str(_resolve_cached_model_load_name(config) or "")) or ""
    ) != pinned:
        return "The adapter's base model changed after the run was checked. Start the run again."
    return None


def _escaping_weights_error(config: dict) -> str | None:
    from utils.security.file_security import escaping_weight_references
    for key in ("model_name", "model_local_path", "model_snapshot_path", "adapter_base_pin"):
        if config.get(key) and escaping_weight_references(config[key]):
            return "This model's weight index names files outside its folder."
    return None


def _untrainable_model_format_error(config: dict) -> str | None:
    model_format = str(config.get("model_format") or "").strip().lower()
    if model_format == "gguf":
        return "GGUF models are inference-only and cannot be trained."
    if model_format == "adapter":
        # LoRA / CPT may continue from a saved adapter; full finetune still cannot (#13140).
        # The embedding and decision runners wrap a fresh adapter, so they still need a base.
        if config.get("training_type") in ("LoRA/QLoRA", "Continued Pretraining") and not (
            config.get("is_embedding", False) or config.get("is_decision", False)
        ):
            from utils.paths import is_local_path

            if not is_local_path(str(config.get("model_name") or "")):
                return (
                    "Continuing from a Hugging Face adapter repository is not supported yet. "
                    "Download it to a local folder and select that folder."
                )
            if config.get("is_dataset_image") or config.get("is_dataset_audio"):
                return (
                    "Continuing a saved adapter on an image or audio dataset is not supported "
                    "yet. Start from its base model instead."
                )
            return None
        return "Adapter models are inference-only and cannot be trained as base models."
    return None


def _lora_uses_loftq(config: dict) -> bool:
    """A resume rebuilds its adapter before loading the checkpoint, so it reruns LoftQ only when the checkpoint was
    trained with it: runs saved while Studio never applied LoftQ kept the original base weights."""
    if not config.get("use_loftq"):
        return False
    resume = config.get("resume_from_checkpoint")
    if not resume:
        return True
    import json

    from core.training.resume import get_resume_checkpoint_path

    checkpoint = Path(get_resume_checkpoint_path(resume) or resume)
    try:
        saved = json.loads((checkpoint / "adapter_config.json").read_text(encoding = "utf-8"))
    except (OSError, ValueError):
        return False
    return isinstance(saved, dict) and saved.get("init_lora_weights") == "loftq"


def _resolve_cached_model_load_name(config: dict) -> str:
    return config.get("model_snapshot_path") or config["model_name"]


def _worker_hf_token(config: dict):
    """Must stay three-valued: collapsing to None made a supplied API key count as the ambient caller."""
    from hub.utils.hf_tokens import hf_token_arg
    return hf_token_arg(
        config.get("hf_token"), allow_ambient_token = config.get("allow_ambient", True)
    )


def _effective_training_load_in_4bit(
    config: dict, model_load_target: str, hf_token: str | None
) -> bool:
    from .provenance import effective_training_load_in_4bit
    return effective_training_load_in_4bit(config, model_load_target, hf_token)


def _training_weight_prefetch(config, trainer, hf_token, event_queue):
    """Resolve only the ordinary SFT checkpoint; uncertain routes keep the existing sequential load."""
    try:
        from .model_weight_prefetch import ModelWeightPrefetch, prefetch_repo
        from utils.hf_cache_settings import get_hf_cache_paths
        from utils.hardware import dataset_map_num_proc

        repo = prefetch_repo(
            config,
            trainer,
            load_in_4bit = lambda: _effective_training_load_in_4bit(
                config, _resolve_cached_model_load_name(config), hf_token
            ),
            capabilities = sys.modules.get("unsloth.device_type"),
            offline = _model_offline_mode_enabled(),
            formatting_can_fork = (
                not config.get("dataset_streaming", False) and (dataset_map_num_proc() or 1) > 1
            ),
        )
        if repo is not None:
            return ModelWeightPrefetch(
                repo,
                token = hf_token,
                cache_dir = get_hf_cache_paths().hub_cache,
                emit = event_queue.put,
                should_stop = lambda: trainer.should_stop,
            )
    except Exception as error:  # noqa: BLE001
        logger.info("Skipping model checkpoint prefetch: %s", error)
    return None


def _drop_model_pin(config: dict) -> str:
    config["model_snapshot_path"] = None
    return config["model_name"]


def _drop_model_pin_for_fallback(config: dict, hf_token: str | None) -> str:
    from utils.transformers_version import get_transformers_activation_tier

    active_target = _resolve_cached_model_load_name(config)
    fallback_target = config["model_name"]
    if not config.get("model_revision"):
        active_tier = get_transformers_activation_tier(active_target, hf_token)
        fallback_tier = get_transformers_activation_tier(fallback_target, hf_token)
        if active_tier != fallback_tier:
            raise RuntimeError(
                "The cached model is incomplete and its Hugging Face fallback requires "
                f"a different Transformers runtime ({active_tier} to {fallback_tier}). "
                "Remove the incomplete cached model and retry."
            )
    return _drop_model_pin(config)


def _is_model_cache_artifact_error(error: BaseException | None) -> bool:
    """Scoped to the model-cache retry path, so a bare TypeError elsewhere is never retried."""
    from hub.utils.dataset_cache import is_cache_artifact_error

    if is_cache_artifact_error(error):
        return True
    markers = (
        "can't load processor for",
        "can't load image processor for",
        "can't load feature extractor for",
        "stat: path should be string, bytes, os.pathlike or integer, not nonetype",
        "expected str, bytes or os.pathlike object, not nonetype",
        # SentencePiece/BPE tokenizers fail on a missing vocab as a bare AttributeError; treat as cache miss.
        "'nonetype' object has no attribute 'endswith'",
        "'nonetype' object has no attribute 'readlines'",
        "argument should be a str or an os.pathlike object",
        "can't find a vocabulary file at path 'none'",
    )
    seen: set[int] = set()
    current = error
    while current is not None and id(current) not in seen:
        seen.add(id(current))
        if any(marker in str(current).lower() for marker in markers):
            return True
        current = current.__cause__ or current.__context__
    return False


def _package_installs_offline() -> bool:
    # UV_OFFLINE too: uv would refuse the index and install_wheel's pip fallback would not.
    try:
        from utils.utils import package_installs_offline
        return package_installs_offline()
    except Exception:
        return os.environ.get("UV_OFFLINE", "").strip().lower() in {
            "1",
            "true",
            "t",
            "yes",
            "y",
            "on",
        }


def _model_offline_mode_enabled() -> bool:
    try:
        from utils.utils import hf_env_offline
        return hf_env_offline()
    except Exception:
        pass
    return any(
        str(os.environ.get(name, "")).strip().lower() in {"1", "true", "yes", "on"}
        for name in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE")
    )


def _cache_artifact_fallback_allowed(
    config: dict, error: BaseException | None, resource: str
) -> bool:
    require_exact = bool(
        config.get("require_exact_resume_resources")
        or config.get(f"require_exact_{resource}_resource")
    )
    if resource == "dataset":
        from hub.utils.dataset_cache import dataset_cache_fallback_allowed
        return dataset_cache_fallback_allowed(
            error,
            require_exact = require_exact,
            revision = config.get("dataset_revision"),
        )
    if require_exact or _model_offline_mode_enabled():
        return False
    return _is_model_cache_artifact_error(error)


def _model_cache_fallback_error(config: dict, error: BaseException | None) -> RuntimeError | None:
    """Return an actionable error when an incomplete cache cannot be repaired."""
    if not _is_model_cache_artifact_error(error):
        return None
    if config.get("require_exact_resume_resources") or config.get("require_exact_model_resource"):
        return RuntimeError(
            "The exact cached model snapshot is incomplete, so this run cannot "
            "preserve its recorded model resources. Restore the missing model, "
            "tokenizer, or processor files and retry."
        )
    if _model_offline_mode_enabled():
        revision = config.get("model_revision")
        revision_text = f" at revision {revision}" if revision else ""
        return RuntimeError(
            "Offline mode is enabled, but the cached model snapshot is incomplete. "
            "Reconnect to download the missing model, tokenizer, or processor files"
            f"{revision_text}, or select a complete local model."
        )
    return None


def _mlx_revision_fallback_error(config: dict) -> RuntimeError | None:
    """MLX remaps bnb repos to a full-precision base, where the pinned commit has no guaranteed meaning."""
    model_name = str(config.get("model_name") or "")
    revision = config.get("model_revision")
    if (
        revision
        and model_name.startswith("unsloth/")
        and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit"))
    ):
        return RuntimeError(
            "The cached model snapshot is incomplete, but MLX cannot safely retry "
            f"'{model_name}' at revision {revision}: MLX maps its bitsandbytes "
            "weights to a different base repository. Select that full-precision "
            "base model directly, or restore the missing cached files."
        )
    return None


def _require_strict_cached_dataset(config: dict, dataset: Any, split: str) -> Any:
    if (
        config.get("require_exact_resume_resources") or config.get("require_exact_dataset_resource")
    ) and dataset is None:
        raise FileNotFoundError(f"The exact cached dataset split '{split}' is no longer available.")
    return dataset


def _offline_mode_enabled() -> bool:
    return any(
        str(os.environ.get(name, "")).strip().lower() in {"1", "true", "yes", "on"}
        for name in ("HF_HUB_OFFLINE", "HF_DATASETS_OFFLINE")
    )


def _verify_config_pins(config: dict, event_queue: Any) -> bool:
    require_model = bool(
        config.get("require_exact_resume_resources") or config.get("require_exact_model_resource")
    )
    require_dataset = bool(
        config.get("require_exact_resume_resources") or config.get("require_exact_dataset_resource")
    )
    if require_model or require_dataset:
        from core.training.provenance import (
            ExactResumeResourcesUnavailable,
            validate_exact_dataset_pin,
            validate_exact_model_pin,
        )

        try:
            model_snapshot = validate_exact_model_pin(config) if require_model else None
            dataset_snapshot = validate_exact_dataset_pin(config) if require_dataset else None
        except ExactResumeResourcesUnavailable as error:
            event_queue.put(
                {
                    "type": "error",
                    "error": str(error),
                    "stack": "",
                    "ts": time.time(),
                }
            )
            return False
        if model_snapshot is not None:
            config["model_snapshot_path"] = model_snapshot
            config["model_revision"] = Path(model_snapshot).name
        if dataset_snapshot is not None:
            config["dataset_snapshot_path"] = dataset_snapshot

    for message in config.get("cache_pin_warnings") or []:
        _send_status(event_queue, message)
    model_path = config.get("model_snapshot_path")
    require_validated_snapshot = bool(config.get("require_validated_model_snapshot"))
    if require_validated_snapshot and not model_path:
        event_queue.put(
            {
                "type": "error",
                "error": (
                    "The cached model snapshot selected during preflight is no longer available."
                ),
                "stack": "",
                "ts": time.time(),
            }
        )
        return False
    if model_path and not require_model:
        from hub.utils.hf_cache_state import (
            latest_snapshot_from_cache_path,
            with_load_subdirs,
        )
        from utils.utils import canonical_model_repo_id

        pinned_repo_id = config.get("actual_model_repo_id") or canonical_model_repo_id(
            config["model_name"]
        )
        config["model_snapshot_path"] = latest_snapshot_from_cache_path(
            model_path,
            "model",
            pinned_repo_id,
            with_load_subdirs(config["model_name"], ("config.json", "adapter_config.json")),
        )
        if config["model_snapshot_path"] is None:
            if require_validated_snapshot:
                event_queue.put(
                    {
                        "type": "error",
                        "error": (
                            "The cached model snapshot selected during preflight is no "
                            "longer available."
                        ),
                        "stack": "",
                        "ts": time.time(),
                    }
                )
                return False
            if not config.get("model_revision"):
                config["actual_model_repo_id"] = None
        else:
            config["model_revision"] = Path(config["model_snapshot_path"]).name
    dataset_path = config.get("dataset_snapshot_path")
    if dataset_path and not require_dataset:
        from hub.utils.dataset_cache import (
            dataset_cache_path_from_cache_path,
            dataset_snapshot_from_cache_path,
        )

        resolved = dataset_cache_path_from_cache_path(
            dataset_path,
            config.get("hf_dataset") or "",
        )
        snapshot = (
            dataset_snapshot_from_cache_path(
                str(resolved),
                config.get("hf_dataset") or "",
            )
            if resolved is not None
            else None
        )
        if snapshot is not None:
            config["dataset_revision"] = snapshot.name
        config["dataset_snapshot_path"] = str(resolved) if resolved else None
    if (
        config.get("dataset_revision")
        and not config.get("dataset_snapshot_path")
        and _offline_mode_enabled()
    ):
        event_queue.put(
            {
                "type": "error",
                "error": (
                    "The selected dataset snapshot is incomplete and its exact "
                    "revision cannot be downloaded while offline."
                ),
                "stack": "",
                "ts": time.time(),
            }
        )
        return False
    return True


def _mlx_diffusion_lm_error(config: dict, model_load_target: str) -> str | None:
    from utils.models.diffusion_lm import mlx_refusal
    return mlx_refusal(
        model_load_target, _worker_hf_token(config), revision = config.get("model_revision")
    )


def _validate_training_worker_config(config: dict, event_queue: Any) -> bool:
    if not _verify_config_pins(config, event_queue):
        return False
    format_error = _untrainable_model_format_error(config) or _adapter_base_changed_error(config)
    if format_error:
        event_queue.put(
            {
                "type": "error",
                "error": format_error,
                "stack": "",
                "ts": time.time(),
            }
        )
        return False
    return True


def _cached_dataset_row_limit(config: dict) -> int | None:
    slice_end = config.get("dataset_slice_end")
    slice_start = config.get("dataset_slice_start")
    if isinstance(slice_end, bool) or not isinstance(slice_end, int) or slice_end < 0:
        return None
    if slice_start is None:
        slice_start = 0
    if isinstance(slice_start, bool) or not isinstance(slice_start, int):
        return None
    return slice_end + 1 if slice_end >= max(slice_start, 0) else None


def _load_cached_dataset_for_config(
    config: dict,
    split: str | None,
    token: str | None = None,
    *,
    row_limit: int | None = None,
):
    hf_dataset = config.get("hf_dataset")
    local_path = config.get("dataset_snapshot_path")
    if not hf_dataset or not local_path:
        return None
    from hub.utils.dataset_cache import load_cached_hf_dataset

    kwargs = {
        "subset": config.get("subset"),
        "split": split or "train",
        "token": token,
    }
    if row_limit is not None:
        kwargs["row_limit"] = row_limit
    return load_cached_hf_dataset(hf_dataset, local_path, **kwargs)


def _load_hf_train_and_eval_datasets(
    config: dict,
    token: str | None,
    load_dataset: Callable,
    status_callback: Callable[[str], None],
    warning_callback: Callable[[str], None] | None = None,
):
    from core.training.eval_dataset import (
        EVAL_SPLIT_CANDIDATES,
        MIN_EVAL_ROWS,
        STREAMING_EVAL_MAX_SAMPLES,
        evaluation_enabled,
    )

    hf_dataset = config["hf_dataset"]
    subset = config.get("subset")
    train_split = config.get("train_split", "train") or "train"
    eval_split = config.get("eval_split")
    revision = config.get("dataset_revision")
    eval_enabled = evaluation_enabled(config.get("eval_steps"))
    require_exact = bool(
        config.get("require_exact_resume_resources") or config.get("require_exact_dataset_resource")
    )
    dataset = None
    loaded_from_cache = False
    config["_dataset_loaded_from_exact_snapshot"] = False

    def warn(message: str) -> None:
        if warning_callback is not None:
            warning_callback(message)
        else:
            logger.warning(message)

    def load_remote(split: str, streaming: bool = False):
        kwargs = {"split": split, "token": token}
        if subset:
            kwargs["name"] = subset
        if revision:
            kwargs["revision"] = revision
        if streaming:
            kwargs["streaming"] = True
        return load_dataset(hf_dataset, **kwargs)

    if config.get("dataset_streaming"):
        # As on CUDA: eval needs its own split, capped so each pass ends.
        status_callback(f"Streaming dataset: {hf_dataset}")
        dataset = load_remote(train_split, streaming = True)
        eval_dataset = None
        if eval_enabled and eval_split and eval_split != train_split:
            eval_dataset = load_remote(eval_split, streaming = True).take(STREAMING_EVAL_MAX_SAMPLES)
        return dataset, eval_dataset

    if _dataset_local_files_only(config):
        status_callback(f"Loading cached dataset: {hf_dataset}")
        try:
            row_limit = _cached_dataset_row_limit(config)
            if row_limit is None:
                dataset = _load_cached_dataset_for_config(config, train_split, token)
            else:
                dataset = _load_cached_dataset_for_config(
                    config,
                    train_split,
                    token,
                    row_limit = row_limit,
                )
            dataset = _require_strict_cached_dataset(config, dataset, train_split)
            loaded_from_cache = dataset is not None
        except Exception as error:
            if not _cache_artifact_fallback_allowed(config, error, "dataset"):
                raise
            status_callback("Cached dataset unavailable; downloading from the Hub...")

    if dataset is None:
        dataset = load_remote(train_split)

    eval_dataset = None
    explicit_separate_eval = bool(eval_split and eval_split != train_split)
    if eval_enabled and explicit_separate_eval:
        if loaded_from_cache:
            try:
                eval_dataset = _load_cached_dataset_for_config(config, eval_split, token)
                eval_dataset = _require_strict_cached_dataset(
                    config,
                    eval_dataset,
                    eval_split,
                )
            except Exception as error:
                if not _cache_artifact_fallback_allowed(config, error, "dataset"):
                    raise
                status_callback(
                    "Cached eval split unavailable; reloading train and eval from the Hub..."
                )
                remote_train = load_remote(train_split)
                remote_eval = load_remote(eval_split)
                dataset = remote_train
                eval_dataset = remote_eval
                loaded_from_cache = False
        else:
            eval_dataset = load_remote(eval_split)
    elif eval_enabled and not eval_split:
        auto_errors: list[tuple[str, Exception]] = []
        try:
            split_info = getattr(getattr(dataset, "info", None), "splits", None)
            if loaded_from_cache:
                available_splits = list(split_info or ())
            else:
                from datasets import get_dataset_split_names

                split_kwargs = {"path": hf_dataset}
                if subset:
                    split_kwargs["config_name"] = subset
                if revision:
                    split_kwargs["revision"] = revision
                if token:
                    split_kwargs["token"] = token
                available_splits = get_dataset_split_names(**split_kwargs)

            excluded_splits = set(hf_dataset_split_instruction_names(train_split))
            for candidate in EVAL_SPLIT_CANDIDATES:
                if candidate not in available_splits or candidate in excluded_splits:
                    continue
                try:
                    if loaded_from_cache:
                        candidate_dataset = _load_cached_dataset_for_config(
                            config,
                            candidate,
                            token,
                        )
                        candidate_dataset = _require_strict_cached_dataset(
                            config,
                            candidate_dataset,
                            candidate,
                        )
                    else:
                        candidate_dataset = load_remote(candidate)
                except Exception as error:
                    if require_exact:
                        raise
                    auto_errors.append((candidate, error))
                    continue
                if len(candidate_dataset) >= MIN_EVAL_ROWS:
                    eval_dataset = candidate_dataset
                    break
        except Exception as error:
            if require_exact:
                raise
            warn(
                "Automatic eval split detection failed; a held-out split will be created "
                f"from the training data when enough rows are available: {error}"
            )
        else:
            if eval_dataset is None and auto_errors:
                candidate, error = auto_errors[0]
                warn(
                    f"Automatic eval split '{candidate}' could not be loaded; a held-out "
                    "split will be created from the training data when enough rows are "
                    f"available: {error}"
                )

    from core.training.provenance import (
        attest_loaded_dataset,
        exact_dataset_snapshot_path,
    )

    snapshot, _ = attest_loaded_dataset(hf_dataset, dataset, eval_dataset)
    if snapshot is None and loaded_from_cache:
        snapshot = exact_dataset_snapshot_path(
            config.get("dataset_snapshot_path"),
            hf_dataset,
        )
    if snapshot is not None:
        config["dataset_snapshot_path"] = snapshot
        config["_dataset_loaded_from_exact_snapshot"] = True
    return dataset, eval_dataset


def _load_embedding_hf_dataset(
    config: dict, load_dataset: Callable, status_callback: Callable[[str], None]
):
    hf_dataset = str(config.get("hf_dataset") or "").strip()
    if not hf_dataset:
        return None

    subset = config.get("subset") or None
    train_split = config.get("train_split", "train") or "train"
    revision = config.get("dataset_revision")
    token = _worker_hf_token(config)
    dataset = None
    config["_dataset_loaded_from_exact_snapshot"] = False

    if _dataset_local_files_only(config):
        status_callback(f"Loading cached dataset: {hf_dataset}")
        try:
            row_limit = _cached_dataset_row_limit(config)
            if row_limit is None:
                dataset = _load_cached_dataset_for_config(
                    config,
                    train_split,
                    token,
                )
            else:
                dataset = _load_cached_dataset_for_config(
                    config,
                    train_split,
                    token,
                    row_limit = row_limit,
                )
            dataset = _require_strict_cached_dataset(
                config,
                dataset,
                train_split,
            )
            if dataset is not None:
                from core.training.provenance import exact_dataset_snapshot_path
                snapshot = exact_dataset_snapshot_path(
                    config.get("dataset_snapshot_path"),
                    hf_dataset,
                )
                if snapshot is not None:
                    config["dataset_snapshot_path"] = snapshot
                    config["_dataset_loaded_from_exact_snapshot"] = True
        except Exception as error:
            if not _cache_artifact_fallback_allowed(config, error, "dataset"):
                raise
            status_callback("Cached dataset unavailable; downloading from the Hub...")
            dataset = None
            config["_dataset_loaded_from_exact_snapshot"] = False

    if dataset is None:
        load_kwargs = {
            "split": train_split,
            "token": token,
        }
        if revision:
            load_kwargs["revision"] = revision
        dataset = load_dataset(hf_dataset, subset, **load_kwargs)
    from core.training.provenance import attest_loaded_dataset

    snapshot, _ = attest_loaded_dataset(hf_dataset, dataset)
    if snapshot is not None:
        config["dataset_snapshot_path"] = snapshot
        config["_dataset_loaded_from_exact_snapshot"] = True
    return dataset


def _pre_detect_load_in_4bit(config: dict, model_load_target: str, hf_token: str | None) -> bool:
    """Mirrors the real load's 4-bit flip so detection reads the repo the loader fetches."""
    if not bool(config.get("load_in_4bit", True)):
        return False
    from utils.transformers_version import latest_tier_active_for

    return not latest_tier_active_for(model_load_target, hf_token)


def _pre_detect_training_model(
    trainer,
    config: dict,
    model_name: str,
    hf_token: str | None,
    model_load_name: str,
    local_files_only: bool,
    model_revision: str | None = None,
) -> None:
    trainer.pre_detect_and_load_tokenizer(
        model_name = model_name,
        max_seq_length = config["max_seq_length"],
        hf_token = hf_token,
        is_dataset_image = config.get("is_dataset_image", False),
        is_dataset_audio = config.get("is_dataset_audio", False),
        trust_remote_code = config.get("trust_remote_code", False),
        model_load_name = model_load_name,
        local_files_only = local_files_only,
        model_revision = model_revision,
        load_in_4bit = _pre_detect_load_in_4bit(config, model_load_name, hf_token),
    )
    _check_finetune_targets_after_detect(trainer, config)
    _apply_diffusion_lm_overrides(trainer, config)


_LORA_DEFAULT_LR = 2e-4
# Near-duplicate head rows for <|im_end|> / <tool_call> / ... (cosine 0.97): a 2e-4 LoRA swaps them in chat.
_DIFFUSION_LM_LORA_LR = {"dream": 5e-5, "sdar": 5e-5}


def _apply_diffusion_lm_overrides(trainer, config: dict) -> None:
    """Text-diffusion runs never pack: packed rows would share one bidirectional attention window. Set before
    the row bound and online tokenization read the flag, so their accounting matches what trains."""
    profile = getattr(trainer, "diffusion_lm_profile", None)
    if not profile:
        return
    config["diffusion_lm_profile"] = profile
    if config.get("packing"):
        config["packing"] = False
        logger.info("Text-diffusion model (%s): sequence packing turned off", profile)
    lowered = _DIFFUSION_LM_LORA_LR.get(profile)
    try:
        at_default = float(config.get("learning_rate", _LORA_DEFAULT_LR)) == _LORA_DEFAULT_LR
    except (TypeError, ValueError):
        at_default = False  # the trainer reports the invalid value itself
    if lowered is not None and at_default:
        config["learning_rate"] = lowered
        logger.info(
            "Text-diffusion model (%s): learning rate 2e-4 lowered to %g; set any other value to override",
            profile,
            lowered,
        )


_NOTHING_TO_TRAIN = (
    "Nothing to train: select at least one layer family (finetune_language_layers or "
    "finetune_vision_layers) and at least one module type (finetune_attention_modules or "
    "finetune_mlp_modules)."
)


def _finetune_selectors(config: dict) -> tuple[bool, bool, bool, bool]:
    """Omitted keys take the CUDA consumer's defaults, so the guard models runs as they really train."""
    return (
        bool(config.get("finetune_vision_layers", True)),
        bool(config.get("finetune_language_layers", True)),
        bool(config.get("finetune_attention_modules", True)),
        bool(config.get("finetune_mlp_modules", True)),
    )


def _requests_all_linear(config: dict) -> bool:
    """all-linear forces all selectors on in PEFT, so every linear layer trains regardless of selectors."""
    target_modules = config.get("target_modules")
    if isinstance(target_modules, str):
        return target_modules == "all-linear"
    if isinstance(target_modules, (list, tuple)):
        return list(target_modules) == ["all-linear"]
    return False


def _check_finetune_targets_after_detect(trainer, config: dict) -> None:
    """Needs detection to settle the branch, so it runs after pre_detect and before any weights load."""
    if config.get("training_type", "LoRA/QLoRA") != "LoRA/QLoRA":
        return  # Full Finetuning / CPT build adapters from target_modules alone
    if not (getattr(trainer, "is_vlm", False) or getattr(trainer, "is_audio_vlm", False)):
        return  # the text branch ignores these four
    if _requests_all_linear(config):
        return  # get_peft_model turns all five selectors on for the keyword; see below
    vision, language, attention, mlp = _finetune_selectors(config)
    # Mirror get_peft_regex's two guards: one layer family AND one module type.
    if not (vision or language) or not (attention or mlp):
        raise ValueError(_NOTHING_TO_TRAIN)


# On CPT, MLX trains these regardless of the layer-family flags.
_CPT_TARGET_NAMES = frozenset({"embed_tokens", "lm_head"})
_MLX_DEFAULT_LORA_TARGETS = (
    "q_proj",
    "k_proj",
    "v_proj",
    "o_proj",
    "gate_proj",
    "up_proj",
    "down_proj",
)


def _trains_embedding_modules(trainable_names, full_module_keys = ()) -> bool:
    """Whether the embedding or output head trains in full, from a fresh CPT wrap or a continued adapter.

    ``full_module_keys`` are the weights zoo resolved for embed_tokens / lm_head, which cover heads named otherwise.
    """
    for name in trainable_names:
        if name in full_module_keys:
            return True
        parts = set(name.split("."))
        if parts & _CPT_TARGET_NAMES and not parts & {"lora_a", "lora_b"}:
            return True
    return False


def _mlx_quantized_cpt_modules(model, names) -> set:
    """The requested embed_tokens / lm_head this load quantized (a 4-bit checkpoint or load), which zoo cannot
    train in full. A tied head is the embedding, so it goes with it."""
    quantized, found = set(), set()
    for path, module in model.named_modules():
        leaf = path.rsplit(".", 1)[-1]
        if leaf in names:
            found.add(leaf)
            if any(hasattr(child, "scales") for _, child in module.named_modules()):
                quantized.add(leaf)
    if "embed_tokens" in quantized and "lm_head" in names and "lm_head" not in found:
        quantized.add("lm_head")
    return quantized


def _mlx_cpt_embedding_learning_rate(requested, lr_value, *, trains_embeddings):
    """The CUDA CPT rule: lr/10 unless set, and dropped when no embedding module trains."""
    if not trains_embeddings:
        if requested is not None:
            logger.warning(
                "CPT: embedding_learning_rate was provided but neither embed_tokens "
                "nor lm_head is being trained; ignoring the override."
            )
        return None
    if requested is None:
        return lr_value / 10.0
    return float(requested)


def _names_a_cpt_target(target_modules) -> bool:
    """Whether an explicit target list names something that trains on its own."""
    if isinstance(target_modules, str):
        return target_modules in _CPT_TARGET_NAMES
    try:
        return any(name in _CPT_TARGET_NAMES for name in target_modules)
    except TypeError:  # not iterable -> not a list of names, so nothing is guaranteed
        return False


def _check_mlx_finetune_targets(config: dict) -> None:
    """A fused qkv survives the module filter but still gets no adapters when language layers are off."""
    targets = config.get("target_modules")
    if targets:
        if _names_a_cpt_target(targets):
            return
        _, language, attention, mlp = _finetune_selectors(config)
        # MLX defaults vision to False (unlike _finetune_selectors), so legacy configs are not waved through.
        vision = bool(config.get("finetune_vision_layers", False))
        # Vision counts: this runs before VLM detection; _check_mlx_effective_targets handles text later.
        if attention or mlp or language or vision:
            return
        raise ValueError(_NOTHING_TO_TRAIN)
    _, _, attention, mlp = _finetune_selectors(config)
    if not (attention or mlp):
        raise ValueError(_NOTHING_TO_TRAIN)


def _check_mlx_effective_targets(
    config: dict, *, finetune_language: bool, finetune_vision: bool
) -> None:
    """Rechecks post-backfill values; with both layer families off, the run trains nothing."""
    if finetune_language or finetune_vision:
        return
    if _names_a_cpt_target(config.get("target_modules") or ()):
        return
    raise ValueError(_NOTHING_TO_TRAIN)


def _fetch_spark_tts_repo(trainer, event_queue: Any, model_name: str) -> None:
    """Spark-TTS encodes its dataset with the BiCodec weights in the model repo, so the repo is fetched before the
    dataset step (#13458), under the model-load lifecycle so a stalled Xet download still respawns over HTTP."""
    from utils.hf_xet_fallback import start_watchdog

    _send_status(event_queue, "Loading model...")
    event_queue.put({"type": "model_load_started", "ts": time.time()})
    # The download lands under the resolved repo, not the "Spark-TTS-0.5B/LLM" alias.
    watchdog_stop = start_watchdog(
        repo_ids = list(dict.fromkeys([model_name, trainer.spark_tts_hub_repo()])),
        on_stall = lambda msg: event_queue.put({"type": "stall", "message": msg, "ts": time.time()}),
        xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
    )
    try:
        trainer.resolve_spark_tts_repo(
            on_model_resolved = lambda repo_id: event_queue.put(
                {"type": "model_load_resolved", "repo_id": repo_id, "ts": time.time()}
            ),
        )
    finally:
        watchdog_stop.set()
        event_queue.put({"type": "model_load_completed", "ts": time.time()})


def _reload_dataset_with_remote_model_tokenizer(
    trainer,
    config: dict,
    model_name: str,
    hf_token: str | None,
    reload_dataset: Callable[[], tuple],
    model_revision: str | None = None,
):
    _pre_detect_training_model(
        trainer,
        config,
        model_name,
        hf_token,
        model_name,
        False,
        model_revision,
    )
    return reload_dataset()


def _strip_unsloth_bnb_4bit_suffix(model_name: str) -> str:
    """unsloth.models.loader._strip_unsloth_bnb_4bit_suffix, copied rather than imported: that
    module pulls in torch, and this runs before the worker is allowed to."""
    stripped = model_name
    for suffix in ("-unsloth-bnb-4bit", "-bnb-4bit"):
        if len(stripped) >= len(suffix) and stripped.lower().endswith(suffix):
            stripped = stripped[: -len(suffix)]
    return stripped


def _model_load_security_error(config: dict, load_target: str, hf_token: str | None) -> dict | None:
    from utils.models.model_config import get_base_model_from_lora_identifier
    from utils.security import (
        evaluate_file_security,
        evaluate_remote_code_consent_for_targets,
        load_scan_target,
        security_load_subdirs,
    )

    requested_targets = [load_target]
    try:
        base_model = get_base_model_from_lora_identifier(load_target, hf_token)
        if base_model:
            requested_targets.append(base_model)
    except Exception as error:
        logger.debug("Could not resolve LoRA base for security scan: %s", error)

    # Scan mirrors the loader substitutes too, else fetched bytes skip the malware/consent scan.
    # Must not import torch/unsloth here; only the torch loader path uses this mapper.
    try:
        from core.training.training import should_use_mlx_training_backend
        from utils.models.unsloth_mirror import unsloth_public_mirror

        torch_loader_path = not config.get("is_embedding", False) and (
            not should_use_mlx_training_backend()
        )
        if (
            torch_loader_path
            and not _model_local_files_only(config)
            and not config.get("model_revision")
        ):
            # Fallbacks only go 4-bit -> 16-bit, so a 16-bit config never scans the 4-bit mirror.
            _modes = (True, False) if config.get("load_in_4bit", True) else (False,)
            # Expand every seed: a remote LoRA's base has its own mirror that gets downloaded.
            for _seed in list(dict.fromkeys(requested_targets)):
                for _mode in _modes:
                    mirrored = unsloth_public_mirror(_seed, _mode)
                    if mirrored and mirrored != _seed:
                        requested_targets.append(mirrored)
                        # Without prequantized models (ROCm bnb < 0.49.2) the loader strips the 4-bit suffix; scan that repo too.
                        stripped = _strip_unsloth_bnb_4bit_suffix(mirrored)
                        if stripped != mirrored and stripped != _seed:
                            requested_targets.append(stripped)
    except Exception as error:  # noqa: BLE001
        logger.debug("Could not resolve the mirror for the security scan: %s", error)

    from utils.utils import hf_env_offline

    primary_name = config["model_name"]
    local_only_load = hf_env_offline()
    consent_load_subdirs: dict[str, tuple] = {}
    targets: list[str] = []
    for requested_target in dict.fromkeys(requested_targets):
        load_subdirs = security_load_subdirs(requested_target, hf_token)
        if requested_target == load_target and requested_target != primary_name:
            load_subdirs = tuple(
                dict.fromkeys((*load_subdirs, *security_load_subdirs(primary_name, hf_token)))
            )
        target, load_subdirs = load_scan_target(requested_target, load_subdirs)
        if target not in consent_load_subdirs:
            targets.append(target)
            consent_load_subdirs[target] = ()
        load_subdirs = tuple(dict.fromkeys((*consent_load_subdirs[target], *load_subdirs)))
        consent_load_subdirs[target] = load_subdirs
        decision = evaluate_file_security(
            target,
            hf_token = hf_token,
            load_subdirs = load_subdirs,
            local_only_load = local_only_load,
        )
        if decision.blocked:
            return {
                "error": decision.reason,
                "error_kind": "malware_blocked",
                "security": decision.response_payload(),
            }

    if not config.get("trust_remote_code", False):
        return None

    decision = evaluate_remote_code_consent_for_targets(
        targets,
        hf_token = hf_token,
        trust_remote_code = True,
        approved_fingerprint = config.get("approved_remote_code_fingerprint"),
        subject = config.get("subject"),
        load_subdirs_by_target = consent_load_subdirs,
    )
    if not decision.blocked:
        return None
    return {
        "error": (
            f"{decision.blocked_message()}\n\n{decision.findings_summary}"
            if decision.max_severity
            else decision.blocked_message()
        ),
        "error_kind": "remote_code_blocked",
        "remote_code": decision.response_payload(),
    }


_FLASH_ATTN_RUNTIME_MIN_SEQ_LEN = 32768
_FLASH_ATTN_SKIP_ENV = "UNSLOTH_STUDIO_SKIP_FLASHATTN_INSTALL"
_FAST_PATH_HOOKS_SKIP_ENV = "UNSLOTH_STUDIO_SKIP_FAST_PATH_HOOKS"

# Module scope so the torch.library.Library registration isn't GC'd mid-run.
_WINDOWS_ROCM_GROUPED_MM_LIB = None


def _install_grouped_mm_cpu_fallback(torch_mod, logger, label):
    """Works around a null HIP _grouped_mm kernel on RDNA4 and ROCm <= 7.12; keep the Library referenced."""
    import warnings as _warnings

    _gm_lib = torch_mod.library.Library("aten", "IMPL")

    def _grouped_mm_safe_impl(
        self,
        mat2,
        offs = None,
        bias = None,
        out_dtype = None,
    ):
        """Python mm/bmm fallback for _grouped_mm on gfx120X (null HIP kernel, ROCm <= 7.12)."""
        _t = torch_mod
        if offs is None:
            # No offsets: 2-D -> mm, 3-D batched -> bmm (unconditional mm broke 3-D MoE).
            if self.dim() == 3 and mat2.dim() == 3:
                result = _t.bmm(self.contiguous(), mat2.contiguous())
            elif self.dim() == 3 and mat2.dim() == 2:
                result = _t.matmul(self.contiguous(), mat2.contiguous())
            elif self.dim() == 2 and mat2.dim() == 3:
                result = _t.matmul(self.contiguous(), mat2.contiguous())
            else:
                result = _t.mm(self.contiguous(), mat2.contiguous())
        else:
            # Grouped: offs[i] is the exclusive end-row of group i.
            offs_list = offs.tolist()
            pieces = []
            prev = 0
            for idx, end in enumerate(offs_list):
                end = int(end)
                a_part = self[prev:end].contiguous()
                b_part = mat2[idx].contiguous() if mat2.dim() == 3 else mat2.contiguous()
                pieces.append(_t.mm(a_part, b_part))
                prev = end
            # Include trailing rows not covered by offs.
            if prev < self.shape[0]:
                a_tail = self[prev:].contiguous()
                b_tail = mat2[-1].contiguous() if mat2.dim() == 3 else mat2.contiguous()
                pieces.append(_t.mm(a_tail, b_tail))
            result = (
                _t.cat(pieces, dim = 0)
                if pieces
                else _t.zeros(0, mat2.shape[-1], device = self.device, dtype = self.dtype)
            )
        if bias is not None:
            result = result + bias
        if out_dtype is not None:
            result = result.to(out_dtype)
        elif result.dtype != self.dtype:
            result = result.to(self.dtype)
        return result

    with _warnings.catch_warnings():
        _warnings.simplefilter("ignore")
        _gm_lib.impl("_grouped_mm", _grouped_mm_safe_impl, "CUDA")
    logger.info(
        "%s: patched _grouped_mm CUDA dispatch (null HIP kernel on gfx120X, "
        "ROCm <= 7.12 -- bypassed with Python mm fallback)",
        label,
    )
    return _gm_lib


# Subprocesses don't inherit os.add_dll_directory; replicate main.py's ROCm DLL setup.
_ROCM_DLL_HANDLES: list = []
if sys.platform == "win32":

    def _add_rocm_dll_dirs_worker() -> None:
        _candidates: list[str] = []
        for _var in ("HIP_PATH", "ROCM_PATH"):
            _val = os.environ.get(_var)
            if _val:
                _candidates.append(os.path.join(_val, "bin"))
        _default_root = os.path.join(
            os.environ.get("ProgramFiles", r"C:\Program Files"), "AMD", "ROCm"
        )

        def _ver_key(name: str) -> tuple:
            # Numeric tuple key so "10.0" sorts after "7.0".
            parts = []
            for chunk in name.split("."):
                try:
                    parts.append((0, int(chunk)))
                except ValueError:
                    parts.append((1, chunk))
            return tuple(parts)

        try:
            if os.path.isdir(_default_root):
                for _ver in sorted(os.listdir(_default_root), key = _ver_key, reverse = True):
                    _bin = os.path.join(_default_root, _ver, "bin")
                    if os.path.isdir(_bin):
                        _candidates.append(_bin)
        except OSError:
            pass
        for _d in _candidates:
            if os.path.isdir(_d):
                try:
                    _ROCM_DLL_HANDLES.append(os.add_dll_directory(_d))
                except (OSError, AttributeError):
                    pass

    _add_rocm_dll_dirs_worker()
    del _add_rocm_dll_dirs_worker


def _decision_has_llm_backbone(model_load_target: str, hf_token: str | None) -> bool:
    # Clef ships joint_head_config.json next to its Qwen3.5 backbone; Laya never does.
    marker = "joint_head_config.json"
    try:
        if (Path(model_load_target) / marker).is_file():
            return True
        from huggingface_hub import HfApi

        siblings = HfApi(token = hf_token).model_info(model_load_target).siblings or ()
        return any(getattr(s, "rfilename", None) == marker for s in siblings)
    except Exception:
        return False


_hipcc_gcc_install_dir = hipcc_gcc_install_dir


def _is_importable(import_name: str) -> bool:
    # Invalidate finder caches so a package installed earlier in this process is seen.
    importlib.invalidate_caches()
    try:
        __import__(import_name)
        return True
    except Exception as exc:
        # Wrong-arch wheels raise OSError/RuntimeError, not ImportError.
        logger.debug("%s is not importable (%s: %s)", import_name, type(exc).__name__, exc)
        return False


# Kept in step with install_python_stack._FLASH_ATTN_IMPORT_PROBE_TIMEOUT.
_IMPORT_PROBE_TIMEOUT = 300


def _is_importable_isolated(import_name: str) -> bool:
    """Probe the import in a child: a wrong-arch wheel can abort or segfault, killing this worker."""
    return _import_probe_isolated(
        import_name, "import importlib, sys; importlib.import_module(sys.argv[1])"
    )


def _extension_importable_isolated(extension: str) -> bool:
    """_is_importable_isolated for a bare torch extension (selective_scan_cuda), which needs
    torch's libraries loaded first."""
    return _import_probe_isolated(
        extension, "import torch, importlib, sys; importlib.import_module(sys.argv[1])"
    )


def _import_probe_isolated(import_name: str, code: str) -> bool:
    try:
        result = _sp.run(
            [sys.executable, "-c", code, import_name],
            stdout = _sp.DEVNULL,
            stderr = _sp.DEVNULL,
            timeout = _IMPORT_PROBE_TIMEOUT,
            # No env override: the probe must see what the real in-process import will.
        )
    except (OSError, _sp.TimeoutExpired) as exc:
        logger.debug("%s import probe did not complete (%s)", import_name, exc)
        return False
    if result.returncode != 0:
        logger.debug("%s import probe exited %s", import_name, result.returncode)
    return result.returncode == 0


def _uninstall_package(pypi_name: str, display_name: str) -> bool:
    """Remove a distribution. True iff it is gone afterwards."""
    result = _sp.run(
        uninstall_command(pypi_name, use_uv = bool(shutil.which("uv"))),
        stdout = _sp.PIPE,
        stderr = _sp.STDOUT,
        text = True,
        encoding = "utf-8",
        errors = "replace",
        env = utf8_child_env(),
    )
    if result.returncode != 0:
        logger.warning("Could not remove the rejected %s install:\n%s", display_name, result.stdout)
        return False
    return True


def _distribution_present(pypi_name: str) -> bool:
    """Metadata alone decides: leftover metadata makes unsloth import a native module that may abort."""
    importlib.invalidate_caches()
    try:
        importlib.metadata.distribution(pypi_name)
        return True
    except Exception:
        return False


def _reject_install(event_queue: Any, pypi_name: str, display_name: str, reason: str) -> None:
    """A rejected install left in place is still imported by the metadata gate, so remove it."""
    if not _distribution_present(pypi_name):
        return
    logger.warning("%s %s", display_name, reason)
    if _uninstall_package(pypi_name, display_name):
        _send_status(event_queue, f"{display_name} is not usable on this GPU; removed it")
    else:
        _send_status(
            event_queue,
            f"{display_name} is not usable on this GPU and could not be removed; "
            f"uninstall {pypi_name} manually before training",
        )


def _install_package_wheel_first(
    *,
    event_queue: Any,
    import_name: str,
    display_name: str,
    pypi_name: str,
    required: bool = False,
    extension: str | None = None,
    **kwargs: Any,
) -> bool:
    """Install a wheel-first fast-path package; any unsuccessful exit discards what it left; offline
    changes nothing; with extension, an extension-free copy is replaced, not accepted."""
    from utils.ssm_runtime import _unblock_import, forget_imported_modules

    _unblock_import(import_name)  # an earlier in-process block; this call probes afresh

    def kernel_usable() -> bool:
        return _is_importable(import_name) and (extension is None or _is_importable(extension))

    if kernel_usable():
        logger.info("%s already installed", display_name)
        return True

    def drop_extension_free_copy() -> None:
        # CUDA route cannot keep an extension-free mamba_ssm (transformers takes its fast path and
        # fails); removal is allowed offline and runs under the same per-env lock as installs.
        if extension is None or not _is_importable(import_name):
            return
        from utils.kernel_install import OPTIONAL_KERNEL_LOCK_WAIT, kernel_install_lock
        with kernel_install_lock(import_name, wait_seconds = OPTIONAL_KERNEL_LOCK_WAIT) as held:
            if not _is_importable(import_name) or _is_importable(extension):
                return
            if not held:
                # Cannot uninstall without the lock: hide it from this process instead.
                from utils.ssm_runtime import block_import, refresh_transformers_kernel_gates

                logger.info(
                    "Hiding the extension-free %s from this run: its install lock is busy",
                    display_name,
                )
                block_import(import_name, extension)
                refresh_transformers_kernel_gates(removed = True)
                return
            logger.info(
                "Removing the extension-free %s: this run needs its CUDA kernels", display_name
            )
            _uninstall_package(pypi_name, display_name)
            forget_imported_modules(import_name, extension)
            if _is_importable(import_name) and not _is_importable(extension):
                # Uninstall failed (read-only env, locked files): hide it from this process.
                from utils.ssm_runtime import block_import, refresh_transformers_kernel_gates
                block_import(import_name, extension)
                refresh_transformers_kernel_gates(removed = True)

    if _model_offline_mode_enabled() or _package_installs_offline():
        logger.info("Skipping %s installation while offline", display_name)
        drop_extension_free_copy()
        return False

    # The runtime's failed-attempt memo (same key): the eager install and the fast-path hook, or a
    # chat load in the same process, never repeat a failed (possibly half-hour) build.
    from utils.ssm_runtime import _FAILED_INSTALLS, _failure_key

    failure_key = _failure_key(import_name, kwargs.get("wheel_url_builder"))
    if failure_key in _FAILED_INSTALLS:
        logger.info("%s already failed to install in this process; not retrying", display_name)
        drop_extension_free_copy()
        return False

    from utils.kernel_install import (
        KERNEL_LOCK_WAIT,
        OPTIONAL_KERNEL_LOCK_WAIT,
        kernel_install_lock,
    )

    # One install per package and env across workers: precheck, install, verify and cleanup are one
    # transaction; an optional acceleration does not hold a run for a peer's build.
    with kernel_install_lock(
        import_name,
        on_wait = lambda: _send_status(
            event_queue, f"Waiting for another process installing {display_name}..."
        ),
        wait_seconds = KERNEL_LOCK_WAIT if required else OPTIONAL_KERNEL_LOCK_WAIT,
    ) as held:
        if not held:
            _send_status(
                event_queue,
                f"Could not take the {display_name} install lock (another process is installing "
                "it, or the lock is unavailable); continuing without it",
            )
            if (
                extension is not None
                and _is_importable(import_name)
                and not _is_importable(extension)
            ):
                # Not ours to uninstall without the lock, but this run must not use it: hidden from
                # this process, transformers takes its torch path instead of selective_scan_fn.
                from utils.ssm_runtime import block_import, refresh_transformers_kernel_gates
                block_import(import_name, extension)
                refresh_transformers_kernel_gates(removed = True)
            return False
        if kernel_usable():
            logger.info("%s installed by another process", display_name)
            if extension is not None:
                forget_imported_modules(import_name, extension)
            return True
        # Importable but without the compiled module: the extension-free wheel, replaced here.
        replacing = extension is not None and _is_importable(import_name)
        if replacing:
            _send_status(
                event_queue, f"Replacing the extension-free {display_name} with its CUDA kernels..."
            )
        installed = False
        try:
            installed = _attempt_package_install(
                event_queue = event_queue,
                import_name = import_name,
                display_name = display_name,
                pypi_name = pypi_name,
                extension = extension,
                replacing = replacing,
                **kwargs,
            )
            return installed
        finally:
            if replacing:
                # The process imported the extension-free copy: the next import must load what is
                # installed now (or nothing, after a failed replacement removes it).
                forget_imported_modules(import_name, extension)
            if not installed:
                _FAILED_INSTALLS.add(failure_key)
                _reject_install(event_queue, pypi_name, display_name, "will not import")
                if extension is not None and _is_importable(import_name):
                    # The extension-free copy could not be removed: hide it from this process.
                    from utils.ssm_runtime import block_import, refresh_transformers_kernel_gates
                    block_import(import_name, extension)
                    refresh_transformers_kernel_gates(removed = True)


def _attempt_package_install(
    *,
    event_queue: Any,
    import_name: str,
    display_name: str,
    pypi_name: str,
    pypi_version: str | None = None,
    filename_prefix: str | None = None,
    release_tag: str | None = None,
    release_base_url: str | None = None,
    wheel_url_builder: Callable[[dict[str, str] | None], str | None] | None = None,
    pypi_spec: str | None = None,
    pypi_status_message: str | None = None,
    allow_source_build: bool = True,
    extension: str | None = None,
    replacing: bool = False,
) -> bool:
    """The install itself. Call it through _install_package_wheel_first, never directly."""

    def _verify() -> bool:
        # Out of process: a bad wheel can take the worker down with it.
        if not _is_importable_isolated(import_name):
            return False
        return extension is None or _extension_importable_isolated(extension)

    # Set when a wheel installed but would not import; see the uninstall before the fallback.
    wheel_rejected = False

    env = probe_torch_wheel_env(timeout = 30, include_windows = True)
    if not env or not (env.get("cuda_version") or env.get("cuda_major") or env.get("hip_version")):
        # These are CUDA / ROCm extensions: CPU-only, XPU, MPS or an unreadable torch builds none.
        _send_status(
            event_queue, f"{display_name} needs a CUDA or ROCm build of torch; skipping it"
        )
        return False
    # Windows is wheel-only: a PyPI fallback there is a source build needing MSVC + the CUDA toolkit.
    wheel_only = sys.platform == "win32" or is_windows_wheel_env(env)
    if wheel_url_builder is not None:
        wheel_url = wheel_url_builder(env)
    else:
        wheel_url = direct_wheel_url(
            filename_prefix = filename_prefix,
            package_version = pypi_version,
            release_tag = release_tag,
            release_base_url = release_base_url,
            env = env,
        )

    wheel_available = url_exists(wheel_url) if wheel_url else False
    if wheel_url is None:
        logger.info("No compatible %s wheel candidate", display_name)
    elif wheel_available is not False:
        # Only a 404 proves nothing is published: an unanswered HEAD (a proxy refusing HEAD but
        # serving the download) still tries the wheel, and falls back only if that fails.
        if wheel_available is None:
            _send_status(
                event_queue,
                f"Could not check the {display_name} prebuilt wheel; trying to download it.",
            )
        _send_status(event_queue, f"Installing {display_name} for faster training...")
        outcome = install_prebuilt(
            wheel_url,
            install = install_wheel,
            verify = _verify,
            on_failed = lambda installer, result: logger.warning(
                "%s failed to install %s wheel:\n%s",
                installer,
                display_name,
                result.stdout,
            ),
            use_uv = bool(shutil.which("uv")),
            run = _sp.run,
            # Over the extension-free copy of the same version, which uv / pip would keep.
            **({"reinstall": True} if replacing else {}),
        )
        if outcome == "installed":
            logger.info("Installed prebuilt %s wheel successfully", display_name)
            return True
        if outcome == "rejected":
            logger.warning(
                "%s wheel installed but is not importable; %s",
                display_name,
                "removing it" if wheel_only or not allow_source_build else "falling back to PyPI",
            )
            wheel_rejected = True
    else:
        logger.info("No published %s wheel found: %s", display_name, wheel_url)

    if wheel_only:
        logger.info("%s: no usable prebuilt wheel for Windows; skipping", display_name)
        return False
    if not allow_source_build:
        _send_status(event_queue, f"No usable prebuilt {display_name} for this torch; skipping it")
        return False
    if pypi_version is not None and not pinned_source_build_supported(env):
        # The pinned causal-conv1d / mamba-ssm sources build at C++17 and torch 2.13+ needs
        # C++20: a certain failure after minutes of compiling.
        _send_status(
            event_queue,
            f"No prebuilt {display_name} for torch {env.get('torch_mm')}, and it cannot be built "
            "from source for it; skipping it",
        )
        return False

    is_hip = env and env.get("hip_version")
    if is_hip and not shutil.which("hipcc"):
        logger.error(
            "%s requires hipcc for source compilation on ROCm. "
            "Install the ROCm HIP SDK: https://rocm.docs.amd.com",
            display_name,
        )
        _send_status(
            event_queue,
            f"{display_name}: hipcc not found (ROCm HIP SDK required)",
        )
        return False

    if pypi_spec is None:
        pypi_spec = f"{pypi_name}=={pypi_version}"

    if pypi_status_message is None:
        if is_hip:
            pypi_status_message = (
                f"Compiling {display_name} from source for ROCm (this may take several minutes)..."
            )
        else:
            pypi_status_message = f"Installing {display_name} from PyPI for faster training..."

    if wheel_rejected or replacing:
        # Remove it rather than install over it (a broken wheel, or the extension-free copy being
        # replaced): pip/uv would report the broken distribution as already satisfying the
        # spec and do nothing. --force-reinstall is not the answer either, since both scope it to the whole resolved
        # transaction, which for flash-attn means torch and the running CUDA stack. A failure here is caught by the
        # _reject_install in the finally below, which is reached from every exit.
        _uninstall_package(pypi_name, display_name)

    _send_status(event_queue, pypi_status_message)

    plain_pypi_install = pypi_version is None
    if plain_pypi_install:
        if shutil.which("uv"):
            pypi_cmd = [
                "uv",
                "pip",
                "install",
                "--python",
                sys.executable,
                pypi_spec,
            ]
        else:
            pypi_cmd = [sys.executable, "-m", "pip", "install", pypi_spec]
    else:
        pypi_cmd = source_build_command(
            pypi_spec, use_uv = bool(shutil.which("uv")), is_hip = bool(is_hip), reinstall = False
        )

    _run_kwargs, _gcc_dir = source_build_run_kwargs(
        is_hip = bool(is_hip), gcc_install_dir = _hipcc_gcc_install_dir
    )
    if _gcc_dir is not None:
        logger.info(
            "HIP source build for %s: appended --gcc-install-dir=%s to HIPCC_COMPILE_FLAGS_APPEND",
            display_name,
            _gcc_dir,
        )

    from utils.kernel_install import run_process_tree

    try:
        # Bounded, and a timeout stops the compilers pip / uv started too.
        result = run_process_tree(pypi_cmd, run = _sp.run, **_run_kwargs)
    except _sp.TimeoutExpired:
        logger.error(
            "%s installation timed out after %ds",
            display_name,
            _run_kwargs.get("timeout"),
        )
        _send_status(
            event_queue,
            f"{display_name} installation timed out after {_run_kwargs.get('timeout')}s",
        )
        return False

    if result.returncode != 0:
        if is_hip:
            error_lines = (result.stdout or "").strip().splitlines()
            snippet = "\n".join(error_lines[-5:]) if error_lines else "(no output)"
            logger.error(
                "Failed to compile %s for ROCm:\n%s",
                display_name,
                result.stdout,
            )
            _send_status(
                event_queue,
                f"Failed to compile {display_name} for ROCm. "
                "Check that hipcc and ROCm development headers are installed.\n"
                f"{snippet}",
            )
        else:
            logger.error(
                "Failed to install %s from PyPI:\n%s",
                display_name,
                result.stdout,
            )
        return False

    # rc=0 is not proof again here: pip/uv exit 0 on "Requirement already satisfied" without installing anything.
    # Returning False is enough; the caller's finally discards it.
    if not _verify():
        logger.warning("%s installed from PyPI but will not import", display_name)
        return False

    if is_hip:
        logger.info("Compiled and installed %s from source for ROCm", display_name)
    else:
        logger.info("Installed %s from PyPI", display_name)
    return True


def _pinned_kernel_kwargs(kernel: PinnedKernel) -> dict[str, str]:
    """_install_package_wheel_first arguments for a pinned kernel release."""
    return {
        "import_name": kernel.import_name,
        "display_name": kernel.display_name,
        "pypi_name": kernel.pypi_name,
        "pypi_version": kernel.package_version,
        "filename_prefix": kernel.import_name,
        "release_tag": kernel.release_tag,
        "release_base_url": kernel.release_base_url,
    }


def _ensure_causal_conv1d_fast_path(
    event_queue: Any,
    model_name: str,
    *,
    required: bool | None = None,
) -> None:
    if required is None:
        required = _model_wants_causal_conv1d(model_name)
    if not required:
        return

    _install_package_wheel_first(event_queue = event_queue, **_pinned_kernel_kwargs(CAUSAL_CONV1D))


def _flash_linear_attention_importable() -> bool:
    """True iff the gated-delta kernels unsloth_zoo vendors and injects as `fla` import."""
    try:
        import fla.modules  # noqa: F401
        import fla.ops.gated_delta_rule  # noqa: F401
        return True
    except Exception as exc:
        logger.warning(
            "flash-linear-attention is not importable; continuing on the pure-torch path: %s",
            exc,
        )
        return False


def _ensure_mamba_ssm(
    event_queue: Any,
    model_name: str,
    *,
    required: bool | None = None,
    essential: bool | None = None,
    triton_only: bool = False,
) -> bool | None:
    # Only Nemotron-H, Falcon-H1, Granite-4.0-H may build mamba-ssm from source, best effort: a
    # missing one is left to the model load; others get prebuilt wheels only.
    if required is None:
        required = _model_is_ssm(model_name)
    if not required:
        return
    if essential is None:
        # Unspecified: only Nemotron-H, Falcon-H1 and Granite-4.0-H may build from source, never
        # other SSMs.
        from utils.ssm_runtime import model_requires_mamba_ssm
        essential = model_requires_mamba_ssm(model_name)

    if sys.platform == "win32" and importlib.util.find_spec("triton") is None:
        # mamba_ssm imports triton at load and the wheel goes in --no-deps.
        _send_status(event_queue, "mamba-ssm needs Triton on Windows (triton-windows); skipping it")
        return

    if triton_only:
        from utils.wheel_utils import mamba_ssm_pure_wheel_url
        logger.info("Mamba-2 model on Triton kernels; setting up the extension-free mamba-ssm")
        # True when it imports; the caller then skips causal-conv1d and the CUDA mamba-ssm.
        return _install_package_wheel_first(
            event_queue = event_queue,
            **_pinned_kernel_kwargs(MAMBA_SSM),
            wheel_url_builder = mamba_ssm_pure_wheel_url,
            allow_source_build = False,
            required = essential,
        )

    if not essential and not _is_importable(MAMBA_SSM.import_name):
        from utils.ssm_runtime import _mamba_ssm_floor_reason
        reason = _mamba_ssm_floor_reason()
        if reason is not None:
            _send_status(event_queue, f"Skipping mamba-ssm, which {reason}")
            return

    logger.info("SSM model detected; setting up mamba-ssm after causal-conv1d")
    from utils.ssm_runtime import MAMBA_SSM_EXTENSION

    _install_package_wheel_first(
        event_queue = event_queue,
        **_pinned_kernel_kwargs(MAMBA_SSM),
        allow_source_build = essential,
        required = essential,
        # Never settle for the extension-free wheel a Mamba-2 load may have left here.
        extension = MAMBA_SSM_EXTENSION,
    )


# Integrated-only Radeon names match as whole tokens (a substring reads RX 7600M as 600M).
_ROCM_APU_NAME_RE = re.compile(r"(?<![\w-])(?:820m|8040s|780m|760m|740m|680m|660m|610m)(?!\w)")


def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
    """Trusts props.is_integrated first (it only upgrades to unified); name match is the last resort."""
    gcn_arch = ""
    for _attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):
        _v = (getattr(props, _attr, "") or "").split(":")[0].strip()
        if _v:
            gcn_arch = _v
            break

    # Driver's is_integrated flag can only upgrade to unified, never downgrade the known APU set.
    if getattr(props, "is_integrated", 0):
        return gcn_arch, True

    if gcn_arch:
        # gfx1152 (Krackan Point) shares system RAM like gfx1150/gfx1151.
        return gcn_arch, gcn_arch.lower() in {"gfx1150", "gfx1151", "gfx1152"}

    # Without arch attrs, match device name, only under _hw.IS_ROCM so NVIDIA 840M cannot hit Krackan.
    dev_lower = (getattr(props, "name", "") or "").lower()
    is_unified = (
        "890m" in dev_lower
        or "880m" in dev_lower
        or "8065s" in dev_lower
        or "8060s" in dev_lower
        or "8050s" in dev_lower
        or "860m" in dev_lower
        or "840m" in dev_lower
        or _ROCM_APU_NAME_RE.search(dev_lower) is not None
    )
    return gcn_arch, is_unified


# Fixed 16 GiB reserve: a flat 20% on 128 GiB Strix Halo is wasteful, 10% starved the OS.
_UNIFIED_OS_RESERVE_BYTES = 16 * 1024**3
_UNIFIED_MAX_RESERVE_FRACTION = 0.20
_DISCRETE_MEM_FRACTION = 0.90
# The name this guard shipped with; still wins on a ROCm host, since setups export it.
_MEM_FRACTION_ENV = "UNSLOTH_ROCM_MEM_FRACTION"
# Backend neutral, so NVIDIA has a cap too (unsloth#8178).
_GPU_MEM_FRACTION_ENV = "UNSLOTH_GPU_MEM_FRACTION"


def _parse_mem_fraction_env(env_value: str | None) -> float | None:
    """``UNSLOTH_ROCM_MEM_FRACTION`` as a float, None when unset or unusable. Shared with the OOM
    guard's log line so it can say whether the override was actually honoured, rather than just
    whether the variable was set."""
    try:
        override = float(env_value)  # None -> TypeError, "" / "  " -> ValueError
    except (TypeError, ValueError):
        return None
    # Two-sided so NaN is rejected.
    return override if 0.0 < override <= 1.0 else None


def _training_vram_budget_fraction(
    budget_gb: float | None,
    denominator_bytes: int,
    current: float = 1.0,
) -> float | None:
    """The memory fraction that holds this process to ``budget_gb``, never looser than ``current``
    (the OOM guard's cap); None when there is no budget or no total to divide by."""
    try:
        budget = float(budget_gb)
    except (TypeError, ValueError):
        return None
    if not (budget > 0) or denominator_bytes <= 0:
        return None
    return min(current, budget * 1024**3 / denominator_bytes)


def _apply_training_vram_budget(
    torch_mod: Any, single_gb: float | None, per_device_gb: list | None, gpu_ids: list | None
) -> dict[int, float]:
    """Cap each visible device at its own budget; returns {ordinal: fraction} for what was set."""
    applied: dict[int, float] = {}
    if not torch_mod.cuda.is_available():
        return applied
    get_fraction = getattr(torch_mod.cuda, "get_per_process_memory_fraction", None)
    divides_by_props = _allocator_divides_by_props_total(getattr(torch_mod, "__version__", ""))
    for index in range(torch_mod.cuda.device_count()):
        budget = _device_budget_gb(single_gb, per_device_gb, index, gpu_ids)
        if not budget:
            continue
        props = torch_mod.cuda.get_device_properties(index)
        denominator = int(getattr(props, "total_memory", 0) or 0)
        if not divides_by_props:
            denominator = int(torch_mod.cuda.mem_get_info(index)[1])
        current = get_fraction(index) if get_fraction is not None else 1.0
        fraction = _training_vram_budget_fraction(budget, denominator, current)
        if fraction is None:
            continue
        torch_mod.cuda.set_per_process_memory_fraction(fraction, index)
        applied[index] = fraction
        logger.info(
            "Training VRAM budget: set_per_process_memory_fraction(%.4f, cuda:%d), "
            "%.1f GiB of %.1f GiB",
            fraction,
            index,
            fraction * denominator / 1024**3,
            denominator / 1024**3,
        )
    return applied


def _device_budget_gb(
    single_gb: float | None,
    per_device_gb: list | None,
    ordinal: int,
    gpu_ids: list | None = None,
) -> float | None:
    """The budget for torch device ``ordinal``. ``per_device_gb`` is indexed by physical GPU id, so
    the ordinal goes through ``gpu_ids`` (the CUDA_VISIBLE_DEVICES narrowing) first; when it is
    given, a missing or null entry means no cap on that card."""
    if not per_device_gb:
        return single_gb
    physical = gpu_ids[ordinal] if gpu_ids and ordinal < len(gpu_ids) else ordinal
    try:
        return per_device_gb[int(physical)]
    except (IndexError, TypeError, ValueError):
        return None


def _offload_plan_shape(config: dict) -> dict:
    """The per-device batch and LoRA rank an Auto offload plan sizes its training reserve for."""
    return {"batch_size": config.get("batch_size"), "lora_rank": config.get("lora_r")}


def _visible_gpu_count() -> int:
    """GPUs the load spreads over when no ids were resolved (a UUID / MIG mask is inherited as is)."""
    try:
        import torch
        return torch.cuda.device_count() if torch.cuda.is_available() else 0
    except Exception:
        return 0


def _with_vram_budget_hint(config: dict, message: str) -> str:
    """Point a run that does not fit at its own VRAM budget, the one cause the generic advice omits."""
    budget = config.get("offload_vram_gb")
    per_device = [gb for gb in config.get("offload_vram_gb_per_device") or [] if gb]
    lower = (message or "").lower()
    if not (budget or per_device) or not any(
        k in lower for k in ("out of memory", "out of vram", "does not fit")
    ):
        return message
    cap = (
        f"a {budget:g} GiB VRAM budget"
        if budget and not per_device
        else "per-GPU VRAM budgets (" + ", ".join(f"{gb:g}" for gb in per_device) + " GiB)"
    )
    return (
        f"{message}\nThis run is capped at {cap}. Raise or clear the "
        "VRAM budget under Training Hyperparameters > Memory."
    )


def _allocator_divides_by_props_total(torch_version: str | None) -> bool:
    """Torch 2.10+ caps at props.total_memory, older torch at hipMemGetInfo; unparsable says True."""
    release = str(torch_version or "").split("+", 1)[0].split(".")
    try:
        major, minor = int(release[0]), int(release[1])
    except (IndexError, ValueError):
        return True
    return (major, minor) >= (2, 10)


def _rocm_memory_fraction(
    total_bytes: int,
    is_unified: bool,
    platform: str,
    env_value: str | None = None,
    denominator_bytes: int | None = None,
) -> float:
    """Valid UNSLOTH_ROCM_MEM_FRACTION overrides; unified Windows gets 1.0 as WDDM excludes the OS share."""
    override = _parse_mem_fraction_env(env_value)
    if override is not None:
        return override

    if not is_unified:
        return _DISCRETE_MEM_FRACTION
    if platform == "win32":
        return 1.0
    if total_bytes <= 0:
        # Missing or None total defaults to 0; with no pool size, keep the historical cap instead of solving.
        return 1.0 - _UNIFIED_MAX_RESERVE_FRACTION

    # Fraction space, not bytes: avoids 0.7999... ULP rounding below the historical 0.80.
    reserve_fraction = min(_UNIFIED_MAX_RESERVE_FRACTION, _UNIFIED_OS_RESERVE_BYTES / total_bytes)
    fraction = 1.0 - reserve_fraction

    if (
        reserve_fraction < _UNIFIED_MAX_RESERVE_FRACTION
        and denominator_bytes
        and denominator_bytes > 0
        and denominator_bytes != total_bytes
    ):
        # Floored so a larger driver total never tightens past the old 0.80; percentage arm is scale-free.
        fraction = max(
            fraction * total_bytes / denominator_bytes,
            1.0 - _UNIFIED_MAX_RESERVE_FRACTION,
        )

    # Past ~160 GiB unified would get a looser cap than discrete; clamp.
    return min(fraction, _DISCRETE_MEM_FRACTION)


def _mem_fraction_env_names(backend: str) -> tuple[str, ...]:
    """Most specific first, so a host already exporting the ROCm name keeps its cap."""
    if backend == "rocm":
        return (_MEM_FRACTION_ENV, _GPU_MEM_FRACTION_ENV)
    return (_GPU_MEM_FRACTION_ENV,)


def _mem_fraction_env_value(backend: str, environ: Any = None) -> tuple[str | None, str | None]:
    """``(raw, name)``, else the first one SET so a warning names what the user exported."""
    if environ is None:
        environ = os.environ
    first_set: tuple[str | None, str | None] = (None, None)
    for name in _mem_fraction_env_names(backend):
        raw = environ.get(name)
        if raw is None:
            continue
        if _parse_mem_fraction_env(raw) is not None:
            return raw, name
        if first_set == (None, None):
            first_set = (raw, name)
    return first_set


def _gpu_memory_fraction(
    total_bytes: int,
    is_unified: bool,
    platform: str,
    backend: str,
    env_value: str | None = None,
    denominator_bytes: int | None = None,
) -> float:
    """An override in ``(0.0, 1.0]`` wins; else ROCm delegates unchanged and the rest
    answer ``1.0``, torch's uncapped default, so setting nothing changes nothing."""
    override = _parse_mem_fraction_env(env_value)
    if override is not None:
        return override
    if backend != "rocm":
        return 1.0
    return _rocm_memory_fraction(total_bytes, is_unified, platform, None, denominator_bytes)


# ── Fast-path hooks ──
# Wrap is_causal_conv1d_available so the first call drives the install.
# UNSLOTH_STUDIO_SKIP_FAST_PATH_HOOKS=1 falls back to the substring path.


def _rebind_in_already_imported_modules(*, attr_name: str, old_obj: Any, new_obj: Any) -> int:
    """Rebind `attr_name -> new_obj` in every module that imported `old_obj`. `from X import Y`
    creates a local binding that reassigning X.Y won't reach. Uses `__dict__.get` to skip lazy
    `__getattr__` aliases."""
    count = 0
    missing = object()
    for mod_name, mod in list(sys.modules.items()):
        if mod is None:
            continue
        module_dict = getattr(mod, "__dict__", None)
        if not isinstance(module_dict, dict):
            continue
        existing = module_dict.get(attr_name, missing)
        if existing is old_obj:
            try:
                setattr(mod, attr_name, new_obj)
                count += 1
            except Exception as exc:
                logger.debug("Could not rebind %s in %s: %s", attr_name, mod_name, exc)
    return count


_TVM_FFI_BROKEN_VERSIONS = ("0.1.10", "0.1.11")


def _install_metadata_scan_cache() -> Any:
    """Share one packages_distributions() scan between transformers and TRL (unsloth's own window
    opens after this worker's first transformers import). Loaded by path: the package would run
    unsloth's whole init. None when the helper is missing (older unsloth) or fails."""
    try:
        spec = importlib.util.find_spec("unsloth")
        for root in list(getattr(spec, "submodule_search_locations", None) or ()):
            path = os.path.join(root, "_import_metadata_cache.py")
            if not os.path.isfile(path):
                continue
            helper_spec = importlib.util.spec_from_file_location(
                "_unsloth_studio_import_metadata_cache", path
            )
            helper = importlib.util.module_from_spec(helper_spec)
            helper_spec.loader.exec_module(helper)
            token = helper.install()
            return None if token is None else (helper, token)
    except Exception:
        pass
    return None


def _uninstall_metadata_scan_cache(installed: Any) -> None:
    if installed is None:
        return
    helper, token = installed
    try:
        helper.uninstall(token)
    except Exception:
        pass


def _guard_fla_tilelang() -> None:
    """Default FLA_TILELANG to 0 on ROCm torch and on an apache-tvm-ffi that faults on sm_100."""
    try:
        import torch as _torch_for_fla
        if (
            getattr(_torch_for_fla.version, "hip", None)
            or "rocm" in getattr(_torch_for_fla, "__version__", "").lower()
        ):
            os.environ.setdefault("FLA_TILELANG", "0")
    except Exception as exc:
        logger.debug("FLA_TILELANG guard skipped: %s", exc)

    # A leftover TileLang plus a broken tvm-ffi crashes the run; steer off it instead of installing.
    try:
        if importlib.util.find_spec("tilelang") is not None:
            tvm_ffi_version = importlib.metadata.version("apache-tvm-ffi")
            if tvm_ffi_version in _TVM_FFI_BROKEN_VERSIONS:
                before = os.environ.get("FLA_TILELANG")
                os.environ.setdefault("FLA_TILELANG", "0")
                # setdefault is a no-op under an override, so only claim what actually happened.
                if before is None:
                    logger.info(
                        "Disabling TileLang: apache-tvm-ffi %s faults under it; FLA_TILELANG is now %s",
                        tvm_ffi_version,
                        os.environ.get("FLA_TILELANG"),
                    )
                else:
                    logger.info(
                        "Keeping FLA_TILELANG=%s set by the environment despite apache-tvm-ffi %s",
                        before,
                        tvm_ffi_version,
                    )
    except Exception as exc:
        logger.debug("FLA_TILELANG guard skipped: %s", exc)


def _install_fast_path_hooks(
    event_queue: Any,
    model_name: str,
    *,
    install_causal_conv1d: bool | None = None,
) -> None:
    """First gate call drives the install; UNSLOTH_STUDIO_SKIP_FAST_PATH_HOOKS=1 uses the substring gate."""
    _guard_fla_tilelang()

    if os.getenv(_FAST_PATH_HOOKS_SKIP_ENV) == "1":
        logger.info("Fast-path hooks disabled via env; using substring fallback")
        return

    try:
        from transformers.utils import import_utils as _iu
    except Exception as exc:
        logger.warning(
            "transformers.utils.import_utils not importable; skipping fast-path hooks: %s",
            exc,
        )
        return

    def _make_wrapper(
        original: Callable[[], bool], install_fn: Callable[[Any], bool], gate_name: str
    ) -> Callable[[], bool]:
        state = {"installed": False}

        def wrapper() -> bool:
            if state["installed"]:
                return original()
            try:
                original.cache_clear()  # defensive; worker subprocess is fresh
            except AttributeError:
                pass
            ok = original()
            if not ok:
                logger.info("Hook fired for %s; triggering install", gate_name)
                try:
                    ok = bool(install_fn(event_queue))
                except Exception as exc:
                    logger.warning("%s install raised: %s; falling back to torch", gate_name, exc)
                    ok = False
                logger.info("%s hook done; available=%s", gate_name, ok)
            state["installed"] = True
            return ok

        wrapper.__wrapped__ = original  # type: ignore[attr-defined]
        wrapper.cache_clear = getattr(original, "cache_clear", lambda: None)  # type: ignore[attr-defined]
        return wrapper

    def _causal_conv1d_install(eq: Any) -> bool:
        ok = _install_package_wheel_first(event_queue = eq, **_pinned_kernel_kwargs(CAUSAL_CONV1D))
        return bool(ok)

    hooks: list[tuple[str, Callable[[Any], bool]]] = []
    if install_causal_conv1d is None:
        install_causal_conv1d = _model_wants_causal_conv1d(model_name)
    if install_causal_conv1d:
        hooks.append(("is_causal_conv1d_available", _causal_conv1d_install))

    for gate_name, install_fn in hooks:
        original = getattr(_iu, gate_name, None)
        if original is None:
            logger.info(
                "%s missing on transformers.utils.import_utils; skipping hook",
                gate_name,
            )
            continue
        wrapped = _make_wrapper(original, install_fn, gate_name)
        setattr(_iu, gate_name, wrapped)
        rebound = _rebind_in_already_imported_modules(
            attr_name = gate_name, old_obj = original, new_obj = wrapped
        )
        logger.info("Installed fast-path hook on %s (rebound %d modules)", gate_name, rebound)


def _should_try_runtime_flash_attn_install(max_seq_length: int) -> bool:
    if os.getenv(_FLASH_ATTN_SKIP_ENV) == "1":
        return False
    if max_seq_length < _FLASH_ATTN_RUNTIME_MIN_SEQ_LEN:
        return False
    return sys.platform.startswith("linux") or sys.platform == "win32"


def _flash_attn_gpu_below_floor() -> bool:
    """Whether the best visible GPU is below sm80, where Unsloth leaves FlashAttention 2 off. Asks
    the torch this worker already imported when there is one (no child process); ROCm never takes
    the NVIDIA floor; no visible GPU is not below it."""
    torch = sys.modules.get("torch")
    if torch is not None:
        try:
            if getattr(torch.version, "hip", None):
                return False
            if not torch.cuda.is_available():
                return False
            count = torch.cuda.device_count()
            return count > 0 and max(
                tuple(torch.cuda.get_device_capability(i)) for i in range(count)
            ) < (8, 0)
        except Exception as exc:
            logger.debug("In-process GPU capability check failed (%s); probing a child", exc)
    from utils.kernel_install import gpu_below_flash_attn_floor

    return gpu_below_flash_attn_floor()


def _ensure_flash_attn_for_long_context(event_queue: Any, max_seq_length: int) -> None:
    if not _should_try_runtime_flash_attn_install(max_seq_length):
        return
    if _is_importable("flash_attn"):
        # Already usable: nothing to download, so no capability probe either.
        logger.info("flash-attn already installed")
        return

    if _flash_attn_gpu_below_floor():
        # Unsloth only turns FlashAttention 2 on from sm80 (a T4 is sm75): nothing to download.
        _send_status(event_queue, "flash-attn needs an sm80 or newer GPU; continuing without it")
        return

    installed = _install_package_wheel_first(
        event_queue = event_queue,
        import_name = "flash_attn",
        display_name = "flash-attn",
        pypi_name = "flash-attn",
        wheel_url_builder = flash_attn_wheel_url,
        # Optional acceleration: skip an unavailable wheel instead of building during training.
        allow_source_build = False,
    )
    if not installed:
        _send_status(event_queue, "Continuing without flash-attn")


def _activate_transformers_version(model_name: str, hf_token: str | None = None) -> None:
    """Activate the correct transformers version BEFORE any ML imports."""
    backend_path = str(Path(__file__).resolve().parent.parent.parent)
    if backend_path not in sys.path:
        sys.path.insert(0, backend_path)

    from utils.transformers_version import activate_transformers_for_subprocess

    activate_transformers_for_subprocess(model_name, hf_token)


def _activate_transformers_version_or_warn(model_name: str, hf_token: str | None = None) -> None:
    """MLX path is non-fatal: failed transformers activation logs a warning, then falls through."""
    try:
        _activate_transformers_version(model_name, hf_token)
    except Exception as exc:
        logger.warning(
            "Failed to activate transformers version for '%s' (MLX); "
            "training may fail if this model requires a specific version. Error: %s",
            model_name,
            exc,
        )


def _mlx_vocab_size(model):
    """Rows of the MLX model's input embedding table (the CUDA path reads get_input_embeddings().num_embeddings), so
    a pre-tokenized id or label past it is refused before the loss indexes it. None when it cannot be found, which
    skips only the range check."""
    for owner in (model, getattr(model, "language_model", None)):
        if owner is None:
            continue
        for embed in (
            getattr(getattr(owner, "model", None), "embed_tokens", None),
            getattr(owner, "embed_tokens", None),
        ):
            # A LoRA-wrapped embedding keeps the table on its inner module.
            for layer in (
                embed,
                getattr(embed, "embedding", None),
                getattr(embed, "base_layer", None),
            ):
                if layer is None:
                    continue
                # QuantizedEmbedding packs its weight, so its row count is the stored num_embeddings.
                rows = getattr(layer, "num_embeddings", None)
                shape = getattr(getattr(layer, "weight", None), "shape", None)
                if rows is None and shape:
                    rows = shape[0]
                if isinstance(rows, int) and not isinstance(rows, bool) and rows > 0:
                    return rows
    return None


def _carry_mlx_chat_template_to_processor(model, tokenizer) -> bool:
    """MLXTrainer saves the loader's processor when it has one, so a text run on a multimodal base checkpoint (Qwen3.5
    Base) saved no template and Chat failed with "chat_template is not set". Give it the template the rows used."""
    processor = getattr(model, "_processor", None)
    if processor is None or processor is tokenizer:
        return False
    trained = tokenizer
    if type(tokenizer).__name__ == "TokenizerWrapper":
        trained = getattr(tokenizer, "_tokenizer", tokenizer)
    template = getattr(trained, "chat_template", None)
    if not template or getattr(processor, "chat_template", None):
        return False
    own = getattr(processor, "tokenizer", None)
    if own is not None and own is not trained and getattr(own, "chat_template", None):
        return False
    processor.chat_template = template
    if own is not None and own is not trained:
        # Also carries get_chat_template's end-token remap.
        processor.tokenizer = trained
    return True


def _mlx_vlm_max_resized_size(width: int, height: int, target: int) -> tuple[int, int]:
    if width <= 0 or height <= 0 or target <= 0:
        return width, height
    largest_side = max(width, height)
    if largest_side <= target:
        return width, height
    # Integer formula matches unsloth_zoo's collator (round() differs by 1px).
    new_w = max(1, (width * target + largest_side // 2) // largest_side)
    new_h = max(1, (height * target + largest_side // 2) // largest_side)
    return new_w, new_h


_MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE = {}


def _mlx_vlm_resized_image_layout(processor = None) -> str | None:
    """Return the numpy image layout expected after Unsloth-side VLM resizing."""
    image_processor = getattr(processor, "image_processor", None)
    if image_processor is None:
        return None
    cls = image_processor.__class__
    key = (getattr(cls, "__module__", ""), getattr(cls, "__qualname__", cls.__name__))
    if key in _MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE:
        return _MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE[key]
    copied_image_processor = _copy_mlx_vlm_image_processor(image_processor)
    layout = (
        _probe_mlx_vlm_numpy_image_layout(copied_image_processor)
        if copied_image_processor is not None
        else None
    )
    _MLX_VLM_RESIZED_IMAGE_LAYOUT_CACHE[key] = layout
    return layout


def _copy_mlx_vlm_image_processor(image_processor):
    import copy
    try:
        return copy.deepcopy(image_processor)
    except Exception:
        try:
            return copy.copy(image_processor)
        except Exception:
            return None


def _probe_mlx_vlm_numpy_image_layout(image_processor) -> str | None:
    try:
        import numpy as np
    except ImportError:
        return None

    def _accepts(candidate) -> bool:
        try:
            image_processor(images = [candidate])
            return True
        except TypeError:
            try:
                image_processor([candidate])
                return True
            except Exception:
                return False
        except Exception:
            return False

    # Asymmetric image so CHW-vs-HWC mistakes are visible to processors that skip 3D conversion.
    hwc = np.zeros((64, 96, 3), dtype = np.uint8)
    chw = np.ascontiguousarray(hwc.transpose(2, 0, 1))
    if _accepts(hwc):
        return None
    if _accepts(chw):
        return "chw"
    return None


def _resize_mlx_vlm_image(
    image,
    resize,
    image_layout = None,
):
    if resize is None:
        return image
    try:
        from PIL import Image
        import numpy as np
    except ImportError:
        return image
    if not isinstance(image, Image.Image):
        return image
    image = image.convert("RGB")
    new_size = _mlx_vlm_max_resized_size(*image.size, int(resize))
    if new_size != image.size:
        resampling = getattr(Image, "Resampling", Image).LANCZOS
        image = image.resize(new_size, resampling)
    # Writable RGB ndarray skips mlx-vlm's PIL square-resize and HF non-writable warnings.
    array = np.array(image, copy = True)
    if image_layout == "chw":
        return np.ascontiguousarray(array.transpose(2, 0, 1))
    return array


def _resize_mlx_vlm_images(
    value,
    resize,
    image_layout = None,
):
    if isinstance(value, list):
        return [_resize_mlx_vlm_image(image, resize, image_layout = image_layout) for image in value]
    return _resize_mlx_vlm_image(value, resize, image_layout = image_layout)


_MLX_VIDEO_UNSUPPORTED = (
    "Training on video is not supported on Apple Silicon (MLX) yet. "
    "Train this dataset on an NVIDIA or AMD GPU, or drop the video column."
)


_RESOLVE_IMAGE_COLUMN = object()


def _mlx_audio_rows(
    dataset,
    processor,
    custom_format_mapping = None,
    image_col = _RESOLVE_IMAGE_COLUMN,
    warn = None,
) -> tuple:
    """The CUDA transcription rows, with each clip left decoded so zoo can check its rate, and
    the row's images (a mixed audio + image dataset) beside them. Returns (rows, image column);
    an eval split takes the train split's image column, None for none."""
    from datasets import Audio
    from unsloth_zoo.mlx.utils import audio_extractor_sampling_rate

    from utils.datasets.audio_decode import ensure_audio_decoding
    from utils.datasets.audio_rows import (
        audio_transcription_messages,
        prepare_audio_vlm_image_column,
        resolve_audio_columns,
        resolve_audio_vlm_image_column,
    )
    from utils.datasets.cells import cell_text

    if not ensure_audio_decoding():
        raise RuntimeError(
            "No audio decoder is available: install the audio runtime (librosa and soundfile)."
        )
    columns = resolve_audio_columns(dataset, custom_format_mapping)
    audio_col, text_col = columns["audio_col"], columns["text_col"]
    if not audio_col or not text_col:
        raise ValueError(
            f"Audio dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
        )
    train_split = image_col is _RESOLVE_IMAGE_COLUMN
    if train_split:
        image_col = resolve_audio_vlm_image_column(
            dataset, custom_format_mapping, audio_col, text_col, columns["speaker_col"]
        )
    elif image_col not in dataset.column_names:
        image_col = None
    rate = audio_extractor_sampling_rate(processor) or 16000
    dataset = dataset.cast_column(audio_col, Audio(sampling_rate = rate))
    if image_col:
        dataset = prepare_audio_vlm_image_column(dataset, image_col)
    rows = []
    blank = 0
    for row in dataset:
        text = cell_text(row[text_col])
        if not text.strip():
            blank += 1
            continue
        images = row[image_col] if image_col else None
        if not isinstance(images, list):
            images = [] if images is None else [images]
        images = [image for image in images if image is not None]
        item = {"messages": audio_transcription_messages(row[audio_col], text, len(images))}
        if images:
            item["images"] = images
        rows.append(item)
    if not rows and train_split:
        raise ValueError(f"Every transcript in the '{text_col}' column is blank.")
    if blank:
        (warn or logger.warning)(
            f"Skipped {blank:,} of {blank + len(rows):,} audio rows with a blank transcript."
        )
    return rows, image_col


def _adapt_for_mlx_vlm(
    items,
    resize = None,
    image_layout = None,
):
    """mlx-vlm needs images at top level to build pixel_values, not inside message content."""
    adapted = []
    for item in items:
        images = []
        messages = []
        for msg in item.get("messages", []):
            content = msg.get("content", "")
            if isinstance(content, list):
                new_content = []
                for part in content:
                    if isinstance(part, dict) and part.get("type") == "video":
                        raise ValueError(_MLX_VIDEO_UNSUPPORTED)
                    if isinstance(part, dict) and part.get("type") == "image":
                        img = part.get("image")
                        if img is not None:
                            images.append(
                                _resize_mlx_vlm_image(
                                    img,
                                    resize,
                                    image_layout = image_layout,
                                )
                            )
                        new_content.append({"type": "image"})
                    else:
                        new_content.append(part)
                messages.append({"role": msg["role"], "content": new_content})
            else:
                messages.append(msg)
        out = {"messages": messages}
        if images:
            out["image"] = images[0] if len(images) == 1 else images
        elif "image" in item:
            out["image"] = _resize_mlx_vlm_images(
                item["image"],
                resize,
                image_layout = image_layout,
            )
        elif "images" in item:
            out["images"] = _resize_mlx_vlm_images(
                item["images"],
                resize,
                image_layout = image_layout,
            )
        adapted.append(out)
    return adapted


_MLX_STUDIO_LR_SCHEDULERS = {"linear", "cosine", "constant"}


# Fallback mirroring unsloth_zoo._normalize_mlx_optimizer_name when mlx isn't importable.
_MLX_STUDIO_ADAMW_ALIASES = frozenset(
    (
        "adamw_8bit",
        "paged_adamw_8bit",
        "adamw_bnb_8bit",
        "paged_adamw_32bit",
        "adamw_torch",
        "adamw_torch_fused",
        "paged_adamw",
        "adamw_32bit",
        "adamw_hf",
        "adamw_anyprecision",
        "adamw_apex_fused",
    )
)
_MLX_STUDIO_NATIVE_OPTIMIZERS = ("adafactor", "adamw", "adam", "sgd", "muon", "lion")


def _normalize_mlx_studio_optimizer(value):
    try:
        from unsloth_zoo.mlx.trainer import _normalize_mlx_optimizer_name
        return _normalize_mlx_optimizer_name(value or "adamw_8bit")
    except (ImportError, ValueError):
        # Missing mlx, or an older zoo normalizer: map common adamw_* names locally.
        opt = str(getattr(value, "value", value) or "adamw_8bit").strip().lower()
        opt = opt.rsplit(".", 1)[-1].replace("-", "_")
        if opt in _MLX_STUDIO_ADAMW_ALIASES:
            opt = "adamw"
        if opt not in _MLX_STUDIO_NATIVE_OPTIMIZERS:
            supported = ", ".join(_MLX_STUDIO_NATIVE_OPTIMIZERS)
            raise ValueError(
                f"Unsupported optimizer for MLX training: {value!r}. "
                f"Supported optimizers: {supported}."
            )
        return opt


def _mlx_dora_peft_kwargs(config, get_peft_model):
    """LoRA kwargs a DoRA request adds for MLX, or raise why it cannot run."""
    import inspect

    if not config.get("use_dora"):
        return {}
    try:
        parameter = inspect.signature(get_peft_model).parameters.get("use_dora")
    except (TypeError, ValueError):
        parameter = None
    # A **kwargs catch-all would silently swallow use_dora, so require a named parameter.
    named = parameter is not None and parameter.kind in (
        inspect.Parameter.POSITIONAL_OR_KEYWORD,
        inspect.Parameter.KEYWORD_ONLY,
    )
    if not named:
        raise NotImplementedError(
            "DoRA on Apple Silicon needs an unsloth-zoo whose MLX "
            "get_peft_model takes use_dora, and this install cannot be "
            "confirmed to. The version that predates MLX DoRA accepts the "
            "request and trains plain LoRA instead, so the run stops here "
            "rather than guessing. Update unsloth-zoo, or pick a different "
            "LoRA variant."
        )
    return {"use_dora": True}


def _normalize_mlx_studio_scheduler(value):
    raw = str(value or "linear").strip().lower()
    if raw not in _MLX_STUDIO_LR_SCHEDULERS:
        supported = ", ".join(sorted(_MLX_STUDIO_LR_SCHEDULERS))
        raise ValueError(
            f"Unsupported LR scheduler for MLX training: {value!r}. Supported values: {supported}."
        )
    return raw


def _resolve_mlx_local_dataset_path(dataset_file) -> Path:
    from utils.paths import resolve_dataset_path

    dataset_path = Path(os.path.expanduser(str(dataset_file)))
    if dataset_path.is_absolute():
        return dataset_path
    if dataset_path.exists():
        return dataset_path.resolve()
    return Path(str(resolve_dataset_path(str(dataset_file))))


def _resolve_mlx_local_dataset_files(file_paths: list) -> list[str]:
    """Resolve CLI paths and Unsloth local dataset uploads without importing the GPU trainer."""
    from utils.paths import dataset_files_in_dir

    all_files: list[str] = []
    for dataset_file in file_paths or []:
        file_path_obj = _resolve_mlx_local_dataset_path(dataset_file)

        if file_path_obj.is_dir():
            all_files.extend(str(p) for p in dataset_files_in_dir(file_path_obj))
            continue

        all_files.append(str(file_path_obj))

    return all_files


def _mlx_local_dataset_loader_for_files(files: list[str]) -> str:
    first_ext = Path(files[0]).suffix.lower()
    if first_ext in (".json", ".jsonl"):
        return "json"
    if first_ext == ".csv":
        return "csv"
    if first_ext == ".parquet":
        return "parquet"
    raise ValueError(f"Unsupported dataset format: {files[0]}")


_MLX_WORKER_COMPLETE = "_mlx_worker_complete"


def _start_worker_stop_poller(
    stop_queue,
    on_stop: Callable[[bool], None],
    *,
    completion_type: str | None = None,
    timeout: float = 1.0,
):
    import queue as _queue
    import threading

    cancel_requested = False

    def poll_stop():
        nonlocal cancel_requested
        while True:
            try:
                msg = stop_queue.get(timeout = timeout)
                if not isinstance(msg, dict):
                    continue
                message_type = msg.get("type")
                if completion_type is not None and message_type == completion_type:
                    return
                if message_type != "stop":
                    continue
                if not bool(msg.get("save", True)):
                    cancel_requested = True
                on_stop(not cancel_requested)
                if cancel_requested:
                    return
            except _queue.Empty:
                continue
            except (EOFError, OSError, ValueError):
                return

    stop_thread = account_thread(target = poll_stop, daemon = True)
    stop_thread.start()
    return stop_thread


def _start_mlx_stop_poller(stop_queue):
    stop_save = [True]
    stop_requested = [False]
    trainer_ref = [None]

    def is_stop_requested():
        return stop_requested[0]

    def apply_stop(save: bool) -> None:
        stop_save[0] = save
        stop_requested[0] = True
        trainer = trainer_ref[0]
        if trainer is not None:
            trainer.stop_requested = True
            # Early stopping set load_best_model_at_end; a manual stop keeps the current adapters instead, since the
            # stop checkpoint stores this step's optimizer state next to them.
            args = getattr(trainer, "args", None)
            if getattr(args, "load_best_model_at_end", False):
                args.load_best_model_at_end = False

    stop_thread = _start_worker_stop_poller(
        stop_queue,
        apply_stop,
        completion_type = _MLX_WORKER_COMPLETE,
        timeout = 0.25,
    )
    return stop_save, stop_requested, trainer_ref, is_stop_requested, stop_thread


def _resolve_mlx_output_dir(config, model_name):
    from utils.paths import resolve_output_dir, default_run_dir_name

    output_dir = config.get("output_dir", "")
    if not output_dir:
        output_dir = f"{default_run_dir_name(model_name)}_{int(time.time())}"
        return str(resolve_output_dir(output_dir))
    if config.get("allow_external_output_dir"):
        output_path = Path(output_dir).expanduser()
        if not output_path.is_absolute():
            output_path = Path.cwd() / output_path
        return str(output_path.resolve())
    return str(resolve_output_dir(output_dir))


def _resolve_mlx_training_steps(
    requested_max_steps,
    dataset_size,
    batch_size,
    gradient_accumulation_steps,
    num_epochs,
    world_size,
):
    if requested_max_steps > 0:
        return requested_max_steps
    global_batch_size = batch_size * world_size
    return max(
        1,
        math.ceil(dataset_size / global_batch_size / gradient_accumulation_steps) * num_epochs,
    )


def _slice_streaming_dataset(dataset, slice_start, slice_end):
    """A stream slices lazily, as on CUDA, and has no length to bound."""
    if slice_start:
        dataset = dataset.skip(slice_start)
    if slice_end is not None:
        dataset = dataset.take(max(0, slice_end - (slice_start or 0) + 1))
    return dataset


def _configure_mlx_training_schedule(
    trainer,
    requested_max_steps,
    dataset_size,
    batch_size,
    gradient_accumulation_steps,
    num_epochs,
    warmup_ratio = None,
    eval_steps_ratio = None,
):
    max_steps = _resolve_mlx_training_steps(
        requested_max_steps,
        dataset_size,
        batch_size,
        gradient_accumulation_steps,
        num_epochs,
        trainer.distributed_world_size,
    )
    trainer.args.max_steps = max_steps
    if warmup_ratio is not None:
        trainer.args.warmup_steps = math.ceil(warmup_ratio * max_steps)
    if eval_steps_ratio is not None:
        trainer.args.eval_steps = max(1, math.ceil(eval_steps_ratio * max_steps))
    return max_steps, int(trainer.args.eval_steps)


def _run_mlx_main_process_action(trainer, action, context):
    world_size = int(trainer.distributed_world_size)
    if world_size <= 1:
        return action()

    raise_distributed_failure = getattr(trainer, "_raise_distributed_failure", None)
    if not callable(raise_distributed_failure):
        raise RuntimeError(
            "Unsloth MLX DDP CLI requires MLXTrainer failure coordination. "
            "Upgrade unsloth-zoo to a compatible version."
        )

    result = None
    error = None
    if bool(trainer.is_main_process):
        try:
            result = action()
        except BaseException as exc:
            error = exc
    raise_distributed_failure(error is not None, context, error)
    return result


def _prepare_mlx_output_dir(
    trainer,
    output_dir,
    ensure_dir,
    event_queue = None,
):
    def _prepare():
        result = ensure_dir(Path(output_dir))
        if event_queue is not None:
            _emit_output_dir(event_queue, output_dir)
        return result

    return _run_mlx_main_process_action(
        trainer,
        _prepare,
        "output directory setup",
    )


def _setup_mlx_tracking(trainer, config, output_dir, send):
    wandb_run = None
    tb_writer = None
    if config.get("enable_wandb", False):

        def _setup_wandb():
            try:
                import wandb as _wandb

                wandb_token = config.get("wandb_token")
                if wandb_token:
                    os.environ["WANDB_API_KEY"] = wandb_token
                # Keep the authenticated subject out of W&B run config (mirrors _sanitize_db_config).
                _wandb_sensitive = {"hf_token", "wandb_token", "s3_config", "subject"}
                return _wandb.init(
                    project = config.get("wandb_project") or "unsloth-mlx",
                    config = {k: v for k, v in config.items() if k not in _wandb_sensitive},
                    reinit = True,
                )
            except Exception as e:
                send("status", status_message = f"wandb init failed: {e}")
                return None

        wandb_run = _run_mlx_main_process_action(trainer, _setup_wandb, "Weights & Biases setup")
    if config.get("enable_tensorboard", False):

        def _setup_tensorboard():
            try:
                from tensorboardX import SummaryWriter
            except ImportError:
                try:
                    from torch.utils.tensorboard import SummaryWriter
                except ImportError:
                    SummaryWriter = None
            if SummaryWriter is None:
                send(
                    "status",
                    status_message = "tensorboard unavailable (install tensorboardX)",
                )
                return None
            try:
                tb_dir = config.get("tensorboard_dir") or f"{output_dir}/runs"
                return SummaryWriter(log_dir = tb_dir)
            except Exception as e:
                send("status", status_message = f"tensorboard init failed: {e}")
                return None

        tb_writer = _run_mlx_main_process_action(trainer, _setup_tensorboard, "TensorBoard setup")
    return wandb_run, tb_writer


def _mlx_worker_finalization_state(trainer, snapshot_stop):
    trainer_stopped = bool(trainer.stop_requested)
    local_stop_requested, stop_save = snapshot_stop()
    stopped = trainer_stopped or bool(local_stop_requested)
    cancelled = bool(local_stop_requested and not stop_save)
    if int(trainer.distributed_world_size) > 1:
        any_flag = getattr(trainer, "_distributed_any_flag", None)
        if not callable(any_flag):
            raise RuntimeError(
                "Unsloth MLX DDP CLI requires MLXTrainer stop coordination. "
                "Upgrade unsloth-zoo to a compatible version."
            )
        stopped = bool(any_flag(stopped))
        cancelled = bool(any_flag(cancelled))
    if stopped:
        trainer.stop_requested = True
    return stopped, cancelled


def _synchronize_mlx_before_final_save(trainer, synchronize):
    error = None
    try:
        synchronize()
    except BaseException as exc:
        error = exc
    if int(trainer.distributed_world_size) <= 1:
        if error is not None:
            raise error
        return
    raise_distributed_failure = getattr(trainer, "_raise_distributed_failure", None)
    if not callable(raise_distributed_failure):
        raise RuntimeError(
            "Unsloth MLX DDP CLI requires MLXTrainer failure coordination. "
            "Upgrade unsloth-zoo to a compatible version."
        )
    raise_distributed_failure(error is not None, "final synchronization", error)


def _finalize_mlx_training(
    trainer, snapshot_stop, output_dir, synchronize, send, write_stop_checkpoint
):
    stopped, cancelled = _mlx_worker_finalization_state(trainer, snapshot_stop)
    if cancelled:
        send("complete", output_dir = None, status_message = "Training cancelled")
        return

    if stopped:
        saving_message = "Saving stopped model..."
        completion_message = "Training stopped"
    else:
        saving_message = "Saving model..."
        completion_message = "Training completed"
    send("status", status_message = saving_message)
    _synchronize_mlx_before_final_save(trainer, synchronize)
    _run_mlx_main_process_action(
        trainer,
        lambda: trainer.save_model(output_dir),
        "final model save",
    )
    stopped_after_save, cancelled_after_save = _mlx_worker_finalization_state(
        trainer, snapshot_stop
    )
    if cancelled_after_save:
        send("complete", output_dir = None, status_message = "Training cancelled")
        return
    if stopped_after_save:
        completion_message = "Training stopped"
        checkpoint_ok = _run_mlx_main_process_action(
            trainer,
            write_stop_checkpoint,
            "stop checkpoint save",
        )
        if int(trainer.distributed_world_size) > 1:
            checkpoint_ok = trainer._distributed_any_flag(bool(checkpoint_ok))
        if not checkpoint_ok:
            send(
                "error",
                error = (
                    "Failed to save a resumable checkpoint after stop. "
                    "Model files were saved, but this run cannot be resumed."
                ),
                keep_error_status = True,
                resume_blocked = True,
            )
            return
    if not stopped_after_save and getattr(trainer, "_early_stopped", False):
        # Zoo ends an early-stopped run without stop_requested, so it reads as a plain completion otherwise.
        completion_message = "Training completed (stopped early: eval loss stopped improving)"
    send(
        "complete",
        output_dir = output_dir if bool(trainer.is_main_process) else None,
        status_message = completion_message,
    )


def _resolve_mlx_max_grad_norm(value):
    """Unset stays 0.0 so MLX keeps cheap per-parameter clipping; a requested value is honored."""
    if value is None:
        return 0.0
    try:
        value = float(value)
    except (TypeError, ValueError):
        raise ValueError(
            f"Unsloth MLX: max_grad_norm={value!r} must be a non-negative float or None."
        )
    # inf clears a >= 0 check but never binds, so the run would train unclipped.
    if value < 0 or not math.isfinite(value):
        raise ValueError(
            f"Unsloth MLX: max_grad_norm={value} must be a finite value >= 0 "
            "(use 0 to disable global norm clipping)."
        )
    return value


def _run_mlx_training(event_queue, stop_queue, config):
    """Apple Silicon MLX path via unsloth_zoo's MLXTrainer, no torch; same event_queue protocol."""
    import time
    import math
    from pathlib import Path

    def _send(event_type, **kwargs):
        if event_type == "status" and "message" not in kwargs:
            sm = kwargs.get("status_message")
            if sm is not None:
                kwargs["message"] = sm
        event_queue.put({"type": event_type, "ts": time.time(), **kwargs})

    _stop_save, _stop_requested, _trainer_ref, _is_stop_requested, _stop_thread = (
        _start_mlx_stop_poller(stop_queue)
    )

    def _snapshot_stop():
        return _stop_requested[0], _stop_save[0]

    _send("status", status_message = "Loading MLX libraries...")

    import mlx.core as mx
    from mlx.utils import tree_flatten

    try:
        from unsloth_zoo.mlx.loader import FastMLXModel
        from unsloth_zoo.mlx.trainer import (
            MLXTrainer,
            MLXTrainingConfig,
            train_on_responses_only,
        )
        from unsloth_zoo.mlx.utils import iter_mlx_lora_modules
    except ImportError as e:
        raise ImportError(
            "Unsloth: MLX training requires unsloth-zoo with the MLX modules "
            "(unsloth_zoo.mlx.loader / unsloth_zoo.mlx.trainer). Reinstall via "
            "install.sh on Apple Silicon."
        ) from e
    from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset

    if mx.metal.is_available():
        info = mx.device_info()
        rec_bytes = info.get("max_recommended_working_set_size", 0) or 0
        if rec_bytes > 0:
            memory_cap = int(rec_bytes * 0.85)
            wired_cap = min(int(rec_bytes), memory_cap)
            mx.set_memory_limit(memory_cap)
            mx.set_wired_limit(wired_cap)

    model_name = config["model_name"]
    hf_token = _worker_hf_token(config)
    if hf_token:
        os.environ["HF_TOKEN"] = hf_token
    model_load_name = _resolve_cached_model_load_name(config)
    model_local_only = _model_local_files_only(config)
    model_revision = None if model_local_only else config.get("model_revision")

    if config.get("use_loftq"):
        message = "LoftQ is not supported for MLX training yet."
        _send("error", error = message)
        raise NotImplementedError(message)
    if config.get("is_embedding"):
        message = "Embedding model training is not supported for MLX training yet."
        _send("error", error = message)
        raise NotImplementedError(message)
    # Decided before the model loads, so version skew does not cost a download.
    try:
        mlx_dora_kwargs = _mlx_dora_peft_kwargs(config, FastMLXModel.get_peft_model)
    except NotImplementedError as exc:
        _send("error", error = str(exc))
        raise

    optim_name = _normalize_mlx_studio_optimizer(config.get("optim", "adamw_8bit"))
    lr_scheduler_type = _normalize_mlx_studio_scheduler(config.get("lr_scheduler_type", "linear"))

    # Force text-only for non-image datasets even on vision-capable models (e.g. Qwen3.5-VL on alpaca).
    # Forward resume_from_checkpoint to MLXTrainer.train(); else Resume silently restarts from step 0.
    resume_from_checkpoint = config.get("resume_from_checkpoint") or None
    training_type = config.get("training_type", "LoRA/QLoRA")
    is_cpt = training_type == "Continued Pretraining"
    # As on CUDA, CPT trains on raw text, so an image dataset loads and formats as text.
    is_dataset_image = bool(config.get("is_dataset_image", False)) and not is_cpt
    # CUDA takes its raw-text path (CPT or format "raw") before the audio one.
    is_dataset_audio = bool(config.get("is_dataset_audio", False)) and not (
        is_cpt or config.get("format_type") == "raw"
    )
    use_lora = training_type in ("LoRA/QLoRA", "Continued Pretraining")
    # Before the download/load below: unlike the CUDA path, this needs none of the model.
    # A continued adapter keeps its own targets, and CPT builds its adapter from target_modules alone.
    if use_lora and not is_cpt and config.get("model_format") != "adapter":
        _check_mlx_finetune_targets(config)
    # Normalize seed; explicit None must not reach the seed chain.
    _raw_seed = config.get("random_seed", 3407)
    random_seed = 3407 if _raw_seed is None else int(_raw_seed)
    # `config.get(k, d)` only fills d when key is missing; handle explicit None too.
    _model_seed = config.get("model_random_state")
    model_random_state = random_seed if _model_seed is None else int(_model_seed)
    _lora_seed = config.get("lora_random_state")
    lora_random_state = random_seed if _lora_seed is None else int(_lora_seed)

    security_error = _model_load_security_error(config, model_load_name, hf_token)
    if security_error:
        _send("error", **security_error)
        return

    # Fetch the dataset before the model so a dataset error does not wait on the model download.
    _send("status", status_message = "Loading dataset...")
    if config.get("is_dataset_audio"):
        # Raw-text runs still read rows carrying the Audio column, so install the decoder for them too.
        from utils.datasets.audio_decode import ensure_audio_decoding
        ensure_audio_decoding()
    hf_dataset = config.get("hf_dataset", "")
    dataset_streaming = bool(hf_dataset and config.get("dataset_streaming"))
    config["_dataset_loaded_from_exact_snapshot"] = False

    def _load_local(file_paths):
        from utils.datasets.saved import load_saved_dataset

        if len(file_paths) == 1:
            saved = load_saved_dataset(_resolve_mlx_local_dataset_path(file_paths[0]))
            if saved is not None:
                return saved
        all_files = _resolve_mlx_local_dataset_files(file_paths)
        if not all_files:
            raise ValueError("No local dataset files found")
        from utils.datasets.cells import csv_as_text_kwargs

        loader = _mlx_local_dataset_loader_for_files(all_files)
        return load_dataset(
            loader,
            data_files = all_files,
            split = "train",
            **csv_as_text_kwargs(all_files),
        )

    eval_dataset = None
    if hf_dataset:
        dataset, eval_dataset = _load_hf_train_and_eval_datasets(
            config,
            hf_token,
            load_dataset,
            lambda message: _send("status", status_message = message),
            lambda message: _send("warning", message = message),
        )
    elif config.get("local_datasets"):
        dataset = _load_local(config["local_datasets"])
    elif config.get("s3_config"):
        from core.training.s3_dataset import (
            S3DownloadCancelled,
            prepare_s3_dataset_download,
        )
        _send("status", status_message = "Downloading dataset from S3...")
        try:
            s3_download = prepare_s3_dataset_download(
                config["s3_config"],
                cancel_callback = _is_stop_requested,
            )
            try:
                dataset = _load_local(s3_download.files)
            finally:
                s3_download.cleanup()
        except S3DownloadCancelled:
            _send("complete", output_dir = None, status_message = "Training cancelled")
            return
    else:
        raise ValueError("No dataset specified")

    from core.training.eval_dataset import evaluation_enabled

    eval_enabled = evaluation_enabled(config.get("eval_steps"))
    if eval_enabled and not hf_dataset and config.get("local_eval_datasets"):
        eval_dataset = _load_local(config["local_eval_datasets"])

    if _is_stop_requested():
        _send("complete", output_dir = None, status_message = "Training cancelled")
        return

    _send("status", status_message = f"Loading {model_name}...")
    try:
        model, tokenizer = FastMLXModel.from_pretrained(
            model_load_name,
            load_in_4bit = config.get("load_in_4bit", True),
            full_finetuning = not use_lora,
            text_only = None if is_dataset_image or is_dataset_audio else True,
            token = hf_token,
            trust_remote_code = bool(config.get("trust_remote_code", False)),
            random_state = model_random_state,
            revision = model_revision,
        )
    except Exception as error:
        if not model_local_only:
            raise
        fallback_error = _model_cache_fallback_error(config, error)
        if fallback_error is not None:
            raise fallback_error from error
        if not _cache_artifact_fallback_allowed(config, error, "model"):
            raise
        revision_error = _mlx_revision_fallback_error(config)
        if revision_error is not None:
            raise revision_error from error
        _send(
            "status",
            status_message = (
                f"Cached files for {model_name} are incomplete; retrying from Hugging Face..."
            ),
        )
        model_load_name = _drop_model_pin_for_fallback(config, hf_token)
        # Scan the Hub target we fall back to, not the cached pin already scanned above.
        security_error = _model_load_security_error(config, model_load_name, hf_token)
        if security_error:
            _send("error", **security_error)
            return
        model_local_only = False
        model_revision = config.get("model_revision")
        model, tokenizer = FastMLXModel.from_pretrained(
            model_load_name,
            load_in_4bit = config.get("load_in_4bit", True),
            full_finetuning = not use_lora,
            text_only = None if is_dataset_image or is_dataset_audio else True,
            token = hf_token,
            trust_remote_code = bool(config.get("trust_remote_code", False)),
            random_state = model_random_state,
            revision = model_revision,
        )

    from utils.models.model_identity import restore_hf_cache_repo_identity

    restored_repo_id = restore_hf_cache_repo_identity(
        model,
        model_load_name,
        expected_repo_id = config.get("actual_model_repo_id") or model_name,
    )
    if restored_repo_id:
        logger.info(
            "Restored Hub model identity for saved MLX adapter metadata: %s",
            restored_repo_id,
        )

    loaded_model_for_provenance = model
    is_vlm = bool((is_dataset_image or is_dataset_audio) and getattr(model, "_is_vlm_model", False))
    model._is_vlm_model = is_vlm
    if is_dataset_audio and not is_vlm:
        # zoo trains audio only as input to an audio-capable vision-language model.
        raise ValueError(
            "Audio dataset training on Apple Silicon needs an audio-input model such as "
            "Gemma 3n or Gemma 4; text-to-speech and speech-recognition checkpoints are not "
            "supported on MLX yet."
        )
    vision_image_size = config.get("vision_image_size")
    # DeepSeek OCR uses a coupled preset tuple; skip resize like the Torch path.
    _model_name_lower = str(config.get("model_name", "")).lower()
    _is_deepseek_ocr = "deepseek" in _model_name_lower and "ocr" in _model_name_lower
    if is_vlm and vision_image_size is not None and _is_deepseek_ocr:
        _send(
            "status",
            status_message = (
                "MLX vision image resize ignored for DeepSeek OCR (uses fixed Gundam preset)."
            ),
        )
        vision_image_size = None
    elif is_vlm and vision_image_size is not None:
        vision_image_size = int(vision_image_size)
        _send(
            "status",
            status_message = f"MLX vision image resize: {vision_image_size} (max dimension)",
        )
    # gradient_checkpointing stays a string; get_peft_model and MLXTrainer both accept strings.
    gc_setting = config.get("gradient_checkpointing", "mlx")
    if isinstance(gc_setting, str):
        use_grad_checkpoint = (
            gc_setting if gc_setting.lower() not in ("false", "none", "") else False
        )
    else:
        use_grad_checkpoint = gc_setting

    # A reloaded adapter stays trainable; never stack a second one on it (#13140).
    continues_adapter = use_lora and next(iter_mlx_lora_modules(model), None) is not None
    if continues_adapter:
        _send(
            "status",
            status_message = "Continuing from existing LoRA adapters (kept the adapter's settings)",
        )
    elif is_cpt:
        _send("status", status_message = "Configuring LoRA for continued pretraining...")
        # As on CUDA, embed_tokens / lm_head train as full modules rather than through LoRA.
        user_modules = config.get("target_modules") or []
        leaf = lambda module: str(module).rsplit(".", 1)[-1]  # noqa: E731
        full_modules = {leaf(m) for m in user_modules} & _CPT_TARGET_NAMES
        quantized = _mlx_quantized_cpt_modules(model, full_modules)
        if quantized:
            _send(
                "warning",
                message = (
                    f"This 4-bit load quantized {' and '.join(sorted(quantized))}, which cannot "
                    "train in full, so continued pretraining leaves them frozen. Load the 16-bit "
                    "model to train them."
                ),
            )
            full_modules -= quantized
        peft_kwargs = dict(
            r = config.get("lora_r", 128),
            lora_alpha = config.get("lora_alpha", 32),
            lora_dropout = config.get("lora_dropout", 0.0),
            use_rslora = config.get("use_rslora", False),
            random_state = lora_random_state,
            target_modules = [m for m in user_modules if leaf(m) not in _CPT_TARGET_NAMES]
            or list(_MLX_DEFAULT_LORA_TARGETS),
            modules_to_save = sorted(full_modules) or None,
            use_gradient_checkpointing = use_grad_checkpoint,
        )
    elif use_lora:
        _send("status", status_message = "Configuring LoRA adapters...")
        peft_kwargs = dict(
            r = config.get("lora_r", 16),
            lora_alpha = config.get("lora_alpha", 16),
            lora_dropout = config.get("lora_dropout", 0.0),
            use_rslora = config.get("use_rslora", False),
            init_lora_weights = config.get("init_lora_weights", True),
            random_state = lora_random_state,
            target_modules = config.get("target_modules") or list(_MLX_DEFAULT_LORA_TARGETS),
            use_gradient_checkpointing = use_grad_checkpoint,
        )
        finetune_language = config.get("finetune_language_layers", True)
        finetune_attention = config.get("finetune_attention_modules", True)
        finetune_mlp = config.get("finetune_mlp_modules", True)
        # Audio-only rows give vision LoRA no gradient, and zoo refuses Gemma 3n's linear-free vision tower.
        finetune_vision = (
            config.get("finetune_vision_layers", False) if is_vlm and is_dataset_image else False
        )

        if (finetune_attention or finetune_mlp) and not finetune_language and not finetune_vision:
            finetune_language = True

        # is_vlm and the back-fill's outcome are known now; the preflight could only guess.
        _check_mlx_effective_targets(
            config,
            finetune_language = finetune_language,
            finetune_vision = finetune_vision,
        )

        peft_kwargs["finetune_language_layers"] = finetune_language
        peft_kwargs["finetune_attention_modules"] = finetune_attention
        peft_kwargs["finetune_mlp_modules"] = finetune_mlp
        if is_vlm:
            peft_kwargs["finetune_vision_layers"] = finetune_vision
    if use_lora and not continues_adapter:
        peft_kwargs.update(mlx_dora_kwargs)
        model = FastMLXModel.get_peft_model(model, **peft_kwargs)

    slice_start = config.get("dataset_slice_start")
    slice_end = config.get("dataset_slice_end")

    # max_steps runs cannot reach the whole dataset; bound rows before mapping.
    mlx_raw_text_mode = (
        training_type == "Continued Pretraining" or config.get("format_type") == "raw"
    )
    # mlx.launch shards the batch like DDP and advertises the process count in the env this reads.
    mlx_max_train_rows = max_train_rows_for_config(
        config,
        branch_never_packs = is_vlm and not mlx_raw_text_mode,
        world_size = _data_parallel_world_size(),
    )
    # Resume replays a batch cursor, so the bound must match the checkpoint's marker.
    mlx_max_train_rows, mlx_max_train_rows_seed = row_bound_for_resume(
        resume_from_checkpoint, mlx_max_train_rows, random_seed
    )

    # A bracketed split names rows the same way the numeric fields do.
    mlx_split_names_rows = "[" in (config.get("train_split") or "")
    # The bound keeps a uniform sample of the rows, so a pass over it is this share of a dataset pass.
    mlx_kept_row_fraction = [1.0]

    def _on_bound(kept, total):
        mlx_kept_row_fraction[0] = kept / total
        _send("status", status_message = f"Using {kept} of {total} rows (max_steps run)")

    def _slice(ds):
        if dataset_streaming:
            return _slice_streaming_dataset(ds, slice_start, slice_end)
        if slice_start is not None or slice_end is not None:
            # The user named these rows; the bound below defers to that.
            return slice_dataset_rows(ds, slice_start, slice_end)[0]
        if mlx_split_names_rows:
            return ds
        return bound_dataset_rows(
            ds,
            mlx_max_train_rows,
            mlx_max_train_rows_seed,
            on_bound = _on_bound,
        )

    dataset = _slice(dataset)

    if not _emit_resource_provenance(
        event_queue,
        config,
        loaded_model_for_provenance,
        model_load_target = model_load_name,
        model_load_in_4bit = bool(config.get("load_in_4bit")),
        dataset_loaded_from_exact_snapshot = bool(config.get("_dataset_loaded_from_exact_snapshot")),
    ):
        return

    # Reuse the GPU format pipeline for VLM (OCR/caption/llava/sharegpt+images) and text.
    format_type = "raw" if is_cpt else config.get("format_type", "")
    custom_format_mapping = config.get("custom_format_mapping")
    dataset_final_format = ""
    eval_final_format = ""
    try:
        from utils.datasets import format_and_template_dataset
        def _fmt_progress(status_message = "", **_kw):
            _send("status", status_message = status_message)

        if is_dataset_audio:
            _send("status", status_message = "Formatting audio dataset...")
            _warn = lambda message: _send("warning", message = message)
            dataset, audio_image_col = _mlx_audio_rows(
                dataset, tokenizer, custom_format_mapping, warn = _warn
            )
            if eval_dataset is not None:
                eval_dataset, _ = _mlx_audio_rows(
                    eval_dataset, tokenizer, custom_format_mapping, audio_image_col, warn = _warn
                )
                if not eval_dataset:
                    _send(
                        "warning",
                        message = "The eval dataset is empty after preprocessing, so this run has no evaluation.",
                    )
                    eval_dataset = None
                    eval_enabled = False
            if audio_image_col:
                image_layout = (
                    _mlx_vlm_resized_image_layout(tokenizer)
                    if vision_image_size is not None
                    else None
                )
                dataset = _adapt_for_mlx_vlm(
                    dataset, resize = vision_image_size, image_layout = image_layout
                )
                if eval_dataset is not None:
                    eval_dataset = _adapt_for_mlx_vlm(
                        eval_dataset, resize = vision_image_size, image_layout = image_layout
                    )
        elif is_vlm:
            from utils.datasets.format_detection import detect_video_columns, video_sample

            # Before formatting: conversion would fetch every Hub clip only to be refused.
            first_row = next(iter(dataset), None)
            if isinstance(first_row, dict) and detect_video_columns(
                video_sample(dataset, first_row)
            ):
                raise ValueError(_MLX_VIDEO_UNSUPPORTED)
            _send("status", status_message = "Formatting VLM dataset...")
            vlm_info = format_and_template_dataset(
                dataset,
                model_name = model_name,
                tokenizer = tokenizer,
                is_vlm = True,
                dataset_name = hf_dataset or "local",
                dataset_revision = config.get("dataset_revision"),
                custom_format_mapping = custom_format_mapping,
                progress_callback = _fmt_progress,
            )
            if vlm_info.get("success"):
                vision_image_layout = (
                    _mlx_vlm_resized_image_layout(tokenizer)
                    if vision_image_size is not None
                    else None
                )
                dataset = _adapt_for_mlx_vlm(
                    vlm_info["dataset"],
                    resize = vision_image_size,
                    image_layout = vision_image_layout,
                )
            else:
                errors = vlm_info.get("errors", [])
                raise ValueError(f"VLM dataset format conversion failed: {'; '.join(errors)}")
            if eval_dataset is not None:
                ev_info = format_and_template_dataset(
                    eval_dataset,
                    model_name = model_name,
                    tokenizer = tokenizer,
                    is_vlm = True,
                    dataset_name = hf_dataset or "local",
                    dataset_revision = config.get("dataset_revision"),
                    custom_format_mapping = custom_format_mapping,
                )
                if ev_info.get("success"):
                    vision_image_layout = (
                        _mlx_vlm_resized_image_layout(tokenizer)
                        if vision_image_size is not None
                        else None
                    )
                    eval_dataset = _adapt_for_mlx_vlm(
                        ev_info["dataset"],
                        resize = vision_image_size,
                        image_layout = vision_image_layout,
                    )

        elif format_type:
            _send("status", status_message = f"Formatting dataset ({format_type})...")
            info = format_and_template_dataset(
                dataset,
                model_name = model_name,
                tokenizer = tokenizer,
                is_vlm = False,
                format_type = format_type,
                dataset_name = hf_dataset or "local",
                custom_format_mapping = custom_format_mapping,
                progress_callback = _fmt_progress,
            )
            if info.get("success", True):
                dataset = info.get("dataset", dataset)
            else:
                errors = info.get("errors", [])
                raise ValueError(f"Dataset format conversion failed: {'; '.join(errors)}")
            if info.get("dropped_rows_warning"):
                _send("warning", message = info["dropped_rows_warning"])
            for message in info.get("run_warnings", []):
                _send("warning", message = message)
            dataset_final_format = str(info.get("final_format", "") or "").lower()
            eval_final_format = dataset_final_format
            if eval_dataset is not None:
                ev = format_and_template_dataset(
                    eval_dataset,
                    model_name = model_name,
                    tokenizer = tokenizer,
                    is_vlm = False,
                    format_type = format_type,
                    dataset_name = hf_dataset or "local",
                    custom_format_mapping = custom_format_mapping,
                    split_name = "eval",
                    raw_text_column = info.get("raw_text_column"),
                )
                if ev.get("success", True):
                    eval_dataset = ev.get("dataset", eval_dataset)
                    # A separately supplied eval split is detected on its own and may not be pre-tokenized.
                    eval_final_format = str(ev.get("final_format", "") or "").lower()
                    if hasattr(eval_dataset, "__len__") and len(eval_dataset) == 0:
                        _send(
                            "warning",
                            message = "The eval dataset is empty after preprocessing, so this run has no evaluation.",
                        )
                        eval_dataset = None
                        # A user-supplied split that filters to nothing must not carve one out of train.
                        eval_enabled = False
                else:
                    eval_errors = ev.get("errors", [])
                    raise ValueError(
                        f"Eval dataset format conversion failed: {'; '.join(eval_errors)}"
                    )
                if ev.get("dropped_rows_warning"):
                    _send("warning", message = f"Eval dataset: {ev['dropped_rows_warning']}")
                for message in ev.get("run_warnings", []):
                    _send("warning", message = message)
    except ImportError:
        _send("status", status_message = "Format helper unavailable, using raw dataset")

    if not is_vlm and dataset_final_format:
        # get_chat_template returns a new tokenizer unless it is an mlx-lm wrapper, so the rows' template stayed in
        # the formatter; mirror the CUDA trainer and re-apply it here (no-op for models with a template).
        from utils.datasets.chat_templates import get_training_chat_template
        tokenizer = get_training_chat_template(tokenizer, model_name, dataset_final_format)
        if _carry_mlx_chat_template_to_processor(model, tokenizer):
            logger.info("Saving the training chat template with the processor")

    if eval_enabled and eval_dataset is None:
        from core.training.eval_dataset import (
            MIN_TOTAL_ROWS_FOR_EVAL,
            split_dataset_for_evaluation,
        )
        split_result = split_dataset_for_evaluation(dataset)
        if split_result is None:
            _send(
                "warning",
                message = (
                    f"Evaluation is enabled, but the training dataset has only {len(dataset)} "
                    f"rows; at least {MIN_TOTAL_ROWS_FOR_EVAL} are required to create a "
                    "held-out eval split. Training will continue without evaluation."
                ),
            )
        else:
            dataset, eval_dataset = split_result
            eval_final_format = dataset_final_format

    # zoo's MLX batcher reads input_ids / labels and ignores attention_mask, so masked padding is trimmed here as on
    # CUDA. MLX truncates to max_seq_length itself. Each split goes by its own format: a separately supplied eval
    # split can be pre-tokenized while the training split is text, and the reverse.
    if dataset_final_format == "pretokenized":
        from utils.datasets.pretokenized import prepare_pretokenized_dataset
        dataset = prepare_pretokenized_dataset(
            dataset, max_seq_length = None, vocab_size = _mlx_vocab_size(model), mlx = True
        )
    if eval_dataset is not None and eval_final_format == "pretokenized":
        from utils.datasets.pretokenized import prepare_pretokenized_dataset
        eval_dataset = prepare_pretokenized_dataset(
            eval_dataset, max_seq_length = None, vocab_size = _mlx_vocab_size(model), mlx = True
        )

    requested_max_steps = config.get("max_steps", 0) or 0
    num_epochs = config.get("num_epochs", 3)
    max_seq_length = config.get("max_seq_length", 2048)
    batch_size = config.get("batch_size", 4)
    grad_accum = config.get("gradient_accumulation_steps", 4)

    lr_value = float(config.get("learning_rate", "5e-5" if is_cpt else "2e-4"))
    embedding_lr_value = (
        _mlx_cpt_embedding_learning_rate(
            config.get("embedding_learning_rate"),
            lr_value,
            trains_embeddings = _trains_embedding_modules(
                (name for name, _ in tree_flatten(model.trainable_parameters())),
                getattr(model, "_unsloth_cpt_full_module_weight_keys", None) or (),
            ),
        )
        if is_cpt
        else None
    )

    # Warmup: prefer warmup_steps; fall back to warmup_ratio
    warmup_steps = config.get("warmup_steps")
    warmup_ratio = config.get("warmup_ratio")
    warmup_uses_ratio = warmup_steps is None and warmup_ratio is not None
    if warmup_steps is None:
        warmup_steps = 0 if warmup_uses_ratio else 5

    # Resolve to ~/.unsloth/studio/outputs/ so the export page finds it
    from utils.paths import ensure_dir

    # Resume must land in the original run dir even when config lacks output_dir.
    resume_dir = config.get("output_dir", "") or _output_dir_from_resume_checkpoint(
        resume_from_checkpoint
    )
    output_dir = _resolve_mlx_output_dir(
        {**config, "output_dir": resume_dir} if resume_dir else config, model_name
    )
    raw_eval_steps = config.get("eval_steps", 0)
    if evaluation_enabled(raw_eval_steps):
        eval_steps_value = float(raw_eval_steps)
    else:
        eval_steps_value = 0.0
    eval_steps_ratio = eval_steps_value if 0 < eval_steps_value < 1 else None
    eval_steps_val = 0 if eval_steps_ratio is not None else int(eval_steps_value)

    # Re-validate for direct worker callers; training.py normalizes the main path.
    max_grad_norm = _resolve_mlx_max_grad_norm(config.get("max_grad_norm"))
    max_grad_value = config.get("max_grad_value")
    if max_grad_value is not None:
        max_grad_value = float(max_grad_value)
        if max_grad_value < 0 or not math.isfinite(max_grad_value):
            raise ValueError(
                f"Unsloth MLX: max_grad_value={max_grad_value} must be finite and >= 0 "
                "(0 or None disables elementwise clipping)."
            )
    max_grad_leaf_norm = config.get("max_grad_leaf_norm")
    if max_grad_leaf_norm is not None:
        max_grad_leaf_norm = float(max_grad_leaf_norm)
        if max_grad_leaf_norm < 0 or not math.isfinite(max_grad_leaf_norm):
            raise ValueError(
                f"Unsloth MLX: max_grad_leaf_norm={max_grad_leaf_norm} must be finite and >= 0 "
                "(0 or None disables proportional leaf-norm clipping)."
            )
    weight_decay = config.get("weight_decay", 0.001)
    weight_decay = 0.001 if weight_decay is None else float(weight_decay)

    # A sized dataset stays eager: without a pass length zoo cannot end an epoch on an optimizer step.
    mlx_config_kwargs = dict(
        per_device_train_batch_size = batch_size,
        gradient_accumulation_steps = grad_accum,
        max_steps = requested_max_steps,
        learning_rate = lr_value,
        warmup_steps = warmup_steps,
        lr_scheduler_type = lr_scheduler_type,
        optim = optim_name,
        weight_decay = weight_decay,
        max_grad_norm = max_grad_norm,
        max_grad_value = max_grad_value,
        logging_steps = 1,
        max_seq_length = max_seq_length,
        seed = random_seed,
        use_cce = True,
        compile = True,
        gradient_checkpointing = use_grad_checkpoint,
        packing = bool(config.get("packing", False)),
        streaming = dataset_streaming,
        output_dir = output_dir,
        save_steps = int(config.get("save_steps", 0) or 0),
        eval_steps = eval_steps_val,
    )

    # Also gates the masking skip below, so defined outside the feature-detect block.
    raw_text_mode = training_type == "Continued Pretraining" or format_type == "raw"

    # Feature-detect optional fields so this PR works without the paired zoo bump.
    _supported_fields = getattr(MLXTrainingConfig, "__dataclass_fields__", {})
    if "cast_norm_output_to_input_dtype" in _supported_fields:
        # Explicit None falls back to True (default).
        _raw_cast = config.get("cast_norm_output_to_input_dtype", True)
        mlx_config_kwargs["cast_norm_output_to_input_dtype"] = (
            True if _raw_cast is None else bool(_raw_cast)
        )
    if "dataset_order" in _supported_fields:
        # CUDA's trainer reads a stream in source order; a sized dataset gets its RandomSampler order.
        mlx_config_kwargs["dataset_order"] = "sequential" if dataset_streaming else "torch_randperm"
    if "max_grad_leaf_norm" in _supported_fields:
        mlx_config_kwargs["max_grad_leaf_norm"] = max_grad_leaf_norm
    if "report_grad_norm" in _supported_fields:
        # Grad norm is free under global-norm clipping; switching clip modes would change loss.
        mlx_config_kwargs["report_grad_norm"] = True
    if (
        "early_stopping_patience" in _supported_fields
        and "load_best_model_at_end" in _supported_fields
    ):
        from core.training.early_stopping import early_stopping_patience

        # Zoo counts patience in evaluations and restores the best adapters itself, the CUDA path's contract.
        patience = early_stopping_patience(config.get("early_stopping_patience"))
        if patience and eval_dataset is not None and eval_steps_value > 0:
            mlx_config_kwargs["early_stopping_patience"] = patience
            mlx_config_kwargs["load_best_model_at_end"] = True
    if embedding_lr_value is not None:
        mlx_config_kwargs["embedding_learning_rate"] = embedding_lr_value
    if "append_eos" in _supported_fields:
        # Unsloth SFT formatting owns rendered examples; raw/CPT text still needs MLX to append EOS.
        mlx_config_kwargs["append_eos"] = bool(raw_text_mode)

    trainer = MLXTrainer(
        model = model,
        tokenizer = tokenizer,
        train_dataset = dataset,
        eval_dataset = eval_dataset,
        args = MLXTrainingConfig(**mlx_config_kwargs),
    )
    max_steps, eval_steps_val = _configure_mlx_training_schedule(
        trainer,
        requested_max_steps,
        None if dataset_streaming else len(dataset),
        batch_size,
        grad_accum,
        num_epochs,
        warmup_ratio = warmup_ratio if warmup_uses_ratio else None,
        eval_steps_ratio = eval_steps_ratio,
    )
    _prepare_mlx_output_dir(trainer, output_dir, ensure_dir, event_queue)

    def _record_mlx_row_bound():
        # Pin the subset before any checkpoint lands here; a resume reads it back.
        if not record_row_bound(output_dir, mlx_max_train_rows, mlx_max_train_rows_seed) and (
            mlx_max_train_rows
        ):
            _send(
                "warning",
                message = (
                    f"Could not record the max_steps row bound in {output_dir}: "
                    "resuming this run later will read it as unbounded"
                ),
            )

    _run_mlx_main_process_action(trainer, _record_mlx_row_bound, "max_steps row bound setup")
    _trainer_ref[0] = trainer
    if _is_stop_requested():
        trainer.stop_requested = True

    # Tell the parent eval is configured so the frontend shows the eval chart
    if eval_dataset is not None and eval_steps_val > 0:
        _send("eval_configured")

    # Auto-detect markers from the chat template first, manual table as fallback. Mirror the CUDA skips: raw/CPT text
    # has no chat turns. Check the resolved format too, since format_type="auto" can land on alpaca or raw.
    from utils.datasets.pretokenized import has_custom_labels

    if (
        config.get("train_on_completions", False)
        and dataset_final_format == "pretokenized"
        and not has_custom_labels(dataset)
    ):
        # zoo's MLX masker re-tokenizes text, so token-id rows have nothing it can mask (CUDA masks input_ids).
        _send(
            "warning",
            message = (
                "'Train on completions' is not available for pre-tokenized datasets on Mac: "
                "training will run on full sequences. Add a labels column to choose the trained tokens."
            ),
        )
    if (
        config.get("train_on_completions", False)
        and not raw_text_mode
        and dataset_final_format not in ("raw_text", "pretokenized")
    ):
        _send("status", status_message = "Configuring response-only training...")
        # No catch: an exception here is a real masking failure and must fail the run.
        from utils.datasets.completion_masking import apply_completion_masking

        trainer, masking_applied = apply_completion_masking(
            trainer,
            model_name,
            train_on_responses_only,
            notify = lambda level, message: _send("status", status_message = message),
            dataset_template = "alpaca" if dataset_final_format == "alpaca" else None,
        )
        if not masking_applied:
            # Masking miss changes the run's objective, so warn in the sticky warning list, not a status line.
            _send(
                "warning",
                message = (
                    f"'Train on completions' could not be applied for {model_name}: no "
                    f"instruction/response markers were found. Training will run on full "
                    f"sequences (prompts included)."
                ),
            )

    wandb_run, tb_writer = _setup_mlx_tracking(trainer, config, output_dir, _send)

    _send("status", status_message = f"Training {model_name}...")

    start_step = 0
    if resume_from_checkpoint:
        start_step = _checkpoint_state(Path(resume_from_checkpoint)) or 0

    def _on_step(
        step,
        total,
        loss,
        lr,
        tok_s,
        peak_gb,
        elapsed,
        num_tokens,
        grad_norm = None,
    ):
        eta = session_eta_seconds(elapsed, step, start_step, total) or 0
        _send(
            "progress",
            step = step,
            epoch = (
                round(trainer.state.epoch * mlx_kept_row_fraction[0], 2)
                if trainer.state.epoch
                else 0
            ),
            loss = loss,
            learning_rate = lr,
            total_steps = total,
            elapsed_seconds = elapsed,
            eta_seconds = eta,
            session_start_step = start_step,
            grad_norm = grad_norm,
            num_tokens = num_tokens,
            eval_loss = None,
            status_message = None,
            peak_memory_gb = peak_gb,
        )
        if wandb_run is not None:
            try:
                wandb_run.log(
                    {
                        "train/loss": loss,
                        "train/learning_rate": lr,
                        "train/tokens_per_sec": tok_s,
                        "train/peak_gb": peak_gb,
                        "train/num_tokens": num_tokens,
                        **({"train/grad_norm": grad_norm} if grad_norm is not None else {}),
                    },
                    step = step,
                )
            except Exception:
                pass
        if tb_writer is not None:
            try:
                tb_writer.add_scalar("train/loss", loss, step)
                tb_writer.add_scalar("train/learning_rate", lr, step)
                tb_writer.add_scalar("train/tokens_per_sec", tok_s, step)
                tb_writer.add_scalar("train/peak_gb", peak_gb, step)
                if grad_norm is not None:
                    tb_writer.add_scalar("train/grad_norm", grad_norm, step)
            except Exception:
                pass

    trainer.add_step_callback(_on_step)

    def _on_eval(step, eval_loss, perplexity):
        _send("progress", step = step, eval_loss = eval_loss)
        if wandb_run is not None:
            try:
                wandb_run.log({"eval/loss": eval_loss, "eval/perplexity": perplexity}, step = step)
            except Exception:
                pass
        if tb_writer is not None:
            try:
                tb_writer.add_scalar("eval/loss", eval_loss, step)
                tb_writer.add_scalar("eval/perplexity", perplexity, step)
            except Exception:
                pass

    trainer.add_eval_callback(_on_eval)

    _opt_ref = [None]
    _orig_build_optimizer = getattr(trainer, "_build_optimizer", None)

    if callable(_orig_build_optimizer):

        def _capture_optimizer(total_steps):
            _opt_ref[0] = _orig_build_optimizer(total_steps)
            return _opt_ref[0]

        trainer._build_optimizer = _capture_optimizer

    gc.collect()
    mx.synchronize()
    _save_model = trainer.save_model

    def _skip_internal_final_save(*args, **kwargs):
        raise ValueError("worker owns final save")

    trainer.save_model = _skip_internal_final_save
    try:
        trainer.train(resume_from_checkpoint = resume_from_checkpoint)
    finally:
        trainer.save_model = _save_model

    def _finish_tracking() -> None:
        # Runs on every save/finalize exit so TB/W&B never leak on early return.
        if tb_writer is not None:
            try:
                tb_writer.close()
            except Exception:
                pass
        if wandb_run is not None:
            try:
                wandb_run.finish()
            except Exception:
                pass

    try:
        _finalize_mlx_training(
            trainer,
            _snapshot_stop,
            output_dir,
            mx.synchronize,
            _send,
            lambda: _write_mlx_stop_checkpoint(trainer, _opt_ref[0], output_dir),
        )
    finally:
        _finish_tracking()


def _is_current_process_apple_silicon() -> bool:
    import platform
    return platform.system() == "Darwin" and platform.machine() == "arm64"


def run_mlx_training_process(
    *,
    event_queue: Any,
    stop_queue: Any,
    config: dict,
    transformers_activated: bool = False,
    config_prevalidated: bool = False,
) -> None:
    """MLX worker entrypoint shared by Unsloth subprocesses and the CLI adapter."""
    backend_path = str(Path(__file__).resolve().parent.parent.parent)
    if backend_path not in sys.path:
        sys.path.insert(0, backend_path)

    from utils.hf_xet_fallback import child_should_disable_xet

    if child_should_disable_xet(config):
        os.environ["HF_HUB_DISABLE_XET"] = "1"
        os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"

    if not config_prevalidated and not _validate_training_worker_config(config, event_queue):
        return
    model_load_target = _resolve_cached_model_load_name(config)

    if not transformers_activated:
        # Must precede detect_hardware(): its MLX stack check imports mlx_lm, hence transformers.
        _activate_transformers_version_or_warn(
            model_load_target,
            _worker_hf_token(config),
        )

    from utils.hardware import hardware as _hw

    _hw.detect_hardware()
    if _hw.DEVICE != _hw.DeviceType.MLX:
        event_queue.put(
            {
                "type": "error",
                "error": "MLX training requires Apple Silicon with the MLX backend available.",
                "stack": "",
                "ts": time.time(),
            }
        )
        return

    try:
        try:
            if config.get("is_decision"):
                # Its own pipeline, as on the torch path, behind the same security gate.
                security_error = _model_load_security_error(
                    config, model_load_target, _worker_hf_token(config)
                )
                if security_error:
                    event_queue.put({"type": "error", **security_error, "ts": time.time()})
                else:
                    from core.training.decision_trainer import run_decision_training
                    run_decision_training(event_queue, stop_queue, config)
            else:
                _run_mlx_training(event_queue, stop_queue, config)
        finally:
            try:
                stop_queue.put({"type": _MLX_WORKER_COMPLETE})
            except (EOFError, OSError, ValueError):
                pass
    except BaseException as exc:
        event_queue.put(
            {
                "type": "error",
                "error": str(exc) or type(exc).__name__,
                "stack": traceback.format_exc(limit = 20),
                "ts": time.time(),
            }
        )
    _end_abandoned_decision_download(event_queue)


def _end_abandoned_decision_download(event_queue: Any) -> None:
    trainer = sys.modules.get("core.training.decision_trainer")
    if trainer is not None:
        trainer.end_abandoned_download(event_queue)


def _training_job_is_local(config) -> bool:
    """True when neither model nor dataset needs the Hub; unresolvable means remote (fail closed)."""
    try:
        from utils.paths import is_local_path
    except Exception:
        return False
    if config.get("hf_dataset"):
        return False
    model = config.get("model_name")
    try:
        if not (model and is_local_path(model)):
            return False
        # A local checkpoint can name a remote base to fetch later, so decide from disk, not network.
        base, needs_hub = _recorded_local_base(model)
        if needs_hub:
            return False
        return not base or is_local_path(base)
    except Exception:
        return False


def _recorded_local_base(model_name) -> "tuple[str | None, bool]":
    """``(base, needs_hub)`` for the base this checkpoint records on disk. Delegates to the
    resolver's own disk reads so the gate cannot drift from what activation later resolves. Fail
    closed on an unavailable reader."""
    try:
        from utils.transformers_version import recorded_local_base
        return recorded_local_base(model_name)
    except Exception:
        return None, True


def _download_decision_checkpoint(event_queue: Any, config: dict) -> None:
    from core.systemone import laya_runtime
    from core.systemone.catalog import Checkpoint
    from utils.hf_xet_fallback import start_watchdog
    from utils.paths import is_local_path

    model_name = config["model_name"]
    # An LLM that gets a new decision head downloads through FastModel's own loader in the trainer.
    if is_local_path(model_name) or config.get("decision_layout") == "llm":
        return
    hf_token = _worker_hf_token(config)
    if hf_token:
        os.environ["HF_TOKEN"] = hf_token
    _send_status(event_queue, "Loading decision model...")
    # Under the stall watchdog, so the parent can retry a stalled Xet download over HTTP.
    event_queue.put({"type": "model_load_started", "ts": time.time()})
    watchdog_stop = start_watchdog(
        repo_ids = [model_name],
        on_stall = lambda msg: event_queue.put({"type": "stall", "message": msg, "ts": time.time()}),
        xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
    )
    try:
        laya_runtime._checkpoint_dir(
            Checkpoint(
                "base",
                model_name,
                config.get("model_subfolder") or None,
                "",
                layout = config.get("decision_layout") or "laya",
            )
        )
    except Exception as exc:
        # The trainer's own load reports it.
        logger.info("Could not download %s ahead of the trainer: %s", model_name, exc)
    finally:
        watchdog_stop.set()
        event_queue.put({"type": "model_load_completed", "ts": time.time()})


def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> None:
    """Fresh subprocess entrypoint: events out on event_queue, stop commands in on stop_queue."""
    # Off on Linux: forked map() workers deadlock.
    os.environ["TOKENIZERS_PARALLELISM"] = (
        "true" if sys.platform in ("win32", "darwin") else "false"
    )
    os.environ["PYTHONWARNINGS"] = "ignore"  # before imports
    # Until unsloth is imported below; transformers is first imported well before that.
    metadata_scan_cache = _install_metadata_scan_cache()

    from utils.security.hub_data_files import install as guard_hub_data_files
    from utils.security.dataset_media import allow_dataset_repos, install as guard_dataset_media

    guard_hub_data_files()
    guard_dataset_media()
    allow_dataset_repos(config.get("hf_dataset"))

    if not config.get("allow_ambient", True):
        from hub.utils.hf_tokens import apply_token_to_child_env, hf_token_arg, is_anonymous

        hf_token = hf_token_arg(config.get("hf_token"), allow_ambient_token = False)
        apply_token_to_child_env(os.environ, hf_token)
        if is_anonymous(hf_token):
            os.environ["HF_TOKEN_PATH"] = os.devnull

    # HTTP-fallback respawn: disable Xet before any huggingface_hub import (read at import time).
    from utils.hf_xet_fallback import child_should_disable_xet

    if child_should_disable_xet(config):
        os.environ["HF_HUB_DISABLE_XET"] = "1"
        os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"
        print(
            "Xet transport disabled for this training worker (HF_HUB_DISABLE_XET=1).",
            file = sys.stderr,
            flush = True,
        )

    # Offline auto-detect: skip ~25s of HF retries per call when the hub is unreachable. Skipped for a filesystem-only
    # job: a local checkpoint with a local dataset never reaches the Hub, and probing unconditionally would add
    # seconds to every such startup.
    if "HF_HUB_OFFLINE" not in os.environ and not _training_job_is_local(config):
        _offline = False
        _network_offline = False
        try:
            from utils.utils import hf_dns_dead, hf_env_offline, hf_probe_disabled

            # Hub ignores TRANSFORMERS_OFFLINE, so translate it before probing.
            _offline = hf_env_offline()
            # hf_dns_dead follows HF_ENDPOINT and stands down behind a proxy, so a mirror still counts.
            if not _offline:
                _offline = _network_offline = hf_dns_dead()
            if not _offline and not hf_probe_disabled():
                # DNS can answer without egress; only a connection failure counts, not a 502/503.
                from utils.transformers_version import hf_endpoint_unreachable
                _offline = _network_offline = hf_endpoint_unreachable(
                    gateway_errors_offline = False,
                    proxy_timeouts_offline = False,
                )
        except Exception:
            _offline = _network_offline = False
        if _offline:
            os.environ["HF_HUB_OFFLINE"] = "1"
            os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
            # TRANSFORMERS_OFFLINE alone must not force cache-only datasets.
            if _network_offline:
                os.environ.setdefault("HF_DATASETS_OFFLINE", "1")
            # logger isn't configured yet; print to stderr instead.
            print(
                "Hugging Face endpoint unreachable; HF_HUB_OFFLINE=1 set for this worker.",
                file = sys.stderr,
                flush = True,
            )

    import warnings
    from loggers.config import LogConfig, keep_progress_bars_countable

    if os.getenv("ENVIRONMENT_TYPE", "production") == "production":
        warnings.filterwarnings("ignore")

    # The monitor polls tqdm._instances, so redirect bars instead of disabling them.
    keep_progress_bars_countable()

    LogConfig.setup_logging(
        service_name = "unsloth-studio-training-worker",
        env = os.getenv("ENVIRONMENT_TYPE", "production"),
    )
    # As the inference worker: a recovered traceback reaching fd 2 must not read as the crash.
    from utils.worker_stderr import mark_log_record_continuations

    mark_log_record_continuations()

    gpu_ids = config.get("resolved_gpu_ids")
    if config.get("is_decision") and gpu_ids and len(gpu_ids) > 1:
        gpu_ids = gpu_ids[:1]
        event_queue.put(
            {
                "type": "warning",
                "message": f"Decision models train on one GPU; using GPU {gpu_ids[0]}.",
                "ts": time.time(),
            }
        )
    apply_gpu_ids(gpu_ids, backend = config.get("device_backend"))

    if not _validate_training_worker_config(config, event_queue):
        return
    # Before any backend dispatch: the MLX, decision and embedding runners load these weights too.
    weights_error = _escaping_weights_error(config)
    if weights_error:
        event_queue.put({"type": "error", "error": weights_error, "stack": "", "ts": time.time()})
        return

    model_name = config["model_name"]
    model_load_target = _resolve_cached_model_load_name(config)

    # MLX fast path, before any torch/transformers import.
    backend_path = str(Path(__file__).resolve().parent.parent.parent)
    if backend_path not in sys.path:
        sys.path.insert(0, backend_path)

    from .training import is_apple_silicon_training_platform, should_use_mlx_training_backend

    mlx_backend_requested = is_apple_silicon_training_platform()

    mlx_transformers_activated = False
    if mlx_backend_requested and _is_current_process_apple_silicon():
        # Must precede detect_hardware(): its MLX stack check imports mlx_lm, hence transformers.
        _activate_transformers_version_or_warn(
            model_load_target,
            _worker_hf_token(config),
        )
        mlx_transformers_activated = True

    from utils.hardware import hardware as _hw

    _hw.detect_hardware()
    if _hw.trains_on_cpu(_hw.CHAT_ONLY_REASON):
        # Studio's verdict wins over the inherited UNSLOTH_DISABLE_CPU_FALLBACK=1 and zoo's own CPU refusals.
        os.environ["UNSLOTH_FORCE_CPU"] = "1"
    if mlx_backend_requested or should_use_mlx_training_backend(device = _hw.DEVICE):
        diffusion_error = _mlx_diffusion_lm_error(config, model_load_target)
        if diffusion_error:
            event_queue.put(
                {"type": "error", "error": diffusion_error, "stack": "", "ts": time.time()}
            )
            return
        run_mlx_training_process(
            event_queue = event_queue,
            stop_queue = stop_queue,
            config = config,
            transformers_activated = mlx_transformers_activated,
            config_prevalidated = True,
        )
        return

    try:
        _activate_transformers_version(
            model_load_target,
            _worker_hf_token(config),
        )
    except Exception as exc:
        event_queue.put(
            {
                "type": "error",
                "error": f"Failed to activate transformers version: {exc}",
                "stack": traceback.format_exc(limit = 20),
                "ts": time.time(),
            }
        )
        return

    # NemotronH needs trust_remote_code=True for config parsing bugs (not Llama-Nemotron).
    from utils.security.trusted_org import is_trusted_org_repo

    _NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano")
    _lowered = model_name.lower()
    if (
        any(sub in _lowered for sub in _NEMOTRON_TRUST_SUBSTRINGS)
        and (_lowered.startswith("unsloth/") or _lowered.startswith("nvidia/"))
        # Confirm a genuine first-party Hub repo (not a spoofed "unsloth/" name); authenticated.
        and is_trusted_org_repo(model_name, hf_token = _worker_hf_token(config))
        and not config.get("trust_remote_code", False)
    ):
        config["trust_remote_code"] = True
        logger.info(
            "Auto-enabled trust_remote_code for Nemotron model: %s",
            model_name,
        )

    security_error = _model_load_security_error(
        config,
        _resolve_cached_model_load_name(config),
        _worker_hf_token(config),
    )
    if security_error:
        event_queue.put({"type": "error", **security_error, "ts": time.time()})
        return

    # ── 1b. Install fast-path kernel libraries for the chosen model.
    # 1) causal-conv1d runs eagerly for matching architectures: some SSM modeling files
    #    lazy_load it without calling is_causal_conv1d_available.
    # 2) mamba-ssm + flash-attn keep their substring / size gates.
    # 3) FLA gated-delta kernels: vendored by unsloth_zoo, nothing to install.
    # Laya decision models are ModernBERT encoders: none of these apply.
    # Clef decision models are Qwen3.5 backbones and need the same gated-delta / conv kernels.
    if (
        not config.get("is_decision")
        or config.get("decision_layout") in ("clef", "llm")
        or _decision_has_llm_backbone(model_load_target, _worker_hf_token(config))
    ):
        try:
            from utils.ssm_runtime import resolve_ssm_kernels

            # A continued local adapter loads from its folder, but its base decides the kernels:
            # the authorized pin when the run was checked, else the adapter's recorded base.
            kernels = resolve_ssm_kernels(
                model_name,
                model_load_target,
                _worker_hf_token(config),
                adapter_base = config.get("adapter_base_pin") or None,
                # Whether the load runs a checkpoint's remote code, not just whether it has some.
                trust_remote_code = bool(config.get("trust_remote_code", False)),
            )
            is_ssm = kernels.mamba_ssm
            wants_causal_conv1d = kernels.causal_conv1d
            from utils.ssm_runtime import mamba2_runs_on_triton

            # Mamba-2 on Triton drop-ins: no causal-conv1d, extension-free mamba_ssm wheel.
            triton_only = mamba2_runs_on_triton(kernels)
            if triton_only:
                # The extension-free wheel first. Not published, refused or not importable: this
                # run takes the CUDA-wheel route (causal-conv1d, then mamba-ssm) instead.
                triton_only = bool(
                    _ensure_mamba_ssm(
                        event_queue,
                        model_name,
                        required = is_ssm,
                        essential = kernels.mamba_ssm_required,
                        triton_only = True,
                    )
                )
                if triton_only:
                    wants_causal_conv1d = False
                else:
                    _send_status(
                        event_queue,
                        "The extension-free mamba-ssm wheel is not available; using the CUDA "
                        "kernels instead",
                    )
            _ensure_causal_conv1d_fast_path(
                event_queue,
                model_name,
                required = wants_causal_conv1d,
            )
            _install_fast_path_hooks(
                event_queue,
                model_name,
                install_causal_conv1d = wants_causal_conv1d,
            )
            if not triton_only:
                _ensure_mamba_ssm(
                    event_queue,
                    model_name,
                    required = is_ssm,
                    essential = kernels.mamba_ssm_required,
                )
            _ensure_flash_attn_for_long_context(
                event_queue,
                int(config.get("max_seq_length", 2048)),
            )
        except Exception as exc:
            event_queue.put(
                {
                    "type": "error",
                    "error": (
                        f"Please choose another model to train, since "
                        f"a fast-path kernel library "
                        f"(causal-conv1d / mamba-ssm) failed to install "
                        f"with error: {exc}"
                    ),
                    "stack": traceback.format_exc(limit = 20),
                    "ts": time.time(),
                }
            )
            return

    # No start-method override: Dataset.map() uses `multiprocess`, not stdlib.

    # Windows: importable Triton isn't enough on AMD; clang-cl JIT needs MSVC CRT headers.
    if sys.platform == "win32":
        from core._msvc_env import gate_torch_compile_on_windows
        gate_torch_compile_on_windows(logger)

    # Stub torchao on Windows ROCm; see core/_torchao_stub.py (no RCCL there). Run before transformers/unsloth_zoo.
    from core._torchao_stub import install_torchao_windows_rocm_stub

    install_torchao_windows_rocm_stub()

    # Single-GPU never inits the process group, but transformers/trl import these torch.distributed attrs anyway.
    _td_stubs = {
        "is_initialized": lambda: False,
        "is_available": lambda: False,
        "is_torchelastic_launched": lambda: False,
        "get_rank": lambda: 0,
        "get_world_size": lambda: 1,
        "barrier": lambda: None,
    }

    try:
        import torch.distributed as _td
        for _name, _stub in _td_stubs.items():
            if not hasattr(_td, _name):
                setattr(_td, _name, _stub)
    except Exception:
        _td_mock = types.ModuleType("torch.distributed")
        for _name, _stub in _td_stubs.items():
            setattr(_td_mock, _name, _stub)
        sys.modules["torch.distributed"] = _td_mock
        try:
            import torch as _torch
            _torch.distributed = _td_mock
        except Exception:
            pass

    # Windows ROCm <= 7.12: _grouped_mm null HIP kernel on gfx1200 crashes; JitDecomp bypasses
    # TORCHDYNAMO_DISABLE, so override the CUDA dispatch key. Fixed in rocm7.13.
    global _WINDOWS_ROCM_GROUPED_MM_LIB
    if sys.platform == "win32":
        _torch_for_rocm = sys.modules.get("torch")
        # AMD SDK / Radeon wheels don't always set torch.version.hip.
        _build_version_for_rocm = (
            getattr(_torch_for_rocm, "__version__", "").lower()
            if _torch_for_rocm is not None
            else ""
        )
        _is_win_rocm_torch = bool(
            _torch_for_rocm is not None
            and (
                getattr(getattr(_torch_for_rocm, "version", None), "hip", None)
                or "rocm" in _build_version_for_rocm
            )
        )
        if _is_win_rocm_torch:
            # Belt-and-suspenders; the JitDecomp patch is the real fix, but this covers other paths.
            if "TORCHDYNAMO_DISABLE" not in os.environ:
                os.environ["TORCHDYNAMO_DISABLE"] = "1"
                logger.info("Windows ROCm: torch.compile (dynamo) disabled")

            # bitsandbytes probes hipinfo.exe from PATH; the venv Scripts dir is only on PATH when activated.
            _scripts_dir = os.path.dirname(sys.executable)
            if os.path.isfile(os.path.join(_scripts_dir, "hipInfo.exe")):
                import shutil as _shutil
                if not _shutil.which("hipinfo.exe"):
                    os.environ["PATH"] = _scripts_dir + os.pathsep + os.environ.get("PATH", "")

            # AMD's Windows BNB DLL suffix may not match torch.version.hip; detect the real one. Caller-set BNB_ROCM_VERSION stays authoritative.
            if (
                "BNB_ROCM_VERSION" not in os.environ
                or os.environ.get("UNSLOTH_BNB_ROCM_VERSION_SOURCE") == "sitecustomize"
            ):
                _bnb_rocm_ver = None
                _found_rocm_bnb = False
                try:
                    import glob as _glob
                    import importlib.util as _ilu
                    import re as _re

                    _bnb_spec = _ilu.find_spec("bitsandbytes")
                    if _bnb_spec and _bnb_spec.submodule_search_locations:
                        _all_vers: list[str] = []
                        for _pkg_dir in _bnb_spec.submodule_search_locations:
                            for _dll in _glob.glob(
                                os.path.join(_pkg_dir, "libbitsandbytes_rocm*.dll")
                            ):
                                _found_rocm_bnb = True
                                _m = _re.search(
                                    r"libbitsandbytes_rocm(\d+)\.dll",
                                    os.path.basename(_dll),
                                )
                                if _m:
                                    _all_vers.append(_m.group(1))
                        # Highest numeric suffix wins (glob order isn't sorted).
                        if _all_vers:
                            _bnb_rocm_ver = max(_all_vers, key = lambda v: int(v))
                except Exception:
                    pass
                # Set BNB_ROCM_VERSION only if a ROCm bnb DLL exists, so later fixes can still redetect or opt out.
                if _found_rocm_bnb:
                    _bnb_rocm_ver = _bnb_rocm_ver or os.environ.get("BNB_ROCM_VERSION") or "72"
                    os.environ["BNB_ROCM_VERSION"] = _bnb_rocm_ver
                    os.environ["UNSLOTH_BNB_ROCM_VERSION_SOURCE"] = "detected"
                    logger.info(
                        "Windows ROCm: set BNB_ROCM_VERSION=%s "
                        "(detected from installed BNB wheel; "
                        "overrides torch.version.hip auto-detection)",
                        _bnb_rocm_ver,
                    )

            # Drop only bnb's benign BNB_ROCM_VERSION override notice.
            if os.environ.get("BNB_ROCM_VERSION"):
                import logging as _logging
                _logging.getLogger("bitsandbytes.cextension").addFilter(
                    lambda _r: "environment variable detected" not in _r.getMessage()
                )

            # Kernel-fix gate reads HIP version, else torch.__version__ (AMD SDK wheels leave version.hip unset).
            def _hip_ver_at_least(major: int, minor: int) -> bool:
                _hip_str = getattr(getattr(_torch_for_rocm, "version", None), "hip", None)
                if not _hip_str:
                    # Try the standard "+rocmX.Y.Z" embedded version first.
                    _ver_match = re.search(r"rocm(\d+)\.(\d+)", _build_version_for_rocm)
                    if _ver_match:
                        return (
                            int(_ver_match.group(1)),
                            int(_ver_match.group(2)),
                        ) >= (major, minor)
                    # "+rocmsdk<date>" wheels postdate the gfx120X null-kernel fix (ROCm 7.13); treat as >= 7.13.
                    if "rocmsdk" in _build_version_for_rocm:
                        logger.debug(
                            "Windows ROCm: AMD SDK wheel detected (%r); "
                            "assuming HIP >= %d.%d (rocmsdk wheels post-date "
                            "the gfx120X null-kernel fix)",
                            _build_version_for_rocm,
                            major,
                            minor,
                        )
                        return True
                    return False
                try:
                    _parts = [int(x) for x in str(_hip_str).split(".")[:2]]
                    if len(_parts) < 2:
                        logger.warning(
                            "Windows ROCm: torch.version.hip %r has fewer than "
                            "two components; cannot compare against %d.%d",
                            _hip_str,
                            major,
                            minor,
                        )
                        return False
                    return (_parts[0], _parts[1]) >= (major, minor)
                except ValueError:
                    logger.warning(
                        "Windows ROCm: could not parse torch.version.hip %r as "
                        "a version number; assuming HIP < %d.%d",
                        _hip_str,
                        major,
                        minor,
                    )
                    return False

            # Only on affected versions (ROCm <= 7.12); 7.13+ uses the real GPU kernel.
            if not _hip_ver_at_least(7, 13):
                try:
                    _WINDOWS_ROCM_GROUPED_MM_LIB = _install_grouped_mm_cpu_fallback(
                        _torch_for_rocm, logger, "Windows ROCm"
                    )
                except Exception as _patch_exc:
                    logger.warning(
                        "Windows ROCm: could not patch _grouped_mm — "
                        "training may crash with 0xC0000005: %s",
                        _patch_exc,
                    )
            else:
                logger.info(
                    "Windows ROCm: HIP >= 7.13 — _grouped_mm kernel is functional, "
                    "skipping Python fallback (AMD fixed gfx1200 null kernel in ROCm 7.13)"
                )

    # The win32 guard above misses Linux: RDNA4 (gfx1200/gfx1201) hits the same null HIP _grouped_mm kernel at ROCm <=
    # 7.12 (fixed 7.13, ROCm/TheRock #5284). Gate on arch + HIP.
    # ── 1f-linux. Linux ROCm RDNA4 _grouped_mm null kernel ──
    if sys.platform.startswith("linux") and _hw.IS_ROCM:
        try:
            _torch_lin = sys.modules.get("torch")
            if _torch_lin is not None and _torch_lin.cuda.is_available():
                # Unknown version on gfx120X: assume affected unless rocmsdk.
                _hip_str = str(getattr(getattr(_torch_lin, "version", None), "hip", "") or "")
                _ver = getattr(_torch_lin, "__version__", "").lower()
                _m = re.match(r"(\d+)\.(\d+)", _hip_str) or re.search(r"rocm(\d+)\.(\d+)", _ver)
                if _m:
                    _hip_lt_713 = (int(_m.group(1)), int(_m.group(2))) < (7, 13)
                else:
                    _hip_lt_713 = "rocmsdk" not in _ver
                # Scan every GPU: device_map="balanced" can place layers on a later RDNA4 card.
                _rdna4 = False
                for _i in range(_torch_lin.cuda.device_count()):
                    _props = _torch_lin.cuda.get_device_properties(_i)
                    _lin_arch, _ = _rocm_classify_unified_memory(_props)
                    _lin_name = (getattr(_props, "name", "") or "").lower()
                    if _lin_arch.lower() in ("gfx1200", "gfx1201") or (
                        not _lin_arch and re.search(r"rx\s*90[0-9]0|r9700", _lin_name)
                    ):
                        _rdna4 = True
                        break
                if _rdna4 and _hip_lt_713:
                    _WINDOWS_ROCM_GROUPED_MM_LIB = _install_grouped_mm_cpu_fallback(
                        _torch_lin, logger, "Linux ROCm gfx120X"
                    )
        except Exception as _gm_lin_exc:
            logger.warning("Linux ROCm gfx120X: could not patch _grouped_mm: %s", _gm_lin_exc)

    # ROCm OOM guard: exhausting VRAM can hang the HIP driver instead of raising, so set_per_process_memory_fraction
    # caps the allocator and PyTorch raises OutOfMemoryError first. Unified hosts share GPU+system RAM and need OS
    # headroom, so the cap depends on the classification and the pool size (see _rocm_memory_fraction and
    # _rocm_classify_unified_memory). Skipped if no torch.
    # ── 1g. ROCm OOM guard ──
    if _hw.IS_ROCM:
        try:
            import torch as _torch_mem
            if _torch_mem.cuda.is_available():
                # torch keeps the fraction PER DEVICE: a sharded run left the rest uncapped.
                _unified_seen = False
                for _mem_index in range(_torch_mem.cuda.device_count()):
                    # Classify unified vs discrete (see _rocm_classify_unified_memory's docstring).
                    _props = _torch_mem.cuda.get_device_properties(_mem_index)
                    _dev_name = _props.name
                    _gcn_arch, _is_unified = _rocm_classify_unified_memory(_props)
                    if _is_unified and not _gcn_arch:
                        logger.debug(
                            "ROCm OOM guard: gcnArchName absent -- inferred "
                            "unified memory from device name %r; applying unified cap",
                            _dev_name,
                        )
                    # Windows mem_get_info total is WDDM budget (OS share excluded); Linux needs headroom; torch 2.10+ scales props.total_memory, older hipMemGetInfo.
                    _total_bytes = int(getattr(_props, "total_memory", 0) or 0)
                    _driver_total = 0
                    if not _allocator_divides_by_props_total(
                        getattr(_torch_mem, "__version__", "")
                    ):
                        try:
                            _driver_total = int(_torch_mem.cuda.mem_get_info(_mem_index)[1])
                        except Exception:
                            _driver_total = 0
                    # ROCm name first, so an existing export keeps the cap it had.
                    _env_raw, _env_name = _mem_fraction_env_value("rocm")
                    _env_fraction = _parse_mem_fraction_env(_env_raw)
                    if _env_raw and _env_fraction is None:
                        logger.warning(
                            "ROCm OOM guard: ignoring %s=%r (needs a float in (0.0, 1.0]); "
                            "using the computed cap instead",
                            _env_name,
                            _env_raw,
                        )
                    _mem_fraction = _gpu_memory_fraction(
                        _total_bytes,
                        _is_unified,
                        sys.platform,
                        "rocm",
                        _env_raw,
                        _driver_total or None,
                    )
                    _allowed = (
                        f"{_total_bytes * _mem_fraction / 1024**3:.1f} of "
                        f"{_total_bytes / 1024**3:.1f} GiB allowed"
                        if _total_bytes > 0
                        else "device total unreported by this wheel"
                    )
                    _torch_mem.cuda.set_per_process_memory_fraction(_mem_fraction, _mem_index)
                    logger.info(
                        "ROCm OOM guard: set_per_process_memory_fraction(%.4f, cuda:%d) — "
                        "%s memory host (%s, %s), %s, %s",
                        _mem_fraction,
                        _mem_index,
                        "unified" if _is_unified else "discrete",
                        _dev_name,
                        _gcn_arch or "unknown arch",
                        _allowed,
                        f"from {_env_name}"
                        if _env_fraction is not None
                        else f"computed; override with {_MEM_FRACTION_ENV} or {_GPU_MEM_FRACTION_ENV}",
                    )
                    # Differing totals: the budget printed above is not the one enforced.
                    if (
                        _is_unified
                        and sys.platform != "win32"
                        and _env_fraction is None
                        and _total_bytes > 0
                        and _driver_total > 0
                        and abs(_driver_total - _total_bytes) > _total_bytes // 100
                    ):
                        logger.info(
                            "ROCm OOM guard: props.total_memory is %.1f GiB but this torch caps "
                            "against the driver's %.1f GiB, so the fraction is solved for that "
                            "total and %.1f GiB stays free against the intended %.1f GiB. Adjust "
                            "with %s.",
                            _total_bytes / 1024**3,
                            _driver_total / 1024**3,
                            (_total_bytes - _mem_fraction * _driver_total) / 1024**3,
                            _UNIFIED_OS_RESERVE_BYTES / 1024**3,
                            _MEM_FRACTION_ENV,
                        )
                    _unified_seen = _unified_seen or _is_unified

                # Windows APU WDDM budget is user-raisable but nothing says so; tell the user.
                if _unified_seen and sys.platform == "win32":
                    try:
                        import psutil as _psutil

                        _phys = _psutil.virtual_memory().total
                        _granted = _torch_mem.cuda.mem_get_info(0)[1]
                        if _granted < 0.75 * _phys:
                            logger.info(
                                "Windows grants the GPU %.1f GiB of %.1f GiB "
                                "system RAM (driver/WDDM budget). To raise it: "
                                "increase the BIOS UMA frame buffer size, or "
                                "AMD Software > Performance > Tuning > "
                                "Variable Graphics Memory.",
                                _granted / 1024**3,
                                _phys / 1024**3,
                            )
                    except Exception:
                        pass
        except Exception as _oom_guard_err:
            logger.debug("Could not set GPU memory fraction: %s", _oom_guard_err)

    # A user preference, unlike the driver-forced ROCm arm; XPU and MPS are not wired yet.
    # ── 1h. Explicit GPU memory cap ──
    if not _hw.IS_ROCM:
        _cap_raw, _cap_name = _mem_fraction_env_value("cuda")
        _cap_fraction = _parse_mem_fraction_env(_cap_raw)
        if _cap_raw and _cap_fraction is None:
            logger.warning(
                "Ignoring %s=%r (needs a float in (0.0, 1.0]); leaving GPU memory uncapped",
                _cap_name,
                _cap_raw,
            )
        elif _cap_fraction is not None:
            try:
                import torch as _torch_cap
                if _torch_cap.cuda.is_available():
                    _cap = _gpu_memory_fraction(0, False, sys.platform, "cuda", _cap_raw)
                    # Fraction is per device, so cap every visible GPU.
                    for _cap_index in range(_torch_cap.cuda.device_count()):
                        _torch_cap.cuda.set_per_process_memory_fraction(_cap, _cap_index)
                        _cap_props = _torch_cap.cuda.get_device_properties(_cap_index)
                        _cap_total = int(getattr(_cap_props, "total_memory", 0) or 0)
                        logger.info(
                            "GPU memory cap: set_per_process_memory_fraction(%.4f, cuda:%d) from %s — %s, %s",
                            _cap,
                            _cap_index,
                            _cap_name,
                            getattr(_cap_props, "name", "unknown device"),
                            f"{_cap_total * _cap / 1024**3:.1f} of {_cap_total / 1024**3:.1f} GiB allowed"
                            if _cap_total > 0
                            else "device total unreported by this wheel",
                        )
                else:
                    logger.debug(
                        "%s is set but no torch CUDA device is available; nothing to cap",
                        _cap_name,
                    )
            except Exception as _cap_err:
                logger.debug("Could not set GPU memory fraction: %s", _cap_err)

    # ── 2. Now import ML libraries (fresh in this clean process) ──
    try:
        _send_status(event_queue, "Importing Unsloth...")

        backend_path = str(Path(__file__).resolve().parent.parent.parent)
        if backend_path not in sys.path:
            sys.path.insert(0, backend_path)

        from core.training.trainer import UnslothTrainer
        from utils.paths import (
            ensure_dir,
            resolve_output_dir,
            resolve_tensorboard_dir,
            datasets_root,
            default_run_dir_name,
        )

        import transformers

        logger.info("Subprocess loaded transformers %s", transformers.__version__)
        # unsloth_zoo injects its vendored fla when the trainer imports unsloth above.
        logger.info(
            "flash-linear-attention fast path importable: %s",
            _flash_linear_attention_importable(),
        )
    except Exception as exc:
        event_queue.put(
            {
                "type": "error",
                "error": f"Failed to import ML libraries: {exc}",
                "stack": traceback.format_exc(limit = 20),
                "ts": time.time(),
            }
        )
        return
    finally:
        _uninstall_metadata_scan_cache(metadata_scan_cache)

    if config.get("offload_layers"):
        from utils.models.diffusion_lm import diffusion_lm_profile_for_model

        # The diffusion load has no block swapping; off before the budget below sizes itself to offloading.
        _diffusion = diffusion_lm_profile_for_model(
            model_load_target, _worker_hf_token(config), revision = config.get("model_revision")
        )
        if _diffusion:
            config["offload_layers"] = 0
            event_queue.put(
                {
                    "type": "warning",
                    "message": f"Layer offloading is not available for text-diffusion models ({_diffusion}); "
                    "training without it.",
                    "ts": time.time(),
                }
            )

    # Offload layers auto-size to allocator budget, so two runs on one card each take a share.
    # ── 2b. Training VRAM budget ──
    # Only "auto" sizes to a budget, and only LoRA runs outside decision / embedding offload.
    _wants_budget = bool(
        (config.get("offload_vram_gb") or config.get("offload_vram_gb_per_device"))
        and config.get("offload_layers") == "auto"
        and config.get("training_type", "LoRA/QLoRA") in ("LoRA/QLoRA", "Continued Pretraining")
        and not config.get("is_decision")
        and not config.get("is_embedding")
    )
    # Vision / audio loads on several GPUs cannot offload at load, and LoRA setup allocates adapters before it swaps:
    # a cap would OOM either step, so those runs offload Auto uncapped.
    _budget_unsupported = (
        _wants_budget
        and (len(gpu_ids) if gpu_ids else _visible_gpu_count()) > 1
        and bool(config.get("is_dataset_image") or config.get("is_dataset_audio"))
    )

    if _budget_unsupported:
        logger.info(
            "Training VRAM budget not applied: multi-GPU vision / audio runs offload at LoRA setup"
        )
        # An out-of-memory error then must not blame a cap that was never set.
        config["offload_vram_gb"] = None
        config["offload_vram_gb_per_device"] = None
    elif _wants_budget:
        try:
            import torch as _torch_budget
            _apply_training_vram_budget(
                _torch_budget,
                config.get("offload_vram_gb"),
                config.get("offload_vram_gb_per_device"),
                gpu_ids,
            )
        except Exception as _budget_err:
            logger.warning("Could not apply the training VRAM budget: %s", _budget_err)

    if config.get("is_decision", False):
        try:
            from core.training.decision_trainer import run_decision_training
            run_decision_training(event_queue, stop_queue, config)
        except Exception as exc:
            event_queue.put(
                {
                    "type": "error",
                    "error": str(exc),
                    "stack": traceback.format_exc(limit = 20),
                    "ts": time.time(),
                }
            )
        _end_abandoned_decision_download(event_queue)
        return

    if config.get("batch_members"):
        try:
            from core.training.multi_lora_trainer import run_multi_lora_training
            return run_multi_lora_training(event_queue, stop_queue, config)
        except Exception as exc:
            event_queue.put(
                {
                    "type": "error",
                    "error": str(exc),
                    "stack": traceback.format_exc(limit = 20),
                    "ts": time.time(),
                }
            )
        return

    # Embedding models branch early: FastSentenceTransformer plus MultipleNegativesRankingLoss.
    if config.get("is_embedding", False):
        try:
            _run_embedding_training(event_queue, stop_queue, config)
        except Exception as exc:
            event_queue.put(
                {
                    "type": "error",
                    "error": str(exc),
                    "stack": traceback.format_exc(limit = 20),
                    "ts": time.time(),
                }
            )
        return

    trainer = UnslothTrainer()

    trainer.add_progress_callback(_create_trainer_progress_callback(event_queue))

    def _apply_stop(save: bool) -> None:
        trainer.should_stop = True
        trainer.save_on_stop = save
        logger.info("Stop signal received (save=%s)", save)

    _start_worker_stop_poller(stop_queue, _apply_stop)

    # Pipeline order: detect -> dataset -> model -> prepare -> train, so both never hold VRAM at once.
    try:
        hf_token = _worker_hf_token(config)
        model_load_name = _resolve_cached_model_load_name(config)
        model_local_only = _model_local_files_only(config)
        model_revision = None if model_local_only else config.get("model_revision")
        dataset_local_only = _dataset_local_files_only(config)
        eval_steps = config.get("eval_steps", 0.00)

        hf_dataset = config.get("hf_dataset", "")
        training_type = config.get("training_type", "LoRA/QLoRA")
        is_cpt_for_dataset = training_type == "Continued Pretraining"

        # Filled in below, after the model probe; the closure runs after both.
        max_train_rows = None
        max_train_rows_seed = config.get("random_seed", 3407)

        def _load_training_dataset():
            result = trainer.load_and_format_dataset(
                dataset_source = hf_dataset if hf_dataset and hf_dataset.strip() else None,
                format_type = config.get("format_type", ""),
                local_datasets = config.get("local_datasets") or None,
                local_eval_datasets = config.get("local_eval_datasets") or None,
                custom_format_mapping = config.get("custom_format_mapping"),
                subset = config.get("subset"),
                train_split = config.get("train_split", "train"),
                eval_split = config.get("eval_split"),
                dataset_streaming = config.get("dataset_streaming", False),
                eval_steps = eval_steps,
                dataset_slice_start = config.get("dataset_slice_start"),
                dataset_slice_end = config.get("dataset_slice_end"),
                is_cpt = is_cpt_for_dataset,
                s3_config = config.get("s3_config"),
                dataset_local_files_only = dataset_local_only,
                dataset_local_path = config.get("dataset_snapshot_path"),
                dataset_revision = config.get("dataset_revision"),
                require_exact_resume_resources = bool(
                    config.get("require_exact_resume_resources")
                    or config.get("require_exact_dataset_resource")
                ),
                hf_token = hf_token,
                max_train_rows = max_train_rows,
                max_train_rows_seed = max_train_rows_seed,
            )
            if isinstance(result, tuple):
                loaded_dataset, loaded_eval_dataset = result
            else:
                loaded_dataset = result
                loaded_eval_dataset = None
            if eval_steps is not None and float(eval_steps) <= 0:
                loaded_eval_dataset = None
            snapshot = getattr(trainer, "dataset_snapshot_path", None)
            if snapshot:
                config["dataset_snapshot_path"] = snapshot
            return loaded_dataset, loaded_eval_dataset

        _send_status(event_queue, "Detecting model type...")
        try:
            _pre_detect_training_model(
                trainer,
                config,
                model_name,
                hf_token,
                model_load_name,
                model_local_only,
                model_revision,
            )
        except Exception as error:
            if not model_local_only:
                raise
            fallback_error = _model_cache_fallback_error(config, error)
            if fallback_error is not None:
                raise fallback_error from error
            if not _cache_artifact_fallback_allowed(config, error, "model"):
                raise
            _send_status(
                event_queue,
                f"Cached files for {model_name} are incomplete; retrying from Hugging Face...",
            )
            model_load_name = _drop_model_pin_for_fallback(config, hf_token)
            # Scan the Hub target we fall back to, not the cached pin already scanned above.
            security_error = _model_load_security_error(config, model_load_name, hf_token)
            if security_error:
                event_queue.put({"type": "error", **security_error, "ts": time.time()})
                return
            model_local_only = False
            model_revision = config.get("model_revision")
            _pre_detect_training_model(
                trainer,
                config,
                model_name,
                hf_token,
                model_load_name,
                model_local_only,
                model_revision,
            )
        if trainer.should_stop:
            event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
            return

        # Packing opt-out reads the probed branch, not client flags; streaming and explicit ranges opt out.
        raw_text_mode = is_cpt_for_dataset or config.get("format_type") == "raw"
        branch_never_packs = bool(getattr(trainer, "_audio_type", None)) or (
            bool(getattr(trainer, "is_vlm", False) or getattr(trainer, "is_audio_vlm", False))
            and not raw_text_mode
        )
        # Every replica draws its own batch, so size the subset for this launch, after model probing settles torch and the GPU mask.
        max_train_rows = max_train_rows_for_config(
            config,
            branch_never_packs = branch_never_packs,
            world_size = _data_parallel_world_size(),
        )
        # Resume uses the recorded subset; with no marker it trained unbounded, so bounding now would continue on unrelated rows.
        resumed_rows, max_train_rows_seed = row_bound_for_resume(
            config.get("resume_from_checkpoint"), max_train_rows, max_train_rows_seed
        )
        if resumed_rows != max_train_rows:
            logger.info(
                "Resuming with the row bound recorded at the original start "
                f"({resumed_rows} rows) instead of {max_train_rows}\n"
            )
            if resumed_rows and max_train_rows and resumed_rows < max_train_rows:
                # Fewer-replica subset kept (re-deriving gives unrelated rows); extra ranks may re-read rows.
                logger.info(
                    "That subset was sized for a smaller data-parallel world than this "
                    "one, so the added replicas may re-read rows; start a new run "
                    "instead of resuming to size it for this machine\n"
                )
        max_train_rows = resumed_rows

        if getattr(trainer, "_audio_type", None) == "bicodec":
            _fetch_spark_tts_repo(trainer, event_queue, model_name)
            if trainer.should_stop:
                event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
                return

        _send_status(event_queue, "Loading and formatting dataset...")
        from .model_weight_prefetch import prepare_dataset_with_prefetch

        dataset, eval_dataset = prepare_dataset_with_prefetch(
            _load_training_dataset,
            _training_weight_prefetch(config, trainer, hf_token, event_queue),
        )

        if dataset is None or trainer.should_stop:
            if trainer.should_stop:
                event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
            else:
                event_queue.put(
                    {
                        "type": "error",
                        "error": trainer.training_progress.error or "Failed to load dataset",
                        "stack": "",
                        "ts": time.time(),
                    }
                )
            return

        # Start the tqdm monitor early to capture download + tokenization bars.
        import threading as _th

        _tqdm_stop = _th.Event()

        def _monitor_tqdm():
            from tqdm.auto import tqdm as _tqdm_cls
            while not _tqdm_stop.is_set():
                for bar in list(getattr(_tqdm_cls, "_instances", set())):
                    try:
                        n, total = bar.n or 0, bar.total or 0
                        desc = getattr(bar, "desc", "") or ""
                        if total > 0 and n > 0 and desc:
                            pct = min(int(n * 100 / total), 100)
                            _send_status(event_queue, f"{desc.strip()} {pct}% ({n:,}/{total:,})")
                    except (AttributeError, ReferenceError):
                        pass
                _tqdm_stop.wait(3)

        _tqdm_thread = account_thread(target = _monitor_tqdm, daemon = True)
        _tqdm_thread.start()

        training_type = config.get("training_type", "LoRA/QLoRA")
        is_cpt = training_type == "Continued Pretraining"
        use_lora = training_type in ("LoRA/QLoRA", "Continued Pretraining")
        cpt_trains_embeddings = False

        # Model load uses VRAM; watchdog lets the parent recover a stalled Xet download via respawn.
        _send_status(event_queue, "Loading model...")
        from utils.hf_xet_fallback import start_watchdog

        # Recheck right before the load: the adapter folder may have been edited since start (#13140).
        base_error = _adapter_base_changed_error(config) or _escaping_weights_error(config)
        if base_error:
            event_queue.put({"type": "error", "error": base_error, "stack": "", "ts": time.time()})
            return

        event_queue.put({"type": "model_load_started", "ts": time.time()})
        _load_watchdog_stop = start_watchdog(
            repo_ids = [model_name],
            on_stall = lambda msg: event_queue.put(
                {"type": "stall", "message": msg, "ts": time.time()}
            ),
            xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
        )

        def _report_model_repo(repo_id):
            # Local cache loads have no active Hub download to track.
            if os.path.isdir(os.path.expanduser(repo_id)):
                return
            event_queue.put({"type": "model_load_resolved", "repo_id": repo_id, "ts": time.time()})

        # Latest-sidecar models load 16-bit: bnb 4-bit feeds quantized experts into unvalidated paths.
        try:
            _train_load_in_4bit = _effective_training_load_in_4bit(
                config,
                model_load_name,
                hf_token,
            )
            if config["load_in_4bit"] and not _train_load_in_4bit:
                logger.info(
                    "Latest-transformers sidecar active for %s - forcing a 16-bit "
                    "training load (4-bit is disabled for brand-new architectures)",
                    model_load_name,
                )
            success = trainer.load_model(
                on_model_resolved = _report_model_repo,
                model_name = model_name,
                max_seq_length = config["max_seq_length"],
                load_in_4bit = _train_load_in_4bit,
                full_finetuning = not use_lora,
                hf_token = hf_token,
                is_dataset_image = config.get("is_dataset_image", False),
                is_dataset_audio = config.get("is_dataset_audio", False),
                trust_remote_code = config.get("trust_remote_code", False),
                gpu_ids = config.get("resolved_gpu_ids"),
                model_load_name = model_load_name,
                local_files_only = model_local_only,
                actual_model_repo_id = config.get("actual_model_repo_id"),
                model_revision = model_revision,
                use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
                offload_layers = config.get("offload_layers", 0),
                prefetch_depth = config.get("prefetch_depth", 2),
                offload_plan_shape = _offload_plan_shape(config),
            )
            fallback_error = (
                _model_cache_fallback_error(config, trainer.model_load_error)
                if not success and model_local_only and not trainer.should_stop
                else None
            )
            if fallback_error is not None:
                trainer.training_progress.error = str(fallback_error)
            if (
                not success
                and model_local_only
                and not trainer.should_stop
                and fallback_error is None
                and _cache_artifact_fallback_allowed(config, trainer.model_load_error, "model")
            ):
                _send_status(
                    event_queue,
                    f"Cached files for {model_name} are incomplete; retrying from Hugging Face...",
                )
                model_load_name = _drop_model_pin_for_fallback(config, hf_token)
                # Scan the Hub target we fall back to, not the cached pin already scanned above.
                security_error = _model_load_security_error(config, model_load_name, hf_token)
                if security_error:
                    event_queue.put({"type": "error", **security_error, "ts": time.time()})
                    return
                model_local_only = False
                model_revision = config.get("model_revision")
                trainer.model = None
                trainer.tokenizer = None
                dataset = None
                eval_dataset = None
                gc.collect()
                from utils.hardware import clear_gpu_cache

                clear_gpu_cache()
                _send_status(
                    event_queue,
                    "Reloading and formatting the dataset with the Hub tokenizer...",
                )
                dataset, eval_dataset = _reload_dataset_with_remote_model_tokenizer(
                    trainer,
                    config,
                    model_name,
                    hf_token,
                    _load_training_dataset,
                    model_revision,
                )
                if dataset is None or trainer.should_stop:
                    success = False
                else:
                    success = trainer.load_model(
                        on_model_resolved = _report_model_repo,
                        model_name = model_name,
                        max_seq_length = config["max_seq_length"],
                        load_in_4bit = _train_load_in_4bit,
                        full_finetuning = not use_lora,
                        hf_token = hf_token,
                        is_dataset_image = config.get("is_dataset_image", False),
                        is_dataset_audio = config.get("is_dataset_audio", False),
                        trust_remote_code = config.get("trust_remote_code", False),
                        gpu_ids = config.get("resolved_gpu_ids"),
                        model_load_name = model_load_name,
                        local_files_only = model_local_only,
                        actual_model_repo_id = config.get("actual_model_repo_id"),
                        model_revision = model_revision,
                        use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
                        offload_layers = config.get("offload_layers", 0),
                        prefetch_depth = config.get("prefetch_depth", 2),
                        offload_plan_shape = _offload_plan_shape(config),
                    )
        finally:
            _load_watchdog_stop.set()
            event_queue.put({"type": "model_load_completed", "ts": time.time()})
        if not success or trainer.should_stop:
            if trainer.should_stop:
                event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
            else:
                error_msg = _with_vram_budget_hint(
                    config, trainer.training_progress.error or "Failed to load model"
                )
                event_queue.put(
                    {
                        "type": "error",
                        "error": error_msg,
                        "stack": "",
                        "ts": time.time(),
                    }
                )
            return

        if not _emit_resource_provenance(
            event_queue,
            config,
            trainer.model,
            model_load_target = model_load_name,
            model_load_in_4bit = _train_load_in_4bit,
            dataset_loaded_from_exact_snapshot = bool(
                getattr(trainer, "dataset_loaded_from_exact_snapshot", False)
            ),
        ):
            return

        if eval_dataset is not None:
            event_queue.put(
                {
                    "type": "eval_configured",
                    "ts": time.time(),
                }
            )

        if is_cpt:
            _send_status(event_queue, "Configuring LoRA for continued pretraining...")
            # LoRA on embed_tokens/lm_head never trains, so use modules_to_save.
            _embedding_modules = ("embed_tokens", "lm_head")
            _user_modules = config.get("target_modules") or []
            _leaf = lambda m: str(m).rsplit(".", 1)[-1]  # noqa: E731
            _wants = [m for m in _user_modules if _leaf(m) in _embedding_modules]
            # Either module in modules_to_save fills the embedding_learning_rate group.
            cpt_trains_embeddings = bool(_wants)
            cpt_target_modules = [m for m in _user_modules if _leaf(m) not in _embedding_modules]
            if not cpt_target_modules:
                cpt_target_modules = [
                    "q_proj",
                    "k_proj",
                    "v_proj",
                    "o_proj",
                    "gate_proj",
                    "up_proj",
                    "down_proj",
                ]
            success = trainer.prepare_model_for_training(
                use_lora = True,
                target_modules = cpt_target_modules,
                modules_to_save = _wants or None,
                lora_r = config.get("lora_r", 128),
                lora_alpha = config.get("lora_alpha", 32),
                lora_dropout = config.get("lora_dropout", 0.0),
                use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
                use_rslora = config.get("use_rslora", False),
                use_loftq = _lora_uses_loftq(config),
                use_dora = config.get("use_dora", False),
                random_state = config.get("random_seed", 3407),
            )
        elif use_lora:
            _send_status(event_queue, "Configuring LoRA adapters...")
            success = trainer.prepare_model_for_training(
                use_lora = True,
                finetune_vision_layers = config.get("finetune_vision_layers", True),
                finetune_language_layers = config.get("finetune_language_layers", True),
                finetune_attention_modules = config.get("finetune_attention_modules", True),
                finetune_mlp_modules = config.get("finetune_mlp_modules", True),
                target_modules = config.get("target_modules"),
                lora_r = config.get("lora_r", 16),
                lora_alpha = config.get("lora_alpha", 16),
                lora_dropout = config.get("lora_dropout", 0.0),
                use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
                use_rslora = config.get("use_rslora", False),
                use_loftq = _lora_uses_loftq(config),
                use_dora = config.get("use_dora", False),
                random_state = config.get("random_seed", 3407),
            )
        else:
            _send_status(event_queue, "Preparing model for full finetuning...")
            success = trainer.prepare_model_for_training(
                use_lora = False,
                use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
            )

        if not success or trainer.should_stop:
            if trainer.should_stop:
                event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
            else:
                event_queue.put(
                    {
                        "type": "error",
                        "error": _with_vram_budget_hint(
                            config, trainer.training_progress.error or "Failed to prepare model"
                        ),
                        "stack": "",
                        "ts": time.time(),
                    }
                )
            return

        if is_cpt and not cpt_trains_embeddings:
            # A continued adapter keeps its own modules_to_save, so it may train them anyway.
            cpt_trains_embeddings = _adapter_trains_embeddings(getattr(trainer, "model", None))

        lr_default = "5e-5" if is_cpt else "2e-4"
        try:
            lr_value = float(config.get("learning_rate", lr_default))
        except ValueError:
            event_queue.put(
                {
                    "type": "error",
                    "error": f"Invalid learning rate: {config.get('learning_rate')}",
                    "stack": "",
                    "ts": time.time(),
                }
            )
            return

        # Pydantic already validated embedding_learning_rate (Optional[float], gt=0, lt=1.0).
        embedding_lr_value = config.get("embedding_learning_rate")
        if is_cpt:
            if cpt_trains_embeddings:
                if embedding_lr_value is None:
                    # Default embedding_learning_rate = lr/10 (Unsloth CPT notebook).
                    embedding_lr_value = lr_value / 10.0
                    logger.info(
                        f"CPT: using default embedding_learning_rate={embedding_lr_value:.1e} "
                        f"(lr/10). Set explicitly to override.\n"
                    )
            elif embedding_lr_value is not None:
                logger.warning(
                    "CPT: embedding_learning_rate was provided but neither embed_tokens "
                    "nor lm_head is being trained; ignoring the override.\n"
                )
                embedding_lr_value = None

        resume_from_checkpoint = config.get("resume_from_checkpoint")
        output_dir = config.get("output_dir") or _output_dir_from_resume_checkpoint(
            resume_from_checkpoint
        )
        if not output_dir:
            output_dir = build_default_output_dir_name(
                model_name,
                config.get("project_name"),
            )
        output_dir = str(resolve_output_dir(output_dir))
        ensure_dir(Path(output_dir))
        _emit_output_dir(event_queue, output_dir)
        # Pinned before any checkpoint, so resume reads it back rather than re-deriving it.
        if not record_row_bound(output_dir, max_train_rows, max_train_rows_seed) and max_train_rows:
            # Not fatal, no fallback: the dataset is already bounded; a resume will read this run as unbounded.
            logger.warning(
                f"Could not record the max_steps row bound in {output_dir}: "
                "resuming this run later will read it as unbounded\n"
            )

        tensorboard_dir = config.get("tensorboard_dir")
        if config.get("enable_tensorboard", False):
            tensorboard_dir = str(resolve_tensorboard_dir(tensorboard_dir))
            ensure_dir(Path(tensorboard_dir))

        # Start training directly - no inner thread, we ARE the subprocess.
        dataset_display = config.get("hf_dataset", "") or config.get("uploaded_file", "") or ""
        _send_status(
            event_queue,
            f'Training "{model_name}"'
            + (f"\nDataset = {dataset_display}" if dataset_display else ""),
        )
        max_steps = config.get("max_steps", 0)
        save_steps = config.get("save_steps", 0)

        trainer._train_worker(
            dataset,
            output_dir = output_dir,
            num_epochs = config.get("num_epochs", 3),
            learning_rate = lr_value,
            embedding_learning_rate = embedding_lr_value,
            batch_size = config.get("batch_size", 2),
            gradient_accumulation_steps = config.get("gradient_accumulation_steps", 4),
            warmup_steps = config.get("warmup_steps"),
            warmup_ratio = config.get("warmup_ratio"),
            max_steps = max_steps if max_steps and max_steps > 0 else 0,
            save_steps = save_steps if save_steps and save_steps > 0 else 0,
            weight_decay = config.get("weight_decay", 0.001),
            random_seed = config.get("random_seed", 3407),
            packing = config.get("packing", False),
            train_on_completions = False if is_cpt else config.get("train_on_completions", False),
            enable_wandb = config.get("enable_wandb", False),
            wandb_project = config.get("wandb_project", "unsloth-training"),
            wandb_token = config.get("wandb_token"),
            enable_tensorboard = config.get("enable_tensorboard", False),
            tensorboard_dir = tensorboard_dir,
            eval_dataset = eval_dataset,
            eval_steps = eval_steps,
            early_stopping_patience = config.get("early_stopping_patience", 0),
            max_seq_length = config.get("max_seq_length", 2048),
            vision_image_size = config.get("vision_image_size"),
            optim = config.get("optim", "adamw_8bit"),
            lr_scheduler_type = config.get("lr_scheduler_type", "linear"),
            is_cpt = is_cpt,
            resume_from_checkpoint = resume_from_checkpoint,
        )

        _tqdm_stop.set()

        progress = trainer.get_training_progress()
        drafter_note = ""
        # A Stop during the drafter phase drops only the drafter; the fine-tune is already saved.
        stopped_in_training = trainer.should_stop
        if config.get("drafter") and not progress.error and not trainer.should_stop:
            from core.training.drafter import run_drafter_phase

            manifest = run_drafter_phase(trainer, config, output_dir, event_queue, _send_status)
            if manifest is not None:
                label = "MTP head" if manifest["strategy"] == "mtp" else "EAGLE-3 drafter"
                drafter_note = f"\nTrained {label} saved with the model."
            progress = trainer.get_training_progress()
        elif output_dir:
            from core.training.drafter import discard_drafter_artifacts
            discard_drafter_artifacts(output_dir)
        if progress.error:
            event_queue.put(
                {
                    "type": "error",
                    "error": progress.error,
                    "stack": "",
                    "ts": time.time(),
                }
            )
        else:
            saved_output_dir = (
                None if stopped_in_training and not trainer.save_on_stop else output_dir
            )
            event_queue.put(
                {
                    "type": "complete",
                    "output_dir": saved_output_dir,
                    "status_message": (progress.status_message or "Training completed")
                    + drafter_note,
                    "ts": time.time(),
                }
            )

    except Exception as exc:
        _exc_str = str(exc).lower()
        _is_oom = (
            "out of memory" in _exc_str
            or "hip out of memory" in _exc_str
            or "cuda out of memory" in _exc_str
            or type(exc).__name__ == "OutOfMemoryError"
        )
        if _is_oom:
            _oom_msg = (
                "GPU ran out of VRAM during training.\n"
                "To fix: reduce max_seq_length (e.g. 2048–4096), enable "
                "gradient_checkpointing=True, lower per_device_train_batch_size, "
                "or use a smaller model / higher quantization."
            )
            logger.error("Training stopped: GPU OOM — %s", exc)
            event_queue.put(
                {
                    "type": "error",
                    "error": _with_vram_budget_hint(config, _oom_msg),
                    "stack": traceback.format_exc(limit = 20),
                    "ts": time.time(),
                }
            )
        else:
            event_queue.put(
                {
                    "type": "error",
                    "error": str(exc),
                    "stack": traceback.format_exc(limit = 20),
                    "ts": time.time(),
                }
            )


def _send_status(event_queue: Any, message: str) -> None:
    """Send a status update to the parent process."""
    event_queue.put(
        {
            "type": "status",
            "message": message,
            "ts": time.time(),
        }
    )


def _emit_output_dir(event_queue: Any, output_dir: str) -> None:
    try:
        event_queue.put({"type": "output_dir", "output_dir": output_dir, "ts": time.time()})
    except Exception:
        pass


def _emit_resource_provenance(
    event_queue: Any,
    config: dict,
    model: Any,
    *,
    model_load_target: str,
    model_load_in_4bit: bool,
    dataset_loaded_from_exact_snapshot: bool,
) -> bool:
    """False, after reporting why, when a resumed run loaded a different base than it trained on."""
    from core.training.provenance import (
        build_worker_provenance_event,
        incomplete_worker_provenance_event,
        resumed_model_mismatch,
    )

    try:
        event = build_worker_provenance_event(
            config,
            model,
            model_load_target = model_load_target,
            model_load_in_4bit = model_load_in_4bit,
            dataset_loaded_from_exact_snapshot = dataset_loaded_from_exact_snapshot,
        )
    except Exception:
        logger.warning("Could not attest training resource provenance", exc_info = True)
        event = incomplete_worker_provenance_event("provenance_attestation_failed")
    event["ts"] = time.time()
    expected = config.get("resume_expected_model")
    mismatch = resumed_model_mismatch(expected, event) if expected else None
    if mismatch:
        # Record only the dataset, so the next resume still checks this run's expected base.
        event_queue.put({**event, "model": incomplete_worker_provenance_event()["model"]})
        event_queue.put({"type": "error", "error": mismatch, "stack": "", "ts": time.time()})
        return False
    event_queue.put(event)
    return True


def _mlx_has_checkpoint_at_step(output_dir, step: int) -> bool:
    if step <= 0:
        return False
    from core.training.resume import is_resume_checkpoint_valid
    return is_resume_checkpoint_valid(
        Path(output_dir) / f"checkpoint-{step}", expected_step = step, backend = "mlx"
    )


def _write_mlx_stop_checkpoint(trainer, optimizer, output_dir) -> bool:
    """Writes a resume checkpoint for a stopped MLX run; True if the current step is covered."""
    step = int(getattr(trainer, "_global_step", 0) or 0)
    # A periodic save or a resumed run may already cover the current step.
    if _mlx_has_checkpoint_at_step(output_dir, step):
        return True
    if step <= 0 or optimizer is None:
        return False
    ckpt_dir = Path(output_dir) / f"checkpoint-{step}"
    if ckpt_dir.is_symlink():
        # Refuse a symlinked dir: it could redirect writes outside output_dir.
        logger.error("Refusing to write MLX stop checkpoint through symlink: %s", ckpt_dir)
        return False
    try:
        ckpt_dir.mkdir(parents = True, exist_ok = True)
        from unsloth_zoo.mlx.utils import (
            save_optimizer_state,
            save_trainable_adapters,
            save_trainer_state,
        )

        save_trainable_adapters(trainer.model, str(ckpt_dir))
        save_optimizer_state(optimizer, str(ckpt_dir))
        save_trainer_state(
            {
                "global_step": step,
                "train_loss_history": list(getattr(trainer, "_train_loss_history", [])),
            },
            str(ckpt_dir),
        )
        logger.info("Saved stop checkpoint to %s", ckpt_dir)
    except Exception:
        logger.exception("Failed to write stop checkpoint under %s", output_dir)
    return _mlx_has_checkpoint_at_step(output_dir, step)


def _create_trainer_progress_callback(event_queue: Any) -> Callable[[TrainingProgress], None]:
    """Metrics go out only when changed, since the parent appends every progress event to its histories; status only when non-empty."""

    sent_warnings: set[str] = set()
    last_metrics: list = [None]

    def _on_progress(progress: TrainingProgress) -> None:
        has_train_loss = progress.step > 0 and progress.loss is not None
        has_eval_loss = progress.eval_loss is not None
        # The run summary flag marks the terminal record (early-stopped runs never reach total_steps).
        is_terminal = bool(getattr(progress, "is_run_summary", False)) or (
            progress.total_steps > 0 and progress.step >= progress.total_steps
        )
        # Wall-clock fields excluded so status replays don't look like new measurements.
        metrics = (
            progress.step,
            progress.loss,
            progress.learning_rate,
            progress.grad_norm,
            progress.num_tokens,
            progress.epoch,
            progress.eval_loss,
        )
        is_repeat = metrics == last_metrics[0]
        if (
            (progress.step == 0 and progress.total_steps > 0)
            or has_train_loss
            or has_eval_loss
            or is_terminal
        ) and not is_repeat:
            last_metrics[0] = metrics
            event_queue.put(
                {
                    "type": "progress",
                    "step": progress.step,
                    "epoch": progress.epoch,
                    "loss": progress.loss,
                    "learning_rate": progress.learning_rate,
                    "total_steps": progress.total_steps,
                    "elapsed_seconds": progress.elapsed_seconds,
                    "eta_seconds": progress.eta_seconds,
                    "session_start_step": progress.session_start_step,
                    "grad_norm": progress.grad_norm,
                    "num_tokens": progress.num_tokens,
                    "eval_loss": progress.eval_loss,
                    "status_message": progress.status_message,
                    "offload": getattr(progress, "offload", None),
                    "ts": time.time(),
                }
            )
        if progress.status_message:
            _send_status(event_queue, progress.status_message)
        for message in progress.warnings:
            if message not in sent_warnings:
                sent_warnings.add(message)
                event_queue.put({"type": "warning", "message": message, "ts": time.time()})

    return _on_progress


def _create_embedding_progress_callback(
    event_queue: Any,
    *,
    total_steps: int,
    training_start_time: float,
    should_stop: Callable[[], bool],
):
    """TrainerCallback that reports embedding training progress to the parent. ``should_stop`` is
    polled in on_train_begin and on_step_end, so a stop signal arriving mid-run is seen."""
    from transformers import TrainerCallback
    from core.training.startup_progress import TrainingStartupProgress

    startup = TrainingStartupProgress()

    class _EmbeddingProgressCallback(TrainerCallback):
        _start_step = 0
        _training_start_time = training_start_time

        def on_train_begin(self, args, state, control, **kwargs):
            self._start_step = state.global_step
            if state.global_step > 0:
                self._training_start_time = time.time()
            if should_stop():
                return
            _send_status(event_queue, startup.begin(args, state))

        def on_step_begin(self, args, state, control, **kwargs):
            if not should_stop() and (message := startup.step_begin()):
                _send_status(event_queue, message)

        def on_substep_end(self, args, state, control, **kwargs):
            if not should_stop() and (message := startup.substep_end()):
                _send_status(event_queue, message)

        def on_log(
            self,
            args,
            state,
            control,
            logs = None,
            **kwargs,
        ):
            if not logs:
                return
            # Trainer end-of-run summary (train_runtime, per-second rates, total_flos) is published nowhere else.
            from core.training.trainer import _RESERVED_LOG_KEYS, _TRAINER_SUMMARY_KEYS

            if any(k in logs for k in _TRAINER_SUMMARY_KEYS):
                logger.info(
                    "trainer summary",
                    **{
                        k: v
                        for k, v in logs.items()
                        if isinstance(k, str) and k not in _RESERVED_LOG_KEYS
                    },
                )
            # train_loss in HF's summary is the run mean, not a step loss, so it must never become the final step.
            loss_value = logs.get("loss")
            if loss_value is None and logs.get("train_loss") is not None:
                print(
                    f"Training finished: mean train_loss={logs.get('train_loss')} "
                    f"over {state.global_step} steps",
                    flush = True,
                )
            current_step = state.global_step

            elapsed = time.time() - self._training_start_time
            eta = session_eta_seconds(elapsed, current_step, self._start_step, total_steps)

            event_queue.put(
                {
                    "type": "progress",
                    "step": current_step,
                    "epoch": round(state.epoch, 2) if state.epoch else 0,
                    "loss": loss_value,
                    "learning_rate": logs.get("learning_rate", None),
                    "total_steps": total_steps,
                    "elapsed_seconds": elapsed,
                    "eta_seconds": eta,
                    "session_start_step": self._start_step,
                    "grad_norm": logs.get("grad_norm"),
                    "num_tokens": getattr(state, "num_input_tokens_seen", None),
                    "eval_loss": logs.get("eval_loss"),
                    "status_message": "" if should_stop() else startup.log(logs) or "",
                    "ts": time.time(),
                }
            )

        def on_step_end(self, args, state, control, **kwargs):
            if should_stop():
                logger.info("Embedding training: stop at step %d", state.global_step)
                control.should_training_stop = True
                return control
            if message := startup.step_end(state):
                _send_status(event_queue, message)

    return _EmbeddingProgressCallback()


def _csv_number(cell):
    from utils.datasets.cells import _NA_CELLS

    text = (cell or "").strip()
    if text.lower() in ("true", "false"):
        return float(text.lower() == "true")
    return None if text in _NA_CELLS else float(text)


def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> None:
    """Separate from UnslothTrainer's LLM/VLM/audio paths: embeddings use SentenceTransformerTrainer."""
    import math

    model_name = config["model_name"]
    model_load_name = _resolve_cached_model_load_name(config)
    model_local_only = _model_local_files_only(config)
    model_revision = None if model_local_only else config.get("model_revision")
    training_start_time = time.time()

    _send_status(event_queue, "Importing embedding libraries...")
    try:
        # Recover from a namespace-package shadow (embedding imports unsloth directly).
        from core.import_guards import ensure_real_packages

        ensure_real_packages("unsloth_zoo", "unsloth")
        from unsloth import FastSentenceTransformer, is_bfloat16_supported
        from sentence_transformers import (
            SentenceTransformerTrainer,
            SentenceTransformerTrainingArguments,
        )
        from sentence_transformers.losses import MultipleNegativesRankingLoss
        from sentence_transformers.training_args import BatchSamplers
        from datasets import Dataset
        from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset
        from utils.paths import datasets_root, resolve_output_dir, default_run_dir_name
    except ImportError as e:
        event_queue.put(
            {
                "type": "error",
                "error": f"Failed to import embedding libraries: {e}. "
                "Ensure 'sentence_transformers' and 'unsloth' are installed.",
                "stack": traceback.format_exc(limit = 20),
                "ts": time.time(),
            }
        )
        return

    # datasets loads only here, so quiet its Generating/Map bars now or they land in the structured log.
    try:
        from loggers.config import quiet_third_party_progress_bars
        quiet_third_party_progress_bars()
    except Exception:  # noqa: BLE001 - never let log tidying stop a run
        pass

    _should_stop = False
    _save_on_stop = True

    def _apply_stop(save: bool) -> None:
        nonlocal _should_stop, _save_on_stop
        _save_on_stop = save
        _should_stop = True
        logger.info(
            "Embedding training: stop signal received (save=%s)",
            _save_on_stop,
        )

    _start_worker_stop_poller(stop_queue, _apply_stop)

    _send_status(event_queue, "Loading embedding model...")
    try:
        hf_token = _worker_hf_token(config)
        max_seq_length = config.get("max_seq_length", 512)
        training_type = config.get("training_type", "LoRA/QLoRA")
        use_lora = training_type == "LoRA/QLoRA"

        security_error = _model_load_security_error(config, model_load_name, hf_token)
        if security_error:
            event_queue.put({"type": "error", **security_error, "ts": time.time()})
            return

        try:
            model = FastSentenceTransformer.from_pretrained(
                model_name = model_load_name,
                max_seq_length = max_seq_length,
                full_finetuning = not use_lora,
                token = hf_token,
                revision = model_revision,
                use_exact_model_name = model_revision is not None,
                # Not config[trust_remote_code]: consent scan checks only auto_map, so forwarding consent would run unscanned modules.json code.
                trust_remote_code = False,
            )
        except Exception as error:
            if not model_local_only:
                raise
            fallback_error = _model_cache_fallback_error(config, error)
            if fallback_error is not None:
                raise fallback_error from error
            if not _cache_artifact_fallback_allowed(config, error, "model"):
                raise
            _send_status(
                event_queue,
                f"Cached files for {model_name} are incomplete; retrying from Hugging Face...",
            )
            model_load_name = _drop_model_pin_for_fallback(config, hf_token)
            # Scan the Hub target we fall back to, not the cached pin already scanned above.
            security_error = _model_load_security_error(config, model_load_name, hf_token)
            if security_error:
                event_queue.put({"type": "error", **security_error, "ts": time.time()})
                return
            model_local_only = False
            model_revision = config.get("model_revision")
            model = FastSentenceTransformer.from_pretrained(
                model_name = model_load_name,
                max_seq_length = max_seq_length,
                full_finetuning = not use_lora,
                token = hf_token,
                revision = model_revision,
                use_exact_model_name = model_revision is not None,
                # Deliberately False, as above.
                trust_remote_code = False,
            )
    except Exception as e:
        event_queue.put(
            {
                "type": "error",
                "error": f"Failed to load embedding model '{model_name}': {e}",
                "stack": traceback.format_exc(limit = 20),
                "ts": time.time(),
            }
        )
        return

    loaded_model_for_provenance = model
    if _should_stop:
        event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
        return

    if use_lora:
        _send_status(event_queue, "Configuring LoRA adapters (FEATURE_EXTRACTION)...")
        try:
            gradient_checkpointing = config.get("gradient_checkpointing", False)
            # Normalize "none"/empty to False.
            if gradient_checkpointing in ("none", "", None):
                gradient_checkpointing = False

            from core.training.trainer import (
                LOFTQ_SKIPPED_ON_QUANTIZED,
                _has_quantized_weights,
                _loftq_peft_kwargs,
            )

            use_loftq = _lora_uses_loftq(config)
            # The fast encoder prepares an intrinsically quantized repo for k-bit training, and LoftQ fails on it.
            if use_loftq and _has_quantized_weights(model):
                _send_status(event_queue, LOFTQ_SKIPPED_ON_QUANTIZED)
                use_loftq = False

            model = FastSentenceTransformer.get_peft_model(
                model,
                r = config.get("lora_r", 32),
                target_modules = config.get("target_modules")
                or ["q_proj", "k_proj", "v_proj", "o_proj"],
                lora_alpha = config.get("lora_alpha", 64),
                lora_dropout = config.get("lora_dropout", 0.0),
                bias = "none",
                use_gradient_checkpointing = gradient_checkpointing,
                random_state = config.get("random_seed", 3407),
                use_rslora = config.get("use_rslora", False),
                use_dora = config.get("use_dora", False),
                **_loftq_peft_kwargs(use_loftq),
                task_type = "FEATURE_EXTRACTION",
            )
        except Exception as e:
            event_queue.put(
                {
                    "type": "error",
                    "error": f"Failed to configure LoRA for embedding model: {e}",
                    "stack": traceback.format_exc(limit = 20),
                    "ts": time.time(),
                }
            )
            return

    if _should_stop:
        event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
        return

    _send_status(event_queue, "Loading dataset...")
    try:
        config["_dataset_loaded_from_exact_snapshot"] = False
        hf_dataset = str(config.get("hf_dataset") or "").strip()
        local_datasets = config.get("local_datasets") or []
        csv_text_columns: list[str] = []

        def _load_local_embedding_dataset(dataset_paths: list[str]):
            from utils.paths import dataset_files_in_dir

            all_files: list[str] = []
            for dataset_file in dataset_paths:
                file_path = (
                    dataset_file
                    if os.path.isabs(dataset_file)
                    else os.path.join(
                        str(datasets_root()),
                        dataset_file,
                    )
                )
                if os.path.isdir(file_path):
                    file_path_obj = Path(file_path)
                    all_files.extend(str(p) for p in dataset_files_in_dir(file_path_obj))
                else:
                    all_files.append(file_path)

            if not all_files:
                raise ValueError("No local dataset files found")

            first_ext = Path(all_files[0]).suffix.lower()
            if first_ext in (".json", ".jsonl"):
                loader = "json"
            elif first_ext == ".csv":
                loader = "csv"
            elif first_ext == ".parquet":
                loader = "parquet"
            else:
                raise ValueError(f"Unsupported local dataset format: {all_files[0]}")
            from datasets import Features, Value
            from utils.datasets.cells import csv_as_text_kwargs, typed_csv_columns

            dataset = load_dataset(
                loader,
                data_files = all_files,
                split = "train",
                **csv_as_text_kwargs(all_files),
            )
            if loader != "csv":
                return dataset
            labels = typed_csv_columns(dataset) & {"label", "labels", "score", "scores"}
            for column in labels:
                dataset = dataset.map(
                    lambda cells: {column: [_csv_number(cell) for cell in cells]},
                    input_columns = column,
                    batched = True,
                    features = Features({**dataset.features, column: Value("float64")}),
                )
            csv_text_columns[:] = [
                column for column in dataset.column_names if column not in labels
            ]
            return dataset

        if hf_dataset:
            dataset = _load_embedding_hf_dataset(
                config,
                load_dataset,
                lambda message: _send_status(event_queue, message),
            )
        elif local_datasets:
            dataset = _load_local_embedding_dataset(local_datasets)
        elif config.get("s3_config"):
            from core.training.s3_dataset import (
                S3DownloadCancelled,
                prepare_s3_dataset_download,
            )

            _send_status(event_queue, "Downloading dataset from S3...")
            s3_download = None
            try:
                s3_download = prepare_s3_dataset_download(
                    config["s3_config"],
                    cancel_callback = lambda: _should_stop,
                )
                dataset = _load_local_embedding_dataset(s3_download.files)
            except S3DownloadCancelled:
                event_queue.put(
                    {
                        "type": "complete",
                        "output_dir": None,
                        "status_message": "Training cancelled",
                        "ts": time.time(),
                    }
                )
                return
            finally:
                if s3_download is not None:
                    s3_download.cleanup()
        else:
            event_queue.put(
                {
                    "type": "error",
                    "error": "No dataset specified for embedding training.",
                    "stack": "",
                    "ts": time.time(),
                }
            )
            return

        slice_start = config.get("dataset_slice_start")
        slice_end = config.get("dataset_slice_end")
        if slice_start is not None or slice_end is not None:
            dataset = slice_dataset_rows(dataset, slice_start, slice_end)[0]

        if csv_text_columns:
            kept = dataset.filter(
                lambda *columns: [None not in cells for cells in zip(*columns)],
                input_columns = csv_text_columns,
                batched = True,
            )
            if len(dataset) and not len(kept):
                raise ValueError(f"All {len(dataset):,} rows have an empty text cell.")
            if len(kept) < len(dataset):
                event_queue.put(
                    {
                        "type": "warning",
                        "message": f"Dropped {len(dataset) - len(kept):,} of {len(dataset):,} "
                        "rows because a text cell was empty.",
                        "ts": time.time(),
                    }
                )
            dataset = kept

        logger.info(f"Embedding dataset loaded: {len(dataset)} samples")
    except Exception as e:
        event_queue.put(
            {
                "type": "error",
                "error": f"Failed to load dataset: {e}",
                "stack": traceback.format_exc(limit = 20),
                "ts": time.time(),
            }
        )
        return

    if _should_stop:
        event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
        return

    if not _emit_resource_provenance(
        event_queue,
        config,
        loaded_model_for_provenance,
        model_load_target = model_load_name,
        model_load_in_4bit = False,
        dataset_loaded_from_exact_snapshot = bool(config.get("_dataset_loaded_from_exact_snapshot")),
    ):
        return

    loss = MultipleNegativesRankingLoss(model)

    _send_status(event_queue, "Configuring training...")
    try:
        lr_value = float(config.get("learning_rate", "2e-4"))
    except ValueError:
        event_queue.put(
            {
                "type": "error",
                "error": f"Invalid learning rate: {config.get('learning_rate')}",
                "stack": "",
                "ts": time.time(),
            }
        )
        return

    resume_from_checkpoint = config.get("resume_from_checkpoint")
    output_dir = config.get("output_dir") or _output_dir_from_resume_checkpoint(
        resume_from_checkpoint
    )
    if not output_dir:
        output_dir = build_default_output_dir_name(
            model_name,
            config.get("project_name"),
        )
    output_dir = str(resolve_output_dir(output_dir))
    _emit_output_dir(event_queue, output_dir)

    num_epochs = config.get("num_epochs", 2)
    batch_size = config.get("batch_size", 256)
    gradient_accumulation_steps = config.get("gradient_accumulation_steps", 1)
    max_steps_val = config.get("max_steps", 0)
    save_steps_val = config.get("save_steps", 0)
    warmup_ratio = config.get("warmup_ratio", 0.03)
    warmup_steps_val = config.get("warmup_steps")
    log_frequency = config.get("log_frequency", 50)

    from core.training.trainer import _drop_hf_stdout_callbacks, _hf_stdout_progress_disabled
    from core.training.training import apply_save_strategy

    training_args_kwargs = {
        "output_dir": output_dir,
        "per_device_train_batch_size": batch_size,
        "gradient_accumulation_steps": gradient_accumulation_steps,
        "learning_rate": lr_value,
        "fp16": not is_bfloat16_supported(),
        "bf16": is_bfloat16_supported(),
        "logging_steps": 1,
        "report_to": ["wandb"] if config.get("enable_wandb") else "none",
        "lr_scheduler_type": config.get("lr_scheduler_type", "linear"),
        "batch_sampler": BatchSamplers.NO_DUPLICATES,
        "optim": config.get("optim", "adamw_8bit"),
        "weight_decay": config.get("weight_decay", 0.001),
        "seed": config.get("random_seed", 3407),
        # No terminal (stdout goes to the server log); embedding progress callback already publishes the bar.
        "disable_tqdm": _hf_stdout_progress_disabled(),
    }

    if max_steps_val and max_steps_val > 0:
        training_args_kwargs["max_steps"] = max_steps_val
    else:
        training_args_kwargs["num_train_epochs"] = num_epochs if num_epochs > 0 else 2

    # warmup: prefer warmup_ratio (standard for embedding scripts), else steps
    if warmup_ratio is not None and warmup_ratio > 0:
        training_args_kwargs["warmup_ratio"] = warmup_ratio
    elif warmup_steps_val is not None and warmup_steps_val > 0:
        training_args_kwargs["warmup_steps"] = warmup_steps_val

    apply_save_strategy(training_args_kwargs, save_steps_val)

    args = SentenceTransformerTrainingArguments(**training_args_kwargs)

    if max_steps_val and max_steps_val > 0:
        total_steps = max_steps_val
    else:
        effective_epochs = num_epochs if num_epochs > 0 else 2
        len_dataloader = math.ceil(len(dataset) / batch_size)
        steps_per_epoch = max(len_dataloader // gradient_accumulation_steps, 1)
        total_steps = steps_per_epoch * effective_epochs

    progress_callback = _create_embedding_progress_callback(
        event_queue,
        total_steps = total_steps,
        training_start_time = training_start_time,
        should_stop = lambda: _should_stop,
    )

    _send_status(event_queue, "Starting embedding training...")
    try:
        trainer = SentenceTransformerTrainer(
            model = model,
            train_dataset = dataset,
            loss = loss,
            args = args,
            callbacks = [progress_callback],
        )
        # disable_tqdm only swaps ProgressCallback for PrinterCallback, both to stdout.
        _drop_hf_stdout_callbacks(trainer)

        trainer.train(resume_from_checkpoint = resume_from_checkpoint)
    except Exception as e:
        event_queue.put(
            {
                "type": "error",
                "error": f"Embedding training failed: {e}",
                "stack": traceback.format_exc(limit = 20),
                "ts": time.time(),
            }
        )
        return

    if _should_stop and not _save_on_stop:
        event_queue.put(
            {
                "type": "complete",
                "output_dir": None,
                "status_message": "Training cancelled",
                "ts": time.time(),
            }
        )
        return

    _send_status(event_queue, "Saving model...")
    try:
        if _should_stop and _save_on_stop:
            trainer._save_checkpoint(trainer.model, trial = None)
        model.save_pretrained(output_dir)
        model.tokenizer.save_pretrained(output_dir)
        logger.info("Embedding model saved to %s", output_dir)
    except Exception as e:
        logger.error("Failed to save embedding model: %s", e)
        event_queue.put(
            {
                "type": "error",
                "error": f"Training completed but failed to save: {e}",
                "stack": traceback.format_exc(limit = 20),
                "ts": time.time(),
            }
        )
        return

    event_queue.put(
        {
            "type": "complete",
            "output_dir": output_dir,
            "status_message": "Embedding training completed",
            "ts": time.time(),
        }
    )