Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

108 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""
Inference subprocess entry point.

Each session runs in a persistent spawn subprocess, giving a clean interpreter
with no stale module state (solves transformers version-switching). It stays
alive while a model is loaded, taking commands (generate, load, unload) via
mp.Queue, and exits on shutdown or unload. Pattern follows core/training/worker.py.
"""

from __future__ import annotations

import contextlib
import functools
import base64
import inspect
import json
from loggers import get_logger
import os
import queue as _queue
import sys
import time
import traceback
from io import BytesIO
from pathlib import Path
from typing import Any, Optional

logger = get_logger(__name__)
from core.inference.audio_errors import (
    AUDIO_RUNTIME_ERROR_CODE,
    AUDIO_UNSUPPORTED_CODE,
    AudioBackendUnsupportedError,
    AudioRuntimeError,
)
from core.inference.context_refusal import ContextBudgetExceeded
from utils.hardware import apply_gpu_ids, is_apple_silicon

# Fresh spawned interpreter: re-apply the process-wide network injections and CPU thread caps.
from utils.cpu_threads import install_openblas_runtime_cap
from utils.native_tls import activate_native_tls
from utils.happy_eyeballs import activate_happy_eyeballs


_ID_BYTES = 36

_SLOTS = 256


class StopLedger:
    """The request ids the parent has stopped, and whether the worker reads them."""

    def __init__(self, ctx: Any):
        self._lock = ctx.Lock()
        self._slots = ctx.Array("c", _SLOTS * _ID_BYTES, lock = False)
        self._read_by_worker = ctx.Value("b", 0, lock = False)
        self._written = ctx.Value("l", 0, lock = False)

    def worker_reads_this(self) -> None:
        """Said once by the worker, as it enters the loop that reads names from here."""
        self._read_by_worker.value = 1

    def read_by_worker(self) -> bool:
        return bool(self._read_by_worker.value)

    def stop(self, request_id: str) -> bool:
        """Record a stop. False for an id no slot could hold."""
        entry = _entry(request_id)
        if entry is None:
            return False
        with self._lock:
            if not self._holds(entry):
                start = (self._written.value % _SLOTS) * _ID_BYTES
                self._slots[start : start + _ID_BYTES] = entry
                self._written.value += 1
        return True

    def snapshot(self, since: int = -1) -> tuple[int, Optional[set]]:
        written = self._written.value
        if written == since:
            return written, None
        with self._lock:
            written = self._written.value
            raw = bytes(self._slots)
        return written, {
            raw[start : start + _ID_BYTES].rstrip(b"\0").decode("utf-8", "replace")
            for start in range(0, min(written, _SLOTS) * _ID_BYTES, _ID_BYTES)
        }

    def _holds(self, entry: bytes) -> bool:
        for slot in range(min(self._written.value, _SLOTS)):
            start = slot * _ID_BYTES
            if bytes(self._slots[start : start + _ID_BYTES]) == entry:
                return True
        return False


def _entry(request_id: Optional[str]) -> Optional[bytes]:
    """One slot's worth of bytes, or None for an id no slot could hold."""
    if not request_id:
        return None
    entry = str(request_id).encode("utf-8", "replace")
    if len(entry) > _ID_BYTES or b"\0" in entry:
        return None
    return entry.ljust(_ID_BYTES, b"\0")


class PendingTeardowns:
    """How many commands that end everything are on their way to the worker."""

    def __init__(self, ctx: Any):
        self._count = ctx.Value("l", 0)

    def sending(self) -> None:
        with self._count.get_lock():
            self._count.value += 1

    def unsent(self) -> None:
        self._counted_off()

    def taken(self) -> None:
        self._counted_off()

    def _counted_off(self) -> None:
        """One fewer on its way, clamped: the two ends are different processes."""
        with self._count.get_lock():
            self._count.value = max(0, self._count.value - 1)

    def any_in_flight(self) -> bool:
        return self._count.value > 0


class RowRefused(Exception):
    """This batch will not take this reply, and is exactly as it was."""


def narrow_load_reason(cmd: dict) -> Optional[str]:
    """Why no batch of either kind may take this command, or None."""
    width = int(cmd.get("parallel_slots") or 1)
    if width <= 1:
        return "this load decodes one reply at a time"
    rows = len(cmd.get("rows") or [])
    if rows > width:
        return f"the load decodes {width} replies at once, and this asks for {rows}"
    return None


activate_native_tls()
activate_happy_eyeballs()
install_openblas_runtime_cap()

_SHARE_OBJECT_MAX_BYTES = 1 << 20
_SHARE_OBJECT_ERROR_SIZE = -1

# studio/backend root, prepended to sys.path so the spawned subprocess can import utils/core.
_BACKEND_PATH = str(Path(__file__).resolve().parent.parent.parent)


def _ensure_backend_on_path() -> None:
    if _BACKEND_PATH not in sys.path:
        sys.path.insert(0, _BACKEND_PATH)


def _native_audio_security_targets_or_error(
    model_name: str, hf_token: str | None, resp_queue: Any
) -> list[str] | None:
    try:
        from core.inference.native_audio import native_audio_security_targets
        return native_audio_security_targets(model_name, hf_token = hf_token)
    except Exception as exc:
        _send_response(
            resp_queue,
            {
                "type": "error",
                "error": f"Failed to inspect native audio security metadata: {exc}",
                "stack": traceback.format_exc(limit = 20),
            },
        )
        return None


def _recorded_local_base(model_name) -> "tuple[str | None, bool]":
    """Reuses the resolver's disk reads; an unavailable reader counts as needing the Hub (fails closed)."""
    try:
        _ensure_backend_on_path()
        from utils.transformers_version import recorded_local_base
        return recorded_local_base(model_name)
    except Exception:
        return None, True


def _hub_targets_are_local(*targets) -> bool:
    """True when every non-empty target is local; an unresolvable one or unavailable is_local_path counts as remote, since a skipped probe costs retry backoff."""
    try:
        _ensure_backend_on_path()
        from utils.paths import is_local_path
    except Exception:
        return False
    for target in targets:
        if not target:
            continue
        try:
            if not (isinstance(target, str) and is_local_path(target)):
                return False
        except Exception:
            return False
    return True


def _activate_transformers_version(model_name: str, hf_token: str | None = None) -> None:
    """Activate the correct transformers version BEFORE any ML imports."""
    _ensure_backend_on_path()

    from utils.transformers_version import activate_transformers_for_subprocess

    activate_transformers_for_subprocess(model_name, hf_token)


def _decode_image(image_base64: str):
    from PIL import Image
    image_data = base64.b64decode(image_base64)
    return Image.open(BytesIO(image_data))


def _active_model_is_deepseek_ocr(backend) -> bool:
    from core.inference.model_ids import is_deepseek_ocr

    models = getattr(backend, "models", None)
    if not isinstance(models, dict):
        return False
    return is_deepseek_ocr(
        (models.get(getattr(backend, "active_model_name", None)) or {}).get("model")
    )


def _resize_image(img, max_size: int = 800):
    """Resize image while maintaining aspect ratio."""
    if img is None:
        return None
    if img.size[0] > max_size or img.size[1] > max_size:
        from PIL import Image

        ratio = min(max_size / img.size[0], max_size / img.size[1])
        new_size = (int(img.size[0] * ratio), int(img.size[1] * ratio))
        return img.resize(new_size, Image.Resampling.LANCZOS)
    return img


def _send_response(resp_queue: Any, response: dict) -> None:
    """Send a response to the parent process; stamps ``ts`` if absent."""
    response.setdefault("ts", time.time())
    try:
        resp_queue.put(response)
    except (OSError, ValueError) as exc:
        logger.error("Failed to send response: %s", exc)


def _exit_after_shutdown(resp_queue: Any) -> None:
    """Models are unloaded and acked; interpreter shutdown can stall 10s+ on Windows CUDA (#16330)."""
    try:
        resp_queue.close()
        resp_queue.join_thread()
    except Exception:
        pass
    for stream in (sys.stdout, sys.stderr):
        try:
            stream.flush()
        except Exception:
            pass
    try:
        from utils.worker_stderr import stop_worker_stderr_mirror
        stop_worker_stderr_mirror()
    except Exception:
        pass
    os._exit(0)


def _encode_share_object(obj: Any) -> bytes:
    data = json.dumps(obj, separators = (",", ":"), ensure_ascii = False).encode("utf-8")
    if len(data) > _SHARE_OBJECT_MAX_BYTES:
        raise ValueError("Distributed object share payload is too large")
    return data


def _decode_share_object(data: Any) -> Any:
    return json.loads(bytes(data.tolist()).decode("utf-8"))


def _clean_token(value: str | None) -> str | None:
    """Normalize an HF token: blank or whitespace-only becomes None."""
    return value if value and value.strip() else None


def _config_hf_token(config: dict) -> str | bool | None:
    if config.get("anonymous_hf_access"):
        return False
    return _clean_token(config.get("hf_token"))


def _apply_worker_hf_token_environment(config: dict) -> None:
    from hub.utils.hf_tokens import apply_token_to_child_env
    apply_token_to_child_env(os.environ, _config_hf_token(config))


def _build_model_config(config: dict):
    from utils.models import ModelConfig
    from utils.models.model_config import shared_hub_model_info

    model_name = config["model_name"]
    with shared_hub_model_info():
        mc = ModelConfig.from_identifier(
            model_id = model_name,
            hf_token = _config_hf_token(config),
            gguf_variant = config.get("gguf_variant"),
        )
    if not mc:
        raise ValueError(f"Invalid model identifier: {model_name}")
    return mc


_NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano")


def _needs_nemotron_trust(model_name: str, hf_token: str | None = None) -> bool:
    """Needs an unsloth/ or nvidia/ namespace and a genuine Hub repo (not a local path or spoofed name) so Llama-Nemotron is never matched."""
    mn = model_name.lower()
    if not (
        any(sub in mn for sub in _NEMOTRON_TRUST_SUBSTRINGS)
        and (mn.startswith("unsloth/") or mn.startswith("nvidia/"))
    ):
        return False

    from utils.security.trusted_org import is_trusted_org_repo

    return is_trusted_org_repo(model_name, hf_token = hf_token)


def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool:
    """Recorded unsloth_load_in_4bit wins; else lora full precision, qlora 4-bit; unknown method forces off unless base is -bnb-4bit; missing config keeps value."""
    from utils.models.checkpoints import is_full_finetune_output

    if load_in_4bit and not mc.is_lora and is_full_finetune_output(mc.path):
        logger.info("Full fine-tune output has no quantization_config — setting load_in_4bit=False")
        return False
    if not (mc.is_lora and mc.path):
        return load_in_4bit

    adapter_cfg_path = Path(mc.path) / "adapter_config.json"
    if not adapter_cfg_path.exists():
        return load_in_4bit

    import json

    try:
        with open(adapter_cfg_path, encoding = "utf-8-sig") as f:
            adapter_cfg = json.load(f)
        trained_in_4bit = adapter_cfg.get("unsloth_load_in_4bit")
        if isinstance(trained_in_4bit, bool):
            if trained_in_4bit != load_in_4bit:
                logger.info(
                    "adapter_config.json says unsloth_load_in_4bit=%s — setting load_in_4bit=%s",
                    trained_in_4bit,
                    trained_in_4bit,
                )
            return trained_in_4bit
        training_method = adapter_cfg.get("unsloth_training_method")
        if training_method == "lora" and load_in_4bit:
            logger.info("adapter_config.json says lora — setting load_in_4bit=False")
            return False
        if training_method == "qlora" and not load_in_4bit:
            logger.info("adapter_config.json says qlora — setting load_in_4bit=True")
            return True
        if (
            not training_method
            and mc.base_model
            and "-bnb-4bit" not in mc.base_model.lower()
            and load_in_4bit
        ):
            logger.info(
                "No training method, base model has no -bnb-4bit — setting load_in_4bit=False"
            )
            return False
    except Exception as e:
        logger.warning("Could not read adapter_config.json: %s", e)
    return load_in_4bit


SSM_RESTART_REQUIRED = "ssm_runtime_restart_required"


def _ensure_ssm_kernels(
    targets: list,
    resp_queue: Any,
    hf_token: str | None = None,
    trust_remote_code: bool | None = None,
) -> bool:
    """Install the SSM kernels the given model(s) lazy-import in from_pretrained; no-op for
    non-SSM models, idempotent. *targets* are ``(probe_identifier, config_target)`` pairs: the
    kernels are decided from the config at ``config_target`` (the resolved base, or a local
    adapter's recorded base), the name only when there is none. Call BEFORE importing
    transformers, which snapshots its optional-backend gates at import.

    Returns True on success. Returns False after sending a 'loaded' failure: on a fatal mamba-ssm
    failure, or when a kernel was newly installed after this process already imported the model's
    modeling module, whose captured kernel functions only a fresh worker picks up (the
    orchestrator respawns once on ``SSM_RESTART_REQUIRED``).
    """
    try:
        from utils.ssm_runtime import modeling_modules_needing_restart, provision_ssm_kernels
    except Exception as exc:
        logger.debug("ssm_runtime unavailable (%s); skipping SSM kernel pre-install", exc)
        return True

    _ssm_status = lambda m: _send_response(resp_queue, {"type": "status", "message": m})
    try:
        for probe, config_target in dict.fromkeys(t for t in targets if t and t[0]):
            # trust_remote_code decides whether an auto_map model loads its remote code, and so
            # whether a missing mamba-ssm refuses the load (see ssm_runtime.mamba_ssm_is_fatal).
            kernels, installed = provision_ssm_kernels(
                probe,
                config_target or probe,
                hf_token,
                trust_remote_code = trust_remote_code,
                status_cb = _ssm_status,
            )
            stale = (
                modeling_modules_needing_restart(kernels.model_types, kernels.remote_code)
                if installed
                else []
            )
            if stale:
                from utils.ssm_runtime import REMOVED_PREFIX

                # "removed mamba_ssm" entries mark an uninstall, which changes the kernels as much
                # as an install does; say which happened.
                changes = ", ".join(
                    f"Removed {name[len(REMOVED_PREFIX):]}"
                    if name.startswith(REMOVED_PREFIX)
                    else f"Installed {name}"
                    for name in installed
                )
                logger.info(
                    "%s after %s was imported; restarting the worker to pick it up",
                    changes,
                    ", ".join(stale),
                )
                _send_response(
                    resp_queue,
                    {
                        "type": "loaded",
                        "success": False,
                        "message": (
                            f"{changes} for this model; restarting the model worker to "
                            "pick up the change."
                        ),
                        "error_kind": SSM_RESTART_REQUIRED,
                    },
                )
                return False
        return True
    except Exception as exc:
        _send_response(
            resp_queue,
            {
                "type": "loaded",
                "success": False,
                "message": (
                    f"This model needs SSM kernel libraries (causal-conv1d / "
                    f"mamba-ssm) that could not be installed: {exc}"
                ),
                "error_kind": "ssm_runtime_install_failed",
            },
        )
        return False


def _run_security_gates(
    targets: list,
    *,
    trust_remote_code: bool,
    hf_token: str | None,
    approved_fingerprint: str | None,
    resp_queue: Any,
    compute_subdirs: bool = True,
    subject: str | None = None,
) -> bool:
    """compute_subdirs=False keeps the preflight from importing transformers before SSM kernels install."""
    targets = list(dict.fromkeys(t for t in targets if t))

    # Poisoned pickles deserialize even without trust_remote_code: scan every load (LoRA: the base).
    from utils.security import evaluate_file_security

    from utils.security import load_scan_target

    if compute_subdirs:
        from utils.security import security_load_subdirs

    scoped_targets: list[str] = []
    consent_load_subdirs: dict[str, tuple] = {}
    for requested_target in targets:
        _subdirs = security_load_subdirs(requested_target, hf_token) if compute_subdirs else ()
        target, _subdirs = load_scan_target(requested_target, _subdirs)
        if target not in consent_load_subdirs:
            scoped_targets.append(target)
            consent_load_subdirs[target] = ()
        _subdirs = tuple(dict.fromkeys((*consent_load_subdirs[target], *_subdirs)))
        consent_load_subdirs[target] = _subdirs

    for target in scoped_targets:
        _subdirs = consent_load_subdirs[target]
        _fs = evaluate_file_security(target, hf_token = hf_token, load_subdirs = _subdirs)
        if _fs.blocked:
            _send_response(
                resp_queue,
                {
                    "type": "loaded",
                    "success": False,
                    "message": _fs.reason,
                    "error_kind": "malware_blocked",
                    "security": _fs.response_payload(),
                },
            )
            return False

    # Block CRITICAL/HIGH auto_map code unless pinned-approved; adapter + base share one fingerprint.
    if trust_remote_code:
        from utils.security import evaluate_remote_code_consent_for_targets
        _rc = evaluate_remote_code_consent_for_targets(
            scoped_targets,
            hf_token = hf_token,
            trust_remote_code = True,
            approved_fingerprint = approved_fingerprint,
            subject = subject,
            load_subdirs_by_target = consent_load_subdirs,
        )
        if _rc.blocked:
            _send_response(
                resp_queue,
                {
                    "type": "loaded",
                    "success": False,
                    "message": _rc.blocked_message(),
                    "error_kind": "remote_code_blocked",
                    "remote_code": _rc.response_payload(),
                },
            )
            return False

    return True


def _worker_reclaimable_gpu_gb(config: dict) -> dict[str, float] | None:
    """Live allocator-owned VRAM this disposable worker will release."""
    resolved_gpu_ids = config.get("resolved_gpu_ids")
    device_backend = str(config.get("device_backend") or "")
    if not resolved_gpu_ids or device_backend not in ("cuda", "xpu"):
        return None
    try:
        import torch

        device_module = torch.cuda if device_backend == "cuda" else torch.xpu
        memory_reserved = getattr(device_module, "memory_reserved", None)
        if not callable(memory_reserved):
            return None
        return {
            str(int(physical_id)): int(memory_reserved(local_ordinal)) / float(1024**3)
            for local_ordinal, physical_id in enumerate(resolved_gpu_ids)
        }
    except Exception as exc:
        logger.warning("Could not report worker-owned GPU memory: %s", exc)
        return None


def _load_download_repos(
    mc,
    load_in_4bit: bool,
    backend,
    companions = (),
) -> list[str]:
    from hub.utils.paths import is_valid_repo_id
    from utils.paths import is_local_path
    from utils.security.file_security import load_scan_target
    from utils.third_party_source import SPEECH_CODEC_REPOSITORIES

    audio_type = getattr(mc, "audio_type", None)
    repos = [str(mc.identifier), *(str(repo) for repo in companions)]
    base = getattr(mc, "base_model", None)
    if base:
        repos.append(str(base))
        mapped = None
        if getattr(backend, "device", None) == "mlx":
            from core.inference.model_ids import mlx_bnb_base_repo
            mapped = mlx_bnb_base_repo(str(base))
        else:
            try:
                from unsloth.models import loader, loader_utils

                quantized = (
                    load_in_4bit
                    and not getattr(mc, "is_audio", False)
                    and getattr(loader, "ALLOW_BITSANDBYTES", True)
                )
                # Unknown names resolve to themselves; get_model_name would fetch the remote mapper for nothing.
                mapped = str(base)
                name = mapped.lower()
                tables = (
                    loader_utils.FLOAT_TO_INT_MAPPER,
                    loader_utils.INT_TO_FLOAT_MAPPER,
                    loader_utils.MAP_TO_UNSLOTH_16bit,
                )
                if name in loader_utils.BAD_MAPPINGS or any(name in t for t in tables):
                    mapped = loader_utils.get_model_name(str(base), load_in_4bit = quantized)
                if mapped and not getattr(loader, "ALLOW_PREQUANTIZED_MODELS", True):
                    mapped = loader._strip_unsloth_bnb_4bit_suffix(mapped)
            except Exception:
                mapped = None
        if mapped:
            repos.append(str(mapped))
    repos.extend(SPEECH_CODEC_REPOSITORIES.get(audio_type, ()))
    hub_ids: list[str] = []
    for repo in repos:
        repo, _subdirs = load_scan_target(repo, ())
        if is_valid_repo_id(repo) and not is_local_path(repo) and repo not in hub_ids:
            hub_ids.append(repo)
    return hub_ids


def _hub_cache_dir() -> Optional[str]:
    try:
        from utils.hf_cache_settings import get_hf_cache_paths
        return str(get_hf_cache_paths().hub_cache)
    except Exception:
        return None


# The token env before a load scrubbed it; the next load restores it.
_TOKEN_ENV_BEFORE_ANONYMOUS_LOAD: Optional[dict] = None


def _token_env_keys() -> tuple:
    from hub.utils.hf_tokens import _HF_TOKEN_ENV_KEYS
    return (*_HF_TOKEN_ENV_KEYS, "HF_HUB_DISABLE_IMPLICIT_TOKEN")


def _restore_token_environment() -> None:
    """Undo an earlier load's anonymous scrub (the token may have been replaced since)."""
    global _TOKEN_ENV_BEFORE_ANONYMOUS_LOAD
    saved, _TOKEN_ENV_BEFORE_ANONYMOUS_LOAD = _TOKEN_ENV_BEFORE_ANONYMOUS_LOAD, None
    for key, value in (saved or {}).items():
        if value is None:
            os.environ.pop(key, None)
        else:
            os.environ[key] = value


def _drop_a_rejected_token(config: dict) -> None:
    """The Hub refused this load's token while anonymous reads worked: load the rest anonymously."""
    global _TOKEN_ENV_BEFORE_ANONYMOUS_LOAD
    from hub.utils.hf_tokens import saved_token_rejected

    token = _config_hf_token(config)
    if token is not False and saved_token_rejected(token):
        config["anonymous_hf_access"] = True
        if _TOKEN_ENV_BEFORE_ANONYMOUS_LOAD is None:
            _TOKEN_ENV_BEFORE_ANONYMOUS_LOAD = {k: os.environ.get(k) for k in _token_env_keys()}
        _apply_worker_hf_token_environment(config)
        logger.warning(
            "Hugging Face rejected the token for %s; loading it without the token.",
            config.get("model_name"),
        )


def _in_token_rejection_scope(handler):
    """Run one load in its own rejected-token scope, so a verdict never outlives it."""

    @functools.wraps(handler)
    def scoped(*args, **kwargs):
        from hub.utils.hf_tokens import token_rejection_scope
        with token_rejection_scope():
            return handler(*args, **kwargs)

    return scoped


def _loaded_auto_map(backend, mc) -> bool | None:
    """Whether the opened MLX snapshot declares remote code; None when it is not the requested repo (adapter, precision folder, bnb substitute)."""
    if getattr(backend, "device", None) != "mlx" or mc.is_lora or getattr(mc, "gguf_variant", None):
        return None
    try:
        from core.inference.mlx_inference import _snapshot_dir
        from core.inference.model_ids import mlx_host_bnb_base_repo
        from utils.security.consent import _config_has_auto_map

        if mlx_host_bnb_base_repo(mc.identifier):
            return None
        snapshot = _snapshot_dir(getattr(backend, "_model", None), mc.identifier)
        return _config_has_auto_map(snapshot) if snapshot else None
    except Exception as exc:
        logger.warning("Could not read remote-code declarations from the loaded files: %s", exc)
        return None


@_in_token_rejection_scope
def _handle_load(backend, config: dict, resp_queue: Any) -> None:
    _restore_token_environment()
    try:
        mc = _build_model_config(config)
        _drop_a_rejected_token(config)

        hf_token = _config_hf_token(config)
        load_in_4bit = _resolve_lora_4bit(mc, config.get("load_in_4bit", True))

        # Latest-tier models load 16-bit: bnb 4-bit feeds quantized experts into unvalidated paths.
        if load_in_4bit:
            from utils.transformers_version import latest_tier_active_for
            if latest_tier_active_for(config["model_name"], hf_token):
                load_in_4bit = False
                logger.info(
                    "Latest-transformers sidecar active for %s - forcing a 16-bit "
                    "load (4-bit is disabled for brand-new architectures)",
                    config["model_name"],
                )

        trust_remote_code = config.get("trust_remote_code", False)
        if not trust_remote_code and _needs_nemotron_trust(config["model_name"], hf_token = hf_token):
            trust_remote_code = True
            logger.info(
                "Auto-enabled trust_remote_code for Nemotron model: %s", config["model_name"]
            )

        # Must run before the SSM install so a blocked model never triggers a native kernel build.
        from core.inference.native_audio import native_audio_security_targets

        targets = native_audio_security_targets(
            config["model_name"], getattr(mc, "audio_type", None), hf_token
        )
        if mc.is_lora and getattr(mc, "base_model", None):
            targets.append(str(mc.base_model))
        if not _run_security_gates(
            targets,
            trust_remote_code = trust_remote_code,
            hf_token = hf_token,
            approved_fingerprint = config.get("approved_remote_code_fingerprint"),
            resp_queue = resp_queue,
            subject = config.get("subject"),
        ):
            return

        # Install SSM kernels for a LoRA's base (resolved only now) and in-process loads; skip on MLX; probe the base, not the adapter id or local path.
        if getattr(backend, "device", None) != "mlx":
            from utils.ssm_runtime import ssm_probe_identifier

            _ssm_base = (
                str(mc.base_model) if (mc.is_lora and getattr(mc, "base_model", None)) else None
            )
            ssm_targets = [
                (
                    ssm_probe_identifier(config["model_name"], _ssm_base),
                    _ssm_base or config["model_name"],
                )
            ]
            if not _ensure_ssm_kernels(ssm_targets, resp_queue, hf_token, trust_remote_code):
                return

        # Heartbeat keeps the parent's deadline alive; a no-progress Xet download is a stall that triggers HTTP respawn; watch model and LoRA base repos.
        from core.inference.model_ids import mlx_bnb_substitutions
        from utils.hf_xet_fallback import start_watchdog

        watch_repos = [mc.identifier]
        base = getattr(mc, "base_model", None)
        if base and str(base) != mc.identifier:
            watch_repos.append(str(base))

        # Watch the repositories Zoo downloads after substitution.
        if getattr(backend, "device", None) == "mlx":
            substitutions = mlx_bnb_substitutions(watch_repos)
            replacements = dict(substitutions)
            watch_repos = list(dict.fromkeys(replacements.get(repo, repo) for repo in watch_repos))
            for requested, mlx_base in substitutions:
                _send_response(
                    resp_queue,
                    {
                        "type": "status",
                        "message": (
                            f"MLX cannot read bitsandbytes 4-bit weights; "
                            f"downloading {mlx_base} instead of {requested}"
                        ),
                    },
                )

        _send_response(
            resp_queue,
            {
                "type": "downloads",
                "repo_ids": _load_download_repos(mc, load_in_4bit, backend, targets),
                "xet_disabled": os.environ.get("HF_HUB_DISABLE_XET") == "1",
                "hub_cache": _hub_cache_dir(),
            },
        )
        heartbeat_stop = start_watchdog(
            repo_ids = watch_repos,
            on_stall = lambda msg: _send_response(resp_queue, {"type": "stall", "message": msg}),
            on_heartbeat = lambda msg: _send_response(resp_queue, {"type": "status", "message": msg}),
            xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
        )
        try:
            load_kwargs = {
                "config": mc,
                "max_seq_length": config.get("max_seq_length", 2048),
                "load_in_4bit": load_in_4bit,
                "hf_token": hf_token,
                "trust_remote_code": trust_remote_code,
                "gpu_ids": config.get("resolved_gpu_ids"),
            }
            if config.get("audio_codec_path") is not None:
                load_kwargs["audio_codec_path"] = config["audio_codec_path"]
            if getattr(backend, "device", None) == "mlx":
                load_kwargs["parallel_mode"] = config.get("mlx_parallel_mode")
                load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")
                load_kwargs["kv_quant"] = config.get("mlx_kv_quant")
                load_kwargs["int8_prefill"] = bool(config.get("mlx_int8_prefill"))
                load_kwargs["ane_prefill"] = bool(config.get("mlx_ane_prefill"))
                load_kwargs["fast_verify"] = bool(config.get("mlx_fast_verify"))
                load_kwargs["offload"] = tuple(config.get("mlx_offload") or (None, None))
                load_kwargs["speculative_type"] = config.get("speculative_type")
                load_kwargs["spec_draft_n_max"] = config.get("spec_draft_n_max")
                load_kwargs["spec_draft_model"] = config.get("spec_draft_model")
                load_kwargs["spec_drafters_allowed"] = config.get("spec_drafters_allowed")
                load_kwargs["chat_template_override"] = config.get("chat_template_override")
            from utils.security.file_security import escaping_weight_references

            # Right before the load: the folder may have been edited since the route checked it.
            lora_base = getattr(mc, "base_model", None) if mc.is_lora else None
            for folder in (config["model_name"], lora_base):
                if folder and escaping_weight_references(folder):
                    raise ValueError("This model's weight index names files outside its folder.")
            success = backend.load_model(**load_kwargs)
        finally:
            heartbeat_stop.set()

        if success:
            model_info = {
                "identifier": mc.identifier,
                "display_name": mc.display_name,
                "is_vision": mc.is_vision,
                "is_lora": mc.is_lora,
                "is_gguf": False,
                # MLX backend sets device="mlx"; lets the UI tag MLX models.
                "is_mlx": getattr(backend, "device", None) == "mlx",
                "loaded_auto_map": _loaded_auto_map(backend, mc),
                "is_audio": getattr(mc, "is_audio", False),
                "audio_type": getattr(mc, "audio_type", None),
                "has_audio_input": getattr(mc, "has_audio_input", False),
                "can_batch": _load_can_batch(backend),
            }
            if _active_model_is_deepseek_ocr(backend):
                from core.inference.model_ids import DEEPSEEK_OCR_MAX_IMAGE_SIDE

                # The route resizes before the IPC hop, so it needs the raised cap too.
                model_info["max_image_side"] = DEEPSEEK_OCR_MAX_IMAGE_SIDE
            _bm = getattr(backend, "models", {}) or {}
            _entry = (
                _bm.get(mc.identifier) or _bm.get(getattr(backend, "active_model_name", None)) or {}
            )
            for _ctx_field in (
                "context_length",
                "native_context_length",
                "max_context_length",
                "requested_context_length",
                "context_length_fitted",
                "mlx_context_budget",
            ):
                try:
                    _ctx_value = _entry.get(_ctx_field)
                    if _ctx_value is not None:
                        model_info[_ctx_field] = int(_ctx_value)
                except Exception as _ctx_exc:
                    logger.warning("%s forward failed: %s", _ctx_field, _ctx_exc)
            # Tri-state: None means the backend does not answer, not False.
            if _entry.get("context_length_enforced") is not None:
                model_info["context_length_enforced"] = bool(_entry["context_length_enforced"])
            # Backend post-load audio and video classification outranks pre-load config.
            model_info.update(
                {
                    k: _entry[k]
                    for k in (
                        "is_audio",
                        "audio_type",
                        "has_audio_input",
                        "has_video_input",
                        "speaks",
                        "audio_family",
                        "audio_options",
                        "gguf_variant",
                        "precision_folder",
                        "audio_workflows",
                        "audio_reference_text",
                        "audio_required_inputs",
                        "audio_clone",
                        "audio_options_by_workflow",
                        "audio_workflow_tasks",
                        "audio_server_task",
                        "audio_convert",
                        "audio_convert_route",
                        "audio_convert_rules",
                        "audio_edit",
                        "audio_music",
                        "audio_cpp_backend",
                        "audio_device_fallback",
                        "audio_request_inputs",
                    )
                    if k in _entry
                }
            )
            # MLX decides at load: an mlx-lm text build reads no images, an mlx-vlm-only tower does.
            if model_info["is_mlx"] and isinstance(_entry.get("is_vision"), bool):
                model_info["is_vision"] = _entry["is_vision"]
            # Resolved MLX runtime knobs; only the backend knows what it honored.
            model_info.update(
                {
                    k: _entry[k]
                    for k in (
                        "context_unbounded_when_batched",
                        "mlx_kv_bits",
                        "mlx_kv_bits_requested",
                        "mlx_kv_quant",
                        "mlx_kv_quant_requested",
                        "mlx_kv_quant_eligibility",
                        "mlx_kv_quant_reason",
                        "mlx_kv_quant_note",
                        "mlx_int8_prefill",
                        "mlx_int8_prefill_requested",
                        "mlx_int8_prefill_reason",
                        "mlx_ane_prefill",
                        "mlx_ane_prefill_requested",
                        "mlx_ane_prefill_reason",
                        "mlx_fast_verify",
                        "mlx_fast_verify_requested",
                        "mlx_fast_verify_reason",
                        "mlx_offload",
                        "chat_template_override_requested",
                        "chat_template_override_reason",
                        "speculative_type",
                        "spec_draft_n_max",
                        "spec_draft_model",
                        "spec_drafter_kind",
                        "spec_fallback_reason",
                    )
                    if k in _entry
                }
            )
            # Forward chat_template_info so the parent can classify capabilities.
            try:
                _tpl_info = _entry.get("chat_template_info")
                _mapped_tpl = None
                if isinstance(_tpl_info, dict) and not model_info["is_mlx"]:
                    from core.inference.chat_template_helpers import mapped_chat_template
                    _mapped_tpl = mapped_chat_template(
                        _entry, getattr(backend, "active_model_name", None) or mc.identifier
                    )
                if isinstance(_tpl_info, dict):
                    model_info["chat_template_info"] = {
                        "has_template": bool(_tpl_info.get("has_template", False)),
                        "template": _tpl_info.get("template"),
                        "format_type": _tpl_info.get("format_type", "generic"),
                        "template_name": _tpl_info.get("template_name"),
                        "special_tokens": _tpl_info.get("special_tokens", {}) or {},
                        # The IMAGE-turn body; the whitelist is the only way out.
                        "processor_template": _tpl_info.get("processor_template"),
                        "renders_image": _tpl_info.get("renders_image"),
                        "accepts_multiple_images": _tpl_info.get("accepts_multiple_images"),
                        # The body a text render installs at generate time, when the model is mapped.
                        "mapped_template": _mapped_tpl,
                    }
            except Exception as _tpl_exc:
                logger.warning("chat_template_info forward failed: %s", _tpl_exc)
            _send_response(
                resp_queue,
                {
                    "type": "loaded",
                    "success": True,
                    "model_info": model_info,
                },
            )
        else:
            _send_response(
                resp_queue,
                {
                    "type": "loaded",
                    "success": False,
                    "error": "Failed to load model",
                },
            )

    except Exception as exc:
        from core.inference.audio_cpp_failures import failure_of
        failure = failure_of(exc)
        _send_response(
            resp_queue,
            {
                "type": "loaded",
                "success": False,
                "error": str(exc),
                "stack": traceback.format_exc(limit = 20),
                **({"failure": failure.payload()} if failure is not None else {}),
            },
        )


def _handle_narrow_context(backend, cmd: dict, resp_queue: Any) -> None:
    try:
        fields = backend.narrow_context(cmd.get("max_seq_length"))
    except Exception as exc:
        logger.warning("Could not narrow the context in place: %s", exc)
        fields = None
    _send_response(resp_queue, {"type": "context_narrowed", "fields": fields})


def _drain_skip_generate(
    cmd: dict,
    resp_queue: Any,
    drain_event,
    *,
    audio: bool = False,
) -> bool:
    """Skips a queued generate during unload after an immediate terminal reply, and reports it skipped so the caller neither clears the cancel nor runs it."""
    if drain_event is None or not drain_event.is_set():
        return False
    request_id = cmd.get("request_id", "")
    logger.info("Skipping generate for request %s: unload draining", request_id)
    response = {
        "type": "audio_error" if audio else "gen_done",
        "request_id": request_id,
        "cancelled": True,
    }
    if audio:
        response["error"] = "Audio generation cancelled"
    else:
        response["stats"] = None
    _send_response(resp_queue, response)
    return True


def _abandon_held_commands(held: list, resp_queue: Any) -> None:
    while held:
        _abandon_one(held.pop(0), resp_queue)


class _Stops:
    def __init__(self, stop_ledger, resp_queue, batch, held: list):
        self._ledger = stop_ledger
        self._resp_queue = resp_queue
        self._batch = batch
        self._held = held
        self._written = 0
        self._stopped: set = set()

    def answer(self) -> None:
        self._refresh()
        self._batch.drop_stopped(self)
        for cmd in [held for held in self._held if held.get("request_id", "") in self]:
            self._held.remove(cmd)
            _abandon_one(cmd, self._resp_queue)

    def _refresh(self) -> None:
        if self._ledger is None:
            return
        self._written, stopped = self._ledger.snapshot(self._written)
        if stopped is not None:
            self._stopped = stopped

    def __contains__(self, request_id) -> bool:
        return request_id in self._stopped

    def unread(self) -> bool:
        return self._ledger is not None and self._ledger.snapshot(self._written)[1] is not None


class _StopWhileItRuns:
    def __init__(self, cancel_event, stops, request_id: str):
        self._cancel_event = cancel_event
        self._stops = stops
        self._request_id = request_id

    def is_set(self) -> bool:
        if self._cancel_event is not None and self._cancel_event.is_set():
            return True
        self._stops.answer()
        return self._request_id in self._stops


_TEARDOWN_COMMANDS = frozenset({"cancel", "reset", "unload", "shutdown"})


def _teardown_skip(cmd: dict, resp_queue: Any, pending_teardowns) -> bool:
    if pending_teardowns is None or not pending_teardowns.any_in_flight():
        return False
    _abandon_one(cmd, resp_queue)
    return True


def _stopped_before_it_ran(cmd: dict, resp_queue: Any, stopped) -> bool:
    stopped.answer()
    if cmd.get("request_id", "") not in stopped:
        return False
    _abandon_one(cmd, resp_queue)
    return True


def _abandon_one(cmd: dict, resp_queue: Any) -> None:
    logger.info(
        "Abandoning held %s for request %s",
        cmd.get("type", ""),
        cmd.get("request_id", ""),
    )
    _send_response(
        resp_queue,
        {
            "type": "gen_done",
            "request_id": cmd.get("request_id", ""),
            "cancelled": True,
            "stats": None,
        },
    )


def _prepare_generate_audio(cmd, resp_queue: Any, cancel_event, drain_event) -> bool:
    """Checks the unload drain on both sides of the cancel clear so a racing unload's cancel survives; parent signals cancel only after audio_started."""
    if _drain_skip_generate(cmd, resp_queue, drain_event, audio = True):
        return False
    cancel_event.clear()
    if _drain_skip_generate(cmd, resp_queue, drain_event, audio = True):
        return False
    _send_response(
        resp_queue,
        {
            "type": "audio_started",
            "request_id": cmd.get("request_id", ""),
        },
    )
    return True


def _backend_declares(
    backend,
    name: str,
    method: str = "generate_chat_response",
) -> bool:
    """A signature check, not a capability claim: a **kwargs backend reads False; failing closed only drops the option, never crashes."""
    generate = getattr(backend, method, None)
    if generate is None:
        return False
    try:
        return name in inspect.signature(generate).parameters
    except (TypeError, ValueError):
        return False


def _dispatch_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
    if cmd.get("rows"):
        _handle_generate_rows(backend, cmd, resp_queue, cancel_event)
    else:
        _handle_generate(backend, cmd, resp_queue, cancel_event)


def _decoded_images(backend, encoded) -> list:
    resize = _resize_image
    if _active_model_is_deepseek_ocr(backend):
        from core.inference.model_ids import DEEPSEEK_OCR_MAX_IMAGE_SIDE
        resize = functools.partial(_resize_image, max_size = DEEPSEEK_OCR_MAX_IMAGE_SIDE)
    return [resize(_decode_image(one)) for one in encoded or () if one]


def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:
    image = next(iter(_decoded_images(backend, [cmd.get("image_base64")])), None)
    images = _decoded_images(backend, cmd.get("images_base64"))

    gen_kwargs = {
        "messages": cmd["messages"],
        "system_prompt": cmd.get("system_prompt", ""),
        "image": image,
        "images": images,
        "temperature": cmd.get("temperature", 0.7),
        "top_p": cmd.get("top_p", 0.9),
        "top_k": cmd.get("top_k", 40),
        "min_p": cmd.get("min_p", 0.0),
        "max_new_tokens": cmd.get("max_new_tokens", 256),
        "repetition_penalty": cmd.get("repetition_penalty", 1.0),
        "presence_penalty": cmd.get("presence_penalty", 0.0),
        "cancel_event": cancel_event,
    }

    for opt_key in (
        "tools",
        "enable_thinking",
        "reasoning_effort",
        "preserve_thinking",
        "continue_final_message",
    ):
        if opt_key in cmd:
            gen_kwargs[opt_key] = cmd[opt_key]

    if cmd.get("image_ordinal") is not None and _backend_declares(backend, "image_ordinal"):
        gen_kwargs["image_ordinal"] = cmd["image_ordinal"]

    for gated in (
        "seed",
        "frequency_penalty",
        "logit_bias",
        "stop",
        "tool_protocol_active",
        "response_format",
        "reasoning_is_extracted",
    ):
        if gated in cmd and _backend_declares(backend, gated):
            gen_kwargs[gated] = cmd[gated]
    if cmd.get("videos_base64"):
        if not _backend_declares(backend, "videos"):
            raise RuntimeError("The loaded backend does not read video.")
        gen_kwargs["videos"] = cmd["videos_base64"]
    # Only a raw completion carries these; its batch rows are planned from them.
    for raw in ("prompt_ids", "penalty_window", "grammar", "n_probs"):
        if raw in cmd:
            gen_kwargs[raw] = cmd[raw]

    return gen_kwargs


def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
    """cancel_event is polled between tokens, never during prefill; batching passes a record plus the shared event, one-at-a-time the shared event alone."""
    request_id = cmd.get("request_id", "")
    # Scoped to this request: cleared when it ends.
    frames = bool(cmd.get("diffusion_frames")) and hasattr(backend, "diffusion_frame_sink")
    if frames:
        backend.diffusion_frame_sink = lambda frame: _send_response(
            resp_queue, {**frame, "request_id": request_id}
        )

    try:
        gen_kwargs = _generation_kwargs(backend, cmd, cancel_event)

        use_adapter = cmd.get("use_adapter")
        if "prompt_ids" in cmd:
            generator = _completion_generator(backend, cmd, cancel_event)
        elif use_adapter is not None:
            generator = backend.generate_with_adapter_control(
                use_adapter = use_adapter,
                **gen_kwargs,
            )
        else:
            generator = backend.generate_chat_response(**gen_kwargs)

        logger.info("Starting text generation for request_id=%s", request_id)

        try:
            for cumulative_text in generator:
                if cancel_event.is_set():
                    logger.info("Generation cancelled for request %s", request_id)
                    break

                _send_response(
                    resp_queue,
                    {
                        "type": "token",
                        "request_id": request_id,
                        "text": cumulative_text,
                    },
                )
        finally:
            close = getattr(generator, "close", None)
            if callable(close):
                close()

        _send_response(
            resp_queue,
            {
                "type": "gen_done",
                "request_id": request_id,
                # Stats: usage/timings from MLX and safetensors, "truncated" from safetensors; None if neither.
                "stats": getattr(backend, "last_generation_stats", None),
            },
        )
        logger.info("Finished text generation for request_id=%s", request_id)

    except Exception as exc:
        logger.error("Generation error: %s", exc, exc_info = True)
        _send_response(resp_queue, _generation_error_payload(request_id, exc))
    finally:
        if frames:
            backend.diffusion_frame_sink = None


_COMPLETION_OPTIONS = (
    "seed",
    "frequency_penalty",
    "logit_bias",
    "stop",
    "penalty_window",
    "grammar",
    "n_probs",
)


def _completion_generator(backend, cmd: dict, cancel_event):
    """A raw-prompt completion, which only a backend with ``generate_completion`` serves."""
    if not callable(getattr(backend, "generate_completion", None)):
        raise RuntimeError("The loaded backend does not serve raw-prompt completions.")
    return backend.generate_completion(
        cmd["prompt_ids"],
        temperature = cmd.get("temperature", 0.7),
        top_p = cmd.get("top_p", 0.9),
        top_k = cmd.get("top_k", 40),
        min_p = cmd.get("min_p", 0.0),
        max_new_tokens = cmd.get("max_new_tokens", 256),
        repetition_penalty = cmd.get("repetition_penalty", 1.0),
        presence_penalty = cmd.get("presence_penalty", 0.0),
        cancel_event = cancel_event,
        **{key: cmd[key] for key in _COMPLETION_OPTIONS if key in cmd},
    )


def _handle_resolve_prompt(backend, cmd: dict, resp_queue: Any) -> None:
    """Resolve a raw completion prompt to token ids with the loaded tokenizer."""
    reply = {"type": "resolve_prompt_response", "request_id": cmd.get("request_id")}
    resolve = getattr(backend, "resolve_completion_prompt", None)
    if not callable(resolve):
        reply["error"] = "The loaded backend does not serve raw-prompt completions."
    else:
        try:
            reply.update(resolve(cmd.get("prompt")))
        except ValueError as exc:
            reply["error"], reply["invalid"] = str(exc), True
        except Exception as exc:
            reply["error"] = str(exc)
    _send_response(resp_queue, reply)


_PROMPT_CACHE_RESUME_METHODS = {
    "save_prompt_cache": ("save_prompt_cache_for_resume", "directory"),
    "restore_prompt_cache": ("restore_prompt_cache_for_resume", "manifest"),
}


def _handle_prompt_cache_resume(
    backend,
    cmd: dict,
    resp_queue: Any,
    busy: bool = False,
) -> None:
    """Every outcome answers: the parent otherwise waits out its timeout."""
    method, argument = _PROMPT_CACHE_RESUME_METHODS[cmd["type"]]
    reply = {"type": f"{cmd['type']}_response", "request_id": cmd.get("request_id")}
    run = getattr(backend, method, None)
    if busy:
        reply["error"] = "A generation is in progress."
    elif not callable(run):
        reply["error"] = "The loaded backend keeps no prompt cache."
    else:
        try:
            reply["result"] = run(cmd.get(argument))
        except Exception as exc:
            reply["error"] = str(exc)
    _send_response(resp_queue, reply)


def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:
    """Count prompt tokens for the loaded model and reply with the total."""
    try:
        count = backend.count_chat_tokens(
            cmd.get("messages") or [],
            cmd.get("system_prompt") or "",
            tools = cmd.get("tools"),
            enable_thinking = cmd.get("enable_thinking"),
            reasoning_effort = cmd.get("reasoning_effort"),
            preserve_thinking = cmd.get("preserve_thinking"),
            images = _decoded_images(backend, cmd.get("images_base64")),
        )
    except Exception as exc:
        _send_response(
            resp_queue,
            {
                "type": "count_tokens_response",
                "request_id": cmd.get("request_id"),
                "error": str(exc),
            },
        )
        return
    _send_response(
        resp_queue,
        {
            "type": "count_tokens_response",
            # Unaddressed replies are dropped and the caller waits out its timeout.
            "request_id": cmd.get("request_id"),
            "input_tokens": int(count),
            "model": backend.active_model_name,
        },
    )


def _decline_count_tokens(
    cmd: dict,
    resp_queue: Any,
    reason: str = "Counting is not supported on the transformers backend.",
) -> None:
    """Answer a count this backend cannot serve; dropping it costs the caller its timeout."""
    _send_response(
        resp_queue,
        {
            "type": "count_tokens_response",
            "request_id": cmd.get("request_id"),
            "error": reason,
        },
    )


def _load_can_batch(backend) -> bool:
    """Whether this load serves several replies at once at all."""
    fixed = getattr(backend, "batch_unavailable_reason", None)
    resident = getattr(backend, "resident_unavailable_reason", None)
    return (callable(fixed) and fixed([{}, {}]) is None) or (
        callable(resident) and resident({}) is None
    )


def _rows_apart(backend, requests, cancel_event, stats: list):
    """A declined batch served reply by reply, as the batch's own (row, snapshot) events."""
    for row, request in enumerate(requests):
        if not cancel_event.is_set():
            generator = (
                _completion_generator(backend, request, cancel_event)
                if "prompt_ids" in request
                else backend.generate_chat_response(
                    **{
                        name: value
                        for name, value in request.items()
                        if _backend_declares(backend, name)
                    },
                    cancel_event = cancel_event,
                )
            )
            try:
                for cumulative_text in generator:
                    if cancel_event.is_set():
                        break
                    yield row, cumulative_text
            finally:
                close = getattr(generator, "close", None)
                if callable(close):
                    close()
            stats[row] = getattr(backend, "last_generation_stats", None)
        yield row, None


def _handle_generate_rows(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
    request_id = cmd.get("request_id", "")
    rows = cmd.get("rows") or []

    try:
        shared = _generation_kwargs(backend, cmd, cancel_event)
        shared.pop("cancel_event", None)
        requests = [{**shared, **row} for row in rows]

        reason = narrow_load_reason(cmd)
        if reason is None:
            unavailable = getattr(backend, "batch_unavailable_reason", None)
            reason = unavailable(requests) if callable(unavailable) else "backend cannot batch"
        if reason is not None:
            logger.info(
                "Declining %d replies in one command for request_id=%s: %s",
                len(requests),
                request_id,
                reason,
            )
            stats = [None] * len(requests)
            events = _rows_apart(backend, requests, cancel_event, stats)
        else:
            logger.info(
                "Starting batched generation for request_id=%s rows=%d",
                request_id,
                len(requests),
            )
            stats = None
            events = backend.generate_chat_batch(requests, cancel_event = cancel_event)
        try:
            for row, snapshot in events:
                if snapshot is None:
                    reported = backend.last_batch_generation_stats if stats is None else stats
                    message = {"type": "row_done", "stats": reported[row]}
                else:
                    message = {"type": "token", "text": snapshot}
                _send_response(resp_queue, {**message, "request_id": request_id, "row": row})
        finally:
            close = getattr(events, "close", None)
            if callable(close):
                close()

        _send_response(
            resp_queue,
            {"type": "gen_done", "request_id": request_id, "stats": None},
        )

    except Exception as exc:
        logger.error("Multi-reply generation error: %s", exc, exc_info = True)
        _send_response(resp_queue, _generation_error_payload(request_id, exc))


def _admitted_width(cmd: dict) -> int:
    return max(1, int(cmd.get("parallel_slots") or 1))


def _held_head_leaves_the_hold(batch: "_ResidentBatch", held: list) -> bool:
    """Whether the head comes off the hold on this pass; the batch can still turn it away."""
    if not held:
        return False
    if not batch.rows_in_flight:
        return True
    head = held[0]
    return head.get("type") == "generate" and batch.unavailable_reason(head) is None


class _MLXIdleWarmth:
    def __init__(self):
        self.enabled = os.environ.get("UNSLOTH_MLX_GPU_KEEP_WARM", "1").lower() in (
            "1",
            "true",
            "yes",
        )
        self.deadline = 0.0

    def active(self):
        self.deadline = time.monotonic() + 60.0

    def clear(self):
        self.deadline = 0.0

    def timeout(self, loaded):
        return 0.5 if self.enabled and loaded and time.monotonic() < self.deadline else 1.0

    def tick(self, loaded):
        if self.timeout(loaded) != 0.5:
            return
        try:
            import mlx.core as mx

            # Avoid the first-command stall after the GPU enters its idle power state.
            mx.eval(mx.zeros((1,), dtype = mx.float32) + 1)
        except Exception as exc:
            self.enabled = False
            logger.warning("MLX GPU keep-warm disabled after an idle tick failed: %s", exc)


class _ResidentBatch:
    """The replies an MLX worker is decoding at once."""

    def __init__(self, backend, resp_queue: Any):
        self.backend = backend
        self.resp_queue = resp_queue
        self.session = None
        self.width = None
        self._owed: dict = {}
        self._refused: set = set()

    @property
    def rows_in_flight(self) -> int:
        return self.session.rows_in_flight if self.session is not None else 0

    def _live(self, request_id: str) -> list:
        if self.session is None:
            return []
        return [handle for handle in self.session.handles if handle[0] == request_id]

    def unavailable_reason(self, cmd: dict) -> Optional[str]:
        if cmd.get("request_id", "") in self._refused:
            return "this batch has already refused these replies"
        if cmd.get("use_adapter") is not None:
            return "the reply asks for a particular adapter state"
        reason = narrow_load_reason(cmd)
        if reason is not None:
            return reason
        if self.width is not None and self.width != _admitted_width(cmd):
            # Joining at the old width would refill the batch so the new width is never reached.
            return "the open batch is decoding at a different width"
        if (
            self.width is not None
            and self.rows_in_flight + len(cmd.get("rows") or [None]) > self.width
        ):
            return "the open batch is full"
        probe = getattr(self.backend, "resident_unavailable_reason", None)
        if not callable(probe):
            return "this backend has no batch a reply can join"
        rows = cmd.get("rows") or []
        for request in [{**cmd, **row} for row in rows] if rows else [cmd]:
            reason = probe(request)
            if reason is not None:
                return reason
        return None

    def admit(self, cmd: dict, cancel_event) -> bool:
        request_id = cmd.get("request_id", "")
        rows = cmd.get("rows") or []
        shared = _generation_kwargs(self.backend, cmd, cancel_event)
        shared.pop("cancel_event", None)
        requests = [{**shared, **row} for row in rows] if rows else [shared]

        if self.session is None:
            self.width = _admitted_width(cmd)
            try:
                self.session = self.backend.open_resident_batch(width = self.width)
            except Exception as unopened:
                logger.warning(
                    "No batch could be opened for request_id=%s, decoding it alone: %s",
                    request_id,
                    unopened,
                )
                self.width = None
                return False
        handles = [(request_id, row if rows else None) for row in range(len(requests))]
        self._owed[request_id] = None
        prefixes = []
        try:
            for request, handle in zip(requests, handles):
                prefixes.append((handle, self.session.admit(request, handle)))
        except RowRefused as refusal:
            if not self._forget(request_id, withdraw = True):
                self._fail_all(refusal)
                raise
            logger.info("Request_id=%s cannot join this batch: %s", request_id, refusal)
            self._refused.add(request_id)
            self._close_if_empty()
            return False
        except BaseException as exc:
            if self._forget(request_id, withdraw = True):
                self._close_if_empty()
            else:
                self._fail_all(exc)
            raise
        logger.info(
            "Admitted request_id=%s (%d replies) to a batch of %d",
            request_id,
            len(requests),
            self.rows_in_flight,
        )
        for handle, prefix in prefixes:
            if prefix:
                self._send_token(handle, prefix)
        return True

    def cancel(self, request_id: str) -> bool:
        if request_id not in self._owed:
            return False
        live = self._live(request_id)
        if not live:
            return False
        logger.info("Cancelling request_id=%s in a batch of %d", request_id, self.rows_in_flight)
        try:
            for handle, snapshot in self.session.withdraw(sorted(live, key = _handle_order)):
                self._report(handle, snapshot)
        except Exception as exc:
            logger.error("Batched cancellation error: %s", exc, exc_info = True)
            self._fail_all(exc)
            return True
        self._close_if_empty()
        return True

    def cancel_all(self) -> None:
        for request_id in list(self._owed):
            self.cancel(request_id)

    def drop_stopped(self, stopped) -> None:
        for request_id in list(self._owed):
            if request_id in stopped:
                self.cancel(request_id)

    def step(self, waiting = None) -> None:
        if self.session is None or not self.session.rows_in_flight:
            return
        try:
            for handle, snapshot in self.session.step(waiting):
                if not isinstance(snapshot, BaseException):
                    self._report(handle, snapshot)
                elif not self._fail_row(handle[0], snapshot):
                    return
        except Exception as exc:
            logger.error("Batched generation error: %s", exc, exc_info = True)
            self._fail_all(exc)
            return
        self._close_if_empty()

    def _fail_row(self, request_id: str, exc: BaseException) -> bool:
        """Answer a request whose own decode failed, the batch decoding on. False where it gave up the batch."""
        if request_id not in self._owed:
            return True
        logger.error("A batched reply failed: %s", exc, exc_info = exc)
        stack = "".join(traceback.format_exception(exc, limit = 20))
        taken_back = self._forget(request_id, withdraw = True)
        # _forget already dropped it from _owed, so _fail_all would not answer it.
        _send_response(
            self.resp_queue, {**_generation_error_payload(request_id, exc), "stack": stack}
        )
        if not taken_back:
            self._fail_all(exc)
        return taken_back

    def _fail_all(self, exc: BaseException) -> None:
        """Give up on the batch: every request in it is owed an error, not silence."""
        stack = traceback.format_exc(limit = 20)
        for request_id in list(self._owed):
            self._fail(request_id, exc, stack)
        self.close()

    def _close_if_empty(self) -> None:
        if self.session is not None and not self.session.rows_in_flight:
            self.close()

    def close(self) -> None:
        ending = (
            {handle[0] for handle in self.session.ending} if self.session is not None else set()
        )
        for request_id in [r for r in self._owed if r in ending]:
            self._end(request_id, cancelled = True)
        session, self.session = self.session, None
        self.width = None
        self._owed.clear()
        self._refused.clear()
        if session is None:
            return
        try:
            session.close()
        except Exception:
            logger.error("Could not close the resident batch", exc_info = True)

    def _report(self, handle, snapshot) -> None:
        request_id, row = handle
        if request_id not in self._owed:
            self._take_stats(handle)
            return
        if snapshot is not None:
            self._send_token(handle, snapshot)
            return
        if row is not None:
            _send_response(
                self.resp_queue,
                {
                    "type": "row_done",
                    "request_id": request_id,
                    "row": row,
                    "stats": self._take_stats(handle),
                },
            )
        if not self._live(request_id):
            self._end(request_id)

    def _send_token(self, handle, text) -> None:
        request_id, row = handle
        event = {"type": "token", "request_id": request_id, "text": text}
        if row is not None:
            event["row"] = row
        _send_response(self.resp_queue, event)

    def _end(
        self,
        request_id: str,
        *,
        cancelled: bool = False,
    ) -> None:
        """Answer a request's gen_done; cancelled when it ends with its batch, not a reply."""
        stats = self._take_stats((request_id, None))
        self._forget(request_id, withdraw = False)
        event = {"type": "gen_done", "request_id": request_id, "stats": stats}
        if cancelled:
            event["cancelled"] = True
        _send_response(self.resp_queue, event)
        logger.info("Ended request_id=%s (cancelled=%s)", request_id, cancelled)

    def _fail(self, request_id: str, exc: BaseException, stack: str) -> None:
        self._forget(request_id, withdraw = False)
        _send_response(
            self.resp_queue, {**_generation_error_payload(request_id, exc), "stack": stack}
        )

    def _forget(self, request_id: str, *, withdraw: bool) -> bool:
        """Stop answering for a request. False where taking its rows back failed."""
        live = self._live(request_id)
        self._owed.pop(request_id, None)
        taken_back = True
        if withdraw and live:
            try:
                for _handle, _snapshot in self.session.withdraw(sorted(live, key = _handle_order)):
                    pass
            except Exception:
                logger.error("Could not withdraw rows for request_id=%s", request_id, exc_info = True)
                taken_back = False
        for handle in live:
            self._take_stats(handle)
        return taken_back

    def _take_stats(self, handle):
        return self.session.take_stats(handle) if self.session is not None else None


def _handle_order(handle):
    _request_id, row = handle
    return -1 if row is None else row


def _handle_share_object(backend, cmd: dict, resp_queue: Any) -> None:
    """Share a small Python object across MLX distributed ranks."""
    request_id = cmd.get("request_id", "")
    group = getattr(backend, "_distributed_group", None)
    rank = int(getattr(backend, "_distributed_rank", 0) or 0)
    world_size = int(getattr(backend, "_distributed_world_size", 1) or 1)
    obj = cmd.get("object")

    try:
        if group is None or world_size <= 1:
            shared = obj
        else:
            import mlx.core as mx
            if rank == 0:
                if obj is None:
                    mx.eval(mx.distributed.all_sum(mx.array(0), group = group))
                    shared = None
                else:
                    try:
                        data = mx.array(_encode_share_object(obj), dtype = mx.uint8)
                    except Exception:
                        mx.eval(
                            mx.distributed.all_sum(
                                mx.array(_SHARE_OBJECT_ERROR_SIZE),
                                group = group,
                            )
                        )
                        raise
                    mx.eval(mx.distributed.all_sum(mx.array(data.size), group = group))
                    mx.eval(mx.distributed.all_sum(data, group = group))
                    shared = obj
            else:
                size = int(mx.distributed.all_sum(mx.array(0), group = group).item())
                if size == _SHARE_OBJECT_ERROR_SIZE:
                    raise RuntimeError("Failed to share distributed object")
                if size == 0:
                    shared = None
                else:
                    data = mx.zeros(size, dtype = mx.uint8)
                    data = mx.distributed.all_sum(data, group = group)
                    shared = _decode_share_object(data)
        _send_response(
            resp_queue,
            {
                "type": "shared",
                "request_id": request_id,
                "object": shared,
            },
        )
    except Exception as exc:
        _send_response(
            resp_queue,
            {
                "type": "share_error",
                "request_id": request_id,
                "error": str(exc),
                "stack": traceback.format_exc(limit = 20),
            },
        )


def _audio_runtime(backend) -> dict:
    fields = getattr(backend, "runtime_fields", None)
    if not callable(fields):
        return {}
    try:
        return {"audio_runtime": dict(fields())}
    except Exception as exc:  # noqa: BLE001 - status detail, never fails the request
        logger.debug("audio runtime fields unavailable: %s", exc)
        return {}


def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
    """Handle TTS audio generation, returning WAV bytes and sample_rate; a separation returns stem paths under output_dir so its audio never crosses the queue."""
    request_id = cmd.get("request_id", "")
    try:
        logger.info("Starting audio generation for request_id=%s", request_id)
        if cmd.get("workflow") == "separate":
            separate = getattr(backend, "separate_audio", None)
            if separate is None:
                raise AudioBackendUnsupportedError("This model cannot separate audio.")
            outputs = separate(
                source_path = cmd["audio_inputs"]["source"],
                output_dir = cmd["output_dir"],
                options = cmd.get("audio_options"),
                cancel_event = cancel_event,
            )
            _send_response(
                resp_queue,
                {
                    "type": "audio_done",
                    "request_id": request_id,
                    "outputs": outputs,
                    "sample_rate": 44100,
                    **_audio_runtime(backend),
                },
            )
            logger.info("Finished audio separation for request_id=%s", request_id)
            return
        params = inspect.signature(backend.generate_audio_response).parameters
        takes_any = any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
        extra = (
            {"audio_options": cmd["audio_options"]}
            if cmd.get("audio_options") and (takes_any or "audio_options" in params)
            else {}
        )
        for key in ("workflow", "audio_inputs", "reference_text", "speed", "convert", "edit"):
            if cmd.get(key) is None:
                continue
            if takes_any or key in params:
                extra[key] = cmd[key]
            elif key == "audio_inputs":
                raise AudioRuntimeError("This model cannot clone a voice.", status = 400)
        for key in ("music", "output_dir"):
            if cmd.get(key) is not None:
                extra[key] = cmd[key]
        wav_bytes, sample_rate = backend.generate_audio_response(
            text = cmd["text"],
            temperature = cmd.get("temperature", 0.6),
            top_p = cmd.get("top_p", 0.95),
            top_k = cmd.get("top_k", 50),
            min_p = cmd.get("min_p", 0.0),
            max_new_tokens = cmd.get("max_new_tokens", 2048),
            repetition_penalty = cmd.get("repetition_penalty", 1.0),
            use_adapter = cmd.get("use_adapter"),
            cancel_event = cancel_event,
            instructions = cmd.get("instructions"),
            language = cmd.get("language"),
            seed = cmd.get("seed"),
            **extra,
        )

        # Send WAV bytes as base64 (bytes can't go through mp.Queue directly).
        done = {
            "type": "audio_done",
            "request_id": request_id,
            "wav_base64": base64.b64encode(wav_bytes).decode("ascii"),
            "sample_rate": sample_rate,
            **_audio_runtime(backend),
            "stats": getattr(backend, "last_generation_stats", None),
        }
        take_status_patch = getattr(backend, "take_status_patch", None)
        status_patch = take_status_patch() if callable(take_status_patch) else None
        if isinstance(status_patch, dict) and status_patch:
            done["status_patch"] = status_patch
        _send_response(resp_queue, done)
        logger.info("Finished audio generation for request_id=%s", request_id)

    except Exception as exc:
        logger.error("Audio generation error: %s", exc, exc_info = True)
        response = {
            "type": "audio_error",
            "request_id": request_id,
            "error": str(exc),
            # Flag a shared-event cancel (unload, training, arbiter) so the orchestrator does not report HTTP 500.
            "cancelled": bool(cancel_event is not None and cancel_event.is_set()),
            "stack": traceback.format_exc(limit = 20),
            **_audio_runtime(backend),
        }
        if isinstance(exc, AudioRuntimeError):
            response["code"] = AUDIO_RUNTIME_ERROR_CODE
            response["status"] = exc.status
        elif isinstance(exc, AudioBackendUnsupportedError):
            response["code"] = AUDIO_UNSUPPORTED_CODE
            response["hint"] = exc.hint
        from core.inference.audio_cpp_failures import failure_of

        if failure_of(exc) is not None:
            response["failure"] = failure_of(exc).payload()
        _send_response(resp_queue, response)


def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
    """Handle audio input generation (ASR/Whisper) — streams text tokens back."""
    request_id = cmd.get("request_id", "")

    try:
        import numpy as np

        # "audio_data" is the older single-clip list form.
        if "audio_clips" in cmd:
            # Copy: frombuffer views are read-only.
            clips = [np.frombuffer(clip, dtype = np.float32).copy() for clip in cmd["audio_clips"]]
        else:
            clips = [np.array(cmd["audio_data"], dtype = np.float32)]
        audio_array = clips[0]
        # Passed only when present, so single-clip calls are unchanged.
        extra_audio_kwargs = {"extra_audio_arrays": clips[1:]} if len(clips) > 1 else {}

        audio_type = cmd.get("audio_type")

        if audio_type == "whisper":
            if not hasattr(backend, "generate_whisper_response"):
                # MLX has no ASR path; report it instead of a raw AttributeError.
                raise RuntimeError("Whisper transcription is not supported on the MLX backend yet.")
            generator = backend.generate_whisper_response(
                audio_array = audio_array,
                use_adapter = cmd.get("use_adapter"),
                cancel_event = cancel_event,
                **extra_audio_kwargs,
            )
        else:
            audio_kwargs = {
                "messages": cmd.get("messages", []),
                "system_prompt": cmd.get("system_prompt", ""),
                "audio_array": audio_array,
                "temperature": cmd.get("temperature", 0.7),
                "top_p": cmd.get("top_p", 0.9),
                "top_k": cmd.get("top_k", 40),
                "min_p": cmd.get("min_p", 0.0),
                "max_new_tokens": cmd.get("max_new_tokens", 512),
                "repetition_penalty": cmd.get("repetition_penalty", 1.0),
                "cancel_event": cancel_event,
            }
            # Forward only when present, as the "generate" branch does.
            use_adapter = cmd.get("use_adapter")
            if use_adapter is not None:
                audio_kwargs["use_adapter"] = use_adapter
            # MLX-only here too, for the reason the text branch gates it.
            if "stop" in cmd and _backend_declares(
                backend, "stop", "generate_audio_input_response"
            ):
                audio_kwargs["stop"] = cmd["stop"]
            if cmd.get("speak"):
                audio_kwargs["speak"] = True
            if cmd.get("tools") is not None:
                audio_kwargs["tools"] = cmd["tools"]
            generator = backend.generate_audio_input_response(**audio_kwargs, **extra_audio_kwargs)

        logger.info("Starting audio input generation for request_id=%s", request_id)

        for text_chunk in generator:
            if cancel_event.is_set():
                logger.info("Audio input generation cancelled for request %s", request_id)
                break

            if isinstance(text_chunk, dict):
                _send_response(
                    resp_queue, {"type": "speech", "request_id": request_id, **text_chunk}
                )
                continue
            _send_response(
                resp_queue,
                {
                    "type": "token",
                    "request_id": request_id,
                    "text": text_chunk,
                },
            )

        _send_response(
            resp_queue,
            {
                "type": "gen_done",
                "request_id": request_id,
                # Same channel as the text path; the ASR backends reset it per run.
                "stats": getattr(backend, "last_generation_stats", None),
            },
        )
        logger.info("Finished audio input generation for request_id=%s", request_id)

    except Exception as exc:
        logger.error("Audio input generation error: %s", exc, exc_info = True)
        _send_response(resp_queue, _generation_error_payload(request_id, exc))


def _generation_error_payload(request_id, exc) -> dict:
    """Carries a context refusal's counts so the parent can rebuild the typed error."""
    payload = {
        "type": "gen_error",
        "request_id": request_id,
        "error": str(exc),
        # Client-safe refusals would otherwise reach the caller as a generic 500.
        "public": bool(getattr(exc, "public", False)),
        "openai_param": getattr(exc, "openai_param", None),
        "stack": traceback.format_exc(limit = 20),
    }
    if isinstance(exc, ContextBudgetExceeded):
        payload["context_budget"] = {
            "request_tokens": exc.request_tokens,
            "context_tokens": exc.context_tokens,
        }
    return payload


def _handle_unload(backend, cmd: dict, resp_queue: Any) -> None:
    model_name = cmd.get("model_name", "")
    try:
        if model_name and model_name in backend.models:
            backend.unload_model(model_name)
        elif backend.active_model_name:
            backend.unload_model(backend.active_model_name)

        _send_response(
            resp_queue,
            {
                "type": "unloaded",
                "model_name": model_name,
            },
        )
    except Exception as exc:
        logger.error("Unload error: %s", exc)
        _send_response(
            resp_queue,
            {
                "type": "unloaded",
                "model_name": model_name,
                "error": str(exc),
            },
        )


def run_inference_process(
    *,
    cmd_queue: Any,
    resp_queue: Any,
    cancel_event,
    config: dict,
    drain_event = None,
    stop_ledger = None,
    pending_teardowns = None,
) -> None:
    """Runs the command loop until shutdown; drain_event is never cleared so cancels survive the queue; stop_ledger and pending_teardowns are read, not received."""
    _apply_worker_hf_token_environment(config)
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
    os.environ["PYTHONWARNINGS"] = "ignore"  # Suppress warnings at C-level before imports

    if config.get("disable_xet"):
        os.environ["HF_HUB_DISABLE_XET"] = "1"
        logger.info("Xet transport disabled (HF_HUB_DISABLE_XET=1)")

    # Offline auto-detect, run before any HF import: child_env scrubs the parent's scoped guard, so probe here; skipped for filesystem-only loads.
    _probe_model = config["model_name"]
    _probe_base, _probe_needs_hub = _recorded_local_base(_probe_model)
    _probed_offline = None
    if "HF_HUB_OFFLINE" not in os.environ and (
        _probe_needs_hub or not _hub_targets_are_local(_probe_model, _probe_base)
    ):
        try:
            _ensure_backend_on_path()
            from utils.utils import hf_dns_dead, hf_env_offline, hf_probe_disabled

            _offline = hf_env_offline()
            if not _offline:
                _offline = hf_dns_dead()
            if not _offline and is_apple_silicon():
                from utils.transformers_version import read_activation_files_ahead
                read_activation_files_ahead(_probe_model, _config_hf_token(config))
            if not _offline and not hf_probe_disabled():
                from utils.transformers_version import hf_endpoint_unreachable

                # Lifetime flag: only a definite no-egress answer counts, not a transient 502 or slow proxy, which must not strand the worker offline.
                _offline = hf_endpoint_unreachable(
                    gateway_errors_offline = False,
                    proxy_timeouts_offline = False,
                )
                _probed_offline = _offline
            if _offline:
                os.environ["HF_HUB_OFFLINE"] = "1"
                os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
                logger.warning(
                    "Hugging Face endpoint unreachable; HF_HUB_OFFLINE=1 for this worker."
                )
        except Exception:
            pass  # fail open: the load decides as it does today

    import warnings
    from loggers.config import LogConfig

    if os.getenv("ENVIRONMENT_TYPE", "production") == "production":
        warnings.filterwarnings("ignore")

    LogConfig.setup_logging(
        service_name = "unsloth-studio-inference-worker",
        env = os.getenv("ENVIRONMENT_TYPE", "production"),
    )
    # Must follow setup_logging: unmarked stdlib lastResort tracebacks look like a crash to the parent.
    from utils.worker_stderr import mark_log_record_continuations

    mark_log_record_continuations()

    apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend"))

    model_name = config["model_name"]

    # Select native backend before the MLX fast-path and before importing Unsloth.
    from core.inference.native_audio import is_native_audio_model

    # The route marks an audio.cpp GGUF, which a bare Hub id does not reveal without a header read.
    _native_audio_worker = bool(config.get("audio_cpp")) or is_native_audio_model(model_name)

    # Before detect_hardware(), whose probe leaves a CUDA context; the route skips the arbiter because this load reserves no GPU.
    if _native_audio_worker:
        from core.inference.audio_device import (
            audio_device_forces_cpu,
            mask_accelerators_for_cpu_audio,
        )
        if audio_device_forces_cpu(config.get("audio_device")):
            mask_accelerators_for_cpu_audio(os.environ)
            logger.info("Audio model '%s' pinned to CPU RAM; accelerators hidden", model_name)

    _ensure_backend_on_path()

    if is_apple_silicon():
        # Non-fatal: fall through with the installed version, but log the cause rather than swallow it.
        try:
            _activate_transformers_version(model_name, _config_hf_token(config))
        except Exception as exc:
            logger.warning(
                "Failed to activate transformers version for '%s' (MLX inference); "
                "inference may fail if this model requires a specific version. Error: %s",
                model_name,
                exc,
            )

    from utils.hardware import hardware as _hw
    from utils.utils import hf_env_offline

    hub_answers = contextlib.ExitStack()
    model_info_ahead = None
    if (
        is_apple_silicon()
        and not _native_audio_worker
        and not hf_env_offline()
        and (_probe_needs_hub or not _hub_targets_are_local(_probe_model, _probe_base))
    ):
        from utils.models.model_config import read_hub_model_info_ahead, shared_hub_model_info
        hub_answers.enter_context(shared_hub_model_info())
        model_info_ahead = read_hub_model_info_ahead(model_name, _config_hf_token(config))

    _hw.detect_hardware()
    if _hw.DEVICE != _hw.DeviceType.MLX or _native_audio_worker:
        hub_answers.close()
    if _hw.DEVICE == _hw.DeviceType.MLX and not _native_audio_worker:
        try:
            from core.inference.mlx_inference import MLXInferenceBackend, _init_mlx_distributed
            from utils.models.diffusion_lm import mlx_refusal
            from utils.models.model_config import shared_hub_model_info
            from utils.utils import pinned_hf_reachability

            hub_answers.enter_context(pinned_hf_reachability(_probed_offline))
            hub_answers.enter_context(shared_hub_model_info())
            if model_info_ahead is not None:
                model_info_ahead.join()
            refusal = mlx_refusal(model_name, _config_hf_token(config))
            if refusal:
                raise RuntimeError(refusal)
            backend = MLXInferenceBackend()
            if config.get("mlx_distributed"):
                group, rank, size = _init_mlx_distributed()
                config["_mlx_distributed_group"] = group
                if size <= 1:
                    # A singleton group (no MLX distributed support or bad launch env) hangs nonzero ranks in share_distributed_object, so fail the load.
                    raise RuntimeError(
                        "MLX distributed launch requested but initialized a singleton "
                        "group (size 1). Ensure the installed MLX has distributed "
                        "support and the launch environment/hostfile is valid, or run "
                        "without distributed."
                    )
                logger.info(
                    "MLX distributed initialized in worker: rank=%s size=%s mode=%s",
                    rank,
                    size,
                    config.get("mlx_parallel_mode"),
                )
            _send_response(
                resp_queue,
                {"type": "status", "message": "Loading model..."},
            )
            _handle_load(backend, config, resp_queue)
        except Exception as exc:
            _send_response(
                resp_queue,
                {
                    "type": "error",
                    "error": f"MLX inference init failed: {exc}",
                    "stack": traceback.format_exc(limit = 20),
                },
            )
            return
        finally:
            hub_answers.close()

        logger.info("MLX inference subprocess ready, entering command loop")
        batch = _ResidentBatch(backend, resp_queue)
        warmth = _MLXIdleWarmth()
        warmth.active()
        deferred: list[dict] = []
        stops = _Stops(stop_ledger, resp_queue, batch, deferred)
        waiting = lambda: stops.unread() or not cmd_queue.empty()
        if stop_ledger is not None:
            stop_ledger.worker_reads_this()
        while True:
            stops.answer()
            tearing_down = pending_teardowns is not None and pending_teardowns.any_in_flight()
            if not tearing_down:
                if batch.rows_in_flight:
                    warmth.active()
                batch.step(waiting)
            from_deferred = False
            if _held_head_leaves_the_hold(batch, deferred):
                cmd = deferred.pop(0)
                from_deferred = True
            else:
                try:
                    cmd = cmd_queue.get(
                        timeout = 0.0
                        if (batch.rows_in_flight or deferred) and not tearing_down
                        else warmth.timeout(getattr(backend, "active_model_name", None))
                    )
                except _queue.Empty:
                    if (
                        not batch.rows_in_flight
                        and not deferred
                        and not (
                            pending_teardowns is not None and pending_teardowns.any_in_flight()
                        )
                        and not (drain_event is not None and drain_event.is_set())
                    ):
                        warmth.tick(getattr(backend, "active_model_name", None))
                    continue
                except (EOFError, OSError):
                    batch.close()
                    return
            if cmd is None:
                continue
            cmd_type = cmd.get("type", "")
            if cmd_type in ("load", "unload", "reset", "cancel", "shutdown"):
                warmth.clear()
            if pending_teardowns is not None and cmd_type in _TEARDOWN_COMMANDS:
                pending_teardowns.taken()
            try:
                if cmd_type == "generate":
                    if _drain_skip_generate(cmd, resp_queue, drain_event):
                        continue
                    if deferred and not from_deferred:
                        deferred.append(cmd)
                        continue
                    reason = batch.unavailable_reason(cmd)
                    if reason is None:
                        if _teardown_skip(cmd, resp_queue, pending_teardowns):
                            continue
                        if _stopped_before_it_ran(cmd, resp_queue, stops):
                            continue
                        if batch.admit(cmd, None):
                            continue
                        reason = "it does not prepare like the replies in the batch"
                    if batch.rows_in_flight:
                        logger.info(
                            "Holding request_id=%s until the batch takes it or drains: %s",
                            cmd.get("request_id", ""),
                            reason,
                        )
                        if from_deferred:
                            deferred.insert(0, cmd)
                        else:
                            deferred.append(cmd)
                        continue
                    batch.close()
                    cancel_event.clear()
                    # Re-check drain after the clear: the parent sets drain_event then cancel_event, so a skip here avoids running the outgoing model and stalling the switch.
                    if _drain_skip_generate(cmd, resp_queue, drain_event):
                        continue
                    if _teardown_skip(cmd, resp_queue, pending_teardowns):
                        continue
                    if _stopped_before_it_ran(cmd, resp_queue, stops):
                        continue
                    _dispatch_generate(
                        backend,
                        cmd,
                        resp_queue,
                        _StopWhileItRuns(cancel_event, stops, cmd.get("request_id", "")),
                    )
                elif cmd_type == "generate_audio_input":
                    if _drain_skip_generate(cmd, resp_queue, drain_event):
                        continue
                    if batch.rows_in_flight or (deferred and not from_deferred):
                        deferred.append(cmd)
                        continue
                    batch.close()
                    cancel_event.clear()
                    if _drain_skip_generate(cmd, resp_queue, drain_event):
                        continue
                    if _teardown_skip(cmd, resp_queue, pending_teardowns):
                        continue
                    if _stopped_before_it_ran(cmd, resp_queue, stops):
                        continue
                    _handle_generate_audio_input(
                        backend,
                        cmd,
                        resp_queue,
                        _StopWhileItRuns(cancel_event, stops, cmd.get("request_id", "")),
                    )
                elif cmd_type == "generate_audio":
                    # Codec checkpoints still reach this loop; answer or the parent waits 120s.
                    _send_response(
                        resp_queue,
                        {
                            "type": "audio_error",
                            "request_id": cmd.get("request_id"),
                            "error": "Text-to-speech is not supported on the MLX backend yet.",
                            # Lets the parent raise a typed error, not a generic 500.
                            "code": AUDIO_UNSUPPORTED_CODE,
                            # Only some TTS families ship GGUF, so the non-MLX host is the general fix; GGUF is conditional.
                            "hint": (
                                "Run it on a non-MLX host, or load a GGUF build of it "
                                "if one is published -- llama.cpp carries the "
                                "snac/bicodec/dac decoders."
                            ),
                        },
                    )
                elif cmd_type == "count_tokens":
                    if batch.rows_in_flight:
                        _decline_count_tokens(cmd, resp_queue, "A generation is in progress.")
                    else:
                        _handle_count_tokens(backend, cmd, resp_queue)
                elif cmd_type == "resolve_prompt":
                    _handle_resolve_prompt(backend, cmd, resp_queue)
                elif cmd_type in _PROMPT_CACHE_RESUME_METHODS:
                    _handle_prompt_cache_resume(
                        backend, cmd, resp_queue, busy = bool(batch.rows_in_flight)
                    )
                elif cmd_type == "share_object":
                    _handle_share_object(backend, cmd, resp_queue)
                elif cmd_type == "load":
                    batch.cancel_all()
                    batch.close()
                    _abandon_held_commands(deferred, resp_queue)
                    if backend.active_model_name:
                        backend.unload_model(backend.active_model_name)
                    _handle_load(backend, cmd, resp_queue)
                elif cmd_type == "narrow_context":
                    batch.cancel_all()
                    batch.close()
                    _abandon_held_commands(deferred, resp_queue)
                    _handle_narrow_context(backend, cmd, resp_queue)
                elif cmd_type == "unload":
                    batch.cancel_all()
                    batch.close()
                    _abandon_held_commands(deferred, resp_queue)
                    _handle_unload(backend, cmd, resp_queue)
                elif cmd_type == "cancel":
                    batch.cancel_all()
                    _abandon_held_commands(deferred, resp_queue)
                    cancel_event.set()
                elif cmd_type == "reset":
                    batch.cancel_all()
                    batch.close()
                    cancel_event.set()
                    _abandon_held_commands(deferred, resp_queue)
                    backend.reset_generation_state()
                    _send_response(resp_queue, {"type": "reset_ack"})
                elif cmd_type == "gpu_memory":
                    _send_response(
                        resp_queue,
                        {
                            "type": "gpu_memory",
                            "request_id": cmd.get("request_id"),
                            "reclaimable_gpu_gb": _worker_reclaimable_gpu_gb(config),
                        },
                    )
                elif cmd_type == "status":
                    _send_response(
                        resp_queue,
                        {
                            "type": "status_response",
                            "active_model": backend.active_model_name,
                            "models": {
                                k: {kk: vv for kk, vv in v.items() if kk != "model"}
                                for k, v in backend.models.items()
                            },
                            "loading": list(backend.loading_models),
                        },
                    )
                elif cmd_type == "shutdown":
                    batch.close()
                    return
                else:
                    # As in the GPU loop: dropping a command silently costs the caller its whole timeout.
                    logger.warning("Unknown MLX command type: %s", cmd_type)
                    _send_response(
                        resp_queue,
                        {
                            "type": "error",
                            "request_id": cmd.get("request_id"),
                            "error": f"Unknown command type: {cmd_type}",
                        },
                    )
            except Exception as exc:
                logger.error("MLX command error (%s): %s", cmd_type, exc)
                _payload = _generation_error_payload(cmd.get("request_id"), exc)
                if cmd_type != "generate":
                    _payload["type"] = "error"
                _send_response(resp_queue, _payload)
            finally:
                if cmd_type in ("generate", "generate_audio_input"):
                    warmth.active()
        return

    # Gate before the torchao stub; AMD Triton JIT also needs MSVC CRT headers (#7595).
    if sys.platform == "win32":
        from core._msvc_env import gate_torch_compile_on_windows
        gate_torch_compile_on_windows(logger)

    # Stub torchao on Windows ROCm before ANY transformers import (incl. _resolve_base_model).
    from core._torchao_stub import install_torchao_windows_rocm_stub

    install_torchao_windows_rocm_stub()

    # Resolve the base once before activation, gates and install; _lora_base is set only for a genuine adapter, never a full fine-tune's base.
    import json as _json

    _ensure_backend_on_path()
    from utils.transformers_version import (
        _remote_lora_base,
        _resolve_base_model,
        remote_lora_base_scope,
    )

    _hf_token = _config_hf_token(config)
    _lora_base = None
    _local_adapter_cfg = Path(model_name) / "adapter_config.json"
    if _local_adapter_cfg.is_file():
        try:
            _lora_base = (
                _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8-sig")).get(
                    "base_model_name_or_path"
                )
                or None
            )
        except Exception:
            _lora_base = None
    # One adapter_config.json answer serves this lookup and activation's own.
    with remote_lora_base_scope():
        if not _lora_base:
            _lora_base = _remote_lora_base(model_name, hf_token = _hf_token)
        # LoRA base, else a full fine-tune's recorded base (its name reveals SSM/sidecar arch).
        _base = _lora_base or _resolve_base_model(model_name)

        try:
            _activate_transformers_version(_base, _hf_token)
        except Exception as exc:
            _send_response(
                resp_queue,
                {
                    "type": "error",
                    "error": f"Failed to activate transformers version: {exc}",
                    "stack": traceback.format_exc(limit = 20),
                },
            )
            return

    # Gates (model and genuine LoRA base only, never a full fine-tune's base), then SSM kernels, both before the transformers import, which snapshots backends.
    _gate_targets = _native_audio_security_targets_or_error(model_name, _hf_token, resp_queue)
    if _gate_targets is None:
        return
    if _lora_base:
        _gate_targets.append(_lora_base)
    _trust_remote_code = config.get("trust_remote_code", False) or _needs_nemotron_trust(
        model_name, hf_token = _hf_token
    )
    if not _run_security_gates(
        _gate_targets,
        trust_remote_code = _trust_remote_code,
        hf_token = _hf_token,
        approved_fingerprint = config.get("approved_remote_code_fingerprint"),
        resp_queue = resp_queue,
        compute_subdirs = False,  # stay transformers-free until the SSM kernels are installed
        subject = config.get("subject"),
    ):
        return
    # Probe the resolved base, not the adapter id / local path.
    from utils.ssm_runtime import ssm_probe_identifier

    _ssm_targets = [(ssm_probe_identifier(model_name, _base), _base or model_name)]
    if not _ensure_ssm_kernels(_ssm_targets, resp_queue, _hf_token, _trust_remote_code):
        return

    try:
        _send_response(
            resp_queue,
            {
                "type": "status",
                "message": (
                    "Importing native audio runtime..."
                    if _native_audio_worker
                    else "Importing Unsloth..."
                ),
            },
        )

        _ensure_backend_on_path()

        if _native_audio_worker:
            from core.inference.native_audio import is_audio_cpp_audio_model
            if config.get("audio_cpp") or is_audio_cpp_audio_model(model_name):
                # Weights run in audiocpp_server; this worker only proxies to it.
                from core.inference.audio_cpp_backend import AudioCppBackend as InferenceBackend
            else:
                from core.inference.native_audio import NativeAudioBackend as InferenceBackend
        else:
            # Recover from any namespace-package shadow before importing Unsloth.
            from core.import_guards import ensure_real_packages
            ensure_real_packages("unsloth_zoo", "unsloth")

            from core.inference.inference import InferenceBackend

        import transformers

        logger.info("Subprocess loaded transformers %s", transformers.__version__)

    except Exception as exc:
        _send_response(
            resp_queue,
            {
                "type": "error",
                "error": f"Failed to import ML libraries: {exc}",
                "stack": traceback.format_exc(limit = 20),
            },
        )
        return

    try:
        # Native audio picks its device in __init__, so the preference goes there.
        backend = (
            InferenceBackend(device_preference = config.get("audio_device"))
            if _native_audio_worker
            else InferenceBackend()
        )

        _send_response(
            resp_queue,
            {
                "type": "status",
                "message": "Loading model...",
            },
        )

        _handle_load(backend, config, resp_queue)

    except Exception as exc:
        _send_response(
            resp_queue,
            {
                "type": "error",
                "error": f"Failed to initialize inference backend: {exc}",
                "stack": traceback.format_exc(limit = 20),
            },
        )
        return

    # Command loop; the parent sets mp.Event cancel_event to cancel generation, with no queue polling.
    logger.info("Inference subprocess ready, entering command loop")

    while True:
        try:
            cmd = cmd_queue.get(timeout = 1.0)
        except _queue.Empty:
            continue
        except (EOFError, OSError):
            logger.info("Command queue closed, shutting down")
            return

        if cmd is None:
            continue

        cmd_type = cmd.get("type", "")
        logger.info("Received command: %s", cmd_type)

        try:
            if cmd_type == "generate":
                if _drain_skip_generate(cmd, resp_queue, drain_event):
                    continue
                cancel_event.clear()
                # Re-check drain after the clear: the parent sets drain_event then cancel_event, so a skip here avoids running the outgoing model and stalling the switch.
                if _drain_skip_generate(cmd, resp_queue, drain_event):
                    continue
                _dispatch_generate(backend, cmd, resp_queue, cancel_event)

            elif cmd_type == "count_tokens":
                _decline_count_tokens(cmd, resp_queue)

            elif cmd_type == "resolve_prompt":
                _handle_resolve_prompt(backend, cmd, resp_queue)

            elif cmd_type in _PROMPT_CACHE_RESUME_METHODS:
                _handle_prompt_cache_resume(backend, cmd, resp_queue)

            elif cmd_type == "share_object":
                _handle_share_object(backend, cmd, resp_queue)

            elif cmd_type == "load":
                if backend.active_model_name:
                    backend.unload_model(backend.active_model_name)
                _handle_load(backend, cmd, resp_queue)

            elif cmd_type == "generate_audio":
                if not _prepare_generate_audio(cmd, resp_queue, cancel_event, drain_event):
                    continue
                _handle_generate_audio(backend, cmd, resp_queue, cancel_event)

            elif cmd_type == "generate_audio_input":
                cancel_event.clear()
                _handle_generate_audio_input(backend, cmd, resp_queue, cancel_event)

            elif cmd_type == "unload":
                _handle_unload(backend, cmd, resp_queue)

            elif cmd_type == "cancel":
                cancel_event.set()
                logger.info("Cancel command received")

            elif cmd_type == "reset":
                cancel_event.set()
                backend.reset_generation_state()
                _send_response(
                    resp_queue,
                    {
                        "type": "reset_ack",
                    },
                )

            elif cmd_type == "gpu_memory":
                _send_response(
                    resp_queue,
                    {
                        "type": "gpu_memory",
                        "request_id": cmd.get("request_id"),
                        "reclaimable_gpu_gb": _worker_reclaimable_gpu_gb(config),
                    },
                )

            elif cmd_type == "status":
                _send_response(
                    resp_queue,
                    {
                        "type": "status_response",
                        "active_model": backend.active_model_name,
                        "models": {
                            name: {
                                "is_vision": info.get("is_vision", False),
                                "is_lora": info.get("is_lora", False),
                                "context_length": info.get("context_length"),
                            }
                            for name, info in backend.models.items()
                        },
                        "loading": list(backend.loading_models),
                    },
                )

            elif cmd_type == "shutdown":
                logger.info("Shutdown command received, exiting")
                for name in list(backend.models.keys()):
                    try:
                        backend.unload_model(name)
                    except Exception:
                        pass
                _send_response(
                    resp_queue,
                    {
                        "type": "shutdown_ack",
                    },
                )
                _exit_after_shutdown(resp_queue)

            else:
                logger.warning("Unknown command type: %s", cmd_type)
                _send_response(
                    resp_queue,
                    {
                        "type": "error",
                        "error": f"Unknown command type: {cmd_type}",
                    },
                )

        except Exception as exc:
            logger.error("Error handling command '%s': %s", cmd_type, exc, exc_info = True)
            _send_response(
                resp_queue,
                {
                    "type": "error",
                    "error": f"Command '{cmd_type}' failed: {exc}",
                    "stack": traceback.format_exc(limit = 20),
                },
            )