Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

104 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""In-process Laya runtime behind ``POST /v1/systemone``: one checkpoint resident at a time."""

from __future__ import annotations

import dataclasses
import functools
import gc
import importlib.util
import logging
import sys
import threading
import time
from pathlib import Path
from typing import Any

from .catalog import CHECKPOINTS, GGUF_COMPANIONS, LOCAL_NAME, MLX_COMPANIONS, Checkpoint
from .laya_worker import LayaWorker, LayaWorkerError

logger = logging.getLogger(__name__)

LOAD_WAIT_S = 20.0
RUN_WAIT_S = 30.0
# Bounded: each waiter holds a shared threadpool worker, so an unbounded queue stalls every sync route.
MAX_PENDING = 8
FAILURE_BACKOFF_S = 60.0
_REQUIRED_DIRS = ("encoder", "tokenizer")
# laya 0.3.5 ships inside Studio (see vendor/README.md), so the Decision API never installs anything.
_VENDORED_LAYA = Path(__file__).resolve().parent.parent.parent / "vendor" / "laya"

_state_lock = threading.Lock()
_run_lock = threading.Lock()
_admission = threading.BoundedSemaphore(MAX_PENDING)
_agent = None
_loaded: Checkpoint | None = None
_device_name: str | None = None
_loader: threading.Thread | None = None
_loading: Checkpoint | None = None
# A llama.cpp or MLX engine load that holds (or is taking) the GPU claim, until its agent is published or dropped.
_gpu_load = False
_failure: tuple[Checkpoint, str, float] | None = None
_import_lock = threading.Lock()
# Bumped whenever the resident agent changes, so a stale idle timer never acts on a newer one.
_generation = 0
# Bumped by every unload from outside a load, so a load that finishes after one never publishes.
_unloads = 0
_last_used = 0.0
_idle_timer: threading.Timer | None = None
_fallback_reason: str | None = None
# A Clef a training run unloaded, reloading once the run ends; an unload cancels it instead of waiting.
_reloading: Checkpoint | None = None
# Why the reload's load was stopped ("training" or "gpu"), recorded when it happens: a short run can be over by the join.
_reload_stopped_by: str | None = None
# The unload generation the running load started for: an unload during its Hub lookup still stops the worker.
_load_generation = threading.local()
TRAINING_POLL_S = 2.0
# A llama.cpp decision server unloads after this long without a request.
IDLE_UNLOAD_S = 300.0
# Binaries (path, mtime) whose server answered without a "decisions" output.
_incapable: set[tuple[str, int]] = set()
# (binary key, llama.cpp form of a Laya export) the build refused to load.
_unloadable: set[tuple[tuple[str, int], Checkpoint]] = set()
_LAYA_OPTION_TOKENS = 48
_LOAD_REFUSALS = ("unknown model architecture", "error loading model vocabulary")


class Unavailable(Exception):
    def __init__(
        self,
        status: int,
        error_type: str,
        message: str,
        retry_after: float | None = None,
    ):
        super().__init__(message)
        self.status = status
        self.error_type = error_type
        self.message = message
        self.retry_after = retry_after


def _device() -> str:
    from utils.systemone_settings import get_device as preferred_device

    if preferred_device() != "gpu":
        return "cpu"
    from utils.hardware.hardware import DeviceType, get_device

    device = get_device()
    if device == DeviceType.MLX:
        if _mlx_available():
            return "mlx"
        import torch
        return "mps" if torch.backends.mps.is_available() else "cpu"
    candidate = {DeviceType.CUDA: "cuda", DeviceType.XPU: "xpu"}.get(device, "cpu")
    if candidate == "cpu":
        return candidate
    from utils.torch_device_probe import device_can_allocate

    return candidate if device_can_allocate(candidate) else "cpu"


def _training_active() -> bool:
    try:
        from core.training import get_training_backend
        return bool(get_training_backend().is_training_active())
    except Exception:
        return False


def _mlx_available() -> bool:
    try:
        import unsloth_zoo.mlx.decision  # noqa: F401
    except ImportError:
        return False
    return True


_WEIGHT_FILES = ("rl_agent_config.json", "model.safetensors")


def _wanted(path: str, subfolder: str | None) -> bool:
    prefix = f"{subfolder}/" if subfolder else ""
    if not path.startswith(prefix):
        return False
    rest = path[len(prefix) :]
    return rest in _WEIGHT_FILES or rest.startswith(tuple(f"{name}/" for name in _REQUIRED_DIRS))


LLAMA_CPP = "llama.cpp"
MLX = "mlx"
GGUF = "gguf"


def _is_native(checkpoint) -> bool:
    return getattr(checkpoint, "backend", "pytorch") == LLAMA_CPP


def _is_mlx(checkpoint) -> bool:
    return getattr(checkpoint, "backend", None) == MLX


def _arbitrated(served) -> bool:
    """A llama.cpp server or an MLX engine model: the GPU arbiter can end it, and it unloads when idle."""
    return getattr(served, "backend", None) in (LLAMA_CPP, MLX)


def _engine_available() -> bool:
    """Apple Silicon with an unsloth-zoo that serves every decision family, and the GPU as where decoders run."""
    try:
        from utils.hardware import hardware

        # Only a finished detection, as clef_unsupported_reason(wait = False): a settings read never waits on it.
        if not hardware.DETECTION_COMPLETE.is_set() or hardware.DEVICE != hardware.DeviceType.MLX:
            return False
        if not _native_gpu():
            return False
        from unsloth_zoo.mlx.decision import DecisionRequestError  # noqa: F401
    except Exception:
        return False
    return True


def _zoo_serves(family: str) -> bool:
    # An unsloth-zoo older than this catalog lacks d1 / mlx_vlm: such entries fall back to llama.cpp or are unavailable.
    families = getattr(sys.modules.get("unsloth_zoo.mlx.decision"), "FAMILIES", None)
    return families is None or family in families


def _mlx_target(checkpoint: Checkpoint) -> Checkpoint | None:
    """What unsloth-zoo's MLX engine serves in process on Apple Silicon: a merged Clef's own folder, or the source repo of a GGUF entry."""
    if not _engine_available():
        return None
    if _is_mlx(checkpoint):
        return (
            checkpoint
            if (checkpoint.is_local or CHECKPOINTS.get(checkpoint.name) == checkpoint)
            and _zoo_serves(_mlx_family(checkpoint))
            else None
        )
    if checkpoint.layout == "clef":
        if checkpoint.is_local:
            from utils.models.model_config import clef_folder_kind
            if clef_folder_kind(Path(checkpoint.source).expanduser()) != "merged":
                return None
        return dataclasses.replace(checkpoint, backend = MLX)
    companion = MLX_COMPANIONS.get(checkpoint.name)
    if (
        companion is None
        or CHECKPOINTS.get(checkpoint.name) != checkpoint
        or not _zoo_serves(companion.family)
    ):
        return None
    return dataclasses.replace(
        checkpoint,
        source = companion.repo,
        download_bytes = sum(repo.download_bytes for repo in _mlx_repos(companion)),
        backend = MLX,
        revision = companion.revision,
    )


def _mlx_reads(checkpoint: Checkpoint, media: str) -> bool:
    """Whether the MLX form of an entry reads `media` ("images" or "audio"): a Clef with that tower and its processor, on an unsloth-zoo that reads it."""
    if _mlx_target(checkpoint) is None:
        return False
    if checkpoint.layout == GGUF:
        companion = MLX_COMPANIONS.get(checkpoint.name)
        return (
            companion is not None
            and companion.family == "d1"
            and (media == "images" or checkpoint.audio)
        )
    if checkpoint.layout != "clef":
        return False
    try:
        from unsloth_zoo.mlx.decision import ClefModel
    except Exception:
        return False
    if not getattr(ClefModel, f"takes_{media}", False):
        return False
    if not checkpoint.is_local:
        return media == "images" or checkpoint.audio
    import json

    folder = Path(checkpoint.source).expanduser()
    try:
        config = json.loads((folder / "config.json").read_text(encoding = "utf-8"))
    except (OSError, ValueError):
        return False
    if not isinstance(config, dict):
        return False
    # Qwen3-Omni: towers under thinker; Phi-4-mm: `embd_layer`; Nemotron Omni: sound tower; LFM3-Omni: audio encoder.
    parts = (config, config.get("thinker_config"), config.get("embd_layer"))
    towers = (
        ("vision_config", "image_embd_layer")
        if media == "images"
        else ("audio_config", "audio_encoder_config", "sound_config", "audio_embd_layer")
    )
    return any(
        isinstance(part, dict) and part.get(tower) is not None for part in parts for tower in towers
    ) and any(
        (folder / name).is_file() for name in ("processor_config.json", "preprocessor_config.json")
    )


def _mlx_reads_images(checkpoint: Checkpoint) -> bool:
    return _mlx_reads(checkpoint, "images")


def _mlx_repos(companion) -> tuple:
    return (companion.base, companion) if companion.base else (companion,)


def _mlx_family(checkpoint: Checkpoint) -> str:
    if checkpoint.layout != "clef":
        return MLX_COMPANIONS[checkpoint.name].family
    from utils.models.model_config import is_d1_folder
    return (
        "d1"
        if checkpoint.is_local and is_d1_folder(Path(checkpoint.source).expanduser())
        else "clef"
    )


def _wants_gpu(checkpoint) -> bool:
    """Whether loading this takes the GPU through the arbiter; an MLX encoder sits beside a chat model, as Laya does."""
    if _is_mlx(checkpoint):
        return _mlx_family(checkpoint) != "laya"
    return _is_native(checkpoint) and _native_gpu()


def _repo_dir(repo, local_only: bool) -> Path:
    from huggingface_hub import snapshot_download

    from utils.hf_cache_settings import active_hf_hub_cache
    from utils.utils import hf_env_offline

    def fetch(revision, local: bool) -> Path | None:
        try:
            folder = Path(
                snapshot_download(
                    repo.repo,
                    revision = revision,
                    cache_dir = active_hf_hub_cache(),
                    local_files_only = local,
                    allow_patterns = list(repo.files),
                )
            )
        except Exception:
            if not local:
                raise
            return None
        return folder if all((folder / name).is_file() for name in repo.files) else None

    # A settings download fetches the repo's main, which serves where the pinned revision is not complete.
    folder = fetch(repo.revision, True) or fetch(None, True)
    if folder is None and not (local_only or hf_env_offline()):
        folder = fetch(repo.revision, False)
    if folder is None:
        raise FileNotFoundError(f"{repo.repo} is not downloaded")
    return folder


def _repo_cached(repo) -> bool:
    try:
        _repo_dir(repo, local_only = True)
    except Exception:
        return False
    return True


def _mlx_dirs(checkpoint: Checkpoint, local_only: bool) -> tuple[Path, Path | None]:
    """(model folder, base folder) of an MLX target."""
    if checkpoint.layout == "clef":
        return _clef_dir(checkpoint, local_only), None
    folders = [_repo_dir(repo, local_only) for repo in _mlx_repos(MLX_COMPANIONS[checkpoint.name])]
    return folders[-1], (folders[0] if len(folders) > 1 else None)


def _mlx_choice(
    checkpoint: Checkpoint,
    unread,
    questions,
    preference: str,
    seen: bool = False,
) -> Checkpoint | None:
    """Auto on Apple Silicon prefers MLX unless only llama.cpp is at hand or MLX cannot read the images (`unread`); `seen` = it can."""
    from .native_worker import request_gap

    if preference != "auto" or unread or request_gap(questions or {}) is not None:
        return None
    if (target := _mlx_target(checkpoint)) is None:
        return None
    if _loaded == target and _agent is not None:
        return target
    native = _native_target(checkpoint)
    if seen and native is not None and not _native_reads_images(native):
        # Without a vision projector the llama.cpp form would refuse the images MLX reads.
        return target
    if native is not None and _native_unavailable(checkpoint, native) is None:
        # A resident llama.cpp server keeps answering, and a downloaded GGUF serves before the MLX form is fetched.
        if _loaded == native and _agent is not None:
            return None
        if not native.is_local and is_cached(native) and not is_cached(target):
            return None
    return target


def _local_folder(checkpoint: Checkpoint) -> Path:
    root = Path(checkpoint.source).expanduser()
    return root / checkpoint.subfolder if checkpoint.subfolder else root


def _native_target(checkpoint: Checkpoint) -> Checkpoint | None:
    """The llama.cpp form of an entry: ggml-org's GGUF for a stock Clef, a current export for a Clef or Laya folder, a GGUF entry itself."""
    if checkpoint.layout == GGUF:
        return None if _is_mlx(checkpoint) else checkpoint
    if _is_native(checkpoint):
        return None
    if checkpoint.is_local:
        from .gguf_export_contract import served_files

        files = served_files(_local_folder(checkpoint), checkpoint.layout)
        if files is None:
            return None
        return dataclasses.replace(
            checkpoint, backend = LLAMA_CPP, revision = _files_revision(files[1:])
        )
    companion = GGUF_COMPANIONS.get(checkpoint.name)
    if companion is None or CHECKPOINTS.get(checkpoint.name) != checkpoint:
        return None
    return dataclasses.replace(
        checkpoint,
        source = companion.repo,
        download_bytes = companion.download_bytes,
        backend = LLAMA_CPP,
    )


def _files_revision(paths) -> str:
    parts = []
    for path in paths:
        if path is None:
            continue
        try:
            st = path.stat()
        except OSError:
            st = None
        parts.append(f"{path}:{st.st_ino}:{st.st_mtime_ns}:{st.st_size}" if st else str(path))
    return "|".join(parts)


def _native_files(checkpoint: Checkpoint, local_only: bool) -> tuple[Path, Path | None]:
    if checkpoint.is_local:
        from .gguf_export_contract import served_files

        files = served_files(_local_folder(checkpoint), checkpoint.layout)
        if files is None:
            raise FileNotFoundError(f"No current GGUF export in {checkpoint.source}")
        return files[1], files[2]
    from huggingface_hub import hf_hub_download

    from utils.hf_cache_settings import active_hf_hub_cache
    from utils.utils import hf_env_offline

    companion = GGUF_COMPANIONS[checkpoint.name]
    cache = active_hf_hub_cache()
    paths = []
    for index, name in enumerate(n for n in (companion.model, companion.mmproj) if n):
        try:
            path = hf_hub_download(
                companion.repo,
                name,
                revision = companion.revision,
                cache_dir = cache,
                local_files_only = local_only or hf_env_offline(),
            )
        except Exception:
            # Settings downloads fetch the repo's main: the same blob there is the pinned file.
            sha256 = companion.sha256[index] if index < len(companion.sha256) else None
            if (path := _cached_blob(cache, companion.repo, name, sha256)) is None:
                raise
        paths.append(Path(path))
    return paths[0], (paths[1] if companion.mmproj else None)


def _cached_blob(cache, repo: str, name: str, sha256: str | None) -> Path | None:
    """``name`` in any cached snapshot of ``repo`` whose blob is ``sha256``, else None."""
    if not sha256:
        return None
    if cache is None:
        from huggingface_hub.constants import HF_HUB_CACHE
        cache = HF_HUB_CACHE
    snapshots = Path(cache) / f"models--{repo.replace('/', '--')}" / "snapshots"
    try:
        candidates = sorted(snapshots.glob(f"*/{name}"))
    except OSError:
        return None
    for path in candidates:
        try:
            if path.resolve(strict = True).name == sha256:
                return path
        except (OSError, RuntimeError):
            continue
    return None


def _binary_key(binary: str) -> tuple[str, int]:
    try:
        return binary, Path(binary).stat().st_mtime_ns
    except OSError:
        return binary, 0


def _native_unavailable(checkpoint: Checkpoint, native: Checkpoint | None) -> str | None:
    if native is None:
        if checkpoint.is_local:
            return "This Clef folder has no current GGUF export."
        return f"{checkpoint.name} has no GGUF build."
    from .native_worker import resolve_binary

    binary = resolve_binary()
    if binary is None:
        return "llama-server is not installed."
    if _binary_key(binary) in _incapable:
        return "This llama.cpp build cannot serve decision models; update llama.cpp in Unsloth."
    if (_binary_key(binary), native) in _unloadable:
        return "This llama.cpp build cannot load this GGUF export."
    return None


def select(
    checkpoint: Checkpoint,
    images = None,
    questions = None,
    preference: str | None = None,
    state_images: bool = False,
    audio = None,
) -> tuple[Checkpoint, str | None]:
    """(what serves this request, why Auto did not pick llama.cpp); raises for what nothing here can serve.

    Auto takes llama.cpp when the entry has a GGUF and Studio's llama-server serves decisions, else PyTorch;
    on Apple Silicon it answers through unsloth-zoo's MLX engine first.
    Images are read by llama.cpp, and by MLX for a Clef. Audio is heard only by MLX, for a Clef with an audio tower.
    Laya is served by PyTorch unless llama.cpp is chosen for a folder with a current export, a GGUF entry otherwise by llama.cpp.
    """
    from utils.systemone_settings import get_backend

    from .native_worker import request_gap

    # Only route-checked images reach MLX: state images would be decoded unvalidated in this process.
    unread = bool(state_images or (images and not _mlx_reads_images(checkpoint)))
    if audio:
        # No other runtime hears, so nothing else may answer as if the clips were not sent.
        if not _mlx_reads(checkpoint, "audio"):
            raise Unavailable(
                400,
                "api_usage_error",
                f"{checkpoint.name} does not hear audio here; audio needs a Clef with an audio tower, served by MLX on Apple Silicon.",
            )
        if (preference or get_backend()) not in ("auto", MLX):
            raise Unavailable(
                400,
                "api_usage_error",
                "Audio is served only by MLX; set the Decision API runtime to Auto or MLX.",
            )
        return _select_mlx(checkpoint, unread), None
    if _is_mlx(checkpoint):
        if (preference or get_backend()) not in ("auto", MLX):
            raise Unavailable(
                400,
                "api_usage_error",
                f"{checkpoint.name} is served only by MLX; set the Decision API runtime to Auto or MLX.",
            )
        return _select_mlx(checkpoint, unread), None
    if (preference or get_backend()) == MLX and checkpoint.layout in ("clef", GGUF):
        return _select_mlx(checkpoint, unread), None
    mlx = _mlx_choice(
        checkpoint, unread, questions, preference or get_backend(), bool(images) and not unread
    )
    if mlx is not None:
        return mlx, None
    if checkpoint.layout == GGUF:
        return _select_gguf(checkpoint, questions, preference or get_backend()), None
    preference = preference or get_backend()
    if checkpoint.layout != "clef":
        if images:
            raise Unavailable(
                400, "api_usage_error", "Images need a Clef model; Laya reads text only."
            )
        if preference != LLAMA_CPP or _native_target(checkpoint) is None:
            return checkpoint, None
    if preference == "pytorch":
        if images:
            raise Unavailable(
                400,
                "api_usage_error",
                "Images are served only by llama.cpp; set the Decision API runtime to Auto or llama.cpp.",
            )
        return checkpoint, None
    native = _native_target(checkpoint)
    reason = _native_unavailable(checkpoint, native)
    if reason is None:
        reason = request_gap(questions or {})
        if reason is None:
            # Auto keeps a resident PyTorch Clef for text rather than reloading the model.
            resident = _loaded == checkpoint and _agent is not None
            if preference == "auto" and not images:
                if resident:
                    return checkpoint, None
                if (why := _auto_keeps_pytorch(checkpoint, native)) is not None:
                    return checkpoint, why
            return native, None
        if preference == LLAMA_CPP or images:
            raise Unavailable(422, "invalid_request_error", reason)
    if preference == LLAMA_CPP:
        status, kind = (
            (503, "model_unavailable") if native is not None else (400, "api_usage_error")
        )
        raise Unavailable(
            status, kind, f"{reason} Set the Decision API runtime to Auto or PyTorch."
        )
    if images:
        raise Unavailable(400, "api_usage_error", f"Images are served only by llama.cpp: {reason}")
    return checkpoint, reason


def _select_mlx(checkpoint: Checkpoint, unread) -> Checkpoint:
    """The MLX runtime answers through the engine or not at all, whatever llama.cpp has loaded or downloaded."""
    if unread:
        raise Unavailable(
            400,
            "api_usage_error",
            f"The MLX runtime does not read these images for {checkpoint.name}; set the Decision API runtime to Auto or llama.cpp.",
        )
    if not _engine_available():
        reason = "The MLX runtime needs Apple Silicon with the Decision API on the GPU."
    elif (target := _mlx_target(checkpoint)) is not None:
        return target
    else:
        reason = f"{checkpoint.name} has no MLX form."
    if _is_mlx(checkpoint):
        raise Unavailable(
            400, "api_usage_error", f"{checkpoint.name} is served only by MLX. {reason}"
        )
    raise Unavailable(400, "api_usage_error", f"{reason} Set the Decision API runtime to Auto.")


def _select_gguf(checkpoint: Checkpoint, questions, preference: str) -> Checkpoint:
    from .native_worker import request_gap

    if preference == "pytorch":
        raise Unavailable(
            400,
            "api_usage_error",
            f"{checkpoint.name} is a GGUF served only by llama.cpp; set the Decision API runtime "
            "to Auto or llama.cpp.",
        )
    if (reason := _native_unavailable(checkpoint, checkpoint)) is not None:
        raise Unavailable(503, "model_unavailable", f"{checkpoint.name} needs llama.cpp: {reason}")
    if (reason := request_gap(questions or {})) is not None:
        raise Unavailable(422, "invalid_request_error", reason)
    return checkpoint


def _gpu_taken() -> bool:
    """A chat, image or video model is resident on the GPU (a decision server does not count)."""
    from core.inference import gpu_arbiter
    return any(owner != gpu_arbiter.DECISIONS for owner in gpu_arbiter.residents())


def _torch_gpu(checkpoint: Checkpoint) -> bool:
    if _is_mlx(checkpoint) or _is_native(checkpoint):
        return False
    if checkpoint.layout == "clef":
        return True
    from utils.systemone_settings import get_device as preferred_device
    from utils.hardware.hardware import DeviceType, get_device

    return preferred_device() == "gpu" and get_device() in (DeviceType.CUDA, DeviceType.XPU)


def _gpu_load_in_flight() -> bool:
    from core.inference import gpu_arbiter
    from core.inference.gpu_residency import chat_load_in_flight

    if chat_load_in_flight():
        return True
    # Held by the acquire that may have just ended the idle worker.
    if not gpu_arbiter._lock.acquire(blocking = False):
        return True
    try:
        # A CPU-only image or video load releases its GPU claim, so it does not count.
        return any(
            owner in gpu_arbiter._residents and gpu_arbiter._media_loading(owner)
            for owner in (gpu_arbiter.DIFFUSION, gpu_arbiter.VIDEO)
        )
    except Exception:
        return False
    finally:
        gpu_arbiter._lock.release()


def _auto_keeps_pytorch(checkpoint: Checkpoint, native: Checkpoint) -> str | None:
    """Why Auto answers text on PyTorch although llama.cpp could: as before llama.cpp existed here."""
    from .catalog import clef_unsupported_reason

    if _loaded == native and _agent is not None:
        return None
    if clef_unsupported_reason(wait = False) is not None:
        return None
    if _native_gpu() and _gpu_taken():
        return "Another model is using the GPU."
    if not native.is_local and not is_cached(native) and is_cached(checkpoint):
        return "The GGUF is not downloaded; the cached PyTorch checkpoint serves."
    return None


def effective_backend(checkpoint) -> tuple[str | None, str | None]:
    """(backend a text request would use, Auto's reason for PyTorch) for the settings page."""
    if not isinstance(checkpoint, Checkpoint):
        return None, None
    try:
        target, reason = select(checkpoint)
    except Unavailable as exc:
        return None, exc.message
    return target.backend, reason


def mlx_available() -> bool:
    """Whether the MLX runtime can be chosen here."""
    return _engine_available()


def mlx_ready(checkpoint) -> bool:
    """Whether the MLX engine could serve this entry here under the runtime setting."""
    from utils.systemone_settings import get_backend
    return (
        isinstance(checkpoint, Checkpoint)
        and get_backend() in ("auto", MLX)
        and _mlx_target(checkpoint) is not None
    )


def has_native_form(checkpoint) -> bool:
    return isinstance(checkpoint, Checkpoint) and _native_target(checkpoint) is not None


def native_ready(checkpoint) -> bool:
    """Whether llama.cpp could serve this entry here under the runtime setting."""
    if not has_native_form(checkpoint):
        return False
    from utils.systemone_settings import get_backend

    native = _native_target(checkpoint)
    return get_backend() in ("auto", LLAMA_CPP) and _native_unavailable(checkpoint, native) is None


def input_modalities(checkpoint) -> list[str]:
    """What this entry reads here: images too when llama.cpp serves it with a vision projector, or MLX a Clef that has its tower; audio when MLX serves a Clef that hears."""
    if mlx_ready(checkpoint):
        media = [
            kind
            for kind, media in (("image", "images"), ("audio", "audio"))
            if _mlx_reads(checkpoint, media)
        ]
        if media:
            return ["text", *media]
    if not native_ready(checkpoint):
        return ["text"]
    return ["text", "image"] if _native_reads_images(_native_target(checkpoint)) else ["text"]


def _native_reads_images(native: Checkpoint) -> bool:
    if native.is_local:
        from .gguf_export_contract import served_files
        files = served_files(_local_folder(native), native.layout)
        return files is not None and files[2] is not None
    return bool(GGUF_COMPANIONS[native.name].mmproj)


def _native_gpu() -> bool:
    from utils.systemone_settings import clef_device
    return clef_device() == "gpu"


def _clef_complete(folder: Path) -> bool:
    import json

    from utils.models.model_config import clef_folder_kind

    kind = clef_folder_kind(folder)
    if kind is None:
        return False
    if kind == "adapter":
        # The base LLM the adapters sit on is fetched by the loader, like any Unsloth LoRA.
        return (folder / "adapter_model.safetensors").is_file()
    index = folder / "model.safetensors.index.json"
    if not index.is_file():
        return (folder / "model.safetensors").is_file()
    try:
        shards = set(json.loads(index.read_text(encoding = "utf-8"))["weight_map"].values())
    except (OSError, ValueError, KeyError, TypeError):
        return False
    return all((folder / shard).is_file() for shard in shards)


def _clef_dir(checkpoint: Checkpoint, local_only: bool) -> Path:
    if checkpoint.is_local:
        root = Path(checkpoint.source).expanduser()
    else:
        from huggingface_hub import snapshot_download

        from utils.hf_cache_settings import active_hf_hub_cache
        from utils.utils import hf_env_offline

        root = Path(
            snapshot_download(
                checkpoint.source,
                cache_dir = active_hf_hub_cache(),
                local_files_only = local_only or hf_env_offline(),
            )
        )
    if not _clef_complete(root):
        raise FileNotFoundError(f"No complete Clef checkpoint at {root}")
    return root


def _checkpoint_dir(checkpoint: Checkpoint, *, local_only: bool = False) -> Path:
    if checkpoint.layout == "clef":
        return _clef_dir(checkpoint, local_only)
    if checkpoint.name == LOCAL_NAME and not checkpoint.is_local:
        raise FileNotFoundError(
            f"UNSLOTH_SYSTEMONE_MODEL={checkpoint.source} is neither a known model nor a directory"
        )
    if checkpoint.is_local:
        root = Path(checkpoint.source).expanduser()
    else:
        from huggingface_hub import snapshot_download

        from utils.hf_cache_settings import active_hf_hub_cache
        from utils.utils import hf_env_offline

        prefix = f"{checkpoint.subfolder}/" if checkpoint.subfolder else ""
        root = Path(
            snapshot_download(
                checkpoint.source,
                cache_dir = active_hf_hub_cache(),
                local_files_only = local_only or hf_env_offline(),
                allow_patterns = [prefix + name for name in _WEIGHT_FILES]
                + [f"{prefix}{name}/*" for name in _REQUIRED_DIRS],
            )
        )
    folder = root / checkpoint.subfolder if checkpoint.subfolder else root
    # Without these, laya falls back to downloading the base encoder from the Hub on every load.
    missing = [name for name in _REQUIRED_DIRS if not (folder / name).is_dir()]
    if missing:
        raise FileNotFoundError(
            f"Laya checkpoint at {folder} is missing {', '.join(n + '/' for n in missing)}"
        )
    return root


def _laya():
    """Loads the vendored laya by file path, so a venv-installed laya cannot replace this copy."""
    if (module := sys.modules.get("laya")) is not None:
        return module
    with _import_lock:
        if (module := sys.modules.get("laya")) is not None:
            return module
        init = _VENDORED_LAYA / "__init__.py"
        spec = importlib.util.spec_from_file_location(
            "laya", init, submodule_search_locations = [str(_VENDORED_LAYA)]
        )
        module = importlib.util.module_from_spec(spec)
        sys.modules["laya"] = module
        try:
            spec.loader.exec_module(module)
        except BaseException:
            for name in [n for n in sys.modules if n == "laya" or n.startswith("laya.")]:
                del sys.modules[name]
            raise
        for name in [n for n in sys.modules if n == "laya" or n.startswith("laya.")]:
            sys.modules[name].open = _utf8_open
        return module


def _utf8_open(
    file,
    mode = "r",
    buffering = -1,
    encoding = None,
    errors = None,
    newline = None,
    closefd = True,
    opener = None,
):
    """Defaults text mode to UTF-8 (laya bare open uses the locale); vendored files stay byte-identical."""
    if encoding is None and "b" not in mode:
        encoding = "utf-8"
    return open(
        file,
        mode,
        buffering,
        encoding = encoding,
        errors = errors,
        newline = newline,
        closefd = closefd,
        opener = opener,
    )


def is_cached(checkpoint: Checkpoint) -> bool:
    if _is_mlx(checkpoint):
        try:
            _mlx_dirs(checkpoint, local_only = True)
        except Exception:
            return False
        return True
    if _is_native(checkpoint):
        try:
            _native_files(checkpoint, local_only = True)
        except Exception:
            return False
        return True
    try:
        root = _checkpoint_dir(checkpoint, local_only = True)
    except Exception:
        return False
    if checkpoint.layout == "clef":
        return True
    folder = root / checkpoint.subfolder if checkpoint.subfolder else root
    return all((folder / name).is_file() for name in _WEIGHT_FILES)


def _mlx_plan(checkpoint: Checkpoint) -> dict[str, Any]:
    # One repo per plan, the base first; the settings page asks again once that download is done.
    cached = is_cached(checkpoint)
    repos = _mlx_repos(MLX_COMPANIONS[checkpoint.name])
    repo = repos[-1] if cached else next((r for r in repos if not _repo_cached(r)), repos[-1])
    return {
        "repo": repo.repo,
        "revision": repo.revision,
        "files": sorted(repo.files),
        "size_bytes": repo.download_bytes,
        "cached": cached,
        "error": None,
    }


def download_plan(checkpoint: Checkpoint, preference: str | None = None) -> dict[str, Any]:
    if not _is_mlx(checkpoint) and (checkpoint.layout == GGUF or not _is_native(checkpoint)):
        try:
            checkpoint = select(checkpoint, preference = preference)[0]
        except Unavailable as exc:
            from utils.systemone_settings import get_backend

            # The MLX runtime has no other form to fall back to, so nothing is offered for download.
            if (preference or get_backend()) == MLX:
                return {
                    "repo": None,
                    "files": [],
                    "size_bytes": 0,
                    "cached": False,
                    "error": exc.message,
                }
    if _is_mlx(checkpoint) and checkpoint.layout == GGUF:
        return _mlx_plan(checkpoint)
    cached = is_cached(checkpoint)
    plan = {"repo": None, "files": [], "size_bytes": 0, "cached": cached, "error": None}
    if checkpoint.name == LOCAL_NAME or checkpoint.is_local:
        if not cached:
            kind = "Clef" if checkpoint.layout == "clef" else "Laya"
            plan["error"] = f"No complete {kind} checkpoint at {checkpoint.source}"
        return plan
    plan["repo"] = checkpoint.source
    plan["size_bytes"] = checkpoint.download_bytes
    if _is_native(checkpoint):
        companion = GGUF_COMPANIONS[checkpoint.name]
        plan["revision"] = companion.revision
        plan["files"] = [name for name in (companion.model, companion.mmproj) if name]
        return plan
    if cached:
        return plan
    try:
        from huggingface_hub import HfApi
        entries = HfApi().list_repo_tree(checkpoint.source, recursive = True)
        files = [
            (entry.path, getattr(entry, "size", 0) or 0)
            for entry in entries
            if hasattr(entry, "size")
            and (checkpoint.layout == "clef" or _wanted(entry.path, checkpoint.subfolder))
        ]
    except Exception as exc:
        plan["error"] = f"Could not list {checkpoint.source}: {type(exc).__name__}"
        return plan
    plan["files"] = sorted(path for path, _ in files)
    plan["size_bytes"] = sum(size for _, size in files) or checkpoint.download_bytes
    return plan


def _release_memory() -> None:
    gc.collect()
    # MLX keeps freed buffers in its allocator cache until told otherwise; mlx below 0.24.1 names it mx.metal.clear_cache.
    if (mx := sys.modules.get("mlx.core")) is not None:
        clear = getattr(mx, "clear_cache", None) or getattr(
            getattr(mx, "metal", None), "clear_cache", None
        )
        if clear is not None:
            clear()
    # Torch's allocator too; only already-initialised backends, so a CPU-only Studio never opens a device context.
    if (torch := sys.modules.get("torch")) is not None:
        try:
            if torch.cuda.is_initialized():
                torch.cuda.empty_cache()
            if hasattr(torch, "xpu") and torch.xpu.is_initialized():
                torch.xpu.empty_cache()
            if torch.backends.mps.is_available():
                torch.mps.empty_cache()
        except Exception:
            logger.debug("Could not clear the Decision API device cache", exc_info = True)


def _close(agent, release_gpu: bool = True) -> None:
    # A Clef agent is a worker process; ending it returns its GPU memory.
    if agent is not None and hasattr(agent, "close"):
        agent.close()
    if release_gpu and _arbitrated(agent) and getattr(agent, "gpu", False):
        from core.inference import gpu_arbiter
        def unclaimed() -> bool:
            # A newer server loading or resident while this one closed keeps the claim.
            with _state_lock:
                return not _gpu_load and not (
                    _arbitrated(_loaded) and getattr(_agent, "gpu", False)
                )

        gpu_arbiter.release_if(gpu_arbiter.DECISIONS, unclaimed)


def _unloaded() -> bool:
    unloads = getattr(_load_generation, "unloads", None)
    return unloads is not None and unloads != _unloads


def _evict(external: bool = False) -> None:
    global _agent, _loaded, _device_name, _generation, _unloads
    with _run_lock:
        # Every load evicts once its files are on disk: one unloaded meanwhile stops here.
        if _unloaded():
            raise RuntimeError("It was unloaded while it downloaded.")
        agent, _agent, _loaded, _device_name = _agent, None, None, None
        _generation += 1
        _unloads += external
    _close(agent)
    del agent
    _release_memory()


class _MLXAgent:
    """The part of ``laya.Agent`` this module drives, over the unsloth-zoo MLX decision model."""

    device = "mlx"

    def __init__(
        self,
        folder: Path,
        fp16_checkpoint: bool = False,
    ):
        import inspect
        import json

        import mlx.core as mx
        from transformers import AutoTokenizer
        from unsloth_zoo.mlx.decision import load_decision_model

        laya = _laya()
        clamp_temperature = laya.common.clamp_temperature
        self.folder = folder
        self.cfg = json.loads((folder / "rl_agent_config.json").read_text(encoding = "utf-8"))
        self.tok = AutoTokenizer.from_pretrained(str(folder / "tokenizer"))
        self.temperature = [
            clamp_temperature(t) for t in self.cfg.get("temperature", [1.0, 1.0, 1.0])
        ]
        self.temperature_by_options = {
            k: clamp_temperature(v) for k, v in self.cfg.get("temperature_by_options", {}).items()
        }
        self._to_internal = laya.agent.Agent._to_internal
        # An unsloth-zoo without compute_dtype casts the whole model instead, so it stays fp32.
        mixed = "compute_dtype" in inspect.signature(load_decision_model).parameters
        self.dtype = mx.float16 if fp16_checkpoint and mixed and not _fp32_forced() else mx.float32
        self.model = load_decision_model(folder, **({"compute_dtype": self.dtype} if mixed else {}))


_ENGINE_TOKENS = 16384


class _EngineAgent:
    """An unsloth-zoo MLX decision model that builds its own prompts from the request."""

    device = backend = MLX

    def __init__(self, folder: Path, family: str, base: Path | None):
        from unsloth_zoo.mlx.decision import load_decision_model

        self.gpu = family != "laya"
        self.model = load_decision_model(folder, family = family, base_model = base)
        # An encoder reads one question at a time, and its window is the encoder's (Julia-1: 8192).
        self.positions = None
        encoder = folder / "encoder" / "config.json"
        if family == "laya" and encoder.is_file():
            import json
            self.positions = json.loads(encoder.read_text(encoding = "utf-8")).get(
                "max_position_embeddings"
            )

    def close(self) -> None:
        # A request that still holds this agent must not keep the weights on the GPU another owner now has.
        self.model = None
        _release_memory()

    def decide(
        self,
        state,
        questions: dict[str, dict[str, Any]],
        images = None,
        audio = None,
    ) -> dict[str, Any]:
        from unsloth_zoo.mlx.decision import DecisionRequestError, DecisionUnsupportedError

        # As on llama.cpp, a question without instructions is asked by its id.
        questions = {
            name: {**q, "instructions": str(name)} if q.get("instructions") in (None, "") else q
            for name, q in questions.items()
        }
        import json

        # The engine has no length limit of its own. This bounds its memory by the request as sent: each family lays the
        # state and questions out its own way, so what the model reads can differ from this count by the layout's overhead.
        text = state if isinstance(state, str) else json.dumps(state, ensure_ascii = False)
        text += json.dumps(questions, ensure_ascii = False)
        processor = getattr(self.model, "processor", None)
        tokenizer = (
            getattr(self.model, "tokenizer", None)
            or getattr(processor, "tokenizer", None)
            or (processor if hasattr(processor, "encode") else None)
        )
        if tokenizer is not None:
            ids = tokenizer.encode(text, add_special_tokens = False)
            if len(getattr(ids, "ids", ids)) > _ENGINE_TOKENS:
                raise Unavailable(
                    422,
                    "invalid_request_error",
                    f"State and questions are longer than {_ENGINE_TOKENS} tokens. Shorten them.",
                )
            if self.positions:
                # [cls] question [sep] options (with the question, at most max_head) [sep] state [sep], as unsloth-zoo lays it out.
                state_text = (
                    state if isinstance(state, str) else json.dumps(state, ensure_ascii = False)
                )
                read = tokenizer.encode(state_text, add_special_tokens = False)
                limit = int(self.positions) - int(getattr(self.model, "max_head", 0)) - 4
                if len(getattr(read, "ids", read)) > limit:
                    raise Unavailable(
                        422,
                        "invalid_request_error",
                        f"The state is longer than the {limit} tokens this model reads beside a question. Shorten it.",
                    )
        try:
            # Only what was sent is passed, so an unsloth-zoo that takes less still answers text.
            media = (images, audio) if audio else (images,) if images else ()
            return self.model.answer(state, questions, *media)
        except DecisionUnsupportedError as exc:
            raise Unavailable(501, "not_supported_error", str(exc)) from None
        except DecisionRequestError as exc:
            raise Unavailable(422, "invalid_request_error", str(exc)) from None


def _load_mlx(checkpoint: Checkpoint):
    global _gpu_load
    from core.inference import gpu_arbiter

    # Fetched before the eviction, so a failed download leaves the resident model serving.
    folder, base = _mlx_dirs(checkpoint, local_only = False)
    _evict()
    gpu = _wants_gpu(checkpoint)
    if gpu:
        with _state_lock:
            _gpu_load = True
        try:
            gpu_arbiter.acquire_for(gpu_arbiter.DECISIONS, allow_evict = False)
        except gpu_arbiter.GpuOwnerBusyError:
            raise RuntimeError(_GPU_BUSY.format(checkpoint.name)) from None
    agent = None
    try:
        agent = _EngineAgent(folder, _mlx_family(checkpoint), base)
        if _training_active():
            raise RuntimeError(
                f"{checkpoint.name} needs the GPU, which a training run took while it loaded."
            )
    except BaseException:
        if agent is not None:
            agent.close()
        if gpu:
            gpu_arbiter.release(gpu_arbiter.DECISIONS)
        raise
    return agent, MLX


def _load_checkpoint(checkpoint: Checkpoint):
    from utils.systemone_settings import get_device as preferred_device

    if _is_mlx(checkpoint):
        return _load_mlx(checkpoint)
    if _is_native(checkpoint):
        return _load_native(checkpoint)
    root = _checkpoint_dir(checkpoint)
    if checkpoint.layout == "clef":
        from .clef_runtime import ClefAgent, ClefWorkerError

        _evict()
        agent = ClefAgent(root, cancelled = lambda: _clef_preempted() or _unloaded())
        # Training may have started while this loaded, and Clef has no CPU fallback.
        if _clef_preempted():
            agent.close()
            raise ClefWorkerError(
                f"{checkpoint.name} needs the GPU, which a training run took while it loaded."
            )
        return agent, "cuda"
    _laya()

    # Evict only once the new checkpoint is on disk, so a long or failed download leaves the resident model serving.
    _evict()
    # While training holds the GPU, a GPU model loads on CPU, as dictation does; see _misplaced.
    for_training = _training_active() and preferred_device() == "gpu"
    device = "cpu" if for_training else _device()
    if device in ("cuda", "xpu"):
        try:
            agent = LayaWorker(
                root, checkpoint, device, cancelled = lambda: _training_active() or _unloaded()
            )
            return agent, agent.device
        except LayaWorkerError:
            if _unloaded() or not _training_active():
                raise
        device, for_training = "cpu", True
    agent, placed = _build_laya(root, checkpoint, device, for_training)
    if placed != "cpu" and _training_active():
        del agent
        _release_memory()
        return _build_laya(root, checkpoint, "cpu", True)
    return agent, placed


def _build_laya(
    root: Path,
    checkpoint: Checkpoint,
    device: str,
    for_training: bool = False,
):
    folder = root / checkpoint.subfolder if checkpoint.subfolder else root
    fp16_checkpoint = checkpoint.name in CHECKPOINTS or _stored_fp16(folder)
    if device == "mlx":
        return _MLXAgent(folder, fp16_checkpoint), device
    if device not in ("cuda", "cpu"):
        return _load_laya(str(root), subfolder = checkpoint.subfolder, device = device), device
    import torch

    # Built on CPU and cast before the move, so the device never holds laya's fp32 copy.
    weights, _ = _precision(torch.device(device), fp16_checkpoint)
    agent = _load_laya(
        str(root), subfolder = checkpoint.subfolder, device = "cpu", embedding_dtype = weights
    )
    _place(agent, torch.device(device), fp16_checkpoint)
    agent.__dict__["_unsloth_for_training"] = for_training
    return agent, str(agent.device.type)


def _is_laya_export(checkpoint: Checkpoint) -> bool:
    return checkpoint.layout == "laya" and _is_native(checkpoint)


def _stock_laya(checkpoint) -> bool:
    return checkpoint.layout == "laya" and not checkpoint.is_local


def _laya_config(checkpoint: Checkpoint) -> dict[str, Any]:
    import json
    config = _local_folder(checkpoint) / "rl_agent_config.json"
    return json.loads(config.read_text(encoding = "utf-8"))


@functools.lru_cache(maxsize = 4)
def _export_counter(checkpoint: Checkpoint):
    """(token count, head_max_len) for an exported Laya folder, read without torch."""
    import json

    from tokenizers import Tokenizer

    folder = _local_folder(checkpoint) / "tokenizer"
    mask = json.loads((folder / "tokenizer_config.json").read_text(encoding = "utf-8"))["mask_token"]
    tokenizer = Tokenizer.from_file(str(folder / "tokenizer.json"))

    def count(text: str) -> int:
        return len(tokenizer.encode(text.replace(mask, " "), add_special_tokens = False).ids)

    return count, int(_laya_config(checkpoint).get("head_max_len", 192))


def _export_criterion(value) -> str:
    import json
    if isinstance(value, str):
        return value
    return json.dumps(value, ensure_ascii = False, separators = (", ", ": "), default = str)


def _export_options(question: dict[str, Any]) -> list[str]:
    kind, criteria = question["type"], question.get("criteria")
    if kind == "choice":
        return [
            key if value in (None, "") else f"{key}: {_export_criterion(value)}"
            for key, value in criteria.items()
        ]
    if kind == "score":
        return [f"level {i}: {_export_criterion(c)}" for i, c in enumerate(criteria)]
    criteria = criteria or {}
    no, yes = criteria.get("false"), criteria.get("true")
    return [
        f"false: {_export_criterion(no)}"
        if no not in (None, "")
        else "false: no, the statement does not hold",
        f"true: {_export_criterion(yes)}"
        if yes not in (None, "")
        else "true: yes, the statement holds",
    ]


def _cut_problem(sizes: list[int], head: int, head_max_len: int) -> str | None:
    """What build_sequence would cut: past the option budget each option keeps its share, instructions what is left."""
    budget = head_max_len - 16
    if sum(sizes) > budget and max(sizes) > max(4, budget // len(sizes)):
        return f"options would be cut short to fit Laya's {budget}-token option budget. Use fewer or shorter criteria"
    if head > max(8, head_max_len - sum(sizes)):
        return (
            f"instructions would be cut short to fit beside its options in Laya's {head_max_len}-token "
            "question budget. Use shorter instructions or fewer criteria"
        )
    return None


@functools.lru_cache(maxsize = 256)
def _export_cut(checkpoint: Checkpoint, name: str, key: str) -> str | None:
    import json

    question = json.loads(key)
    count, head_max_len = _export_counter(checkpoint)
    sizes = [
        1 + min(_LAYA_OPTION_TOKENS, count(f" {option}")) for option in _export_options(question)
    ]
    # As native_worker._wire_questions sends it and laya's _to_internal reads it.
    instructions = question.get("instructions")
    if instructions in (None, ""):
        instructions = name
    elif not isinstance(instructions, str):
        instructions = json.dumps(instructions)
    return _cut_problem(sizes, count(f"{question['type']} question: {instructions}"), head_max_len)


def _refuse_cut_questions(checkpoint: Checkpoint, questions: dict[str, dict[str, Any]]) -> None:
    """llama.cpp cuts what does not fit Laya's question budget without saying so; refused here as PyTorch refuses it."""
    import json

    try:
        _export_counter(checkpoint)
    except Exception as exc:
        raise Unavailable(
            503, "model_unavailable", f"Could not read the tokenizer of {checkpoint.name}: {exc}"
        ) from None
    for name, question in questions.items():
        key = json.dumps(question, ensure_ascii = False)
        if (problem := _export_cut(checkpoint, str(name), key)) is not None:
            raise Unavailable(422, "invalid_request_error", f"Question {name!r} {problem}.")


def _load_native(checkpoint: Checkpoint):
    global _gpu_load
    from core.inference import gpu_arbiter
    from utils.systemone_settings import get_native_ctx

    from .native_worker import (
        NativeClefAgent,
        NativeError,
        NativeIncapable,
        NativeLoadError,
        resolve_binary,
    )

    # Fetched (pinned revision) before the eviction, so a failed download leaves the resident model serving.
    model, mmproj = _native_files(checkpoint, local_only = False)
    _evict()
    gpu = _native_gpu()
    if gpu:
        with _state_lock:
            _gpu_load = True
        try:
            gpu_arbiter.acquire_for(gpu_arbiter.DECISIONS, allow_evict = False)
        except gpu_arbiter.GpuOwnerBusyError:
            raise NativeError(_GPU_BUSY.format("a decision model through llama.cpp")) from None
    try:
        agent = NativeClefAgent(
            model,
            mmproj,
            checkpoint.name,
            gpu = gpu,
            clef_answers = checkpoint.layout == "clef",
            ctx = min(get_native_ctx(), int(_laya_config(checkpoint).get("max_len", 512)))
            if checkpoint.layout == "laya"
            else get_native_ctx(),
            cancelled = lambda: (gpu and _training_active()) or _unloaded(),
        )
        if gpu and _training_active():
            agent.close()
            raise NativeError(
                f"{checkpoint.name} needs the GPU, which a training run took while it loaded."
            )
    except BaseException as exc:
        if isinstance(exc, NativeError) and (binary := resolve_binary()):
            if isinstance(exc, NativeIncapable):
                _incapable.add(_binary_key(binary))
            elif (
                isinstance(exc, NativeLoadError)
                and _is_laya_export(checkpoint)
                and any(refusal in str(exc) for refusal in _LOAD_REFUSALS)
            ):
                _unloadable.add((_binary_key(binary), checkpoint))
        if gpu:
            gpu_arbiter.release(gpu_arbiter.DECISIONS)
        raise
    return agent, agent.device


_GPU_BUSY = "Unload the resident chat, image or video model before serving {} on the GPU."


_build_lock = threading.Lock()


def _load_laya(
    path: str,
    *,
    embedding_dtype = None,
    **kwargs,
):
    """Builds the vocab embedding with skip_init in the serving dtype, avoiding a huge random init."""
    laya = _laya()
    hook = getattr(laya, "agent", None)
    if not hasattr(hook, "build_model"):
        return laya.load(path, **kwargs)
    with _build_lock:
        original = hook.build_model

        def build_model(cfg, encoder_dir = None):
            return _build_model(cfg, encoder_dir, original, embedding_dtype)

        hook.build_model = build_model
        try:
            return laya.load(path, **kwargs)
        finally:
            hook.build_model = original


def _build_model(
    cfg,
    encoder_dir,
    original,
    embedding_dtype = None,
):
    """laya.common.build_model, with the vocabulary embedding allocated but not initialised."""
    import os

    import torch
    from transformers import AutoConfig, AutoModel

    if not encoder_dir or not os.path.exists(encoder_dir):
        return original(cfg, encoder_dir = encoder_dir)
    config = AutoConfig.from_pretrained(encoder_dir)
    vocab_size, pad_token_id = config.vocab_size, getattr(config, "pad_token_id", None)
    # ModernBERT reads pad_token_id only for this embedding; others keep it (RoBERTa's position ids).
    if config.model_type != "modernbert" or vocab_size <= 1:
        return original(cfg, encoder_dir = encoder_dir)
    # One-row placeholder: row 0 stands in for pad, so the check can tell config sized the embedding.
    placeholder_pad = None if pad_token_id is None else 0
    config.vocab_size, config.pad_token_id = 1, placeholder_pad
    try:
        encoder = AutoModel.from_config(config, attn_implementation = "sdpa")
    finally:
        config.vocab_size, config.pad_token_id = vocab_size, pad_token_id
    placeholder = encoder.get_input_embeddings()
    if (
        type(placeholder) is not torch.nn.Embedding
        or placeholder.num_embeddings != 1
        or placeholder.padding_idx != placeholder_pad
        or encoder.config.vocab_size != vocab_size
    ):
        # Not laid out like ModernBERT: build it laya's way.
        del encoder, placeholder
        return original(cfg, encoder_dir = encoder_dir)
    encoder.set_input_embeddings(
        torch.nn.utils.skip_init(
            torch.nn.Embedding,
            vocab_size,
            placeholder.embedding_dim,
            padding_idx = pad_token_id,
            max_norm = placeholder.max_norm,
            norm_type = placeholder.norm_type,
            scale_grad_by_freq = placeholder.scale_grad_by_freq,
            sparse = placeholder.sparse,
            dtype = embedding_dtype or placeholder.weight.dtype,
        )
    )
    return _laya().common.DecisionModel(
        encoder, cfg.get("head_layers", 2), len(cfg.get("act_costs", {})) + 1
    )


def _stored_fp16(folder: Path) -> bool:
    """Whether the checkpoint's weights are saved as float16, as all three published Laya checkpoints are."""
    import json
    import math
    import struct

    try:
        with open(folder / "model.safetensors", "rb") as f:
            header = json.loads(f.read(struct.unpack("<Q", f.read(8))[0]))
    except (OSError, ValueError, struct.error):
        return False
    sizes: dict[str, int] = {}
    for name, meta in header.items():
        if name != "__metadata__" and isinstance(meta, dict):
            sizes[meta["dtype"]] = sizes.get(meta["dtype"], 0) + math.prod(meta["shape"])
    return bool(sizes) and max(sizes, key = sizes.get) == "F16"


def _fp32_forced() -> bool:
    import os
    return os.environ.get("UNSLOTH_SYSTEMONE_FP32", "") == "1"


def _precision(device, fp16_checkpoint: bool):
    """An fp16 checkpoint stays fp16 (exact); otherwise bf16 where native, else fp16."""
    import platform

    import torch

    if _fp32_forced():
        return None, None
    if device.type == "cuda":
        if fp16_checkpoint:
            return torch.float16, torch.float16
        from core.inference.rocm_bf16 import is_rocm_torch, rocm_bf16_supported

        if is_rocm_torch(torch):
            bf16 = rocm_bf16_supported(torch, device.index)
        else:
            # By capability: pre-Ampere NVIDIA reports is_bf16_supported() through slow emulation.
            bf16 = torch.cuda.get_device_capability(device)[0] >= 8
        dtype = torch.bfloat16 if bf16 else torch.float16
        return dtype, dtype
    # CPU only where the ISA has native fp16/bf16 (AVX512-FP16 / AMX); measured on x86 only.
    if device.type == "cpu" and platform.machine().lower() in ("x86_64", "amd64"):
        mkldnn = getattr(torch.ops, "mkldnn", None)
        try:
            if fp16_checkpoint and mkldnn._is_mkldnn_fp16_supported():
                return torch.float16, torch.float16
            if not fp16_checkpoint and mkldnn._is_mkldnn_bf16_supported():
                return torch.bfloat16, torch.bfloat16
        except (AttributeError, RuntimeError):
            pass
    return None, None


def _fp32_output(module, args, output):
    return output.float()


def _cast_matmul_weights(model, dtype) -> None:
    """Casts only Linear and Embedding weights; norms stay fp32 as CPU layer_norm rejects low precision."""
    import torch.nn as nn
    for module in model.modules():
        if isinstance(module, (nn.Linear, nn.Embedding)):
            module.to(dtype)
        elif isinstance(module, nn.MultiheadAttention):
            module.in_proj_weight.data = module.in_proj_weight.data.to(dtype)
            if module.in_proj_bias is not None:
                module.in_proj_bias.data = module.in_proj_bias.data.to(dtype)
        if isinstance(module, nn.Embedding) and not getattr(module, "_unsloth_fp32_output", False):
            module.register_forward_hook(_fp32_output)
            module._unsloth_fp32_output = True


def _place(agent, device, fp16_checkpoint: bool) -> None:
    """Move a CPU-built laya agent to ``device`` in the precision :func:`_precision` picks for it."""
    import torch

    agent.__dict__["_unsloth_fp16_checkpoint"] = fp16_checkpoint
    # Captured graphs point at the current weights; a move or recast replaces them.
    agent.__dict__.pop("_unsloth_graphs", None)
    weights, compute = _precision(device, fp16_checkpoint)
    # Cast on the CPU side of the move: before moving to an accelerator, after coming back from one.
    if device.type == "cpu":
        agent.model.to(device)
    if weights is None:
        agent.model.float()
    else:
        _cast_matmul_weights(agent.model, weights)
    if device.type != "cpu":
        try:
            agent.model.to(device)
        except (RuntimeError, torch.cuda.OutOfMemoryError):
            # Same fallback as laya's own loader: a device that cannot hold the model leaves it on CPU.
            logger.warning("Laya could not be placed on %s; running it on CPU", device)
            _place(agent, torch.device("cpu"), fp16_checkpoint)
            _release_memory()
            return
    agent.device, agent.dtype = device, compute or torch.float32


def _hub_download_active(checkpoint: Checkpoint) -> bool:
    # A Hub download job writing the same repo would share blobs with our snapshot_download.
    if checkpoint.is_local:
        return False
    try:
        from hub.utils.download_registry import get_models_registry
        registry = get_models_registry()
        if not any(registry.active_job_refs(repo) for repo in _sources(checkpoint)):
            return False
    except Exception:
        return False
    return not is_cached(checkpoint)


def _sources(checkpoint: Checkpoint) -> tuple[str, ...]:
    """The repos a load reads: an MLX adapter's base as well as its own."""
    if _is_mlx(checkpoint) and checkpoint.layout == GGUF:
        return tuple(repo.repo for repo in _mlx_repos(MLX_COMPANIONS[checkpoint.name]))
    return (checkpoint.source,)


def loading_repo_ids() -> tuple[str, ...]:
    with _state_lock:
        if _loading is not None and not _loading.is_local:
            return _sources(_loading)
    return ()


def _load(checkpoint: Checkpoint, unloads: int | None = None) -> None:
    global _agent, _loaded, _device_name, _loading, _failure, _generation, _last_used, _gpu_load
    started = time.monotonic()
    if unloads is None:
        unloads = _unloads
    if _unloads != unloads:
        with _state_lock:
            _loading = None
        logger.info("System One dropped %s: it was unloaded before it loaded", checkpoint.name)
        return
    _load_generation.unloads = unloads
    try:
        agent, device = _load_checkpoint(checkpoint)
    except Exception as exc:
        message = f"Could not load {checkpoint.name}: {type(exc).__name__}: {exc}"
        with _state_lock:
            stale = _unloads != unloads
            if not stale:
                _failure = (checkpoint, message, time.monotonic() + FAILURE_BACKOFF_S)
            _loading = None
            _gpu_load = False
        if stale:
            logger.info("System One dropped %s: it was unloaded while it loaded", checkpoint.name)
        else:
            logger.warning("System One load failed: %s", message, exc_info = True)
        return
    finally:
        del _load_generation.unloads
    with _state_lock:
        stale = _unloads != unloads
        if not stale:
            _agent, _loaded, _device_name = agent, checkpoint, str(getattr(agent, "device", device))
            _loading = None
        _failure = None
        _gpu_load = False
        _generation += 1
        _last_used = time.monotonic()
        generation = _generation
    if stale:
        logger.info("System One dropped %s: it was unloaded while it loaded", checkpoint.name)
        # _loading stays set until its memory is back, so no other load claims the GPU first.
        try:
            _close(agent)
            del agent
            _release_memory()
        finally:
            with _state_lock:
                _loading = None
        return
    if _arbitrated(checkpoint):
        _schedule_idle_unload(generation)
    logger.info(
        "System One loaded %s on %s in %.1fs",
        checkpoint.name,
        _device_name,
        time.monotonic() - started,
    )


def _schedule_idle_unload(generation: int, delay: float | None = None) -> None:
    global _idle_timer
    timer = threading.Timer(
        IDLE_UNLOAD_S if delay is None else delay, _idle_unload, args = (generation,)
    )
    timer.daemon = True
    with _state_lock:
        if _idle_timer is not None:
            _idle_timer.cancel()
        _idle_timer = timer
    timer.start()


def _idle_unload(generation: int) -> None:
    global _agent, _loaded, _device_name, _generation, _unloads
    if not _run_lock.acquire(blocking = False):
        # A decision is running; look again once the idle window could have passed.
        _schedule_idle_unload(generation)
        return
    try:
        if _generation != generation or not _arbitrated(_loaded):
            return
        idle = time.monotonic() - _last_used
        if idle < IDLE_UNLOAD_S:
            _schedule_idle_unload(generation, IDLE_UNLOAD_S - idle)
            return
        agent, _agent, _loaded, _device_name = _agent, None, None, None
        _generation += 1
        _unloads += 1
    finally:
        _run_lock.release()
    logger.info("System One unloaded the idle decision model")
    _close(agent)


def evict_for_gpu() -> None:
    """GPU arbiter evictor: ends an idle llama.cpp server or MLX engine model; refuses while it loads or decides."""
    global _agent, _loaded, _device_name, _generation, _unloads
    from core.inference.gpu_arbiter import DECISIONS, GpuOwnerBusyError

    with _state_lock:
        if _arbitrated(_loading):
            raise GpuOwnerBusyError(DECISIONS)
    if not _run_lock.acquire(blocking = False):
        raise GpuOwnerBusyError(DECISIONS)
    try:
        if not _arbitrated(_loaded):
            return
        agent, _agent, _loaded, _device_name = _agent, None, None, None
        _generation += 1
        _unloads += 1
    finally:
        _run_lock.release()
    # The arbiter hands ownership on itself, under its own lock.
    _close(agent, release_gpu = False)


def evict_idle_worker() -> None:
    global _agent, _loaded, _device_name, _generation, _unloads
    from .clef_runtime import ClefAgent

    if not _run_lock.acquire(blocking = False):
        return
    try:
        with _state_lock:
            if _loading is not None or not isinstance(_agent, (LayaWorker, ClefAgent)):
                return
            agent, _agent, _loaded, _device_name = _agent, None, None, None
            _generation += 1
            _unloads += 1
    finally:
        _run_lock.release()
    logger.info("System One unloaded the idle decision model to free the GPU")
    _close(agent)


def _holds_gpu() -> bool:
    # A worker that fell back to CPU still holds its CUDA or XPU context until it ends.
    return _device_name not in (None, "cpu") or isinstance(_agent, LayaWorker)


def _misplaced() -> bool:
    # The resident model follows the same rule: off the GPU while training holds it, back on it after.
    if getattr(_agent, "_unsloth_for_training", False):
        return not _training_active()
    return _holds_gpu() and _training_active()


def _clef_blocked_by_training(checkpoint: Checkpoint) -> None:
    if checkpoint.layout == "laya" and not _is_laya_export(checkpoint):
        return
    from .catalog import clef_unsupported_reason

    native = _arbitrated(checkpoint)
    # llama.cpp also serves Clef on CPU, Metal or Vulkan, and MLX on Apple Silicon; the PyTorch worker needs CUDA or ROCm.
    if not native and (reason := clef_unsupported_reason()) is not None:
        raise Unavailable(400, "api_usage_error", reason)
    if (_is_native(checkpoint) and not _native_gpu()) or not _training_active():
        return
    # Clef has no CPU fallback: it waits for the GPU instead of taking it from the run.
    # Only a resident on the GPU: a llama.cpp server on CPU keeps serving during the run.
    if (
        _loaded is not None
        and (_loaded.layout != "laya" or _is_laya_export(_loaded))
        and _device_name not in (None, "cpu")
    ):
        _evict(external = True)
    raise Unavailable(
        503,
        "model_unavailable",
        f"{checkpoint.name} needs the GPU, which a training run is using; it answers again when the run ends.",
        retry_after = 30,
    )


def _ensure_loading(checkpoint: Checkpoint, unloads: int | None = None) -> threading.Thread | None:
    global _loader, _loading
    # Asked before _state_lock: the training backend takes its own lock.
    _clef_blocked_by_training(checkpoint)
    misplaced = _misplaced()
    with _state_lock:
        if _loaded == checkpoint and _agent is not None and not misplaced:
            return None
        if _wants_gpu(checkpoint):
            if _gpu_taken():
                busy = _GPU_BUSY.format(
                    checkpoint.name if _is_mlx(checkpoint) else "a decision model through llama.cpp"
                )
                raise Unavailable(409, "gpu_busy", busy, retry_after = 1)
        if _failure and _failure[0] == checkpoint and time.monotonic() < _failure[2]:
            raise Unavailable(
                503,
                "model_unavailable",
                _failure[1],
                retry_after = max(1.0, _failure[2] - time.monotonic()),
            )
        if _loading is not None:
            if _loading != checkpoint:
                raise Unavailable(
                    503, "model_loading", f"{_loading.name} is loading", retry_after = 5
                )
            if _loader is not None and _loader.is_alive():
                return _loader
            raise Unavailable(503, "model_loading", f"{checkpoint.name} is loading", retry_after = 5)
        # Claimed before the registry probe, so a Hub download admitted from now on sees this load.
        _loading = checkpoint
        if unloads is None:
            unloads = _unloads
    # Checked after the claim, so an eviction that lands before it is still seen.
    if _torch_gpu(checkpoint) and _gpu_load_in_flight():
        with _state_lock:
            _loading = None
        raise Unavailable(
            409,
            "gpu_busy",
            f"Another model is loading on the GPU; {checkpoint.name} loads once it has.",
            retry_after = 1,
        )
    # Outside _state_lock: the registry calls loading_repo_ids() while holding its own lock.
    if _hub_download_active(checkpoint):
        with _state_lock:
            _loading = None
        raise Unavailable(503, "model_loading", f"{checkpoint.name} is downloading", retry_after = 5)
    with _state_lock:
        _loader = threading.Thread(
            target = _load, args = (checkpoint, unloads), name = "systemone-load", daemon = True
        )
        _loader.start()
        return _loader


def _agent_for(checkpoint: Checkpoint, wait: float):
    unloads = _unloads
    loader = _ensure_loading(checkpoint)
    if loader is not None:
        loader.join(wait)
        # An unload while this waited (a settings change) wins over a load of the old checkpoint.
        if loader.is_alive() or _unloads != unloads or _ensure_loading(checkpoint) is not None:
            raise Unavailable(
                503, "model_loading", f"{checkpoint.name} is still loading", retry_after = 5
            )
    return _agent


def _to_laya(question: dict[str, Any], name = "") -> dict[str, Any]:
    import json

    instructions = question.get("instructions")
    if instructions in (None, ""):
        instructions = str(name)
    elif not isinstance(instructions, str):
        instructions = json.dumps(instructions, ensure_ascii = False)
    out = {"type": question["type"], "instructions": instructions}
    if question.get("criteria") is not None:
        out["criteria"] = question["criteria"]
    return out


# Long states tokenized in growing prefixes; margin keeps kept tokens clear of the cut (prefix tokenization differs).
_PREFIX_MARGIN = 64
_HEAD_CACHE_SIZE = 1024


def _state_ids(tok, state, room: int) -> tuple[list[int], bool]:
    serialize_state = _laya().common.serialize_state

    text = serialize_state(state).replace(tok.mask_token, " ")
    chars = max(4096, room * 16)
    while chars < len(text):
        ids = tok(text[:chars], add_special_tokens = False)["input_ids"]
        if len(ids) > room + _PREFIX_MARGIN:
            return ids[:room], True
        chars *= 4
    ids = tok(text, add_special_tokens = False)["input_ids"]
    return ids[:room], len(ids) > room


def _head(agent, question: dict[str, Any], max_len: int, head_max_len: int):
    import json

    common = _laya().common
    build_sequence = common.build_sequence
    render_options = common.render_options

    cache = agent.__dict__.setdefault("_unsloth_heads", {})
    key = json.dumps(question, ensure_ascii = False)
    if key not in cache:
        internal = agent._to_internal(question)
        tok = agent.tok
        options = [" " + o.replace(tok.mask_token, " ") for o in render_options(internal)]
        sizes = [
            1 + min(_LAYA_OPTION_TOKENS, len(tok(o, add_special_tokens = False)["input_ids"]))
            for o in options
        ]
        ins = str(internal["ins"]).replace(tok.mask_token, " ")
        head = tok(f"{internal['t']} question: {ins}", add_special_tokens = False)["input_ids"]
        if (problem := _cut_problem(sizes, len(head), head_max_len)) is not None:
            raise ValueError(problem)
        ids, markers = build_sequence(tok, "", internal, max_len, head_max_len)
        if len(markers) != len(sizes):
            raise ValueError(
                f"options exceed the Laya context window ({max_len} tokens). "
                "Use fewer or shorter criteria"
            )
        if len(cache) >= _HEAD_CACHE_SIZE:
            cache.clear()
        cache[key] = (ids, markers, internal)
    return cache[key]


def _predict(agent, state, questions: dict[str, dict[str, Any]]) -> tuple[dict[str, Any], bool]:
    import numpy as np

    common = _laya().common
    QTYPES = common.QTYPES
    confidence_from_probs = common.confidence_from_probs

    max_len = int(agent.cfg.get("max_len", 512))
    head_max_len = int(agent.cfg.get("head_max_len", 192))
    names = list(questions)
    heads = []
    for name in names:
        try:
            heads.append(_head(agent, questions[name], max_len, head_max_len))
        except ValueError as exc:
            raise ValueError(f"Question {name!r} {exc}.") from None
    room = max(0, max_len - min(len(ids) for ids, _, _ in heads))
    state_ids, state_cut = _state_ids(agent.tok, state, room)
    items, truncated = [], False
    for ids, markers, internal in heads:
        keep = max(0, max_len - len(ids))
        truncated = truncated or state_cut or len(state_ids) > keep
        items.append(
            {
                "ids": (ids[:-1] + state_ids[:keep] + ids[-1:])[:max_len],
                "markers": markers,
                "qtype": QTYPES[internal["t"]],
            }
        )
    logits, usage = _forward(agent, items)
    answers = {}
    for row, (name, (_, markers, internal)) in enumerate(zip(names, heads)):
        k = len(markers)
        p = _probabilities(agent, logits[row], k, QTYPES[internal["t"]])
        confidence = round(confidence_from_probs(p, k), 4)
        if internal["t"] == "choice":
            keys = list(internal["crit"])
            answers[name] = {
                "type": "choice",
                "choice": keys[int(p.argmax())],
                "probabilities": {key: round(float(v), 4) for key, v in zip(keys, p)},
                "confidence": confidence,
            }
        elif internal["t"] == "score":
            answers[name] = {
                "type": "score",
                "score": round(float((np.arange(k) * p).sum()), 4),
                "legend": {str(i): c for i, c in enumerate(internal["crit"])},
                "probabilities": {str(i): round(float(v), 4) for i, v in enumerate(p)},
                "confidence": confidence,
            }
        else:
            answers[name] = {
                "type": "noul",
                "noul": round(float(p[1]), 4),
                "confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),
            }
    return {"answers": answers, "usage": {"input_tokens": usage, "output_tokens": 0}}, truncated


def _forward(agent, items: list[dict[str, Any]]):
    global _agent, _loaded, _device_name
    import numpy as np
    import torch

    batch = _collate(items, agent.tok.pad_token_id)
    if agent.device == "mlx":
        try:
            logits = _mlx_logits(agent, batch)
            if not np.isfinite(logits).all():
                import mlx.core as mx
                if agent.dtype != mx.float32:
                    logger.warning("Laya overflowed in float16; continuing in float32")
                    agent.model.set_dtype(mx.float32)
                    agent.dtype = mx.float32
                    logits = _mlx_logits(agent, batch)
            return logits, int(batch["attention_mask"].sum())
        except RuntimeError as exc:
            # MLX reports exhausted memory as "[malloc] Unable to allocate ..." or "Insufficient Memory".
            reason = str(exc).lower()
            if "memory" not in reason and "allocate" not in reason:
                raise
        # Past the handler, so the traceback no longer keeps the MLX arrays alive while the CPU copy loads.
        logger.warning("Laya ran out of GPU memory; moving it to CPU")
        agent.model = None
        _release_memory()
        try:
            cpu_device, fp16_checkpoint = torch.device("cpu"), _stored_fp16(Path(agent.folder))
            cpu = _load_laya(
                str(agent.folder),
                device = "cpu",
                embedding_dtype = _precision(cpu_device, fp16_checkpoint)[0],
            )
            _place(cpu, cpu_device, fp16_checkpoint)
        except Exception:
            # Callers hold _run_lock, so drop the half-moved agent here; the next request loads it again.
            _agent = _loaded = _device_name = None
            raise
        agent.model, agent.device, agent.dtype = cpu.model, cpu.device, cpu.dtype
        _device_name = "cpu"
    try:
        logits = _run_model(agent, batch)
    except (RuntimeError, torch.cuda.OutOfMemoryError) as exc:
        # Same fallback as laya's Agent.predict: a GPU that runs out of memory moves the model to CPU.
        reason = str(exc).lower()
        if agent.device.type == "cpu" or ("memory" not in reason and "cuda" not in reason):
            raise
        logger.warning("Laya ran out of GPU memory; moving it to CPU")
        _place(agent, torch.device("cpu"), agent.__dict__.get("_unsloth_fp16_checkpoint", False))
        _device_name = "cpu"
        _release_memory()
        logits = _run_model(agent, batch)
    if agent.dtype == torch.float16 and not bool(torch.isfinite(logits).all()):
        # fp16 activations overflowed; the fp16 weights widen to fp32 exactly, so rerun at full precision.
        logger.warning("Laya overflowed in float16; continuing in float32")
        # The graphs read the fp16 weights that .float() is about to replace.
        agent.__dict__.pop("_unsloth_graphs", None)
        agent.model.float()
        agent.dtype = torch.float32
        logits = _run_model(agent, batch)
    return logits.float().cpu().numpy(), int(batch["attention_mask"].sum())


def _chunks(batch, budget: int):
    """``batch`` as consecutive row slices of at most ``budget`` padded tokens, each trimmed to its longest row.

    ``budget=None`` keeps the whole batch."""
    rows, tokens = batch["input_ids"].shape
    step = rows if budget is None else max(1, budget // tokens)
    if rows <= step:
        yield batch
        return
    for start in range(0, rows, step):
        part = {name: batch[name][start : start + step] for name in _INPUTS}
        keep = int(part["attention_mask"].sum(1).max())
        part["input_ids"] = part["input_ids"][:, :keep]
        part["attention_mask"] = part["attention_mask"][:, :keep]
        yield part


def _mlx_logits(agent, batch):
    """One MLX request, forwarded in chunks and concatenated on the host."""
    import numpy as np
    return np.concatenate(
        [agent.model.logits(part) for part in _chunks(batch, _chunk_budget("mlx"))]
    )


def _run_model(agent, batch):
    import torch

    if not _marker_head(agent.model):
        return _run_chunk(agent, batch)
    budget = _chunk_budget(agent.device)
    # cat copies, so no result is a view into a CUDA graph's output buffer, which its next replay overwrites.
    return torch.cat([_run_chunk(agent, part) for part in _chunks(batch, budget)])


def _chunk_budget(device):
    """Padded tokens per forward, or None for one pass; CPUs never split, as forward overhead dominates."""
    kind = getattr(device, "type", device)
    budget = _CHUNK_TOKENS.get(kind)
    if kind != "cuda" or budget is None:
        return budget
    import torch

    try:
        total = torch.cuda.get_device_properties(device).total_memory
    except (RuntimeError, AssertionError):
        return budget
    per_token = _TOKEN_BYTES["hip" if torch.version.hip else "cuda"]
    return max(budget, int(total * _GPU_SHARE) // per_token)


def _run_chunk(agent, batch):
    import os

    import torch

    device = agent.device
    fast = _marker_head(agent.model)
    if (
        fast
        and device.type == "cuda"
        and os.environ.get("UNSLOTH_SYSTEMONE_CUDA_GRAPHS", "") != "0"
        # ROCm reports "cuda" too; as in diffusion_cuda_graph.py, graphs are CUDA only.
        and not torch.version.hip
    ):
        graphs = agent.__dict__.get("_unsloth_graphs")
        if graphs is None:
            graphs = agent.__dict__["_unsloth_graphs"] = _CUDAGraphs(agent)
        with torch.inference_mode():
            logits = graphs.run(batch)
        if logits is not None:
            return logits
    with torch.inference_mode(), _autocast(agent):
        args = [batch[name].to(device) for name in _INPUTS]
        if fast:
            # No padding in the batch: no mask, so SDPA may pick its fastest kernel.
            return _decision_logits(
                agent.model, *args, padded = not bool(batch["attention_mask"].all())
            )
        logits, _ = agent.model(*args)
    return logits


_INPUTS = ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")
# Smallest padded rows x tokens per forward; a larger request runs as several (see _chunk_budget). None: one forward.
_CHUNK_TOKENS = {"cuda": 16384, "mlx": 8192, "cpu": None}
# Activation bytes per padded token of the worst request (B200 fp16 41 KiB, gfx1151 137 KiB), with headroom.
_TOKEN_BYTES = {"cuda": 48 << 10, "hip": 160 << 10}
# Share of the GPU's total memory one Decision API forward may take before the request is split.
_GPU_SHARE = 0.05
# Padded rows x tokens above which a batch runs eagerly (B200: graphs win up to 8 x 1024, lose at 16 x 512).
_GRAPH_TOKENS = 8192


def _autocast(agent, cache_enabled = True):
    import torch
    device = agent.device
    return torch.autocast(
        device_type = device.type,
        dtype = agent.dtype,
        enabled = device.type in ("cuda", "cpu") and agent.dtype != torch.float32,
        cache_enabled = cache_enabled,
    )


def _collate(items: list[dict[str, Any]], pad_id: int) -> dict[str, Any]:
    """laya's ``collate_items`` for one request (same tensors), filled with numpy instead of per-row copies."""
    import numpy as np
    import torch

    n = len(items)
    lengths = np.fromiter((len(it["ids"]) for it in items), dtype = np.int64, count = n)
    counts = np.fromiter((len(it["markers"]) for it in items), dtype = np.int64, count = n)
    tokens = np.arange(int(lengths.max()))[None, :] < lengths[:, None]
    options = np.arange(int(counts.max()))[None, :] < counts[:, None]
    ids = np.full(tokens.shape, pad_id, dtype = np.int64)
    ids[tokens] = np.fromiter(
        (t for it in items for t in it["ids"]), dtype = np.int64, count = int(lengths.sum())
    )
    positions = np.zeros(options.shape, dtype = np.int64)
    positions[options] = np.fromiter(
        (m for it in items for m in it["markers"]), dtype = np.int64, count = int(counts.sum())
    )
    return {
        "input_ids": torch.from_numpy(ids),
        "attention_mask": torch.from_numpy(tokens.astype(np.int64)),
        "marker_pos": torch.from_numpy(positions),
        "marker_mask": torch.from_numpy(options),
        "qtype": torch.tensor([it["qtype"] for it in items]),
    }


def _marker_head(model) -> bool:
    """Whether :func:`_decision_logits` can stand in for ``model(...)``: laya's pre-norm ReLU head and GELU scorer."""
    import os

    import torch.nn as nn

    if os.environ.get("UNSLOTH_SYSTEMONE_FAST", "") == "0":
        return False
    cached = model.__dict__.get("_unsloth_marker_head")
    if cached is not None:
        return cached
    head, scorer = getattr(model, "head", None), getattr(model, "scorer", None)
    ok = (
        isinstance(head, nn.TransformerEncoder)
        and len(head.layers) > 0
        and head.norm is None
        and all(
            type(layer) is nn.TransformerEncoderLayer
            and layer.norm_first
            and layer.activation_relu_or_gelu == 1
            and type(layer.self_attn) is nn.MultiheadAttention
            and layer.self_attn.batch_first
            and layer.self_attn._qkv_same_embed_dim
            and layer.self_attn.in_proj_bias is not None
            for layer in head.layers
        )
        and isinstance(scorer, nn.Sequential)
        and [type(m) for m in scorer] == [nn.LayerNorm, nn.Linear, nn.GELU, nn.Linear]
        and scorer[2].approximate == "none"
    )
    model.__dict__["_unsloth_marker_head"] = ok
    return ok


def _feed_forward(layer, x):
    import torch.nn.functional as F

    y = F.layer_norm(x, x.shape[-1:], layer.norm2.weight, layer.norm2.bias, layer.norm2.eps)
    y = F.relu(F.linear(y, layer.linear1.weight, layer.linear1.bias), inplace = True)
    return x.add_(F.linear(y, layer.linear2.weight, layer.linear2.bias))


def _head_layer(
    layer,
    x,
    mask,
    markers = None,
):
    """Eval-mode pre-norm encoder layer; with markers, only those rows, as keys still span every token."""
    import torch
    import torch.nn.functional as F

    attn = layer.self_attn
    batch, length, width = x.shape
    heads = attn.num_heads
    y = F.layer_norm(x, (width,), layer.norm1.weight, layer.norm1.bias, layer.norm1.eps)
    if markers is None:
        q, k, v = (
            F.linear(y, attn.in_proj_weight, attn.in_proj_bias)
            .view(batch, length, 3, heads, width // heads)
            .permute(2, 0, 3, 1, 4)
        )
    else:
        index = markers[:, :, None].expand(-1, -1, width)
        k, v = (
            F.linear(y, attn.in_proj_weight[width:], attn.in_proj_bias[width:])
            .view(batch, length, 2, heads, width // heads)
            .permute(2, 0, 3, 1, 4)
        )
        q = F.linear(
            torch.gather(y, 1, index), attn.in_proj_weight[:width], attn.in_proj_bias[:width]
        )
        q = q.view(batch, -1, heads, width // heads).transpose(1, 2)
        x = torch.gather(x, 1, index)
    # Bool attn_mask is True where attention is allowed; SDPA applies dropout_p even in eval.
    out = F.scaled_dot_product_attention(q, k, v, attn_mask = mask, dropout_p = 0.0)
    out = out.transpose(1, 2).reshape(batch, -1, width)
    x = x.add_(F.linear(out, attn.out_proj.weight, attn.out_proj.bias))
    return _feed_forward(layer, x)


def _decision_logits(
    model,
    input_ids,
    attention_mask,
    marker_pos,
    marker_mask,
    qtype,
    *,
    padded = True,
):
    """Computes only the logits the scorer reads: the last head layer runs at the option markers alone."""
    import torch.nn.functional as F

    h = model.encoder(input_ids = input_ids, attention_mask = attention_mask).last_hidden_state
    h = h + model.type_emb(qtype)[:, None, :]
    mask = attention_mask.bool()[:, None, None, :] if padded else None
    layers = model.head.layers
    for layer in layers[:-1]:
        h = _head_layer(layer, h, mask)
    m = _head_layer(layers[-1], h, mask, marker_pos.clamp(min = 0))
    norm, up, _, down = model.scorer
    m = F.layer_norm(m, m.shape[-1:], norm.weight, norm.bias, norm.eps)
    logits = (
        F.linear(F.gelu(F.linear(m, up.weight, up.bias)), down.weight, down.bias)
        .squeeze(-1)
        .float()
    )
    return logits.masked_fill(~marker_mask, -1e4)


class _CUDAGraphs:
    """One captured :func:`_decision_logits` per padded (rows, tokens, options) bucket, sharing one memory pool.

    A short request is bound by kernel launches (about 2.5 ms of GPU work in a 12 ms forward); replaying a
    graph launches it all at once. Past ``_GRAPH_TOKENS`` padded tokens the GPU is the bottleneck and
    padding up to a bucket costs more than launches save, so those batches run eagerly. Callers hold
    ``_run_lock``, so one replay runs at a time and each output is read before the next overwrites it.
    """

    ROWS = (1, 2, 4, 8, 16)
    TOKENS = (64, 128, 256, 512, 1024)
    OPTIONS = (2, 4, 8, 16, 32, 64, 128, 256)
    MAX_GRAPHS = 64

    def __init__(self, agent):
        import torch

        self.agent = agent
        # transformers 4.x wraps ModernBERT parts in torch.compile on CUDA, which cannot run under capture.
        config = getattr(getattr(agent.model, "encoder", None), "config", None)
        if getattr(config, "reference_compile", False) is not False:
            config.reference_compile = False
        self.graphs: dict[tuple[int, int, int], tuple[Any, dict[str, Any], Any]] = {}
        self.broken = False
        self.pool = torch.cuda.graph_pool_handle()
        self.pool_bytes = 0
        # One warm-up stream: the allocator caches blocks per stream, so a new stream per capture would strand them.
        self.stream = torch.cuda.Stream(agent.device)
        free, _ = torch.cuda.mem_get_info(agent.device)
        self.max_pool_bytes = min(1 << 30, free // 10)

    @staticmethod
    def _fit(n, sizes):
        return next((size for size in sizes if n <= size), None)

    def _forward(self, static):
        with _autocast(self.agent, cache_enabled = False):
            return _decision_logits(self.agent.model, *(static[name] for name in _INPUTS))

    def _capture(self, key):
        import torch

        rows, tokens, options = key
        device = self.agent.device
        static = {
            "input_ids": torch.zeros((rows, tokens), dtype = torch.long, device = device),
            "attention_mask": torch.ones((rows, tokens), dtype = torch.long, device = device),
            "marker_pos": torch.zeros((rows, options), dtype = torch.long, device = device),
            "marker_mask": torch.ones((rows, options), dtype = torch.bool, device = device),
            "qtype": torch.zeros((rows,), dtype = torch.long, device = device),
        }
        # Warm up off the default stream first, as torch.cuda.graph requires, so lazy init is not captured.
        self.stream.wait_stream(torch.cuda.current_stream(device))
        with torch.cuda.stream(self.stream):
            for _ in range(2):
                self._forward(static)
        torch.cuda.current_stream(device).wait_stream(self.stream)
        graph = torch.cuda.CUDAGraph()
        # thread_local: CUDA work on other Studio threads cannot break (or be broken by) this capture.
        with torch.cuda.graph(graph, pool = self.pool, capture_error_mode = "thread_local"):
            logits = self._forward(static)
        return graph, static, logits

    def run(self, batch):
        """Logits for ``batch`` from a graph replay, or None to run it eagerly."""
        import torch

        rows, tokens = batch["input_ids"].shape
        options = batch["marker_pos"].shape[1]
        key = (
            self._fit(rows, self.ROWS),
            self._fit(tokens, self.TOKENS),
            self._fit(options, self.OPTIONS),
        )
        if self.broken or None in key or key[0] * key[1] > _GRAPH_TOKENS:
            return None
        entry = self.graphs.get(key)
        if entry is None:
            if len(self.graphs) >= self.MAX_GRAPHS or self.pool_bytes >= self.max_pool_bytes:
                return None
            before = torch.cuda.memory_reserved(self.agent.device)
            try:
                entry = self._capture(key)
            except Exception:
                # e.g. transformers 4.x builds ModernBERT's sliding-window mask on the CPU every forward.
                logger.warning(
                    "Laya cannot run in CUDA graphs here; running it eagerly", exc_info = True
                )
                self.broken = True
                return None
            self.graphs[key] = entry
            self.pool_bytes += max(0, torch.cuda.memory_reserved(self.agent.device) - before)
            if self.pool_bytes > self.max_pool_bytes:
                # Over budget: the pool is only released with every graph in it, so drop them all and stay eager.
                logger.warning("Laya CUDA graphs need more memory than allowed; running eagerly")
                self.graphs.clear()
                self.broken = True
                return None
        graph, static, logits = entry
        # Padding tokens are masked; padding rows repeat row 0 so every row has a real token to attend to.
        static["input_ids"].zero_()
        static["attention_mask"].zero_()
        static["marker_pos"].zero_()
        static["marker_mask"].zero_()
        for name in _INPUTS:
            source = batch[name]
            if source.dim() == 1:
                static[name][:rows].copy_(source)
            else:
                static[name][:rows, : source.shape[1]].copy_(source)
            if rows < key[0]:
                static[name][rows:] = static[name][:1]
        graph.replay()
        return logits[:rows, :options]


def _probabilities(agent, row, k: int, qtype: int):
    import numpy as np

    temp_bucket = _laya().common.temp_bucket

    scale = agent.temperature_by_options.get(temp_bucket(qtype, k), agent.temperature[qtype])
    z = row[:k] / scale
    p = np.exp(z - z.max())
    return p / p.sum()


def _wire_answer(answer: dict[str, Any]) -> dict[str, Any]:
    kind = answer["type"]
    if kind == "noul":
        return {"type": "noul", "noul": float(answer["noul"])}
    if kind == "choice":
        return {
            "type": "choice",
            "choice": answer["choice"],
            "confidence": float(answer["confidence"]),
            "probabilities": {k: float(v) for k, v in answer["probabilities"].items()},
        }
    return {
        "type": "score",
        "score": float(answer["score"]),
        "confidence": float(answer["confidence"]),
        "legend": answer["legend"],
        "probabilities": {k: float(v) for k, v in answer["probabilities"].items()},
    }


def decide(
    checkpoint: Checkpoint,
    state,
    questions: dict[str, dict[str, Any]],
    images: list[str] | None = None,
    audio: list[str] | None = None,
) -> dict[str, Any]:
    if not _admission.acquire(blocking = False):
        raise Unavailable(529, "overloaded", "System One is busy; retry shortly", retry_after = 1)
    try:
        return _route(checkpoint, state, questions, images or None, audio or None)
    finally:
        _admission.release()


def _route(
    checkpoint: Checkpoint,
    state,
    questions,
    images,
    audio = None,
) -> dict[str, Any]:
    global _fallback_reason
    from utils.systemone_settings import get_backend

    from .clef_runtime import MAX_LENGTH
    from .native_worker import NativeContextOverflow

    from utils.hardware import hardware

    if checkpoint.layout != "laya" and hardware.is_apple_silicon():
        # A request, unlike a settings read, waits for device detection, so the first one already knows of MLX.
        hardware.get_device()
    state_images = _state_has_images(state)
    target, reason = select(checkpoint, images, questions, state_images = state_images, audio = audio)
    if not _stock_laya(checkpoint):
        _fallback_reason = reason
    try:
        if _is_laya_export(target):
            _refuse_cut_questions(target, questions)
        return _decide(target, state, questions, images, audio)
    except NativeContextOverflow as exc:
        # Auto answers on PyTorch only what its longer window can hold, and only from a download already made.
        from .catalog import clef_unsupported_reason

        if (
            get_backend() != "auto"
            or checkpoint.layout == GGUF
            or images
            or exc.limit >= MAX_LENGTH
            or clef_unsupported_reason() is not None
            or not is_cached(checkpoint)
        ):
            raise Unavailable(422, "invalid_request_error", str(exc)) from None
        _fallback_reason = "The request is longer than the llama.cpp context."
        return _decide(checkpoint, state, questions, None)
    except Unavailable:
        # A llama-server that turned out not to serve decisions: Auto answers this request on PyTorch.
        if not _is_native(target) or get_backend() != "auto" or images:
            raise
        retry, reason = select(checkpoint, images, questions, state_images = state_images)
        if retry == target:
            raise
        _fallback_reason = reason
        return _decide(retry, state, questions, None)


def _state_has_images(state) -> bool:
    """Image parts of a chat-message state, which llama.cpp reads; they keep a request off the MLX engine (as unsloth-zoo scans them)."""
    messages = state.get("messages") if isinstance(state, dict) else state
    for message in messages if isinstance(messages, list) else []:
        content = message.get("content") if isinstance(message, dict) else None
        for part in content if isinstance(content, list) else []:
            if isinstance(part, dict) and part.get("type") == "image_url":
                return True
    return False


def _decide(
    checkpoint: Checkpoint,
    state,
    questions: dict[str, dict[str, Any]],
    images: list[str] | None = None,
    audio: list[str] | None = None,
) -> dict[str, Any]:
    global _last_used, _device_name
    agent = _agent_for(checkpoint, LOAD_WAIT_S)
    if not _run_lock.acquire(timeout = RUN_WAIT_S):
        raise Unavailable(529, "overloaded", "System One is busy; retry shortly", retry_after = 1)
    try:
        if agent is None or _agent is not agent or _loaded != checkpoint:
            raise Unavailable(
                503, "model_loading", f"{checkpoint.name} is reloading", retry_after = 5
            )
        if _is_native(checkpoint):
            result = _decide_native(checkpoint, agent, state, questions, images)
            _last_used = time.monotonic()
            return result
        if _is_mlx(checkpoint):
            result = agent.decide(state, questions, images, audio)
            _last_used = time.monotonic()
            return {
                "model": checkpoint.name,
                "answers": {name: _wire_answer(result["answers"][name]) for name in questions},
                "usage": {"input_tokens": int(result["usage"]["input_tokens"]), "output_tokens": 0},
                "truncated": False,
                "_backend": MLX,
            }
        if checkpoint.layout == "clef":
            return _decide_clef(checkpoint, agent, state, questions)
        laya_questions = {name: _to_laya(q, name) for name, q in questions.items()}
        try:
            if isinstance(agent, LayaWorker):
                result, truncated = agent.predict(state, laya_questions)
                _device_name = agent.device
            else:
                result, truncated = _predict(agent, state, laya_questions)
        except ValueError as exc:
            raise Unavailable(422, "invalid_request_error", str(exc)) from None
        except LayaWorkerError as exc:
            threading.Thread(target = _evict_agent, args = (agent,), daemon = True).start()
            raise Unavailable(503, "model_unavailable", str(exc), retry_after = 5) from None
    finally:
        _run_lock.release()
    return {
        "model": checkpoint.name,
        "answers": {name: _wire_answer(result["answers"][name]) for name in questions},
        "usage": {"input_tokens": int(result["usage"]["input_tokens"]), "output_tokens": 0},
        "truncated": truncated,
    }


def _decide_clef(checkpoint: Checkpoint, agent, state, questions) -> dict[str, Any]:
    from .clef_runtime import ClefWorkerError
    try:
        result = agent.decide(state, questions)
    except ValueError as exc:
        raise Unavailable(422, "invalid_request_error", str(exc)) from None
    except ClefWorkerError as exc:
        # A dead or hung worker is dropped, so the next request starts a fresh one.
        threading.Thread(target = _evict_agent, args = (agent,), daemon = True).start()
        raise Unavailable(503, "model_unavailable", str(exc), retry_after = 5) from None
    return {
        "model": checkpoint.name,
        "answers": {name: _wire_answer(result["answers"][name]) for name in questions},
        "usage": {"input_tokens": int(result["input_tokens"]), "output_tokens": 0},
        "truncated": bool(result["truncated"]),
    }


def _decide_native(checkpoint: Checkpoint, agent, state, questions, images) -> dict[str, Any]:
    from .native_worker import NativeContextOverflow, NativeError
    try:
        result = agent.decide(state, questions, images)
    except NativeContextOverflow:
        raise
    except NativeError as exc:
        if exc.retire:
            # A crashed or misbehaving server is retired; the next request starts a fresh one.
            threading.Thread(target = _evict_agent, args = (agent,), daemon = True).start()
        raise Unavailable(
            exc.status, exc.error_type, str(exc), retry_after = 5 if exc.retire else None
        ) from None
    return {
        "model": checkpoint.name,
        "answers": {name: _wire_answer(result["answers"][name]) for name in questions},
        "usage": {"input_tokens": int(result["input_tokens"]), "output_tokens": 0},
        "truncated": bool(result["truncated"]),
        "_backend": LLAMA_CPP,
    }


def _evict_agent(agent) -> None:
    global _agent, _loaded, _device_name, _generation
    # Waits for _decide to release _run_lock.
    with _run_lock:
        if _agent is not agent:
            return
        _agent = _loaded = _device_name = None
        _generation += 1
    _close(agent)


def status() -> dict[str, Any]:
    with _state_lock:
        failure = _failure if _failure and time.monotonic() < _failure[2] else None
        return {
            "loaded_model": _loaded.name if _loaded else None,
            "loaded_backend": _loaded.backend if _loaded else None,
            # Laya never falls back, so a reason left by an earlier Clef is not its status.
            "fallback_reason": _fallback_reason if _loaded and _loaded.layout != "laya" else None,
            "device": _device_name,
            "holds_gpu": _holds_gpu(),
            "loading_model": _loading.name if _loading else None,
            # Kept for the settings API: laya is vendored, so there is never an install in flight.
            "installing": False,
            "error": failure[1] if failure else None,
            "error_model": failure[0].name if failure else None,
        }


def unload(reload_after_training: bool = False) -> bool:
    global _agent, _loaded, _device_name, _failure, _generation, _unloads
    # _state_lock too: a load publishes under it, so it can never land just after this unload.
    with _run_lock, _state_lock:
        agent, was_loaded, checkpoint = _agent, _agent is not None, _loaded
        _agent = _loaded = _device_name = None
        _failure = None
        _generation += 1
        _unloads += 1
        unloads = _unloads
    _close(agent)
    # Dropped first, so an in-process MLX or MPS Laya is collected before the allocator caches are cleared.
    del agent
    _release_memory()
    if (
        reload_after_training
        and getattr(checkpoint, "layout", None) == "clef"
        and not _arbitrated(checkpoint)
    ):
        threading.Thread(
            target = _reload_after_training,
            args = (checkpoint, unloads),
            name = "systemone-reload",
            daemon = True,
        ).start()
    return was_loaded


def _clef_preempted() -> bool:
    global _reload_stopped_by
    if _training_active():
        reason = "training"
    elif _reloading is not None and _gpu_taken():
        # A reload nobody asked for yields to a chat, image or video model that claims the GPU meanwhile.
        reason = "gpu"
    else:
        return False
    if _reloading is not None:
        _reload_stopped_by = reason
    return True


def _reload_after_training(checkpoint: Checkpoint, unloads: int) -> None:
    global _reloading, _reload_stopped_by, _failure
    while True:
        while _training_active():
            time.sleep(TRAINING_POLL_S)
        with _state_lock:
            if _unloads != unloads or _agent is not None or _loading is not None or _gpu_taken():
                return
            _reloading = checkpoint
            _reload_stopped_by = None
        try:
            loader = _ensure_loading(checkpoint, unloads)
            if loader is not None:
                loader.join()
        except Unavailable:
            pass
        finally:
            with _state_lock:
                _reloading = None
        training = _reload_stopped_by == "training" or _training_active()
        if training or _reload_stopped_by == "gpu" or _gpu_taken():
            # Another run, or a chat, image or video model, took the GPU while this loaded: not a failure to report.
            with _state_lock:
                if _failure is not None and _failure[0] == checkpoint:
                    _failure = None
        if not training:
            break
    agent = _agent
    if agent is not None and _loaded == checkpoint:
        from .clef_runtime import ClefWorkerError

        # The first decision builds the GPU kernels; done now, a request does not wait for it.
        try:
            if not _gpu_taken():
                agent.decide("ready", {"ready": {"type": "noul"}})
        except ClefWorkerError:
            logger.info(
                "Retiring %s: its worker failed the warm-up", checkpoint.name, exc_info = True
            )
            _evict_agent(agent)
            return
        except Exception:
            logger.debug("Could not warm up %s", checkpoint.name, exc_info = True)
        if _gpu_taken():
            logger.info("Retiring %s: another model took the GPU as it reloaded", checkpoint.name)
            _evict_agent(agent)


def shutdown() -> None:
    """Studio shutdown: end a llama.cpp decision server, Laya or Clef worker (in-process models go with the process)."""
    global _agent, _loaded, _device_name, _generation, _unloads
    from .clef_runtime import ClefAgent

    with _state_lock:
        timer = _idle_timer
    if timer is not None:
        timer.cancel()
    with _run_lock:
        _unloads += 1
        if not _is_native(_loaded) and not isinstance(_agent, (LayaWorker, ClefAgent)):
            return
        agent, _agent, _loaded, _device_name = _agent, None, None, None
        _generation += 1
    _close(agent)