Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

40.2 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Malware / unsafe-file gate for model loads. The ``trust_remote_code`` consent gate covers the ``auto_map`` Python vector; this covers the other one, a malicious pickle inside a weight file, which executes during ``from_pretrained`` deserialization even with ``trust_remote_code=False``. It reads Hugging Face's OWN scan (picklescan + ClamAV) via ``model_info(securityStatus=True).security_repo_status``. METADATA-ONLY: it never downloads, opens, or unpickles the flagged files. Policy: a hard, non-approvable block whenever ``filesWithIssues`` lists a non-``safe`` level, regardless of ``scansDone`` (often false even for clean repos); unknown/future levels fail CLOSED so Hub schema drift cannot silently allow a bad verdict, and only a small allowlist of clean / not-yet-scanned levels is non-blocking, while an unavailable status falls back to local inspection for a pinned cache snapshot and otherwise fails open. Scoped to the load-path RCE vector: a root-level (or load-subdir-level) code-executing file, since inert formats (safetensors / gguf / config / text) and subdirectory pickles that no root weight-index references are not loaded and do not block, while an index-referenced shard does, wherever it lives. That blocks real malware (eicar's root ``*.pkl``/``*.dat``) without false-blocking repos like ``nvidia/Nemotron-H-8B-Base-8K`` (flagged NeMo pickles under ``nemo/`` that no index lists). No first-party exemption (scoping is by load path/format, not org), and local paths are skipped (no Hub scan) while a remote ``*.gguf``-named repo is still scanned so a repo cannot dodge the gate by suffixing its name."""

import json
import os
import posixpath
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional

from loggers import get_logger

logger = get_logger(__name__)

# Pickle-format weight files (plain or sharded) that execute code on load; safetensors/gguf are inert. Grouped by weight family so an inert safetensors only suppresses the pickle it replaces.
_PICKLE_WEIGHT_RE = re.compile(
    r"^(model|pytorch_model|adapter_model|consolidated)(-\d+-of-\d+)?"
    r"\.(bin|pt|pth|ckpt|pkl|pickle)$",
    re.IGNORECASE,
)

# Non-blocking levels: clean or not-yet-finished. Anything else blocks, so schema drift fails CLOSED.
_NONBLOCKING_LEVELS = frozenset(
    {"", "safe", "pending", "scanning", "queued", "unscanned", "error", "unknown", "none"}
)

# Suffixes that cannot execute code on load (safetensors, gguf, text/markup/images).
_INERT_SUFFIXES = frozenset(
    {
        ".safetensors",
        ".gguf",
        ".json",
        ".txt",
        ".md",
        ".rst",
        ".yaml",
        ".yml",
        ".png",
        ".jpg",
        ".jpeg",
        ".gif",
        ".webp",
        ".svg",
        ".bmp",
        ".gitattributes",
        ".gitignore",
    }
)

# Source files are not deserialized by a weight load; executable repo code runs only via auto_map, the consent gate's domain, else a flagged helper/train script would false-block.
_SOURCE_SUFFIXES = frozenset({".py", ".pyc", ".pyx", ".pyi"})


# Torch-family weight indexes: from_pretrained feeds each shard they name to load_state_dict, which torch.load()s (pickle) any shard not ending in .safetensors. A pytorch index is superseded by a base safetensors; a safetensors index IS the chosen archive. tf/flax are inert.
_TORCH_INDEX_FILES = ("pytorch_model.bin.index.json", "model.safetensors.index.json")

# Root weight-index files: a flagged subdir pickle is a load vector iff a root index names it.
_TRANSFORMERS_INDEX_FILES = (
    "pytorch_model.bin.index.json",
    "model.safetensors.index.json",
    "tf_model.h5.index.json",
    "flax_model.msgpack.index.json",
)


def _escapes_folder(name: str) -> bool:
    spelled = name.replace("\\", "/")
    if spelled.startswith("/") or (len(spelled) > 1 and spelled[1] == ":"):
        return True
    # weights/../weights/shard.bin stays inside; only a name that normalizes above the folder escapes.
    normalized = posixpath.normpath(spelled)
    return normalized == ".." or normalized.startswith("../")


def _confinement_roots(folder: Path) -> list:
    # Plus a Hub snapshot's models--org--repo folder: its files link into that repo's blobs.
    root = os.path.realpath(folder)
    roots = [root]
    ref = _hf_cache_snapshot_ref(root)
    if ref is not None:
        roots.append(os.path.realpath(ref[2].parent.parent))
    return roots


def _within(path: str, root: str) -> bool:
    path, root = os.path.normcase(path), os.path.normcase(root)
    try:
        return os.path.commonpath([path, root]) == root
    except ValueError:
        return False


def _climbs_through_link(folder: Path, name: str) -> bool:
    # POSIX follows a link before applying "..", so link/../x opens x beside the link's target.
    # Windows (and ntpath.realpath) collapse ".." first, so both sides agree there.
    parts = name.replace("\\", "/").split("/")
    if ".." not in parts:
        return False
    head = "/".join(parts[: len(parts) - parts[::-1].index("..")])
    try:
        landing = os.path.realpath(os.path.join(folder, head))
        roots = _confinement_roots(folder)
    except (OSError, ValueError):
        return True
    return not any(_within(landing, root) for root in roots)


def _lands_outside(folder: Path, module: Path, name: str) -> bool:
    spelled = name.replace("\\", "/")
    if spelled.startswith("/") or name[1:2] == ":":
        return True
    try:
        landing = os.path.realpath(os.path.join(os.path.realpath(module), name))
        roots = _confinement_roots(folder)
    except (OSError, ValueError):
        return True
    return not any(_within(landing, root) for root in roots)


def _same_folder(path: Path, folder: Path) -> bool:
    try:
        return os.path.normcase(os.path.realpath(path)) == os.path.normcase(
            os.path.realpath(folder)
        )
    except (OSError, ValueError):
        return False


def _escapes(folder: Path, name: str) -> bool:
    return _escapes_folder(name) or _climbs_through_link(folder, name)


def _in_subfolder(sub: str, name: str) -> str:
    # A rooted or drive-qualified name ignores the subfolder it is joined onto.
    spelled = name.replace("\\", "/")
    if spelled.startswith("/") or name[1:2] == ":":
        return name
    return posixpath.join(sub.replace("\\", "/"), name)


def _weight_names(folder: Path) -> list:
    names = []
    for index_name in _TORCH_INDEX_FILES:
        try:
            payload = json.loads((folder / index_name).read_text(encoding = "utf-8-sig"))
        except (OSError, ValueError):
            continue
        weight_map = payload.get("weight_map") if isinstance(payload, dict) else None
        if isinstance(weight_map, dict):
            names.extend(v for v in weight_map.values() if isinstance(v, str))
    try:
        config = json.loads((folder / "config.json").read_text(encoding = "utf-8-sig"))
    except (OSError, ValueError):
        config = None
    weights = config.get("transformers_weights") if isinstance(config, dict) else None
    if isinstance(weights, str):
        weights = [weights]
    for name in weights if isinstance(weights, (list, tuple)) else []:
        if not isinstance(name, str):
            continue
        names.append(name)
        # A custom index named here is read like the standard ones.
        if name.endswith(".index.json") and not _escapes(folder, name):
            try:
                payload = json.loads((folder / name).read_text(encoding = "utf-8-sig"))
            except (OSError, ValueError):
                continue
            weight_map = payload.get("weight_map") if isinstance(payload, dict) else None
            if isinstance(weight_map, dict):
                names.extend(v for v in weight_map.values() if isinstance(v, str))
    return names


def escaping_weight_references(folder) -> list:
    """Weight names a local model folder's index files or config.json ``transformers_weights``
    give that point outside the folder: transformers joins each onto the folder as is. Judged by
    the text, so Hub cache snapshots whose files link into the blob store still pass, except that
    a ``..`` is judged where the kernel sends it. A Sentence Transformers folder's modules.json
    module folders are read too, judged against the model folder."""
    try:
        folder = Path(str(folder)).expanduser()
        if not folder.is_dir():
            return []
    except (OSError, ValueError):
        return []
    names = _weight_names(folder)
    outside = []
    try:
        modules = json.loads((folder / "modules.json").read_text(encoding = "utf-8-sig"))
    except (OSError, ValueError):
        modules = None
    for module in modules if isinstance(modules, list) else []:
        sub = module.get("path") if isinstance(module, dict) else None
        if not isinstance(sub, str):
            continue
        # Before the self-folder skip: "..", "../.." and "/" are only dots and slashes too.
        if _escapes(folder, sub):
            names.append(sub)
            continue
        # Sentence Transformers opens Path(folder, sub): "\\" separates on Windows only, so judge
        # both spellings. Only a path normalizing to "." is the model root; "..." is a real folder.
        windows = sub.replace("\\", "/")
        posix = "/".join(part.replace("\\", "_") for part in sub.split("/"))
        spellings = {posixpath.normpath(p) for p in (windows, posix) if p} - {"."}
        if any(_escapes_folder(spelled) for spelled in spellings):
            names.append(sub)
            continue
        if not _same_folder(folder / sub, folder / os.path.normpath(sub)):
            # link/.. or link/../x lands beside the link's target: judge names from there.
            outside.extend(
                _in_subfolder(sub, name)
                for name in _weight_names(folder / sub)
                if _lands_outside(folder, folder / sub, name)
            )
        elif spellings:
            module_names = _weight_names(folder / sub)
            for spelled in spellings:
                names.extend(_in_subfolder(spelled, name) for name in module_names)
    # Spark-TTS / BiCodec load their transformers model from LLM/.
    if (folder / "LLM").is_dir():
        names.extend(_in_subfolder("LLM", name) for name in _weight_names(folder / "LLM"))
    names = [name for name in names if _escapes(folder, name)]
    return list(dict.fromkeys(names + outside))


def _normalize_repo_path(path: str) -> str:
    """Strip ``./`` prefixes and normalize separators for repo-relative comparison."""
    p = (path or "").strip().replace("\\", "/")
    while p.startswith("./"):
        p = p[2:]
    return p


def _file_suffix(path: str) -> str:
    """Lowercase ``.ext`` of the basename, or ``""`` if none."""
    base = _normalize_repo_path(path).rsplit("/", 1)[-1]
    return "." + base.rsplit(".", 1)[1].lower() if "." in base else ""


def _hf_cache_snapshot_ref(local_path: str) -> Optional[tuple[str, str, Path]]:
    """Return provenance for an HF-cache snapshot path, else ``None``. An inactive Unsloth cache loads by its snapshot path but keeps the ``models--org--repo/snapshots/<rev>`` layout, so the gate recovers its provenance and scans that exact commit instead of exempting it."""
    try:
        path = Path(local_path).resolve(strict = False)
    except (OSError, ValueError):
        return None
    for parent in path.parents:
        if parent.name != "snapshots":
            continue
        encoded = parent.parent.name
        if not encoded.startswith("models--"):
            return None
        repo_id = encoded.removeprefix("models--").replace("--", "/")
        if not repo_id:
            return None
        revision = path.relative_to(parent).parts[0]
        return repo_id, revision, parent / revision
    return None


def _load_relative_path(norm: str, load_subdirs) -> str:
    """``norm`` relative to a ``from_pretrained`` load root. Some loads read from a snapshot SUBDIRECTORY (Spark-TTS / BiCodec load ``<snapshot>/LLM``), where a file directly under the subdir is root-level, not nested. Strips the matching load-subdir prefix, or returns ``norm`` unchanged when it is not under one."""
    for subdir in load_subdirs or ():
        prefix = _normalize_repo_path(subdir).strip("/")
        if prefix and norm.startswith(prefix + "/"):
            return norm[len(prefix) + 1 :]
    return norm


def _index_prefixes(load_subdirs) -> tuple:
    """Prefixes to look for weight-index files under: repo root plus each load subdir."""
    prefixes = [""]
    for subdir in load_subdirs or ():
        p = _normalize_repo_path(subdir).strip("/")
        if p:
            prefixes.append(p + "/")
    return tuple(prefixes)


def _indexed_shard_paths(
    model_name: str,
    hf_token: Optional[str],
    load_subdirs = (),
    revision: Optional[str] = None,
):
    """Repo-relative weight paths a load could fetch via weight-index files. Returns a set (empty when the repo ships no index files, a definitive "nothing sharded") or None when the lookup was inconclusive (transient error) so the caller treats a flagged subdir pickle conservatively. Reads only small JSON indexes, never weights. Indexes are looked up at the root and each ``load_subdirs`` root, with ``weight_map`` entries re-prefixed to repo-relative paths; ``revision`` scopes to a cached commit."""
    import json

    try:
        from huggingface_hub import hf_hub_download
        from huggingface_hub.utils import EntryNotFoundError
        from utils.hf_cache_settings import active_hf_hub_cache
        from utils.hf_probe import hf_file_definitely_absent

        from hub.utils.hf_tokens import call_hub_with_anonymous_retry
    except Exception:
        return None

    paths: set = set()
    inconclusive = False
    for prefix in _index_prefixes(load_subdirs):
        for filename in _TRANSFORMERS_INDEX_FILES:
            # Most repos are unsharded, so avoid caching expected 404s for optional indexes.
            if hf_file_definitely_absent(
                model_name, prefix + filename, revision = revision, token = hf_token or None
            ):
                continue
            try:
                index_path = call_hub_with_anonymous_retry(
                    hf_hub_download,
                    hf_token or None,
                    model_name,
                    prefix + filename,
                    revision = revision,
                    cache_dir = active_hf_hub_cache(),
                )
            except EntryNotFoundError:
                continue
            except Exception:
                inconclusive = True
                continue
            try:
                weight_map = (json.loads(open(index_path, encoding = "utf-8-sig").read()) or {}).get(
                    "weight_map"
                ) or {}
                for shard in weight_map.values():
                    shard_norm = _normalize_repo_path(str(shard))
                    # weight_map paths are relative to the index file's directory.
                    if prefix and not shard_norm.startswith(prefix):
                        shard_norm = prefix + shard_norm
                    paths.add(shard_norm)
            except Exception:
                inconclusive = True
    # Any transient failure -> inconclusive (the shard could be listed only by the index we could not read), so fail closed. No index files -> empty set, a definitive "nothing sharded".
    if inconclusive:
        return None
    return paths


# Two-timeout metadata fetch, mirroring hub.workers.hf_download._retry_metadata_fetch.
_REQUEST_TIMEOUT = 10.0
_RETRY_TIMEOUT = 20.0

# Why a load is hard-blocked, strongest first: a finding outranks a check that could not run.
UNSAFE_FILES = "unsafe_files"
CRITICAL_FINDINGS = "critical_findings"
UNSCANNED_WEIGHTS = "unscanned_weights"
MANAGED_REFUSED = "managed_refused"
NOT_ON_SOURCE = "not_on_source"
UNSCANNABLE = "unscannable"
_BLOCK_CAUSES = (
    UNSAFE_FILES,
    CRITICAL_FINDINGS,
    UNSCANNED_WEIGHTS,
    MANAGED_REFUSED,
    NOT_ON_SOURCE,
    UNSCANNABLE,
)


def strongest_block(blocks: list) -> tuple:
    """The ``(cause, target)`` to report from every gate's block; an unnamed cause is unscannable."""
    return min(
        ((cause if cause in _BLOCK_CAUSES else UNSCANNABLE, target) for cause, target in blocks),
        key = lambda block: _BLOCK_CAUSES.index(block[0]),
        default = (None, None),
    )


@dataclass
class FileSecurityDecision:
    """Outcome of the Hub security scan for one model repo."""

    model_name: str
    blocked: bool
    unsafe_files: list = field(default_factory = list)
    reason: str = ""
    block_cause: Optional[str] = None

    def response_payload(self) -> dict:
        """Machine-readable detail merged into the preflight payload the dialog reads."""
        return {
            "unsafe_files": self.unsafe_files,
            "security_blocked": self.blocked,
            "reason": self.reason,
            "block_cause": self.block_cause,
        }


def security_load_subdirs(
    model_name: str,
    hf_token: Optional[str] = None,
    local_files_only: bool = False,
) -> tuple:
    """Snapshot subdirectories a load calls ``from_pretrained`` on, for scoping the scan. Most models load from the root (``()``); Spark-TTS / BiCodec load ``<snapshot>/LLM``, so ``LLM/`` is a load root for them. Metadata-only (tokenizer special tokens), cached. ``local_files_only`` skips the remote tokenizer fetch, and callers deciding whether a cache already on disk is usable must pass it: that work is meant to be pure filesystem, and a hung hub would otherwise block local snapshot resolution."""
    try:
        from utils.models.model_config import detect_audio_type, load_model_defaults
        if (
            detect_audio_type(model_name, hf_token = hf_token, local_files_only = local_files_only)
            == "bicodec"
        ):
            return ("LLM",)
        # Tokenizer detection can fail (network/gated/unresolved alias); the YAML default also pins the audio type, so fall back to it, else a flagged LLM/ pickle reads as an ignored subdir artifact.
        if (load_model_defaults(model_name) or {}).get("audio_type") == "bicodec":
            return ("LLM",)
    except Exception:
        pass
    return ()


def load_scan_target(model_name: str, load_subdirs: tuple) -> tuple:
    """Map a load alias to the ``(repo_id, load_subdirs)`` the load actually fetches. The Spark-TTS / BiCodec alias ``<parent>/LLM`` is downloaded by the trainer as ``unsloth/<parent>`` and loaded from ``LLM/``, so scan that repo with ``LLM`` as a load root (the literal alias 404s and fails open). Everything else is unchanged."""
    try:
        from utils.paths import is_local_path
        if is_local_path(model_name):
            return model_name, load_subdirs
    except Exception:
        return model_name, load_subdirs
    name = (model_name or "").strip().strip("/")
    # Rewrite ONLY a registry-known bicodec alias: "evil/LLM" would scan unsloth/evil and fail open.
    if name.endswith("/LLM") and name.count("/") == 1:
        try:
            from utils.models.model_config import load_model_defaults
            if (load_model_defaults(name) or {}).get("audio_type") == "bicodec":
                parent = name[: -len("/LLM")]
                return f"unsloth/{parent}", tuple(dict.fromkeys((*load_subdirs, "LLM")))
        except Exception:
            pass
    return model_name, load_subdirs


def _fetch_security_status(
    model_name: str,
    hf_token: Optional[str],
    revision: Optional[str] = None,
):
    """``security_repo_status`` (a dict) or None if unavailable. Hub metadata only; retries once on a transient error, then returns None so the caller can apply its local-fallback policy. ``revision`` scopes the scan to a specific cached commit (else the default branch)."""
    from huggingface_hub import model_info as hf_model_info
    from hub.utils.hf_tokens import call_hub_with_anonymous_retry

    token_arg = hf_token if hf_token else False
    last_exc = None
    for attempt, timeout in enumerate((_REQUEST_TIMEOUT, _RETRY_TIMEOUT)):
        try:
            # A refused token still gets the public repo's scan, as the worker's load will.
            info = call_hub_with_anonymous_retry(
                hf_model_info,
                token_arg,
                model_name,
                revision = revision,
                securityStatus = True,
                timeout = timeout,
            )
            return getattr(info, "security_repo_status", None)
        except Exception as exc:
            last_exc = exc
            if attempt == 0:
                continue
    logger.debug(
        "HF security scan unavailable for '%s' (%s).",
        model_name,
        type(last_exc).__name__ if last_exc else "unknown",
    )
    return None


def _st_load_roots(snapshot: Path, load_subdirs = ()) -> list:
    """Directories a SentenceTransformer load deserializes weights from: the snapshot root plus each module path in modules.json and each explicit ``from_pretrained`` load subdirectory. Local, no network. Mirrors the online gate (which ignores unreferenced nested pickles the loader never opens) so the offline gate doesn't over-block."""
    roots = [snapshot]

    def _add_repo_root(raw_path: Any) -> None:
        path = _normalize_repo_path(str(raw_path)).strip("/")
        if not path or ".." in path.split("/"):
            return
        candidate = snapshot.joinpath(*path.split("/"))
        if candidate not in roots:
            roots.append(candidate)

    for subdir in load_subdirs or ():
        _add_repo_root(subdir)
    try:
        import json
        modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8-sig"))
    except (OSError, ValueError):
        return roots
    for module in modules if isinstance(modules, list) else ():
        if not isinstance(module, dict):
            continue
        path = str(module.get("path", "")).strip()
        native_path = Path(path)
        if not path:
            continue
        if (
            native_path.is_absolute()
            or native_path.drive
            or native_path.root
            or ".." in native_path.parts
        ):
            raise OSError("unsafe SentenceTransformer module path")
        candidate = snapshot / native_path
        if candidate not in roots:
            roots.append(candidate)
    return roots


def _snapshot_module_subdirs(snapshot: Path) -> tuple[str, ...]:
    return tuple(root.relative_to(snapshot).as_posix() for root in _st_load_roots(snapshot)[1:])


def _indexed_pickle_shards(index_path: Path, root: Path, snapshot: Path) -> list:
    """Shards a torch weight index points a ``from_pretrained`` load at that load_state_dict would torch.load (pickle): every ``weight_map`` target NOT ending in ``.safetensors``, whatever its stem (an arbitrary name like ``shards/payload`` still deserializes). Resolved relative to the index dir (``root``) like the loader, so a shard in a nested dir is followed (iterdir misses it). Lexical only, never ``Path.resolve()`` (HF snapshot files symlink into ``blobs/``, so resolving escapes the snapshot and false-blocks every shard). Raises OSError -> caller fails CLOSED on an unreadable/invalid index or a target escaping the snapshot."""
    import json
    import os

    try:
        # JSON is UTF-8 by spec; pin it so a non-ASCII index is not misdecoded under Windows' cp1252.
        parsed = json.loads(index_path.read_text(encoding = "utf-8-sig"))
    except (OSError, ValueError) as exc:
        raise OSError(f"unreadable weight index: {index_path}") from exc
    weight_map = parsed.get("weight_map") if isinstance(parsed, dict) else None
    if not isinstance(weight_map, dict):
        return []  # no dict weight_map -> the loader resolves no shards from this index
    snapshot_norm = os.path.normpath(str(snapshot))
    shards = []
    for shard in weight_map.values():
        raw = str(shard)
        if not raw:
            continue
        # Join the RAW weight_map value like from_pretrained's os.path.join: on POSIX a backslash is a literal filename char, so normalizing it would probe a different path than the loader opens.
        joined = os.path.normpath(os.path.join(str(root), raw))
        if joined != snapshot_norm and not joined.startswith(snapshot_norm + os.sep):
            raise OSError(f"weight index escapes the snapshot: {index_path}")
        shard_path = Path(joined)
        # Case-SENSITIVE, mirroring load_state_dict's own endswith(".safetensors"): payload.SAFETENSORS falls to torch.load.
        if not shard_path.name.endswith(".safetensors") and shard_path.is_file():
            shards.append(shard_path)
    return shards


def _loader_resolves(root: Path, name: str) -> bool:
    """True iff from_pretrained would open ``name`` under ``root``. ``is_file()`` honors the platform (case-sensitive on Linux, case-insensitive on Windows/macOS), so it mirrors the loader's own lookup: an oddly-cased decoy counts as an alternative only where the loader would truly open it. A name-fold instead would let an uppercase MODEL.SAFETENSORS suppress the scan on Linux while the loader, asking for the canonical lowercase name, silently falls through to a pickle index."""
    return (root / name).is_file()


def _cached_pickle_weight_files(snapshot: Path, load_subdirs = ()) -> list:
    """Pickle weight files a SentenceTransformer/Transformers load deserializes from snapshot's ST load roots, EXCLUDING those whose weight family also ships an inert safetensors in the same dir (the loader prefers it): a base pickle is suppressed only by a base model.safetensors, an adapter pickle only by adapter_model.safetensors, and an unrelated safetensors is no substitute. Covers both direct-child pickles AND pickle shards referenced by a local weight index (which the loader follows into nested dirs, matching the online gate). Raises OSError, so the caller fails CLOSED, if the snapshot root or a weight index is unreadable, or an index reference escapes the snapshot."""
    blocked = []
    seen = set()

    def _add(path: Path):
        key = str(path)
        if key not in seen:
            seen.add(key)
            blocked.append(path)

    for root in _st_load_roots(snapshot, load_subdirs):
        try:
            entries = [p for p in root.iterdir() if p.is_file()]
        except OSError:
            if root == snapshot:
                raise
            continue
        # Safetensors alternatives the loader would actually resolve, never a bare name-fold, which fails OPEN: a base pickle is replaced only by a base safetensors, and model.safetensors outranks both indexes.
        has_direct_base_safetensors = _loader_resolves(root, "model.safetensors")
        has_base_safetensors = has_direct_base_safetensors or _loader_resolves(
            root, "model.safetensors.index.json"
        )
        has_adapter_safetensors = _loader_resolves(root, "adapter_model.safetensors")
        for path in entries:
            if not _PICKLE_WEIGHT_RE.match(path.name):
                continue
            is_adapter = path.name.lower().startswith("adapter_model")
            has_alternative = has_adapter_safetensors if is_adapter else has_base_safetensors
            if not has_alternative:
                _add(path)
        # A torch weight index makes from_pretrained load nested shards iterdir never sees, torch.loading any not ending in .safetensors, so probe the canonical index name with the loader's own lookup (an oddly-cased artifact it would never open must not block). model.safetensors wins over both.
        for index_name in _TORCH_INDEX_FILES:
            if not _loader_resolves(root, index_name):
                continue
            if has_direct_base_safetensors:
                continue
            if index_name == "pytorch_model.bin.index.json" and has_base_safetensors:
                continue
            for shard_path in _indexed_pickle_shards(root / index_name, root, snapshot):
                _add(shard_path)
    return blocked


def _evaluate_local_snapshot(
    model_name: str,
    snapshot_path: Optional[Path] = None,
    *,
    context: str,
    load_subdirs = (),
) -> FileSecurityDecision:
    """Inspect cached weights when the Hub scan cannot be used."""
    if snapshot_path is None:
        from utils.utils import hf_cache_snapshot_dir
        try:
            snapshot = hf_cache_snapshot_dir(model_name)
        except Exception:
            logger.warning(
                "Local security fallback (%s): could not resolve the cache for '%s'; blocking.",
                context,
                model_name,
            )
            return FileSecurityDecision(
                model_name,
                True,
                reason = f"{context}; could not inspect the local cache",
                block_cause = UNSCANNABLE,
            )
    else:
        try:
            snapshot = snapshot_path.resolve(strict = True)
            if not snapshot.is_dir():
                raise OSError("snapshot is not a directory")
        except (OSError, RuntimeError, ValueError):
            logger.warning(
                "Local security fallback (%s): could not resolve selected snapshot for "
                "'%s'; blocking.",
                context,
                model_name,
            )
            return FileSecurityDecision(
                model_name,
                True,
                reason = f"{context}; could not inspect the selected model snapshot",
                block_cause = UNSCANNABLE,
            )

    if snapshot is None:
        return FileSecurityDecision(
            model_name,
            False,
            reason = f"{context}; nothing cached to load",
        )

    try:
        pickles = _cached_pickle_weight_files(snapshot, load_subdirs)
    except OSError:
        logger.warning(
            "Local security fallback (%s): could not read the cache for '%s'; blocking.",
            context,
            model_name,
        )
        return FileSecurityDecision(
            model_name,
            True,
            reason = f"{context}; could not read the local cache",
            block_cause = UNSCANNABLE,
        )

    if not pickles:
        return FileSecurityDecision(
            model_name,
            False,
            reason = f"{context}; cached weights are inert (safetensors/gguf)",
        )

    # Snapshot-relative posix paths (match the online gate; disambiguate same-named pickles).
    rel_paths = sorted(p.relative_to(snapshot).as_posix() for p in pickles)
    names = ", ".join(rel_paths)
    logger.warning(
        "Blocking load of '%s': cached pickle weight(s) could not be malware-scanned "
        "(%s) and have no safetensors alternative (%s).",
        model_name,
        context,
        names,
    )
    return FileSecurityDecision(
        model_name,
        True,
        unsafe_files = [{"path": rel, "level": "unscanned"} for rel in rel_paths],
        reason = (f"{context}; unscanned pickle weights with no safetensors alternative: {names}"),
        block_cause = UNSCANNED_WEIGHTS,
    )


def _modelscope_serves() -> bool:
    try:
        from utils.hub_settings import MODELSCOPE, active_source
    except Exception:
        return False
    return active_source() == MODELSCOPE


def _evaluate_unscanned_listing(
    model_name: str,
    hf_token: Optional[str],
    load_subdirs = (),
) -> FileSecurityDecision:
    """ModelScope publishes no malware scan, so its "unavailable" is permanent, not transient:
    fail closed on pickle weights, as the offline gate does for a cached snapshot."""
    context = "ModelScope has no malware scan"
    try:
        from huggingface_hub import HfApi
        files = HfApi().list_repo_files(model_name, token = hf_token or False)
    except Exception as exc:
        from utils.hf_probe import is_repo_not_found, repo_not_found_message

        if is_repo_not_found(exc):
            from utils.utils import hf_cache_snapshot_dir_for_repo

            # The hub falls back to this snapshot, so its weights are what a load would read.
            snapshot = hf_cache_snapshot_dir_for_repo(model_name)
            if snapshot is not None:
                return _evaluate_local_snapshot(
                    model_name, snapshot, context = context, load_subdirs = load_subdirs
                )
            logger.warning("Blocking load of '%s': it is not on ModelScope.", model_name)
            return FileSecurityDecision(
                model_name,
                True,
                reason = repo_not_found_message(model_name),
                block_cause = NOT_ON_SOURCE,
            )
        logger.warning(
            "Blocking load of '%s': %s and its files could not be listed.", model_name, context
        )
        return FileSecurityDecision(
            model_name,
            True,
            reason = f"{context}; could not list the repository",
            block_cause = UNSCANNABLE,
        )
    by_dir: dict[str, set] = {}
    for path in files:
        folder, _, name = _normalize_repo_path(str(path)).rpartition("/")
        by_dir.setdefault(folder, set()).add(name)
    blocked = []
    roots = {"", *(_normalize_repo_path(str(d)).strip("/") for d in load_subdirs or ())}
    for folder, names in sorted(by_dir.items()):
        if folder not in roots:
            continue
        prefix = f"{folder}/" if folder else ""
        has_base = bool(names & {"model.safetensors", "model.safetensors.index.json"})
        has_adapter = "adapter_model.safetensors" in names
        for name in sorted(names):
            if _PICKLE_WEIGHT_RE.match(name):
                if not (has_adapter if name.lower().startswith("adapter_model") else has_base):
                    blocked.append(prefix + name)
        # Its shards are named freely and torch.loaded unless they end in .safetensors.
        if "pytorch_model.bin.index.json" in names and not has_base:
            blocked.append(prefix + "pytorch_model.bin.index.json")
    if not blocked:
        return FileSecurityDecision(
            model_name, False, reason = f"{context}; no pickle weights to load"
        )
    listed = ", ".join(blocked)
    logger.warning(
        "Blocking load of '%s': %s and it ships pickle weights (%s).", model_name, context, listed
    )
    return FileSecurityDecision(
        model_name,
        True,
        unsafe_files = [{"path": path, "level": "unscanned"} for path in blocked],
        reason = (
            f"{context}; unscanned pickle weights with no safetensors alternative: {listed}. "
            "Switch the model source to Hugging Face in Settings to load it."
        ),
        block_cause = UNSCANNED_WEIGHTS,
    )


def evaluate_file_security(
    model_name: str,
    hf_token: Optional[str] = None,
    *,
    load_subdirs = (),
    local_only_load: bool = False,
) -> FileSecurityDecision:
    """Block a load when HF's security scan flags unsafe serialized files. Call UNCONDITIONALLY before any load (independent of trust_remote_code): a malicious pickle deserializes during ``from_pretrained`` regardless. Metadata-only; when the scan is unavailable, exact cached snapshots receive a local fail-closed inspection while unresolved remote refs remain fail-open. ``load_subdirs`` names subdirs the load calls ``from_pretrained`` on (e.g. ``("LLM",)`` for Spark-TTS / BiCodec, loading ``<snapshot>/LLM``): a flagged file directly under one is root-level there and blocks, and an index inside it is honored when scoping shards. ``local_only_load`` marks an offline load and skips the Hub request."""
    # Scan the repo the load actually fetches; the literal alias 404s and fails open.
    model_name, load_subdirs = load_scan_target(model_name, tuple(load_subdirs))

    # Local paths have no Hub scan, EXCEPT an HF-cache snapshot whose canonical path encodes a repo id + commit: scan that exact commit so an inactive-cache load can't dodge the gate.
    snapshot_revision = None
    selected_snapshot = None
    try:
        from utils.paths import is_local_path
        if is_local_path(model_name):
            cache_ref = _hf_cache_snapshot_ref(model_name)
            if cache_ref is None:
                return FileSecurityDecision(model_name, False, reason = "local path; no Hub scan")
            model_name, snapshot_revision, selected_snapshot = cache_ref
    except Exception:
        # Cannot classify the path -> do not block on that account.
        return FileSecurityDecision(model_name, False, reason = "path check failed; not blocked")

    # Offline: inspect the local cache and fail closed rather than hang on model_info or fail open.
    if local_only_load:
        return _evaluate_local_snapshot(
            model_name,
            selected_snapshot,
            context = "offline",
            load_subdirs = load_subdirs,
        )

    status = _fetch_security_status(model_name, hf_token, revision = snapshot_revision)
    if not isinstance(status, dict):
        if selected_snapshot is not None:
            return _evaluate_local_snapshot(
                model_name,
                selected_snapshot,
                context = "Hub scan unavailable",
                load_subdirs = load_subdirs,
            )
        if _modelscope_serves():
            return _evaluate_unscanned_listing(model_name, hf_token, load_subdirs)
        return FileSecurityDecision(
            model_name, False, reason = "scan unavailable; allowed (fail-open)"
        )
    if selected_snapshot is not None:
        try:
            load_subdirs = tuple(
                dict.fromkeys((*load_subdirs, *_snapshot_module_subdirs(selected_snapshot)))
            )
        except OSError:
            return _evaluate_local_snapshot(
                model_name,
                selected_snapshot,
                context = "invalid cached model metadata",
                load_subdirs = load_subdirs,
            )

    # Block a non-``safe`` flagged file scoped to the load-path RCE vector. Not gated on ``scansDone`` (often false even when clean), and unknown levels fail closed. Subdir pickles and inert formats are not loaded by from_pretrained and do not block; an unavailable status is fail-open only for an unresolved remote ref.
    unsafe = []
    skipped = []
    maybe_shard = []
    for entry in status.get("filesWithIssues") or []:
        if not isinstance(entry, dict):
            continue
        level = str(entry.get("level", "")).lower()
        if level in _NONBLOCKING_LEVELS:
            continue
        path = entry.get("path", "")
        norm = _normalize_repo_path(path)
        suffix = _file_suffix(norm)
        # Path relative to the load root: a file under a load subdir (e.g. LLM/) is root-level there.
        load_rel = _load_relative_path(norm, load_subdirs)
        if not norm or suffix in _INERT_SUFFIXES or suffix in _SOURCE_SUFFIXES:
            # Inert formats cannot execute on load; source code is the consent gate's domain (auto_map).
            skipped.append({"path": path, "level": level})
        elif "/" not in load_rel:
            unsafe.append({"path": path, "level": level})
        else:
            # Subdir pickle: deserialized only if a weight index references it.
            maybe_shard.append({"path": path, "level": level, "norm": norm})

    if maybe_shard:
        indexed = _indexed_shard_paths(
            model_name, hf_token, load_subdirs, revision = snapshot_revision
        )
        for m in maybe_shard:
            # Block if a root index lists this shard, or if the lookup was inconclusive. A definitive "no index / not listed" stays non-blocking (e.g. NeMo nemo/*.distcp).
            if indexed is None or m["norm"] in indexed:
                unsafe.append({"path": m["path"], "level": m["level"]})
            else:
                skipped.append({"path": m["path"], "level": m["level"]})

    if not unsafe:
        if skipped:
            # Flagged files exist, but none the load deserializes -> allow, but log them.
            logger.info(
                "'%s': Hugging Face flagged files, but none are a load-path RCE "
                "vector (subdir/inert); allowing the load. Flagged: %s",
                model_name,
                ", ".join(f"{s['path']}({s['level']})" for s in skipped),
            )
        return FileSecurityDecision(model_name, False, reason = "no unsafe files in the load path")

    names = ", ".join(u["path"] for u in unsafe if u["path"]) or "unknown files"
    logger.warning(
        "Blocking load of '%s': Hugging Face security scan flagged unsafe files (%s).",
        model_name,
        names,
    )
    return FileSecurityDecision(
        model_name,
        True,
        unsafe_files = unsafe,
        reason = f"Hugging Face security scan flagged unsafe files: {names}",
        block_cause = UNSAFE_FILES,
    )