Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

44.6 KB
Raw Blame
"""Native Spectrum targeted-training scan (#266, v0.71.23).

Implements the Spectrum method (arXiv:2406.06623, *Spectrum: Targeted Training
on Signal to Noise Ratio*) as a **pure-numpy, weight-streaming** scan:

- :func:`compute_snr` is the core kernel — singular-value SNR with a
  Marchenko-Pastur noise threshold. It is **transpose-invariant** (the
  singular values of ``W`` and ``W.T`` are identical and the MP aspect ratio
  ``min(n,m)/max(n,m)`` is symmetric), so a GPT-2 ``Conv1D`` weight stored as
  ``[in, out]`` scores the same as a Linear stored ``[out, in]``. The
  "Conv1D-aware" requirement is therefore about *module naming*
  (:func:`classify_module` recognises ``c_attn``/``c_fc``/``c_proj``), not the
  math.
- :func:`iter_weight_matrices` streams ``.safetensors`` shards **one tensor at
  a time** via ``safetensors.safe_open`` — there is NO full model load, so
  peak RSS is the size of the largest single weight matrix. This is what lets
  a 70B's layer SNR be scanned on a CPU box; the reference implementation
  loads the whole model.
- Scan results are cached at ``~/.soup/spectrum/<slug>.json`` (override via
  ``SOUP_SPECTRUM_CACHE_DIR``, containment-checked under ``$HOME`` / ``$CWD`` /
  ``$TMPDIR`` per the v0.36.0 cache-dir policy).

All heavy deps (numpy / torch / safetensors / huggingface_hub) are imported
lazily inside functions so the module — and the ``soup`` CLI — load without
them. LISA (per-step layer sampling) is split to #267.
"""
from __future__ import annotations

import hashlib
import json
import logging
import math
import os
import re
import shutil
import stat
import tempfile
import time
from dataclasses import dataclass
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Iterator,
    Literal,
    Optional,
    Sequence,
    Tuple,
    Union,
)

if TYPE_CHECKING:  # static types only — numpy stays a lazy runtime import
    from numpy.typing import NDArray

logger = logging.getLogger(__name__)

# DoS / sanity caps.
_MAX_TENSORS = 100_000
_MAX_UNFROZEN_PATTERNS = 50_000
_MAX_CACHE_BYTES = 64 * 1024 * 1024  # 64 MiB
_MAX_CACHE_LAYERS = _MAX_TENSORS
# A single 2-D weight above this element count is skipped (and logged) rather
# than fed to a multi-GB SVD — a backstop against a crafted/oversized
# ``--model``. Legitimate mlp/attn matrices (even a 70B's ~8k×29k) are far
# below this; only giant embeddings / adversarial shapes trip it.
_MAX_MATRIX_ELEMENTS = 2**31
_CACHE_SCHEMA = 1

_MLP_MARKERS = ("mlp", "feed_forward", "ffn")
_ATTN_MARKERS = ("self_attn", "attention")
_VALID_MODULE_TYPES = ("mlp", "attn", "other")

_LAYER_IDX_RE = re.compile(r"(?:^|\.)(?:layers|h)\.\d+\.")
_SLUG_RE = re.compile(r"[^A-Za-z0-9._-]+")
_HF_COMMIT_RE = re.compile(r"^[A-Fa-f0-9]{40}$")
# A local snapshot folder's own cache slot: ``<folder name>@<digest of its location>``.
_LOCAL_SLOT_LABEL_CHARS = 64
_LOCAL_SLOT_DIGEST_CHARS = 32

ModulesArg = Union[str, Sequence[str], None]


@dataclass(frozen=True)
class ModelWeightsPlan:
    """A weight source located before Soup creates a second on-disk copy."""

    model: str
    source_dir: str
    weights_dir: str
    source_bytes: int
    materialized_copy_bytes: int
    materialize_bytes: int
    source_files: Tuple[Tuple[str, int, int], ...]
    source_revision: Optional[str] = None

    @property
    def needs_materialization(self) -> bool:
        return self.materialize_bytes > 0


def _np():
    import numpy as np  # lazy — keep the CLI import light

    return np


# ---------------------------------------------------------------------------
# Dataclasses
# ---------------------------------------------------------------------------
ModuleType = Literal["attn", "mlp", "other"]


@dataclass(frozen=True)
class LayerSNR:
    """Per-weight-matrix SNR record."""

    name: str
    module_type: ModuleType
    group: str  # layer-type signature, e.g. "self_attn.q_proj"
    snr: float
    shape: Tuple[int, int]


@dataclass(frozen=True)
class ScanResult:
    """A full model scan: the model id, the requested modules, the layers."""

    model: str
    modules: str  # canonical "all" or comma-joined sorted types
    layers: Tuple[LayerSNR, ...]


# ---------------------------------------------------------------------------
# SNR kernel (pure numpy)
# ---------------------------------------------------------------------------
def estimate_sigma(singular_values: Any) -> float:
    """Estimate the noise std via the inter-quartile range of singular values.

    ``sigma = IQR / 1.349`` (the IQR-to-std factor for a normal) — the
    Spectrum reference estimator.
    """
    np = _np()
    s = np.asarray(singular_values, dtype=np.float64).ravel()
    if s.size == 0:
        return 0.0
    q75, q25 = np.percentile(s, [75, 25])
    return float((q75 - q25) / 1.349)


def marchenko_pastur_threshold(sigma: float, n: int, m: int) -> float:
    """Marchenko-Pastur singular-value noise edge.

    ``threshold = sigma * sqrt((1 + sqrt(beta))**2)`` with
    ``beta = min(n, m) / max(n, m)`` (the Spectrum reference form;
    ``sqrt((1+x)**2) == 1+x`` for ``x >= 0``). Symmetric in ``n``/``m``.
    """
    if n <= 0 or m <= 0:
        return 0.0
    beta = min(n, m) / max(n, m)
    return float(sigma) * math.sqrt((1.0 + math.sqrt(beta)) ** 2)


def compute_snr(matrix: Any) -> float:
    """Signal-to-noise ratio of a single 2-D weight matrix.

    Singular values above the MP threshold are "signal", the rest "noise";
    ``snr = signal_sum / noise_sum`` and the returned ratio is normalised by
    the largest singular value (the Spectrum ``snr_ratio``). Always finite.
    Transpose-invariant. Raises ``ValueError`` on non-2-D input.
    """
    np = _np()
    # Preserve float32/float64 (the streaming path yields float32 to keep peak
    # RSS ≈ the largest single matrix); upcast float16 / bf16-already-float /
    # int to float32 for a stable SVD.
    a = np.asarray(matrix)
    if a.dtype not in (np.float32, np.float64):
        a = a.astype(np.float32)
    if a.ndim != 2:
        raise ValueError(f"compute_snr expects a 2-D matrix, got ndim={a.ndim}")
    if a.size == 0:
        return 0.0
    s = np.linalg.svd(a, compute_uv=False)  # singular values, descending
    if s.size == 0:
        return 0.0
    max_sv = float(s[0])
    if max_sv <= 0.0 or not math.isfinite(max_sv):
        return 0.0
    sigma = estimate_sigma(s)
    threshold = marchenko_pastur_threshold(sigma, a.shape[0], a.shape[1])
    signal_mask = s > threshold
    signal = float(s[signal_mask].sum()) if signal_mask.any() else 0.0
    # An empty below-threshold sum is 0.0; the guard floors it to 1.0 so the
    # ratio stays finite (mirrors the Spectrum reference's noise default).
    noise = float(s[~signal_mask].sum())
    if noise <= 0.0:
        noise = 1.0
    ratio = (signal / noise) / max_sv
    return float(ratio) if math.isfinite(ratio) else 0.0


# ---------------------------------------------------------------------------
# Module classification + grouping
# ---------------------------------------------------------------------------
def classify_module(name: Any) -> Optional[ModuleType]:
    """Classify a parameter name as ``"attn"`` / ``"mlp"`` / ``"other"``.

    Returns ``None`` for non-``.weight`` params (biases, etc.). Recognises
    both Llama-style (``self_attn.q_proj``, ``mlp.gate_proj``) and GPT-2
    ``Conv1D`` (``attn.c_attn``, ``mlp.c_fc``, ``mlp.c_proj``) naming.
    """
    if not isinstance(name, str) or not name.endswith(".weight"):
        return None
    low = name.lower()
    if any(mk in low for mk in _MLP_MARKERS):
        return "mlp"
    if any(mk in low for mk in _ATTN_MARKERS) or ".attn." in low:
        return "attn"
    return "other"


def layer_type_signature(name: str) -> str:
    """Strip the numeric layer index so same-type layers group together.

    ``model.layers.5.self_attn.q_proj.weight`` -> ``self_attn.q_proj``;
    ``transformer.h.3.mlp.c_fc.weight`` -> ``mlp.c_fc``;
    ``lm_head.weight`` -> ``lm_head`` (non-layer params group by themselves).
    """
    match = _LAYER_IDX_RE.search(name)
    base = name[match.end():] if match else name
    for suffix in (".weight", ".bias"):
        if base.endswith(suffix):
            return base[: -len(suffix)]
    return base


def param_prefix(name: str) -> str:
    """Strip a trailing ``.weight`` — the unfreeze pattern is the name prefix."""
    return name[: -len(".weight")] if name.endswith(".weight") else name


# ---------------------------------------------------------------------------
# Module-filter normalisation
# ---------------------------------------------------------------------------
def _normalize_modules(modules: ModulesArg) -> Optional[frozenset[str]]:
    """Return a frozenset of kept types, or ``None`` meaning "keep all"."""
    if modules is None:
        return None
    if isinstance(modules, str):
        parts = [p.strip().lower() for p in modules.split(",")]
    else:
        parts = [str(p).strip().lower() for p in modules]
    parts = [p for p in parts if p]
    if not parts or "all" in parts:
        return None
    for part in parts:
        if part not in _VALID_MODULE_TYPES:
            raise ValueError(
                f"unknown module type {part!r}; choose from "
                f"{', '.join(_VALID_MODULE_TYPES)}, all"
            )
    return frozenset(parts)


def _canonical_modules_str(modules: ModulesArg) -> str:
    keep = _normalize_modules(modules)
    return "all" if keep is None else ",".join(sorted(keep))


# ---------------------------------------------------------------------------
# Safetensors streaming
# ---------------------------------------------------------------------------
def _framework() -> Tuple[str, Callable[[Any], "NDArray[Any]"]]:
    """Pick a safetensors framework + a to-numpy converter.

    Prefer torch (handles bf16, which numpy cannot represent); fall back to
    the pure-numpy framework when torch is absent (bf16 weights then need
    torch — :func:`iter_weight_matrices` raises a friendly error).
    """
    np = _np()
    try:
        import torch
    except Exception:  # pragma: no cover - torch present in dev/CI
        torch = None
    if torch is not None:
        def to_np(tensor: Any) -> "NDArray[Any]":
            return tensor.detach().to(torch.float32).cpu().numpy()

        return "pt", to_np

    def to_np(arr: Any) -> "NDArray[Any]":
        # Keep the native dtype (compute_snr upcasts float16 → float32); a
        # forced float64 would double peak RSS over the streamed matrix.
        return np.asarray(arr)

    return "np", to_np


def _discover_safetensors(weights_dir: str) -> list[str]:
    from soup_cli.utils.checkpoint_files import checkpoint_files

    real = os.path.realpath(weights_dir)
    if not os.path.isdir(real):
        raise FileNotFoundError(f"weights dir not found: {weights_dir}")
    # #1806: the checkpoint's own files, not every *.safetensors. Mistral's
    # consolidated.safetensors carries the same weights under mistral-inference
    # names; scanning it too emitted unfrozen_parameters that match no
    # parameter of the HF model.
    selected = checkpoint_files(real)
    files = []
    for entry in selected.weights:
        full = os.path.join(real, entry)
        # Skip symlinked shards — defence-in-depth against a link redirecting
        # the read (mirrors the project's no-symlink TOCTOU policy).
        if os.path.islink(full):
            logger.warning("spectrum scan: skipping symlinked shard %s", entry)
            continue
        if not os.path.isfile(full):
            continue
        files.append(full)
    if not files:
        raise FileNotFoundError(selected.why_empty(weights_dir))
    return files


def iter_weight_matrices(
    weights_dir: str, *, modules: ModulesArg = "all"
) -> Iterator[Tuple[str, "NDArray[Any]"]]:
    """Stream 2-D float weight matrices from a model dir, one at a time.

    Yields ``(param_name, np.ndarray)`` for every 2-D ``.weight`` tensor whose
    module type passes the ``modules`` filter. The shape is probed via
    ``get_slice(...).get_shape()`` so 1-D norms, filtered-out modules and
    oversized matrices are NEVER materialised — peak RSS is the largest *kept*
    tensor.
    """
    from safetensors import safe_open

    keep = _normalize_modules(modules)
    framework, to_np = _framework()
    count = 0
    for path in _discover_safetensors(weights_dir):
        with safe_open(path, framework=framework) as handle:
            for key in handle.keys():
                module_type = classify_module(key)
                if module_type is None:
                    continue
                if keep is not None and module_type not in keep:
                    continue
                shape = tuple(handle.get_slice(key).get_shape())
                if len(shape) != 2:
                    continue
                if shape[0] * shape[1] > _MAX_MATRIX_ELEMENTS:
                    logger.warning(
                        "spectrum scan: skipping %s — %dx%d exceeds the "
                        "%d-element SVD cap (scan a subset via --modules "
                        "mlp,attn for very large models)",
                        key, shape[0], shape[1], _MAX_MATRIX_ELEMENTS,
                    )
                    continue
                count += 1
                if count > _MAX_TENSORS:
                    raise ValueError(
                        f"scan exceeded the {_MAX_TENSORS}-tensor cap"
                    )
                try:
                    array = to_np(handle.get_tensor(key))
                except Exception as exc:
                    raise RuntimeError(
                        f"could not read tensor {key!r} as float "
                        f"(bf16 weights need torch — pip install "
                        f"'soup-cli[train]'): {type(exc).__name__}"
                    ) from exc
                yield key, array


def scan_weights_dir(
    weights_dir: str, *, modules: ModulesArg = "all"
) -> Tuple[LayerSNR, ...]:
    """Stream a local model dir and compute the SNR of every kept matrix."""
    out = []
    for name, array in iter_weight_matrices(weights_dir, modules=modules):
        out.append(
            LayerSNR(
                name=name,
                module_type=classify_module(name) or "other",
                group=layer_type_signature(name),
                snr=compute_snr(array),
                shape=(int(array.shape[0]), int(array.shape[1])),
            )
        )
    return tuple(out)


# ---------------------------------------------------------------------------
# Selection
# ---------------------------------------------------------------------------
def select_unfrozen_parameters(
    results: Sequence[LayerSNR],
    *,
    top_percent: float,
    modules: ModulesArg = "all",
) -> list[str]:
    """Pick the top ``top_percent`` % of layers *per module type group*.

    Grouping by :func:`layer_type_signature` keeps the unfreeze balanced
    across module types (top-50% of ``q_proj`` layers, top-50% of
    ``down_proj`` layers, ...). At least one layer is kept per group
    (``ceil``). Returns a sorted, de-duplicated list of parameter-name
    prefixes (``.weight`` stripped) ready for ``training.unfrozen_parameters``.
    """
    if isinstance(top_percent, bool) or not isinstance(top_percent, (int, float)):
        raise ValueError("top_percent must be a number in (0, 100]")
    if not (0 < top_percent <= 100):
        raise ValueError("top_percent must be in (0, 100]")

    keep_types = _normalize_modules(modules)
    groups: dict[str, list[LayerSNR]] = {}
    for record in results:
        if keep_types is not None and record.module_type not in keep_types:
            continue
        groups.setdefault(record.group, []).append(record)

    selected: list[str] = []
    for items in groups.values():
        ordered = sorted(items, key=lambda r: (r.snr, r.name), reverse=True)
        keep_n = max(1, math.ceil(len(ordered) * top_percent / 100.0))
        selected.extend(param_prefix(r.name) for r in ordered[:keep_n])

    if len(selected) > _MAX_UNFROZEN_PATTERNS:
        raise ValueError(
            f"selection exceeded the {_MAX_UNFROZEN_PATTERNS}-pattern cap"
        )
    return sorted(set(selected))


# ---------------------------------------------------------------------------
# Cache + slug
# ---------------------------------------------------------------------------
def model_slug(model: str) -> str:
    """Sanitise a model id into a traversal-safe cache filename stem."""
    if not isinstance(model, str) or not model.strip():
        raise ValueError("model must be a non-empty string")
    base = model.strip().replace("\\", "/").rstrip("/").replace("/", "__")
    slug = _SLUG_RE.sub("_", base).replace("..", "_").strip("._-")
    return (slug or "model")[:128]


def default_spectrum_cache_dir() -> str:
    """``~/.soup/spectrum`` — the scan cache root (no side effects)."""
    return os.path.join(os.path.expanduser("~"), ".soup", "spectrum")


def resolve_cache_dir(cache_dir: Optional[str] = None) -> str:
    """Resolve the cache dir (explicit arg > ``SOUP_SPECTRUM_CACHE_DIR`` > default).

    The env override is rejected if it contains C0 control characters or
    escapes ``$HOME`` / ``$CWD`` / ``$TMPDIR`` (silent fall-through to the
    default — an env var is operator config, not API input). The chosen dir
    is created.
    """
    from soup_cli.utils.paths import is_under

    if cache_dir is not None:
        chosen = os.path.realpath(os.path.expanduser(str(cache_dir)))
        os.makedirs(chosen, exist_ok=True)
        return chosen

    override = os.environ.get("SOUP_SPECTRUM_CACHE_DIR")
    if override and not any(ord(ch) < 0x20 for ch in override):
        candidate = os.path.realpath(os.path.expanduser(override))
        bounds = [
            os.path.realpath(os.path.expanduser("~")),
            os.path.realpath(os.getcwd()),
            os.path.realpath(tempfile.gettempdir()),
        ]
        if any(is_under(candidate, bound) for bound in bounds):
            os.makedirs(candidate, exist_ok=True)
            return candidate

    default = default_spectrum_cache_dir()
    os.makedirs(default, exist_ok=True)
    return default


def _cache_path(model: str, cache_dir: Optional[str]) -> str:
    return os.path.join(resolve_cache_dir(cache_dir), model_slug(model) + ".json")


def _atomic_write_json(payload: dict, path: str) -> str:
    """Atomic JSON write into an already-validated cache dir.

    The cache lives under ``$HOME``/``$TMPDIR`` (not cwd), so it deliberately
    does NOT use ``paths.atomic_write_text`` (which enforces cwd containment);
    the dir is bounded by :func:`resolve_cache_dir` and the filename by
    :func:`model_slug`. mkstemp + ``os.replace`` keeps the write atomic.
    """
    parent = os.path.dirname(os.path.abspath(path)) or "."
    os.makedirs(parent, exist_ok=True)
    fd, tmp = tempfile.mkstemp(prefix=".soup.", suffix=".tmp", dir=parent)
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as handle:
            json.dump(payload, handle, indent=2)
        os.replace(tmp, path)
    finally:
        if os.path.exists(tmp):
            try:
                os.unlink(tmp)
            except OSError:
                pass
    return os.path.realpath(path)


def write_cached_scan(result: ScanResult, *, cache_dir: Optional[str] = None) -> str:
    """Persist a :class:`ScanResult` to the cache. Returns the written path."""
    payload = {
        "schema": _CACHE_SCHEMA,
        "model": result.model,
        "modules": result.modules,
        "layers": [
            {
                "name": ls.name,
                "module_type": ls.module_type,
                "group": ls.group,
                "snr": ls.snr,
                "shape": list(ls.shape),
            }
            for ls in result.layers
        ],
    }
    return _atomic_write_json(payload, _cache_path(result.model, cache_dir))


def read_cached_scan(
    model: str, *, modules: ModulesArg = "all", cache_dir: Optional[str] = None
) -> Optional[ScanResult]:
    """Load a cached scan, or ``None`` on miss / corruption / module mismatch."""
    path = _cache_path(model, cache_dir)
    if not os.path.isfile(path):
        return None
    try:
        if os.path.getsize(path) > _MAX_CACHE_BYTES:
            return None
        with open(path, encoding="utf-8") as handle:
            payload = json.load(handle)
    except (OSError, ValueError):
        return None
    if not isinstance(payload, dict) or payload.get("schema") != _CACHE_SCHEMA:
        return None
    if payload.get("modules") != _canonical_modules_str(modules):
        return None
    raw_layers = payload.get("layers")
    if not isinstance(raw_layers, list) or len(raw_layers) > _MAX_CACHE_LAYERS:
        return None
    try:
        layers = tuple(
            LayerSNR(
                name=str(d["name"]),
                module_type=str(d["module_type"]),
                group=str(d["group"]),
                snr=float(d["snr"]),
                shape=(int(d["shape"][0]), int(d["shape"][1])),
            )
            for d in payload["layers"]
        )
        return ScanResult(
            model=str(payload["model"]),
            modules=str(payload["modules"]),
            layers=layers,
        )
    except (KeyError, TypeError, ValueError, IndexError):
        return None


# ---------------------------------------------------------------------------
# Resolve + scan orchestration
# ---------------------------------------------------------------------------
def _weight_file_manifest(
    directory: str,
    *,
    permit_symlinks: bool,
) -> Tuple[Tuple[str, int, int], ...]:
    """Return ``(basename, size, mtime_ns)`` for the checkpoint's top-level safetensors.

    The checkpoint's files are what the index names (#1806), so a Mistral-style
    ``consolidated.safetensors`` beside the shards is neither sized nor copied.
    """
    from soup_cli.utils.checkpoint_files import checkpoint_files

    root = os.path.realpath(os.path.expanduser(directory))
    if not os.path.isdir(root):
        raise FileNotFoundError(f"weights directory not found: {directory}")
    selected = checkpoint_files(root)
    files = []
    for name in selected.weights:
        path = os.path.join(root, name)
        if os.path.islink(path) and not permit_symlinks:
            raise FileNotFoundError(
                f"weight cache still contains symlinked shard {name!r}"
            )
        if not os.path.isfile(path):
            continue
        stat = os.stat(path)
        files.append((name, int(stat.st_size), int(stat.st_mtime_ns)))
    if not files:
        raise FileNotFoundError(selected.why_empty(directory))
    return tuple(files)


def _materialized_matches_hf_snapshot(
    source_dir: str,
    materialized_dir: str,
    source_files: Tuple[Tuple[str, int, int], ...],
    *,
    source_revision: Optional[str] = None,
) -> Optional[Tuple[Tuple[str, int, int], ...]]:
    """Return the real-file manifest when HF metadata proves blob identity.

    The comparison reads link names and sizes only, so the layout is checked
    last, by the copy step's own listing: a copy is reused only for a snapshot
    that would be copied now. A layout that listing refuses (a linked blob
    store, a linked directory inside the snapshot, a file that leaves the blob
    store or has no blob) answers "no match": the plan then asks for a copy,
    and the copy step raises its refusal before it touches the earlier copy.
    """
    try:
        existing = _weight_file_manifest(materialized_dir, permit_symlinks=False)
    except FileNotFoundError:
        return None
    expected_sizes = {name: size for name, size, _mtime in source_files}
    existing_sizes = {name: size for name, size, _mtime in existing}
    if existing_sizes != expected_sizes:
        return None
    for name, _size, _mtime in source_files:
        source_path = os.path.join(source_dir, name)
        if not os.path.islink(source_path):
            return None
        blob_id = _hf_blob_id(source_path)
        metadata_path = os.path.join(
            materialized_dir,
            ".cache",
            "huggingface",
            "download",
            name + ".metadata",
        )
        try:
            if os.path.getsize(metadata_path) > 4096:
                return None
            with open(metadata_path, encoding="utf-8") as handle:
                lines = handle.read(4097).splitlines()
        except OSError:
            return None
        # local_dir metadata is: commit hash, blob etag, materialisation time.
        if len(lines) < 2 or lines[1].strip() != blob_id:
            return None
        if source_revision is not None and lines[0].strip() != source_revision:
            return None
    try:
        _snapshot_materialization_entries(source_dir, source_revision=source_revision)
    except (ValueError, OSError):
        return None
    return existing


def _hf_blob_id(snapshot_path: str) -> str:
    """The per-repo ``blobs/<etag>`` name a snapshot link points at.

    Not the final target: huggingface_hub >= 1.32 may chain that entry to its
    cache-wide Xet store, whose file is named by the Xet hash instead.
    """
    return os.path.basename(os.readlink(snapshot_path))


def _hf_shared_blob_root(repo_root: str) -> Optional[str]:
    """The cache-wide ``<cache>/blobs`` store of huggingface_hub >= 1.32, if marked.

    Per-repo ``blobs/<etag>`` entries of Xet downloads are relative symlinks into
    it; huggingface_hub writes the ``.huggingface-shared-blobs`` marker file.
    Like huggingface_hub's ``is_shared_blobs_dir``, neither the directory nor the
    marker may be a link: a linked store points somewhere else.
    """
    store = os.path.join(os.path.dirname(repo_root), "blobs")
    marker = os.path.join(store, ".huggingface-shared-blobs")
    try:
        store_stat = os.lstat(store)
        marker_stat = os.lstat(marker)
    except OSError:
        return None
    if not stat.S_ISDIR(store_stat.st_mode) or not stat.S_ISREG(marker_stat.st_mode):
        return None
    reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
    if getattr(store_stat, "st_file_attributes", 0) & reparse:
        return None
    return os.path.realpath(store)


def _is_junction(path: str) -> bool:
    """Whether ``path`` is a Windows junction.

    ``os.path.islink`` is false for a junction and ``os.walk`` descends into
    one, so a junction needs its own check wherever a linked directory is
    refused. This is the comparison ``os.path.isjunction`` makes (that function
    needs Python 3.12), except that an ``OSError`` from ``lstat`` is not turned
    into "not a junction": an entry that cannot be inspected is not waved on.
    The tag is compared, not the reparse-point attribute, so a directory that
    carries some other reparse point is not taken for a link.
    """
    mount_point = getattr(stat, "IO_REPARSE_TAG_MOUNT_POINT", 0xA0000003)
    return getattr(os.lstat(path), "st_reparse_tag", 0) == mount_point


def _hf_repo_blob_root(repo_root: str) -> str:
    """The repo's own ``blobs`` store, which has to be a real directory.

    huggingface_hub creates it with ``os.makedirs``. Like the shared store, a
    linked one (a symlink, or a junction on Windows) points somewhere else, and
    it is refused without being resolved. The link's target is not compared
    with any directory: for a local snapshot path the folder around the
    snapshot is wherever it was put, so there is no limit to hold a target to.
    """
    store = os.path.join(repo_root, "blobs")
    if not os.path.lexists(store):
        raise FileNotFoundError("cached Hugging Face blob store is missing")
    if os.path.islink(store) or _is_junction(store):
        raise ValueError(
            f"cached Hugging Face blob store {store!r} is a link and is not followed: "
            "Soup copies weights only from inside the Hugging Face cache; make blobs a "
            "real directory (to keep the files elsewhere, link the whole repository "
            "folder instead), or pass a directory of regular files"
        )
    if not os.path.isdir(store):
        raise FileNotFoundError("cached Hugging Face blob store is missing")
    return os.path.realpath(store)


def _hf_snapshot_revision(source_dir: str) -> Optional[str]:
    """Commit carried by a canonical ``snapshots/<sha>`` cache directory."""
    source = os.path.realpath(os.path.expanduser(source_dir))
    if os.path.basename(os.path.dirname(source)) != "snapshots":
        return None
    revision = os.path.basename(source)
    return revision if _HF_COMMIT_RE.fullmatch(revision) else None


def _snapshot_materialization_entries(
    source_dir: str,
    *,
    source_revision: Optional[str],
) -> list[Tuple[str, str, Optional[str]]]:
    """Validate and list cached files before any destination is created.

    Canonical Hugging Face snapshots contain symlinks into their sibling
    ``blobs`` directory, whose Xet entries may in turn link into the
    cache-wide shared blob store.  Only links resolving into one of those two
    stores are followed: a crafted snapshot cannot turn Soup's regular-file
    copy into an arbitrary-file disclosure.
    """
    from soup_cli.utils.paths import is_under

    source = os.path.realpath(os.path.expanduser(source_dir))
    if not os.path.isdir(source):
        raise FileNotFoundError(f"cached snapshot directory not found: {source_dir}")

    blob_roots: list[str] = []
    if source_revision is not None:
        repo_root = os.path.dirname(os.path.dirname(source))
        expected = os.path.realpath(
            os.path.join(repo_root, "snapshots", source_revision)
        )
        if source != expected:
            raise ValueError("cached snapshot path does not match its resolved revision")
        blob_roots.append(_hf_repo_blob_root(repo_root))
        shared_root = _hf_shared_blob_root(repo_root)
        if shared_root is not None:
            blob_roots.append(shared_root)

    from soup_cli.utils.checkpoint_files import checkpoint_files

    # #1806: only the checkpoint's own top-level weights are copied (and sized in
    # materialize_bytes), the same set _weight_file_manifest reports, since the
    # copy is checked against that manifest afterwards.
    top_level_weights = set(checkpoint_files(source).weights)

    entries: list[Tuple[str, str, Optional[str]]] = []
    for root, dirnames, filenames in os.walk(source, followlinks=False):
        for dirname in dirnames:
            directory = os.path.join(root, dirname)
            if os.path.islink(directory):
                raise ValueError(
                    f"cached snapshot directory symlink is not allowed: {dirname!r}"
                )
            if _is_junction(directory):
                raise ValueError(
                    f"cached snapshot directory junction is not allowed: {dirname!r}"
                )
        for filename in filenames:
            if not filename.endswith((".safetensors", ".json")):
                continue
            if (
                filename.endswith(".safetensors")
                and root == source
                and filename not in top_level_weights
            ):
                continue
            snapshot_path = os.path.join(root, filename)
            relative = os.path.relpath(snapshot_path, source)
            blob_id = None
            if os.path.islink(snapshot_path):
                resolved = os.path.realpath(snapshot_path)
                if blob_roots and not any(is_under(resolved, r) for r in blob_roots):
                    raise ValueError(
                        f"cached snapshot file {relative!r} points outside the "
                        "Hugging Face blob store"
                    )
                blob_id = _hf_blob_id(snapshot_path)
            else:
                resolved = snapshot_path
            if not os.path.isfile(resolved):
                raise FileNotFoundError(
                    f"cached snapshot file {relative!r} is missing its blob"
                )
            entries.append((relative, resolved, blob_id))
    return entries


def _copy_regular_file(source: str, destination: str) -> None:
    """Copy through a no-follow descriptor so validation cannot be race-swapped."""
    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
    descriptor = os.open(source, flags)
    try:
        source_stat = os.fstat(descriptor)
        if not stat.S_ISREG(source_stat.st_mode):
            raise ValueError(f"cached snapshot source is not a regular file: {source!r}")
        with os.fdopen(descriptor, "rb") as source_handle:
            descriptor = -1
            with open(destination, "xb") as destination_handle:
                shutil.copyfileobj(source_handle, destination_handle)
    finally:
        if descriptor >= 0:
            os.close(descriptor)


def _reject_materialized_target_link(target: str) -> None:
    """Keep cache replacement from following a symlink, junction, or mount."""
    if not os.path.lexists(target):
        return
    target_stat = os.lstat(target)
    if stat.S_ISLNK(target_stat.st_mode):
        raise ValueError("materialized weights path must not be a symlink")
    if os.path.ismount(target):
        raise ValueError("materialized weights path must not be a mount point")
    if os.name == "nt":
        reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
        if getattr(target_stat, "st_file_attributes", 0) & reparse:
            raise ValueError(
                "materialized weights path must not be a reparse point / junction"
            )


def _materialize_cached_snapshot(plan: ModelWeightsPlan) -> str:
    """Copy one already-resolved snapshot without asking the Hub again."""
    from soup_cli.utils.paths import is_under

    if plan.source_revision is None:
        raise ValueError(
            "cannot materialize a symlinked Hub snapshot without its resolved commit"
        )
    requested_target = os.path.abspath(os.path.expanduser(plan.weights_dir))
    _reject_materialized_target_link(requested_target)
    target = os.path.realpath(requested_target)
    weights_root = os.path.realpath(os.path.join(resolve_cache_dir(), "weights"))
    if target == weights_root or not is_under(target, weights_root):
        raise ValueError("materialized weights path must stay inside Soup's cache")

    entries = _snapshot_materialization_entries(
        plan.source_dir,
        source_revision=plan.source_revision,
    )
    os.makedirs(weights_root, exist_ok=True)
    staging = tempfile.mkdtemp(
        prefix=f".{os.path.basename(target)}.",
        dir=os.path.dirname(target),
    )
    try:
        metadata_root = os.path.join(
            staging, ".cache", "huggingface", "download"
        )
        for relative, source_path, blob_id in entries:
            destination = os.path.join(staging, relative)
            os.makedirs(os.path.dirname(destination), exist_ok=True)
            _copy_regular_file(source_path, destination)
            if relative.endswith(".safetensors") and blob_id is not None:
                metadata_path = os.path.join(metadata_root, relative + ".metadata")
                os.makedirs(os.path.dirname(metadata_path), exist_ok=True)
                with open(metadata_path, "w", encoding="utf-8") as handle:
                    handle.write(
                        f"{plan.source_revision or ''}\n{blob_id}\n{time.time()}\n"
                    )

        staged_manifest = _weight_file_manifest(staging, permit_symlinks=False)
        expected_sizes = {name: size for name, size, _mtime in plan.source_files}
        staged_sizes = {name: size for name, size, _mtime in staged_manifest}
        if staged_sizes != expected_sizes:
            raise FileNotFoundError(
                "cached snapshot changed while Soup materialized its weight files"
            )

        if os.path.isdir(target):
            existing = _materialized_matches_hf_snapshot(
                plan.source_dir,
                target,
                plan.source_files,
                source_revision=plan.source_revision,
            )
            if existing is not None:
                return target
            _reject_materialized_target_link(requested_target)
            shutil.rmtree(target)
        elif os.path.lexists(target):
            raise ValueError("materialized weights path must be a directory")
        os.replace(staging, target)
        staging = ""
        return target
    finally:
        if staging and os.path.isdir(staging):
            shutil.rmtree(staging)


def plan_model_weights(model: str) -> ModelWeightsPlan:
    """Locate weights and quantify any copy Soup would need, without making it."""
    if not isinstance(model, str) or not model.strip():
        raise ValueError("model must be a non-empty string")
    if os.path.isdir(model):
        source = os.path.realpath(model)
        manifest = _weight_file_manifest(source, permit_symlinks=True)
        if any(os.path.islink(os.path.join(source, name)) for name, _s, _m in manifest):
            # A local path to an HF cache snapshot: its weights are symlinks into
            # blobs/, which the sharder and the scanner skip. Copy it like the
            # snapshot a Hub id resolves to. It shares that copy's cache slot only
            # when it is that snapshot's own repo folder (_local_snapshot_slot).
            source_revision = _hf_snapshot_revision(source)
            if source_revision is None:
                raise ValueError(
                    f"{model} holds symlinked .safetensors files outside a Hugging "
                    "Face snapshots/<commit> directory; Soup reads weights only "
                    "from regular files: pass the Hub id, the HF snapshot "
                    "directory, or a directory of regular files"
                )
            return _hf_snapshot_plan(
                _hf_cache_repo_id(source) or model,
                source,
                manifest,
                source_revision,
                slot=_local_snapshot_slot(source),
            )
        return ModelWeightsPlan(
            model=model,
            source_dir=source,
            weights_dir=source,
            source_bytes=sum(size for _name, size, _mtime in manifest),
            materialized_copy_bytes=0,
            materialize_bytes=0,
            source_files=manifest,
        )
    source = os.path.realpath(_download_hub_checkpoint(model))
    source_revision = _hf_snapshot_revision(source)
    manifest = _weight_file_manifest(source, permit_symlinks=True)
    source_paths = [os.path.join(source, name) for name, _size, _mtime in manifest]
    if all(not os.path.islink(path) for path in source_paths):
        return ModelWeightsPlan(
            model=model,
            source_dir=source,
            weights_dir=source,
            source_bytes=sum(size for _name, size, _mtime in manifest),
            materialized_copy_bytes=0,
            materialize_bytes=0,
            source_files=manifest,
            source_revision=source_revision,
        )
    return _hf_snapshot_plan(model, source, manifest, source_revision)


def _download_hub_checkpoint(model: str) -> str:
    """Fetch a Hub repo's checkpoint files, and nothing beside them (#1806).

    ``*.safetensors`` fetched Mistral's ``consolidated.safetensors`` next to the
    shards: 29.0 GB for a 14.5 GB model. One call asks for the Hugging Face
    spellings (``model.safetensors``, ``model-*-of-*.safetensors``) and the JSON
    files; one call, so an already complete cache entry resolves offline as
    before (#533). The local rule then reads what came, and a second call is
    made only when it says the checkpoint is not all there: the files an index
    names outside those spellings, or, with no index and neither spelling
    present, every ``*.safetensors`` except ``consolidated*``, which is the
    local fallback rule itself. So what is downloaded is what is counted.
    """
    from soup_cli.utils.checkpoint_files import (
        hub_allow_patterns,
        hub_second_call,
        read_index_weight_map,
        select_checkpoint_files,
    )
    from soup_cli.utils.hubs import snapshot_download
    from soup_cli.utils.run_provenance import base_revision_kwargs

    revision_kwargs = base_revision_kwargs(model)
    source = snapshot_download(
        model, cache_dir=None, allow_patterns=[*hub_allow_patterns(None), "*.json"],
        **revision_kwargs,
    )
    real = os.path.realpath(source)
    selected = select_checkpoint_files(os.listdir(real), read_index_weight_map(real))
    second = hub_second_call(selected)
    if second is not None:
        allow, ignore = second
        source = snapshot_download(
            model,
            cache_dir=None,
            allow_patterns=[*allow, "*.json"],
            ignore_patterns=ignore or None,
            **revision_kwargs,
        )
    return source


def _hf_cache_repo_id(source_dir: str) -> Optional[str]:
    """``org/name`` for a ``models--org--name/snapshots/<sha>`` directory."""
    repo_dir = os.path.basename(os.path.dirname(os.path.dirname(source_dir)))
    if not repo_dir.startswith("models--"):
        return None
    return repo_dir[len("models--"):].replace("--", "/") or None


def _location_key(path: str) -> str:
    """One spelling per directory: links resolved, letter case folded where the OS does."""
    return os.path.normcase(os.path.realpath(path))


def _local_snapshot_slot(source: str) -> str:
    """Cache slot for the regular-file copy of a local ``snapshots/<commit>`` directory.

    The slot of the Hub id ``org/name`` when the snapshot's repo folder is the directory
    that id resolves to in the active Hugging Face cache. A folder anywhere else gets a
    slot of its own, ``<folder name>@<digest of its resolved location>``: a copy is
    reused on matching names and sizes, so two requests may share one only when they
    name the same place. ``@`` cannot occur in a :func:`model_slug`, which keeps a
    folder's slot apart from every slot named after a model.
    """
    repo_root = os.path.dirname(os.path.dirname(source))
    location = _location_key(repo_root)
    repo_id = _hf_cache_repo_id(source)
    if repo_id is not None:
        from soup_cli.utils.hubs import hf_cache_dir

        cache = hf_cache_dir()
        if cache is not None:
            in_cache = os.path.join(cache, os.path.basename(repo_root))
            if _location_key(in_cache) == location:
                return model_slug(repo_id)
    name = os.path.basename(location)
    label = model_slug(name)[:_LOCAL_SLOT_LABEL_CHARS] if name.strip() else "local"
    digest = hashlib.sha256(os.fsencode(location)).hexdigest()
    return f"{label}@{digest[:_LOCAL_SLOT_DIGEST_CHARS]}"


def _hf_snapshot_plan(
    model: str,
    source: str,
    manifest: Tuple[Tuple[str, int, int], ...],
    source_revision: Optional[str],
    *,
    slot: Optional[str] = None,
) -> ModelWeightsPlan:
    """Plan the regular-file copy of a symlinked HF snapshot (reused when current)."""
    from soup_cli.utils.run_provenance import base_revision_kwargs

    if slot is None:
        slot = model_slug(model)
        revision = base_revision_kwargs(model).get("revision")
        if revision:
            slot += "@" + revision
    materialized = os.path.join(
        resolve_cache_dir(), "weights", slot
    )
    existing = _materialized_matches_hf_snapshot(
        source,
        materialized,
        manifest,
        source_revision=source_revision,
    )
    needs_copy = existing is None
    return ModelWeightsPlan(
        model=model,
        source_dir=source,
        weights_dir=materialized,
        source_bytes=sum(size for _name, size, _mtime in manifest),
        materialized_copy_bytes=sum(size for _name, size, _mtime in manifest),
        materialize_bytes=(
            sum(size for _name, size, _mtime in manifest) if needs_copy else 0
        ),
        source_files=manifest if existing is None else existing,
        source_revision=source_revision,
    )


def materialize_model_weights(plan: ModelWeightsPlan) -> str:
    """Create the planned regular-file copy only when the source requires it."""
    if not plan.needs_materialization:
        return plan.weights_dir
    return _materialize_cached_snapshot(plan)


def resolve_model_weights(
    model: str,
    *,
    before_materialize: Optional[Callable[[ModelWeightsPlan], None]] = None,
) -> str:
    """Return a regular-file weight directory, with an optional pre-copy gate."""
    plan = plan_model_weights(model)
    if before_materialize is not None:
        before_materialize(plan)
    return materialize_model_weights(plan)


def scan_model(
    model: str,
    *,
    modules: ModulesArg = "all",
    use_cache: bool = True,
    cache_dir: Optional[str] = None,
) -> ScanResult:
    """Resolve, (cache-)scan a model and return its :class:`ScanResult`."""
    canonical = _canonical_modules_str(modules)
    if use_cache:
        cached = read_cached_scan(model, modules=canonical, cache_dir=cache_dir)
        if cached is not None:
            return cached
    weights_dir = resolve_model_weights(model)
    layers = scan_weights_dir(weights_dir, modules=modules)
    result = ScanResult(model=model, modules=canonical, layers=layers)
    if use_cache:
        try:
            write_cached_scan(result, cache_dir=cache_dir)
        except OSError:
            pass  # cache write failures are non-fatal
    return result