Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Native Spectrum targeted-training scan (#266, v0.71.23).
Implements the Spectrum method (arXiv:2406.06623, *Spectrum: Targeted Training
on Signal to Noise Ratio*) as a **pure-numpy, weight-streaming** scan:
- :func:`compute_snr` is the core kernel — singular-value SNR with a
Marchenko-Pastur noise threshold. It is **transpose-invariant** (the
singular values of ``W`` and ``W.T`` are identical and the MP aspect ratio
``min(n,m)/max(n,m)`` is symmetric), so a GPT-2 ``Conv1D`` weight stored as
``[in, out]`` scores the same as a Linear stored ``[out, in]``. The
"Conv1D-aware" requirement is therefore about *module naming*
(:func:`classify_module` recognises ``c_attn``/``c_fc``/``c_proj``), not the
math.
- :func:`iter_weight_matrices` streams ``.safetensors`` shards **one tensor at
a time** via ``safetensors.safe_open`` — there is NO full model load, so
peak RSS is the size of the largest single weight matrix. This is what lets
a 70B's layer SNR be scanned on a CPU box; the reference implementation
loads the whole model.
- Scan results are cached at ``~/.soup/spectrum/<slug>.json`` (override via
``SOUP_SPECTRUM_CACHE_DIR``, containment-checked under ``$HOME`` / ``$CWD`` /
``$TMPDIR`` per the v0.36.0 cache-dir policy).
All heavy deps (numpy / torch / safetensors / huggingface_hub) are imported
lazily inside functions so the module — and the ``soup`` CLI — load without
them. LISA (per-step layer sampling) is split to #267.
"""
from __future__ import annotations
import hashlib
import json
import logging
import math
import os
import re
import shutil
import stat
import tempfile
import time
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Any,
Callable,
Iterator,
Literal,
Optional,
Sequence,
Tuple,
Union,
)
if TYPE_CHECKING: # static types only — numpy stays a lazy runtime import
from numpy.typing import NDArray
logger = logging.getLogger(__name__)
# DoS / sanity caps.
_MAX_TENSORS = 100_000
_MAX_UNFROZEN_PATTERNS = 50_000
_MAX_CACHE_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_CACHE_LAYERS = _MAX_TENSORS
# A single 2-D weight above this element count is skipped (and logged) rather
# than fed to a multi-GB SVD — a backstop against a crafted/oversized
# ``--model``. Legitimate mlp/attn matrices (even a 70B's ~8k×29k) are far
# below this; only giant embeddings / adversarial shapes trip it.
_MAX_MATRIX_ELEMENTS = 2**31
_CACHE_SCHEMA = 1
_MLP_MARKERS = ("mlp", "feed_forward", "ffn")
_ATTN_MARKERS = ("self_attn", "attention")
_VALID_MODULE_TYPES = ("mlp", "attn", "other")
_LAYER_IDX_RE = re.compile(r"(?:^|\.)(?:layers|h)\.\d+\.")
_SLUG_RE = re.compile(r"[^A-Za-z0-9._-]+")
_HF_COMMIT_RE = re.compile(r"^[A-Fa-f0-9]{40}$")
# A local snapshot folder's own cache slot: ``<folder name>@<digest of its location>``.
_LOCAL_SLOT_LABEL_CHARS = 64
_LOCAL_SLOT_DIGEST_CHARS = 32
ModulesArg = Union[str, Sequence[str], None]
@dataclass(frozen=True)
class ModelWeightsPlan:
"""A weight source located before Soup creates a second on-disk copy."""
model: str
source_dir: str
weights_dir: str
source_bytes: int
materialized_copy_bytes: int
materialize_bytes: int
source_files: Tuple[Tuple[str, int, int], ...]
source_revision: Optional[str] = None
@property
def needs_materialization(self) -> bool:
return self.materialize_bytes > 0
def _np():
import numpy as np # lazy — keep the CLI import light
return np
# ---------------------------------------------------------------------------
# Dataclasses
# ---------------------------------------------------------------------------
ModuleType = Literal["attn", "mlp", "other"]
@dataclass(frozen=True)
class LayerSNR:
"""Per-weight-matrix SNR record."""
name: str
module_type: ModuleType
group: str # layer-type signature, e.g. "self_attn.q_proj"
snr: float
shape: Tuple[int, int]
@dataclass(frozen=True)
class ScanResult:
"""A full model scan: the model id, the requested modules, the layers."""
model: str
modules: str # canonical "all" or comma-joined sorted types
layers: Tuple[LayerSNR, ...]
# ---------------------------------------------------------------------------
# SNR kernel (pure numpy)
# ---------------------------------------------------------------------------
def estimate_sigma(singular_values: Any) -> float:
"""Estimate the noise std via the inter-quartile range of singular values.
``sigma = IQR / 1.349`` (the IQR-to-std factor for a normal) — the
Spectrum reference estimator.
"""
np = _np()
s = np.asarray(singular_values, dtype=np.float64).ravel()
if s.size == 0:
return 0.0
q75, q25 = np.percentile(s, [75, 25])
return float((q75 - q25) / 1.349)
def marchenko_pastur_threshold(sigma: float, n: int, m: int) -> float:
"""Marchenko-Pastur singular-value noise edge.
``threshold = sigma * sqrt((1 + sqrt(beta))**2)`` with
``beta = min(n, m) / max(n, m)`` (the Spectrum reference form;
``sqrt((1+x)**2) == 1+x`` for ``x >= 0``). Symmetric in ``n``/``m``.
"""
if n <= 0 or m <= 0:
return 0.0
beta = min(n, m) / max(n, m)
return float(sigma) * math.sqrt((1.0 + math.sqrt(beta)) ** 2)
def compute_snr(matrix: Any) -> float:
"""Signal-to-noise ratio of a single 2-D weight matrix.
Singular values above the MP threshold are "signal", the rest "noise";
``snr = signal_sum / noise_sum`` and the returned ratio is normalised by
the largest singular value (the Spectrum ``snr_ratio``). Always finite.
Transpose-invariant. Raises ``ValueError`` on non-2-D input.
"""
np = _np()
# Preserve float32/float64 (the streaming path yields float32 to keep peak
# RSS ≈ the largest single matrix); upcast float16 / bf16-already-float /
# int to float32 for a stable SVD.
a = np.asarray(matrix)
if a.dtype not in (np.float32, np.float64):
a = a.astype(np.float32)
if a.ndim != 2:
raise ValueError(f"compute_snr expects a 2-D matrix, got ndim={a.ndim}")
if a.size == 0:
return 0.0
s = np.linalg.svd(a, compute_uv=False) # singular values, descending
if s.size == 0:
return 0.0
max_sv = float(s[0])
if max_sv <= 0.0 or not math.isfinite(max_sv):
return 0.0
sigma = estimate_sigma(s)
threshold = marchenko_pastur_threshold(sigma, a.shape[0], a.shape[1])
signal_mask = s > threshold
signal = float(s[signal_mask].sum()) if signal_mask.any() else 0.0
# An empty below-threshold sum is 0.0; the guard floors it to 1.0 so the
# ratio stays finite (mirrors the Spectrum reference's noise default).
noise = float(s[~signal_mask].sum())
if noise <= 0.0:
noise = 1.0
ratio = (signal / noise) / max_sv
return float(ratio) if math.isfinite(ratio) else 0.0
# ---------------------------------------------------------------------------
# Module classification + grouping
# ---------------------------------------------------------------------------
def classify_module(name: Any) -> Optional[ModuleType]:
"""Classify a parameter name as ``"attn"`` / ``"mlp"`` / ``"other"``.
Returns ``None`` for non-``.weight`` params (biases, etc.). Recognises
both Llama-style (``self_attn.q_proj``, ``mlp.gate_proj``) and GPT-2
``Conv1D`` (``attn.c_attn``, ``mlp.c_fc``, ``mlp.c_proj``) naming.
"""
if not isinstance(name, str) or not name.endswith(".weight"):
return None
low = name.lower()
if any(mk in low for mk in _MLP_MARKERS):
return "mlp"
if any(mk in low for mk in _ATTN_MARKERS) or ".attn." in low:
return "attn"
return "other"
def layer_type_signature(name: str) -> str:
"""Strip the numeric layer index so same-type layers group together.
``model.layers.5.self_attn.q_proj.weight`` -> ``self_attn.q_proj``;
``transformer.h.3.mlp.c_fc.weight`` -> ``mlp.c_fc``;
``lm_head.weight`` -> ``lm_head`` (non-layer params group by themselves).
"""
match = _LAYER_IDX_RE.search(name)
base = name[match.end():] if match else name
for suffix in (".weight", ".bias"):
if base.endswith(suffix):
return base[: -len(suffix)]
return base
def param_prefix(name: str) -> str:
"""Strip a trailing ``.weight`` — the unfreeze pattern is the name prefix."""
return name[: -len(".weight")] if name.endswith(".weight") else name
# ---------------------------------------------------------------------------
# Module-filter normalisation
# ---------------------------------------------------------------------------
def _normalize_modules(modules: ModulesArg) -> Optional[frozenset[str]]:
"""Return a frozenset of kept types, or ``None`` meaning "keep all"."""
if modules is None:
return None
if isinstance(modules, str):
parts = [p.strip().lower() for p in modules.split(",")]
else:
parts = [str(p).strip().lower() for p in modules]
parts = [p for p in parts if p]
if not parts or "all" in parts:
return None
for part in parts:
if part not in _VALID_MODULE_TYPES:
raise ValueError(
f"unknown module type {part!r}; choose from "
f"{', '.join(_VALID_MODULE_TYPES)}, all"
)
return frozenset(parts)
def _canonical_modules_str(modules: ModulesArg) -> str:
keep = _normalize_modules(modules)
return "all" if keep is None else ",".join(sorted(keep))
# ---------------------------------------------------------------------------
# Safetensors streaming
# ---------------------------------------------------------------------------
def _framework() -> Tuple[str, Callable[[Any], "NDArray[Any]"]]:
"""Pick a safetensors framework + a to-numpy converter.
Prefer torch (handles bf16, which numpy cannot represent); fall back to
the pure-numpy framework when torch is absent (bf16 weights then need
torch — :func:`iter_weight_matrices` raises a friendly error).
"""
np = _np()
try:
import torch
except Exception: # pragma: no cover - torch present in dev/CI
torch = None
if torch is not None:
def to_np(tensor: Any) -> "NDArray[Any]":
return tensor.detach().to(torch.float32).cpu().numpy()
return "pt", to_np
def to_np(arr: Any) -> "NDArray[Any]":
# Keep the native dtype (compute_snr upcasts float16 → float32); a
# forced float64 would double peak RSS over the streamed matrix.
return np.asarray(arr)
return "np", to_np
def _discover_safetensors(weights_dir: str) -> list[str]:
from soup_cli.utils.checkpoint_files import checkpoint_files
real = os.path.realpath(weights_dir)
if not os.path.isdir(real):
raise FileNotFoundError(f"weights dir not found: {weights_dir}")
# #1806: the checkpoint's own files, not every *.safetensors. Mistral's
# consolidated.safetensors carries the same weights under mistral-inference
# names; scanning it too emitted unfrozen_parameters that match no
# parameter of the HF model.
selected = checkpoint_files(real)
files = []
for entry in selected.weights:
full = os.path.join(real, entry)
# Skip symlinked shards — defence-in-depth against a link redirecting
# the read (mirrors the project's no-symlink TOCTOU policy).
if os.path.islink(full):
logger.warning("spectrum scan: skipping symlinked shard %s", entry)
continue
if not os.path.isfile(full):
continue
files.append(full)
if not files:
raise FileNotFoundError(selected.why_empty(weights_dir))
return files
def iter_weight_matrices(
weights_dir: str, *, modules: ModulesArg = "all"
) -> Iterator[Tuple[str, "NDArray[Any]"]]:
"""Stream 2-D float weight matrices from a model dir, one at a time.
Yields ``(param_name, np.ndarray)`` for every 2-D ``.weight`` tensor whose
module type passes the ``modules`` filter. The shape is probed via
``get_slice(...).get_shape()`` so 1-D norms, filtered-out modules and
oversized matrices are NEVER materialised — peak RSS is the largest *kept*
tensor.
"""
from safetensors import safe_open
keep = _normalize_modules(modules)
framework, to_np = _framework()
count = 0
for path in _discover_safetensors(weights_dir):
with safe_open(path, framework=framework) as handle:
for key in handle.keys():
module_type = classify_module(key)
if module_type is None:
continue
if keep is not None and module_type not in keep:
continue
shape = tuple(handle.get_slice(key).get_shape())
if len(shape) != 2:
continue
if shape[0] * shape[1] > _MAX_MATRIX_ELEMENTS:
logger.warning(
"spectrum scan: skipping %s — %dx%d exceeds the "
"%d-element SVD cap (scan a subset via --modules "
"mlp,attn for very large models)",
key, shape[0], shape[1], _MAX_MATRIX_ELEMENTS,
)
continue
count += 1
if count > _MAX_TENSORS:
raise ValueError(
f"scan exceeded the {_MAX_TENSORS}-tensor cap"
)
try:
array = to_np(handle.get_tensor(key))
except Exception as exc:
raise RuntimeError(
f"could not read tensor {key!r} as float "
f"(bf16 weights need torch — pip install "
f"'soup-cli[train]'): {type(exc).__name__}"
) from exc
yield key, array
def scan_weights_dir(
weights_dir: str, *, modules: ModulesArg = "all"
) -> Tuple[LayerSNR, ...]:
"""Stream a local model dir and compute the SNR of every kept matrix."""
out = []
for name, array in iter_weight_matrices(weights_dir, modules=modules):
out.append(
LayerSNR(
name=name,
module_type=classify_module(name) or "other",
group=layer_type_signature(name),
snr=compute_snr(array),
shape=(int(array.shape[0]), int(array.shape[1])),
)
)
return tuple(out)
# ---------------------------------------------------------------------------
# Selection
# ---------------------------------------------------------------------------
def select_unfrozen_parameters(
results: Sequence[LayerSNR],
*,
top_percent: float,
modules: ModulesArg = "all",
) -> list[str]:
"""Pick the top ``top_percent`` % of layers *per module type group*.
Grouping by :func:`layer_type_signature` keeps the unfreeze balanced
across module types (top-50% of ``q_proj`` layers, top-50% of
``down_proj`` layers, ...). At least one layer is kept per group
(``ceil``). Returns a sorted, de-duplicated list of parameter-name
prefixes (``.weight`` stripped) ready for ``training.unfrozen_parameters``.
"""
if isinstance(top_percent, bool) or not isinstance(top_percent, (int, float)):
raise ValueError("top_percent must be a number in (0, 100]")
if not (0 < top_percent <= 100):
raise ValueError("top_percent must be in (0, 100]")
keep_types = _normalize_modules(modules)
groups: dict[str, list[LayerSNR]] = {}
for record in results:
if keep_types is not None and record.module_type not in keep_types:
continue
groups.setdefault(record.group, []).append(record)
selected: list[str] = []
for items in groups.values():
ordered = sorted(items, key=lambda r: (r.snr, r.name), reverse=True)
keep_n = max(1, math.ceil(len(ordered) * top_percent / 100.0))
selected.extend(param_prefix(r.name) for r in ordered[:keep_n])
if len(selected) > _MAX_UNFROZEN_PATTERNS:
raise ValueError(
f"selection exceeded the {_MAX_UNFROZEN_PATTERNS}-pattern cap"
)
return sorted(set(selected))
# ---------------------------------------------------------------------------
# Cache + slug
# ---------------------------------------------------------------------------
def model_slug(model: str) -> str:
"""Sanitise a model id into a traversal-safe cache filename stem."""
if not isinstance(model, str) or not model.strip():
raise ValueError("model must be a non-empty string")
base = model.strip().replace("\\", "/").rstrip("/").replace("/", "__")
slug = _SLUG_RE.sub("_", base).replace("..", "_").strip("._-")
return (slug or "model")[:128]
def default_spectrum_cache_dir() -> str:
"""``~/.soup/spectrum`` — the scan cache root (no side effects)."""
return os.path.join(os.path.expanduser("~"), ".soup", "spectrum")
def resolve_cache_dir(cache_dir: Optional[str] = None) -> str:
"""Resolve the cache dir (explicit arg > ``SOUP_SPECTRUM_CACHE_DIR`` > default).
The env override is rejected if it contains C0 control characters or
escapes ``$HOME`` / ``$CWD`` / ``$TMPDIR`` (silent fall-through to the
default — an env var is operator config, not API input). The chosen dir
is created.
"""
from soup_cli.utils.paths import is_under
if cache_dir is not None:
chosen = os.path.realpath(os.path.expanduser(str(cache_dir)))
os.makedirs(chosen, exist_ok=True)
return chosen
override = os.environ.get("SOUP_SPECTRUM_CACHE_DIR")
if override and not any(ord(ch) < 0x20 for ch in override):
candidate = os.path.realpath(os.path.expanduser(override))
bounds = [
os.path.realpath(os.path.expanduser("~")),
os.path.realpath(os.getcwd()),
os.path.realpath(tempfile.gettempdir()),
]
if any(is_under(candidate, bound) for bound in bounds):
os.makedirs(candidate, exist_ok=True)
return candidate
default = default_spectrum_cache_dir()
os.makedirs(default, exist_ok=True)
return default
def _cache_path(model: str, cache_dir: Optional[str]) -> str:
return os.path.join(resolve_cache_dir(cache_dir), model_slug(model) + ".json")
def _atomic_write_json(payload: dict, path: str) -> str:
"""Atomic JSON write into an already-validated cache dir.
The cache lives under ``$HOME``/``$TMPDIR`` (not cwd), so it deliberately
does NOT use ``paths.atomic_write_text`` (which enforces cwd containment);
the dir is bounded by :func:`resolve_cache_dir` and the filename by
:func:`model_slug`. mkstemp + ``os.replace`` keeps the write atomic.
"""
parent = os.path.dirname(os.path.abspath(path)) or "."
os.makedirs(parent, exist_ok=True)
fd, tmp = tempfile.mkstemp(prefix=".soup.", suffix=".tmp", dir=parent)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2)
os.replace(tmp, path)
finally:
if os.path.exists(tmp):
try:
os.unlink(tmp)
except OSError:
pass
return os.path.realpath(path)
def write_cached_scan(result: ScanResult, *, cache_dir: Optional[str] = None) -> str:
"""Persist a :class:`ScanResult` to the cache. Returns the written path."""
payload = {
"schema": _CACHE_SCHEMA,
"model": result.model,
"modules": result.modules,
"layers": [
{
"name": ls.name,
"module_type": ls.module_type,
"group": ls.group,
"snr": ls.snr,
"shape": list(ls.shape),
}
for ls in result.layers
],
}
return _atomic_write_json(payload, _cache_path(result.model, cache_dir))
def read_cached_scan(
model: str, *, modules: ModulesArg = "all", cache_dir: Optional[str] = None
) -> Optional[ScanResult]:
"""Load a cached scan, or ``None`` on miss / corruption / module mismatch."""
path = _cache_path(model, cache_dir)
if not os.path.isfile(path):
return None
try:
if os.path.getsize(path) > _MAX_CACHE_BYTES:
return None
with open(path, encoding="utf-8") as handle:
payload = json.load(handle)
except (OSError, ValueError):
return None
if not isinstance(payload, dict) or payload.get("schema") != _CACHE_SCHEMA:
return None
if payload.get("modules") != _canonical_modules_str(modules):
return None
raw_layers = payload.get("layers")
if not isinstance(raw_layers, list) or len(raw_layers) > _MAX_CACHE_LAYERS:
return None
try:
layers = tuple(
LayerSNR(
name=str(d["name"]),
module_type=str(d["module_type"]),
group=str(d["group"]),
snr=float(d["snr"]),
shape=(int(d["shape"][0]), int(d["shape"][1])),
)
for d in payload["layers"]
)
return ScanResult(
model=str(payload["model"]),
modules=str(payload["modules"]),
layers=layers,
)
except (KeyError, TypeError, ValueError, IndexError):
return None
# ---------------------------------------------------------------------------
# Resolve + scan orchestration
# ---------------------------------------------------------------------------
def _weight_file_manifest(
directory: str,
*,
permit_symlinks: bool,
) -> Tuple[Tuple[str, int, int], ...]:
"""Return ``(basename, size, mtime_ns)`` for the checkpoint's top-level safetensors.
The checkpoint's files are what the index names (#1806), so a Mistral-style
``consolidated.safetensors`` beside the shards is neither sized nor copied.
"""
from soup_cli.utils.checkpoint_files import checkpoint_files
root = os.path.realpath(os.path.expanduser(directory))
if not os.path.isdir(root):
raise FileNotFoundError(f"weights directory not found: {directory}")
selected = checkpoint_files(root)
files = []
for name in selected.weights:
path = os.path.join(root, name)
if os.path.islink(path) and not permit_symlinks:
raise FileNotFoundError(
f"weight cache still contains symlinked shard {name!r}"
)
if not os.path.isfile(path):
continue
stat = os.stat(path)
files.append((name, int(stat.st_size), int(stat.st_mtime_ns)))
if not files:
raise FileNotFoundError(selected.why_empty(directory))
return tuple(files)
def _materialized_matches_hf_snapshot(
source_dir: str,
materialized_dir: str,
source_files: Tuple[Tuple[str, int, int], ...],
*,
source_revision: Optional[str] = None,
) -> Optional[Tuple[Tuple[str, int, int], ...]]:
"""Return the real-file manifest when HF metadata proves blob identity.
The comparison reads link names and sizes only, so the layout is checked
last, by the copy step's own listing: a copy is reused only for a snapshot
that would be copied now. A layout that listing refuses (a linked blob
store, a linked directory inside the snapshot, a file that leaves the blob
store or has no blob) answers "no match": the plan then asks for a copy,
and the copy step raises its refusal before it touches the earlier copy.
"""
try:
existing = _weight_file_manifest(materialized_dir, permit_symlinks=False)
except FileNotFoundError:
return None
expected_sizes = {name: size for name, size, _mtime in source_files}
existing_sizes = {name: size for name, size, _mtime in existing}
if existing_sizes != expected_sizes:
return None
for name, _size, _mtime in source_files:
source_path = os.path.join(source_dir, name)
if not os.path.islink(source_path):
return None
blob_id = _hf_blob_id(source_path)
metadata_path = os.path.join(
materialized_dir,
".cache",
"huggingface",
"download",
name + ".metadata",
)
try:
if os.path.getsize(metadata_path) > 4096:
return None
with open(metadata_path, encoding="utf-8") as handle:
lines = handle.read(4097).splitlines()
except OSError:
return None
# local_dir metadata is: commit hash, blob etag, materialisation time.
if len(lines) < 2 or lines[1].strip() != blob_id:
return None
if source_revision is not None and lines[0].strip() != source_revision:
return None
try:
_snapshot_materialization_entries(source_dir, source_revision=source_revision)
except (ValueError, OSError):
return None
return existing
def _hf_blob_id(snapshot_path: str) -> str:
"""The per-repo ``blobs/<etag>`` name a snapshot link points at.
Not the final target: huggingface_hub >= 1.32 may chain that entry to its
cache-wide Xet store, whose file is named by the Xet hash instead.
"""
return os.path.basename(os.readlink(snapshot_path))
def _hf_shared_blob_root(repo_root: str) -> Optional[str]:
"""The cache-wide ``<cache>/blobs`` store of huggingface_hub >= 1.32, if marked.
Per-repo ``blobs/<etag>`` entries of Xet downloads are relative symlinks into
it; huggingface_hub writes the ``.huggingface-shared-blobs`` marker file.
Like huggingface_hub's ``is_shared_blobs_dir``, neither the directory nor the
marker may be a link: a linked store points somewhere else.
"""
store = os.path.join(os.path.dirname(repo_root), "blobs")
marker = os.path.join(store, ".huggingface-shared-blobs")
try:
store_stat = os.lstat(store)
marker_stat = os.lstat(marker)
except OSError:
return None
if not stat.S_ISDIR(store_stat.st_mode) or not stat.S_ISREG(marker_stat.st_mode):
return None
reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
if getattr(store_stat, "st_file_attributes", 0) & reparse:
return None
return os.path.realpath(store)
def _is_junction(path: str) -> bool:
"""Whether ``path`` is a Windows junction.
``os.path.islink`` is false for a junction and ``os.walk`` descends into
one, so a junction needs its own check wherever a linked directory is
refused. This is the comparison ``os.path.isjunction`` makes (that function
needs Python 3.12), except that an ``OSError`` from ``lstat`` is not turned
into "not a junction": an entry that cannot be inspected is not waved on.
The tag is compared, not the reparse-point attribute, so a directory that
carries some other reparse point is not taken for a link.
"""
mount_point = getattr(stat, "IO_REPARSE_TAG_MOUNT_POINT", 0xA0000003)
return getattr(os.lstat(path), "st_reparse_tag", 0) == mount_point
def _hf_repo_blob_root(repo_root: str) -> str:
"""The repo's own ``blobs`` store, which has to be a real directory.
huggingface_hub creates it with ``os.makedirs``. Like the shared store, a
linked one (a symlink, or a junction on Windows) points somewhere else, and
it is refused without being resolved. The link's target is not compared
with any directory: for a local snapshot path the folder around the
snapshot is wherever it was put, so there is no limit to hold a target to.
"""
store = os.path.join(repo_root, "blobs")
if not os.path.lexists(store):
raise FileNotFoundError("cached Hugging Face blob store is missing")
if os.path.islink(store) or _is_junction(store):
raise ValueError(
f"cached Hugging Face blob store {store!r} is a link and is not followed: "
"Soup copies weights only from inside the Hugging Face cache; make blobs a "
"real directory (to keep the files elsewhere, link the whole repository "
"folder instead), or pass a directory of regular files"
)
if not os.path.isdir(store):
raise FileNotFoundError("cached Hugging Face blob store is missing")
return os.path.realpath(store)
def _hf_snapshot_revision(source_dir: str) -> Optional[str]:
"""Commit carried by a canonical ``snapshots/<sha>`` cache directory."""
source = os.path.realpath(os.path.expanduser(source_dir))
if os.path.basename(os.path.dirname(source)) != "snapshots":
return None
revision = os.path.basename(source)
return revision if _HF_COMMIT_RE.fullmatch(revision) else None
def _snapshot_materialization_entries(
source_dir: str,
*,
source_revision: Optional[str],
) -> list[Tuple[str, str, Optional[str]]]:
"""Validate and list cached files before any destination is created.
Canonical Hugging Face snapshots contain symlinks into their sibling
``blobs`` directory, whose Xet entries may in turn link into the
cache-wide shared blob store. Only links resolving into one of those two
stores are followed: a crafted snapshot cannot turn Soup's regular-file
copy into an arbitrary-file disclosure.
"""
from soup_cli.utils.paths import is_under
source = os.path.realpath(os.path.expanduser(source_dir))
if not os.path.isdir(source):
raise FileNotFoundError(f"cached snapshot directory not found: {source_dir}")
blob_roots: list[str] = []
if source_revision is not None:
repo_root = os.path.dirname(os.path.dirname(source))
expected = os.path.realpath(
os.path.join(repo_root, "snapshots", source_revision)
)
if source != expected:
raise ValueError("cached snapshot path does not match its resolved revision")
blob_roots.append(_hf_repo_blob_root(repo_root))
shared_root = _hf_shared_blob_root(repo_root)
if shared_root is not None:
blob_roots.append(shared_root)
from soup_cli.utils.checkpoint_files import checkpoint_files
# #1806: only the checkpoint's own top-level weights are copied (and sized in
# materialize_bytes), the same set _weight_file_manifest reports, since the
# copy is checked against that manifest afterwards.
top_level_weights = set(checkpoint_files(source).weights)
entries: list[Tuple[str, str, Optional[str]]] = []
for root, dirnames, filenames in os.walk(source, followlinks=False):
for dirname in dirnames:
directory = os.path.join(root, dirname)
if os.path.islink(directory):
raise ValueError(
f"cached snapshot directory symlink is not allowed: {dirname!r}"
)
if _is_junction(directory):
raise ValueError(
f"cached snapshot directory junction is not allowed: {dirname!r}"
)
for filename in filenames:
if not filename.endswith((".safetensors", ".json")):
continue
if (
filename.endswith(".safetensors")
and root == source
and filename not in top_level_weights
):
continue
snapshot_path = os.path.join(root, filename)
relative = os.path.relpath(snapshot_path, source)
blob_id = None
if os.path.islink(snapshot_path):
resolved = os.path.realpath(snapshot_path)
if blob_roots and not any(is_under(resolved, r) for r in blob_roots):
raise ValueError(
f"cached snapshot file {relative!r} points outside the "
"Hugging Face blob store"
)
blob_id = _hf_blob_id(snapshot_path)
else:
resolved = snapshot_path
if not os.path.isfile(resolved):
raise FileNotFoundError(
f"cached snapshot file {relative!r} is missing its blob"
)
entries.append((relative, resolved, blob_id))
return entries
def _copy_regular_file(source: str, destination: str) -> None:
"""Copy through a no-follow descriptor so validation cannot be race-swapped."""
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(source, flags)
try:
source_stat = os.fstat(descriptor)
if not stat.S_ISREG(source_stat.st_mode):
raise ValueError(f"cached snapshot source is not a regular file: {source!r}")
with os.fdopen(descriptor, "rb") as source_handle:
descriptor = -1
with open(destination, "xb") as destination_handle:
shutil.copyfileobj(source_handle, destination_handle)
finally:
if descriptor >= 0:
os.close(descriptor)
def _reject_materialized_target_link(target: str) -> None:
"""Keep cache replacement from following a symlink, junction, or mount."""
if not os.path.lexists(target):
return
target_stat = os.lstat(target)
if stat.S_ISLNK(target_stat.st_mode):
raise ValueError("materialized weights path must not be a symlink")
if os.path.ismount(target):
raise ValueError("materialized weights path must not be a mount point")
if os.name == "nt":
reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
if getattr(target_stat, "st_file_attributes", 0) & reparse:
raise ValueError(
"materialized weights path must not be a reparse point / junction"
)
def _materialize_cached_snapshot(plan: ModelWeightsPlan) -> str:
"""Copy one already-resolved snapshot without asking the Hub again."""
from soup_cli.utils.paths import is_under
if plan.source_revision is None:
raise ValueError(
"cannot materialize a symlinked Hub snapshot without its resolved commit"
)
requested_target = os.path.abspath(os.path.expanduser(plan.weights_dir))
_reject_materialized_target_link(requested_target)
target = os.path.realpath(requested_target)
weights_root = os.path.realpath(os.path.join(resolve_cache_dir(), "weights"))
if target == weights_root or not is_under(target, weights_root):
raise ValueError("materialized weights path must stay inside Soup's cache")
entries = _snapshot_materialization_entries(
plan.source_dir,
source_revision=plan.source_revision,
)
os.makedirs(weights_root, exist_ok=True)
staging = tempfile.mkdtemp(
prefix=f".{os.path.basename(target)}.",
dir=os.path.dirname(target),
)
try:
metadata_root = os.path.join(
staging, ".cache", "huggingface", "download"
)
for relative, source_path, blob_id in entries:
destination = os.path.join(staging, relative)
os.makedirs(os.path.dirname(destination), exist_ok=True)
_copy_regular_file(source_path, destination)
if relative.endswith(".safetensors") and blob_id is not None:
metadata_path = os.path.join(metadata_root, relative + ".metadata")
os.makedirs(os.path.dirname(metadata_path), exist_ok=True)
with open(metadata_path, "w", encoding="utf-8") as handle:
handle.write(
f"{plan.source_revision or ''}\n{blob_id}\n{time.time()}\n"
)
staged_manifest = _weight_file_manifest(staging, permit_symlinks=False)
expected_sizes = {name: size for name, size, _mtime in plan.source_files}
staged_sizes = {name: size for name, size, _mtime in staged_manifest}
if staged_sizes != expected_sizes:
raise FileNotFoundError(
"cached snapshot changed while Soup materialized its weight files"
)
if os.path.isdir(target):
existing = _materialized_matches_hf_snapshot(
plan.source_dir,
target,
plan.source_files,
source_revision=plan.source_revision,
)
if existing is not None:
return target
_reject_materialized_target_link(requested_target)
shutil.rmtree(target)
elif os.path.lexists(target):
raise ValueError("materialized weights path must be a directory")
os.replace(staging, target)
staging = ""
return target
finally:
if staging and os.path.isdir(staging):
shutil.rmtree(staging)
def plan_model_weights(model: str) -> ModelWeightsPlan:
"""Locate weights and quantify any copy Soup would need, without making it."""
if not isinstance(model, str) or not model.strip():
raise ValueError("model must be a non-empty string")
if os.path.isdir(model):
source = os.path.realpath(model)
manifest = _weight_file_manifest(source, permit_symlinks=True)
if any(os.path.islink(os.path.join(source, name)) for name, _s, _m in manifest):
# A local path to an HF cache snapshot: its weights are symlinks into
# blobs/, which the sharder and the scanner skip. Copy it like the
# snapshot a Hub id resolves to. It shares that copy's cache slot only
# when it is that snapshot's own repo folder (_local_snapshot_slot).
source_revision = _hf_snapshot_revision(source)
if source_revision is None:
raise ValueError(
f"{model} holds symlinked .safetensors files outside a Hugging "
"Face snapshots/<commit> directory; Soup reads weights only "
"from regular files: pass the Hub id, the HF snapshot "
"directory, or a directory of regular files"
)
return _hf_snapshot_plan(
_hf_cache_repo_id(source) or model,
source,
manifest,
source_revision,
slot=_local_snapshot_slot(source),
)
return ModelWeightsPlan(
model=model,
source_dir=source,
weights_dir=source,
source_bytes=sum(size for _name, size, _mtime in manifest),
materialized_copy_bytes=0,
materialize_bytes=0,
source_files=manifest,
)
source = os.path.realpath(_download_hub_checkpoint(model))
source_revision = _hf_snapshot_revision(source)
manifest = _weight_file_manifest(source, permit_symlinks=True)
source_paths = [os.path.join(source, name) for name, _size, _mtime in manifest]
if all(not os.path.islink(path) for path in source_paths):
return ModelWeightsPlan(
model=model,
source_dir=source,
weights_dir=source,
source_bytes=sum(size for _name, size, _mtime in manifest),
materialized_copy_bytes=0,
materialize_bytes=0,
source_files=manifest,
source_revision=source_revision,
)
return _hf_snapshot_plan(model, source, manifest, source_revision)
def _download_hub_checkpoint(model: str) -> str:
"""Fetch a Hub repo's checkpoint files, and nothing beside them (#1806).
``*.safetensors`` fetched Mistral's ``consolidated.safetensors`` next to the
shards: 29.0 GB for a 14.5 GB model. One call asks for the Hugging Face
spellings (``model.safetensors``, ``model-*-of-*.safetensors``) and the JSON
files; one call, so an already complete cache entry resolves offline as
before (#533). The local rule then reads what came, and a second call is
made only when it says the checkpoint is not all there: the files an index
names outside those spellings, or, with no index and neither spelling
present, every ``*.safetensors`` except ``consolidated*``, which is the
local fallback rule itself. So what is downloaded is what is counted.
"""
from soup_cli.utils.checkpoint_files import (
hub_allow_patterns,
hub_second_call,
read_index_weight_map,
select_checkpoint_files,
)
from soup_cli.utils.hubs import snapshot_download
from soup_cli.utils.run_provenance import base_revision_kwargs
revision_kwargs = base_revision_kwargs(model)
source = snapshot_download(
model, cache_dir=None, allow_patterns=[*hub_allow_patterns(None), "*.json"],
**revision_kwargs,
)
real = os.path.realpath(source)
selected = select_checkpoint_files(os.listdir(real), read_index_weight_map(real))
second = hub_second_call(selected)
if second is not None:
allow, ignore = second
source = snapshot_download(
model,
cache_dir=None,
allow_patterns=[*allow, "*.json"],
ignore_patterns=ignore or None,
**revision_kwargs,
)
return source
def _hf_cache_repo_id(source_dir: str) -> Optional[str]:
"""``org/name`` for a ``models--org--name/snapshots/<sha>`` directory."""
repo_dir = os.path.basename(os.path.dirname(os.path.dirname(source_dir)))
if not repo_dir.startswith("models--"):
return None
return repo_dir[len("models--"):].replace("--", "/") or None
def _location_key(path: str) -> str:
"""One spelling per directory: links resolved, letter case folded where the OS does."""
return os.path.normcase(os.path.realpath(path))
def _local_snapshot_slot(source: str) -> str:
"""Cache slot for the regular-file copy of a local ``snapshots/<commit>`` directory.
The slot of the Hub id ``org/name`` when the snapshot's repo folder is the directory
that id resolves to in the active Hugging Face cache. A folder anywhere else gets a
slot of its own, ``<folder name>@<digest of its resolved location>``: a copy is
reused on matching names and sizes, so two requests may share one only when they
name the same place. ``@`` cannot occur in a :func:`model_slug`, which keeps a
folder's slot apart from every slot named after a model.
"""
repo_root = os.path.dirname(os.path.dirname(source))
location = _location_key(repo_root)
repo_id = _hf_cache_repo_id(source)
if repo_id is not None:
from soup_cli.utils.hubs import hf_cache_dir
cache = hf_cache_dir()
if cache is not None:
in_cache = os.path.join(cache, os.path.basename(repo_root))
if _location_key(in_cache) == location:
return model_slug(repo_id)
name = os.path.basename(location)
label = model_slug(name)[:_LOCAL_SLOT_LABEL_CHARS] if name.strip() else "local"
digest = hashlib.sha256(os.fsencode(location)).hexdigest()
return f"{label}@{digest[:_LOCAL_SLOT_DIGEST_CHARS]}"
def _hf_snapshot_plan(
model: str,
source: str,
manifest: Tuple[Tuple[str, int, int], ...],
source_revision: Optional[str],
*,
slot: Optional[str] = None,
) -> ModelWeightsPlan:
"""Plan the regular-file copy of a symlinked HF snapshot (reused when current)."""
from soup_cli.utils.run_provenance import base_revision_kwargs
if slot is None:
slot = model_slug(model)
revision = base_revision_kwargs(model).get("revision")
if revision:
slot += "@" + revision
materialized = os.path.join(
resolve_cache_dir(), "weights", slot
)
existing = _materialized_matches_hf_snapshot(
source,
materialized,
manifest,
source_revision=source_revision,
)
needs_copy = existing is None
return ModelWeightsPlan(
model=model,
source_dir=source,
weights_dir=materialized,
source_bytes=sum(size for _name, size, _mtime in manifest),
materialized_copy_bytes=sum(size for _name, size, _mtime in manifest),
materialize_bytes=(
sum(size for _name, size, _mtime in manifest) if needs_copy else 0
),
source_files=manifest if existing is None else existing,
source_revision=source_revision,
)
def materialize_model_weights(plan: ModelWeightsPlan) -> str:
"""Create the planned regular-file copy only when the source requires it."""
if not plan.needs_materialization:
return plan.weights_dir
return _materialize_cached_snapshot(plan)
def resolve_model_weights(
model: str,
*,
before_materialize: Optional[Callable[[ModelWeightsPlan], None]] = None,
) -> str:
"""Return a regular-file weight directory, with an optional pre-copy gate."""
plan = plan_model_weights(model)
if before_materialize is not None:
before_materialize(plan)
return materialize_model_weights(plan)
def scan_model(
model: str,
*,
modules: ModulesArg = "all",
use_cache: bool = True,
cache_dir: Optional[str] = None,
) -> ScanResult:
"""Resolve, (cache-)scan a model and return its :class:`ScanResult`."""
canonical = _canonical_modules_str(modules)
if use_cache:
cached = read_cached_scan(model, modules=canonical, cache_dir=cache_dir)
if cached is not None:
return cached
weights_dir = resolve_model_weights(model)
layers = scan_weights_dir(weights_dir, modules=modules)
result = ScanResult(model=model, modules=canonical, layers=layers)
if use_cache:
try:
write_cached_scan(result, cache_dir=cache_dir)
except OSError:
pass # cache write failures are non-fatal
return result