Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

18.9 KB
Raw Blame
"""``SOUP_LAYER_STREAM_STRIPE_DIRS`` — extra NVMe roots the layer-shard cache stripes over.

The primary cache root (``SOUP_LAYER_STREAM_CACHE_DIR`` or ``~/.soup/layer-stream``) keeps its
meaning exactly, lenient fallback included. This variable is deliberately the opposite: setting
it is an explicit act, so every entry is validated and a bad one is refused by name. A silent
fallback here would train at single-drive speed with nothing said, which is the failure the
feature exists to remove.

Why it exists, measured: ``benchmarks/probe-rtx5070-two-drive-read.md`` — two NVMe drives read
7.65-9.15 GB/s together where one reads ~4.

No top-level torch: imported on the setup path only.
"""

import hashlib
import os
import stat
from dataclasses import dataclass
from typing import Callable, Iterator, List, Mapping, Optional, Sequence, Tuple

from soup_cli.utils.config_bounds import (
    DEFAULT_STREAM_READ_AHEAD,
    MAX_STREAM_READ_AHEAD,
    MIN_STREAM_READ_AHEAD,
)
from soup_cli.utils.paths import is_under, quote_path

STRIPE_DIRS_ENV = "SOUP_LAYER_STREAM_STRIPE_DIRS"

#: With N roots the reader wants N + 1 staging slots (one read in flight per drive plus the
#: slot the consumer holds), and ``stream_read_ahead`` stops at MAX_STREAM_READ_AHEAD, so the
#: primary root plus this many extra roots is the most the depth can serve.
MAX_STRIPE_DIRS = MAX_STREAM_READ_AHEAD - 2

#: Windows device-namespace spellings (``\\?\C:\...``, ``\\.\C:\...``), refused by name.
_DEVICE_NAMESPACE_PREFIXES = ("\\\\?\\", "\\\\.\\")


class StripeRootError(ValueError):
    """A stripe root that breaks a rule; the message names the entry and the rule."""


def effective_read_ahead(configured: int, n_roots: int) -> int:
    """The async reader's depth for ``configured`` over ``n_roots`` drives.

    Every drive needs a read in flight beside the slot the consumer holds, so the DEFAULT depth
    becomes ``n_roots + 1``, capped at MAX_STREAM_READ_AHEAD. "Default" means the configured
    value equals DEFAULT_STREAM_READ_AHEAD — a default is not a decision (``config/schema.py``),
    and a config round-tripped through ``model_dump`` looks explicitly set everywhere. Any
    other value is the operator's and is kept.
    """
    configured = int(configured)
    if n_roots <= 1 or configured != DEFAULT_STREAM_READ_AHEAD:
        return configured
    return max(configured, min(n_roots + 1, MAX_STREAM_READ_AHEAD))


@dataclass(frozen=True)
class ReadAheadDecision:
    """The depth the reader uses, what the operator configured, and over how many drives.

    Carried from the setup to every message that advises lowering the depth: after the N + 1
    bump, "lower it" is a loop — the operator writes 2, which is the default, and it is raised
    again. :meth:`lowering_advice` names a value that the rule does NOT raise back up.
    """

    depth: int
    configured: int
    n_roots: int = 1

    @property
    def raised(self) -> bool:
        """Soup deepened the configured value because the cache spans several drives."""
        return self.depth > self.configured

    def _next_lower(self) -> Optional[int]:
        """The largest configurable value whose effective depth is below this one."""
        for candidate in range(self.depth - 1, MIN_STREAM_READ_AHEAD - 1, -1):
            if effective_read_ahead(candidate, self.n_roots) < self.depth:
                return candidate
        return None

    def lowering_advice(self) -> str:
        """A clause telling the operator how to lower the depth, or ``""`` at the floor.

        One drive keeps the historical wording. With several, the clause names a value that
        actually lowers the depth, says when the depth was raised (and that unsetting the
        variable undoes that), and says when the default value would be re-raised.
        """
        lower = self._next_lower()
        if lower is None:
            return ""
        if self.n_roots <= 1:
            return f"Lower training.stream_read_ahead (currently {self.depth})"
        notes = []
        if self.raised:
            notes.append(
                f"raised from {self.configured} because the layer cache spans "
                f"{self.n_roots} drives"
            )
        if lower < DEFAULT_STREAM_READ_AHEAD < self.depth:
            bumped = effective_read_ahead(DEFAULT_STREAM_READ_AHEAD, self.n_roots)
            notes.append(
                f"with {self.n_roots} drives, {DEFAULT_STREAM_READ_AHEAD} counts as the "
                f"default and becomes {bumped}"
            )
        said = f"Set training.stream_read_ahead to {lower} (currently {self.depth}"
        said += "".join(f"; {note}" for note in notes) + ")"
        if self.raised:
            said += f", unset {STRIPE_DIRS_ENV} or name fewer folders in it"
        return said


def parse_stripe_dirs(raw: Optional[str]) -> Tuple[str, ...]:
    """Split the variable on ``os.pathsep``; blank entries are dropped, not refused."""
    if raw is None:
        return ()
    return tuple(part.strip() for part in raw.split(os.pathsep) if part.strip())


def _nearest_existing(path: str) -> str:
    anchor = os.path.realpath(os.path.expanduser(path))
    while not os.path.exists(anchor):
        parent = os.path.dirname(anchor)
        if parent == anchor:
            raise StripeRootError(f"cannot find an existing folder above {quote_path(path)}")
        anchor = parent
    return anchor


def volume_of(path: str) -> int:
    """The volume ``path`` lives on: ``st_dev``, the volume serial number on Windows."""
    return int(os.stat(_nearest_existing(path)).st_dev)


def _is_link(info: os.stat_result) -> bool:
    """A symlink, or on Windows any reparse point: a junction does not report S_ISLNK (and
    ``os.path.islink`` misses it before 3.12), but it redirects exactly as a symlink does."""
    if stat.S_ISLNK(info.st_mode):
        return True
    reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
    return os.name == "nt" and bool(getattr(info, "st_file_attributes", 0) & reparse)


def is_volume_mount_point(path: str) -> bool:
    """Whether the reparse point at ``path`` is a genuine Windows volume mount point.

    A directory junction and a volume mount point carry the identical
    ``IO_REPARSE_TAG_MOUNT_POINT`` tag (``_is_link`` cannot tell them apart), so the two are
    distinguished by their target instead: ``mountvol`` names a volume mount point's target
    ``\\\\?\\Volume{<guid>}\\`` with nothing after it -- it names a whole volume rather than a
    folder, where a junction can target a folder BENEATH a volume GUID path just as easily as
    one beneath a drive letter or UNC share. Refusing every reparse point would also refuse a
    legitimate volume-mount-point stripe root; this is the narrower rule that does not.
    """
    try:
        target = os.readlink(path)
    except (OSError, ValueError):
        # On Windows, os.readlink raises ValueError ("not a symbolic link"), not OSError,
        # for a reparse point that is neither a symlink nor a junction/mount point.
        return False
    stripped = target.removeprefix("\\\\?\\").removeprefix("\\??\\")
    volume = stripped.rstrip("\\")
    return volume.startswith("Volume{") and volume.endswith("}") and "\\" not in volume


def check_stripe_root(
    entry: str,
    *,
    primary_root: str,
    accepted: Sequence[str] = (),
) -> Tuple[Optional[str], Optional[str]]:
    """Syntax, existence, symlink and overlap rules; returns ``(resolved, reason)``.

    If valid, returns ``(resolved_realpath, None)``.
    If invalid, returns ``(None, reason)`` where ``reason`` does NOT contain the
    environment variable or entry prefix.
    """
    if any(ord(ch) < 0x20 or 0x7F <= ord(ch) <= 0x9F for ch in entry):
        return None, "contains control characters"
    if os.name == "nt" and entry.replace("/", "\\").startswith(_DEVICE_NAMESPACE_PREFIXES):
        return None, (
            "a device-namespace path is not accepted; name the folder by its drive "
            "letter or UNC share"
        )
    expanded = os.path.expanduser(entry)
    # Checked before isabs: Python 3.13 no longer calls a rooted path without a drive
    # ("\data", "/data") absolute on Windows, and 3.11-3.12 do; the message is the same on all.
    if os.name == "nt" and expanded[:1] in ("\\", "/") and not os.path.splitdrive(expanded)[0]:
        return None, "must name a drive letter or a UNC share"
    if not os.path.isabs(expanded):
        return None, "must be an absolute path"
    if os.path.islink(expanded):
        return None, "must not be a symlink"
    if os.name == "nt":
        try:
            entry_info = os.lstat(expanded)
        except OSError:
            entry_info = None
        if (
            entry_info is not None
            and _is_link(entry_info)
            and not is_volume_mount_point(expanded)
        ):
            return None, (
                "is a directory junction, not a real folder or a genuine Windows "
                "volume mount point; Soup would stripe layer writes through it to wherever "
                "it currently points, which a junction (unlike a mounted volume) can be "
                "repointed to at any time"
            )
    if not os.path.isdir(expanded):
        return None, (
            "must be an existing directory. Soup creates only its per-model folder "
            "inside a stripe root, never the root itself, so a drive that is not mounted "
            "cannot be mistaken for an empty one. Reconnect the drive, or remove this entry "
            f"(unset {STRIPE_DIRS_ENV} to re-shard to one root)."
        )
    resolved = os.path.realpath(expanded)
    primary = os.path.realpath(os.path.expanduser(primary_root))
    if is_under(resolved, primary) or is_under(primary, resolved):
        return None, f"overlaps the primary layer-stream cache root {primary}"
    for other in accepted:
        if is_under(resolved, other) or is_under(other, resolved):
            return None, f"overlaps another stripe root, {other}"
    return resolved, None


def validate_stripe_root(entry: str, *, primary_root: str, accepted: Sequence[str] = ()) -> str:
    """Syntax, existence, symlink and overlap rules; returns the entry's realpath.

    The distinct-volume and NVMe rules live in :func:`resolve_stripe_roots`, which the setup
    path runs once. The sharder calls this one to bound its own writes without paying the
    ~9 s disk-kind probe a second time.
    """
    resolved, reason = check_stripe_root(entry, primary_root=primary_root, accepted=accepted)
    if reason is not None:
        raise StripeRootError(f"{STRIPE_DIRS_ENV} entry {quote_path(entry)}: {reason}")
    assert resolved is not None
    return resolved


def iter_early_stripe_roots(
    primary_root: str,
    entries: Sequence[str],
) -> Iterator[Tuple[str, Optional[str], Optional[str]]]:
    """Validate stripe root entries early, yielding ``(entry, resolved, reason)``.

    ``resolved`` is the resolved canonical path when valid, or ``None``.
    ``reason`` is the refusal message without environment variable prefix when invalid,
    or ``None``.
    """
    accepted: List[str] = []
    primary_canonical = os.path.realpath(os.path.expanduser(primary_root))
    owners = {volume_of(primary_canonical): f"the primary cache root {primary_canonical}"}

    for idx, entry in enumerate(entries):
        if len(entries) > MAX_STRIPE_DIRS and idx >= MAX_STRIPE_DIRS:
            yield (
                entry,
                None,
                (
                    f"names {len(entries)} folders; at most {MAX_STRIPE_DIRS} "
                    f"(training.stream_read_ahead stops at {MAX_STREAM_READ_AHEAD}, "
                    f"and every drive needs a staging slot of its own)"
                ),
            )
            continue

        resolved, reason = check_stripe_root(
            entry, primary_root=primary_canonical, accepted=accepted
        )
        if reason is not None:
            yield (entry, None, reason)
            continue

        assert resolved is not None
        volume = volume_of(resolved)
        if volume in owners:
            yield (
                entry,
                None,
                (
                    f"on the same volume as {owners[volume]}. "
                    f"Two roots on one drive read no faster than one and cost a staging slot."
                ),
            )
            continue

        owners[volume] = f"stripe root {resolved}"
        accepted.append(resolved)
        yield (entry, resolved, None)


def validate_early_stripe_roots(
    primary_root: str,
    *,
    environ: Optional[Mapping[str, str]] = None,
) -> Tuple[str, ...]:
    """Syntax, existence, count cap, symlink, overlap and same-volume rules.

    Runs early in the CLI (``soup train``, ``--dry-run``) before expensive dataset
    loading or run creation. Omits the ~9 s NVMe disk-kind probe, which stays in
    :func:`resolve_stripe_roots` on the setup path.
    """
    env = os.environ if environ is None else environ
    entries = parse_stripe_dirs(env.get(STRIPE_DIRS_ENV))
    if not entries:
        return ()
    if len(entries) > MAX_STRIPE_DIRS:
        raise StripeRootError(
            f"{STRIPE_DIRS_ENV} names {len(entries)} folders; at most {MAX_STRIPE_DIRS} "
            f"(training.stream_read_ahead stops at {MAX_STREAM_READ_AHEAD}, and every drive "
            f"needs a staging slot of its own)"
        )
    accepted: List[str] = []
    for entry, resolved, reason in iter_early_stripe_roots(primary_root, entries):
        if reason is not None:
            raise StripeRootError(f"{STRIPE_DIRS_ENV} entry {quote_path(entry)}: {reason}")
        assert resolved is not None
        accepted.append(resolved)
    return tuple(accepted)


def resolve_stripe_roots(
    primary_root: str,
    *,
    disk_kind: Callable[[str], str],
    environ: Optional[Mapping[str, str]] = None,
) -> Tuple[str, ...]:
    """The validated stripe roots, or ``()`` when the variable is unset or blank.

    ``disk_kind`` maps a path to ``nvme`` / ``ssd`` / ``hdd`` / ``unknown``; the setup path
    passes ``resolve_disk_kind(...).kind`` so ``training.stream_disk_kind`` overrides the probe
    here exactly as it does for the primary cache.
    """
    env = os.environ if environ is None else environ
    entries = parse_stripe_dirs(env.get(STRIPE_DIRS_ENV))
    if not entries:
        return ()
    accepted = validate_early_stripe_roots(primary_root, environ=env)
    for entry, resolved in zip(entries, accepted):
        kind = disk_kind(resolved)
        if kind != "nvme":
            raise StripeRootError(
                f"{STRIPE_DIRS_ENV} entry {quote_path(entry)}: the disk tier streams "
                f"from NVMe only, and this volume classifies as {kind!r}. If the probe "
                f"is wrong, training.stream_disk_kind overrides it."
            )
    return accepted


def layer_roots_for(n_layers: int, n_roots: int) -> Tuple[int, ...]:
    """Decoder layer ``i`` lives on root ``i mod n_roots``; ``()`` means one root."""
    if n_roots < 1:
        raise ValueError(f"n_roots must be at least 1; got {n_roots}")
    if n_roots == 1:
        return ()
    return tuple(idx % n_roots for idx in range(n_layers))


# -- the per-model folder inside a stripe root ------------------------------------------------
def primary_cache_identity(shard_dir: str) -> str:
    """The primary cache a stripe folder belongs to: its realpath, case-folded on Windows."""
    return os.path.normcase(os.path.realpath(os.path.abspath(os.path.expanduser(shard_dir))))


def stripe_folder_name(shard_dir: str) -> str:
    """The folder a stripe root holds for the primary cache at ``shard_dir``.

    The model slug plus 12 hex of a hash of the primary cache's realpath. Two primary caches of
    one base (per user, or per ``SOUP_LAYER_STREAM_CACHE_DIR``, or with different dtypes) that
    share one stripe root therefore never share a folder, so neither can read the other's layer
    files as its own or overwrite them on a re-shard. The slug keeps the folder recognisable.
    """
    slug = os.path.basename(os.path.normpath(shard_dir))
    digest = hashlib.sha256(os.fsencode(primary_cache_identity(shard_dir))).hexdigest()[:12]
    return f"{slug}-{digest}"


def stripe_folder_problem(folder: str, root: str) -> Optional[str]:
    """Why ``folder`` must not be read, written or deleted in, or ``None``.

    ``root`` is the VALIDATED stripe root (a realpath) and ``folder`` its per-model child. A
    missing folder is not a problem here — the caller creates it or reports it. A link or
    junction at the folder is, and so is a folder whose realpath is not ``root``'s child: that
    catches a root swapped for a link after it was validated, which lstat on the folder cannot.
    """
    try:
        info = os.lstat(folder)
    except FileNotFoundError:
        return None
    except OSError as exc:
        return f"the stripe folder {folder} cannot be inspected ({exc})"
    if _is_link(info):
        return (
            f"the stripe folder {folder} is a link or junction. Soup reads and writes only a "
            f"real folder it made inside a {STRIPE_DIRS_ENV} entry, so it will not follow one; "
            f"remove the link (Soup never deletes inside a stripe root) or name another folder"
        )
    if not stat.S_ISDIR(info.st_mode):
        return f"the stripe folder {folder} exists and is not a folder"
    expected = os.path.join(root, os.path.basename(folder))
    actual = os.path.realpath(folder)
    if os.path.normcase(actual) != os.path.normcase(expected):
        return (
            f"the stripe folder {folder} resolves to {actual}, not to a folder inside its "
            f"stripe root {root}; a link somewhere on that path changed after it was checked"
        )
    return None


def secure_stripe_folder(folder: str, root: str) -> None:
    """Make ``folder`` (inside the validated ``root``) Soup's own before anything touches it.

    Refuses a link or junction; then creates the folder owner-only, or verifies that an
    existing one belongs to this account (and nobody else can write it) and restricts it
    again. Files there are trusted between runs as the owner-only primary cache's are, so a
    folder that cannot be made private refuses the run by name rather than being used.
    """
    from soup_cli.utils import owner_only_dir

    problem = stripe_folder_problem(folder, root)
    if problem is not None:
        raise StripeRootError(f"{STRIPE_DIRS_ENV}: {problem}.")
    try:
        owner_only_dir.ensure_owner_only_dir(folder)
    except OSError as exc:
        raise StripeRootError(
            f"{STRIPE_DIRS_ENV}: the stripe folder {folder} cannot be made private to the "
            f"account running Soup: {exc}. Its layer files are trusted between runs, as the "
            f"owner-only primary cache's are, so Soup refuses rather than use a folder another "
            f"account could change. Remove it (Soup never deletes inside a stripe root) or "
            f"point {STRIPE_DIRS_ENV} at a folder this account owns."
        ) from exc
    problem = stripe_folder_problem(folder, root)
    if problem is not None:
        raise StripeRootError(f"{STRIPE_DIRS_ENV}: {problem}.")