Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

81.0 KB
Raw Blame
"""training.stream_layers — layer streaming planner (v0.72.0 BETA).

The pure half: tier choice, pinned-vs-pageable decision, the architecture
allowlist, and the VRAM / throughput arithmetic. **No top-level torch** — this
module sits on the light CLI's import path so `soup profile` and friends can
forecast a streaming run without pulling in the training stack.

The runtime half (buffer pool, weight source, prefetch scheduler, layer
wrapper) lives in ``layer_stream_runtime.py``; the checkpoint sharder lives in
``layer_shard.py``.

Model of the mechanism: the frozen base lives in CPU RAM and is streamed into a
small pool of pre-allocated VRAM buffers one decoder layer at a time. Vocabulary
embeddings and an untied output head share a second, single-slot pool. Peak VRAM
is therefore bounded by one decoder layer plus one vocabulary matrix rather than
the whole model. Only the LoRA adapters, their gradients and optimizer state
stay resident.
"""

import math
from dataclasses import dataclass
from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Tuple, Union

from rich.panel import Panel

# Buffer, read-ahead and task bounds live in a dependency-free leaf so the schema
# can import them without this module's rich subtree (#780). Re-exported here, not
# redeclared, so the schema bound and the runtime message are the same object.
from soup_cli.utils.config_bounds import (
    DEFAULT_STREAM_BUFFERS,
    DEFAULT_STREAM_READ_AHEAD,
    MAX_STREAM_BUFFERS,
    MAX_STREAM_READ_AHEAD,  # noqa: F401
    MIN_STREAM_BUFFERS,
    MIN_STREAM_READ_AHEAD,  # noqa: F401
    ROLLOUT_STREAM_TASKS,  # noqa: F401
    SUPPORTED_STREAM_TASKS,  # noqa: F401
)

# Quantisation names are re-exported, NOT redeclared: layer_shard owns the shard
# format and these same strings key its cache-invalidation check. Two copies
# could drift and leave this module's pre-flight RAM estimate disagreeing with
# what was actually written to disk. (layer_shard has no top-level torch either,
# so the light CLI import path is unaffected.)
from soup_cli.utils.layer_shard import QUANT_NONE, SUPPORTED_STREAM_QUANTS

# --- tiers (plan 5.1) -----------------------------------------------------
TIER_RAM = "ram"
TIER_DISK = "disk"
STREAM_SOURCES = ("auto", "ram", "disk")

#: store_bytes must be under this fraction of free RAM to claim the RAM tier.
RAM_TIER_HEADROOM = 0.7
#: store_bytes plus resident extras must also stay under this fraction of total
#: physical RAM. Pinned host memory is unevictable while shard reads pressure the
#: page cache, so a run can OOM even when the dynamic MemAvailable check passes (#622).
#: **A chosen safety margin, not a measured bound** — unlike the measured constants
#: below, no benchmark derives 0.55. It is also not a tmpfs limit: Linux defaults
#: /dev/shm to 50% of RAM and a container's is far smaller, so a store that clears
#: this ceiling can still overflow the shared-memory mount it is allocated from.
#: Reading that real limit (``os.statvfs("/dev/shm")``, ``RLIMIT_MEMLOCK``) is the
#: follow-up this ceiling stands in for, not something it measures (#644 review).
PHYSICAL_RAM_TIER_HEADROOM = 0.55
PHYSICAL_RAM_TIER_HEADROOM_PERCENT = round(PHYSICAL_RAM_TIER_HEADROOM * 100)

# Measured throughput a page-locked RAM store buys over a pageable one
# (benchmarks/gate-h100-validation.md): 6.56x on real Qwen2.5-32B NF4, 7.41x on
# a 32-layer synthetic. Stated out loud whenever pinning is disabled so the cost
# is not absorbed silently — a silent fallback spends the entire margin.
PIN_THROUGHPUT_GAIN_REAL = 6.56
PIN_THROUGHPUT_GAIN_SYNTHETIC = 7.41

#: FLOPs per parameter per token. 6 == WITH gradient checkpointing
#: (2 forward + 2 recompute + 2 dL/dx; base weight-grads are skipped because
#: the base is frozen). Streaming always checkpoints, so this is never 4.
FLOPS_PER_PARAM_PER_TOKEN = 6

#: Families whose decoder layout has been proven bit-exact against a resident
#: run of the same numerics (v0.72.0 llama/qwen2/qwen3; v0.72.3 GATE 1 added the
#: rest, under bf16 AND NF4). An allowlist rather than a heuristic because a
#: half-supported architecture streams weights into the wrong module and
#: mis-trains silently instead of crashing.
#:
#: ``gemma3_text`` is present but ``gemma3`` is NOT: a real ``google/gemma-3-*``
#: reports ``model_type='gemma3'`` for the vision-capable wrapper, and streaming
#: a multimodal wrapper as though it were a causal LM is exactly the failure the
#: allowlist exists to prevent.
#:
#: ``deepseek_v3`` is the DeepSeek-V3 family (MLA attention, DeepSeekMoE with
#: a shared expert, the first ``first_k_dense_replace`` layers dense). Kimi K2
#: ships the same decoder under ``model_type='kimi_k2'``, which transformers
#: does not register. The streamed model's config loader (``utils/kimi_k2.py``)
#: reads such a config into the native ``DeepseekV3Config`` behind a guard, so a
#: K2 that passes it streams as ``deepseek_v3``; the name ``kimi_k2`` is not in
#: this list, and a config still carrying it did not pass that guard.
SUPPORTED_STREAM_ARCHS = (
    "llama",
    "qwen2",
    "qwen3",
    "qwen4_exp",
    "mistral",
    "gemma",
    "gemma2",
    "gemma3_text",
    "phi",
    "phi3",
    "deepseek_v3",
)

# Model types whose text decoders reuse an admitted streaming family.  Each
# alias needs its own resident-vs-streamed parity control; mapping a model type
# by name alone is not enough to establish that its decoder graph is safe.
_STREAM_ARCH_ALIASES = {
    "qwen3_5": "qwen3",
    "qwen3_5_text": "qwen3",
    "qwen3_5_moe": "qwen3",
    "qwen3_5_moe_text": "qwen3",
    "qwen4_exp_text": "qwen4_exp",
}

#: The loss path's own arithmetic, in VRAM bytes per logit element. **Measured
#: stage by stage (issue #327), not derived.** ``ForCausalLMLoss`` upcasts to
#: fp32, hands the view to ``cross_entropy`` and returns; the residency at each
#: stage, per element, excluding the source logits tensor:
#:
#:   after the loss returns   4  — only log-softmax's saved fp32 output survives;
#:                                 the fp32 upcast is freed with the local
#:   backward peak           12  — that saved output plus the two transient fp32
#:                                 gradient buffers (nll -> log-softmax)
#:
#: So the peak holds THREE fp32 logits-shaped buffers, not "upcast + softmax +
#: grad" as v0.72.3 recorded — the totals agree, the decomposition did not.
#: Measured 12.000000 exactly, spread 0.00e+00 over 3 repeats, at every
#: (vocab, tokens) probed, and **byte-identical under torch 2.5.1 and 2.13.0
#: and under trl 0.19.1 and 0.26.2** — this term is not stack-dependent.
LOGITS_LOSS_BYTES_PER_ELEMENT = 12

#: One further bf16 logits-shaped tensor, charged unconditionally.
#:
#: This is the whole of the inter-stack disagreement in issue #327. The v0.72.3
#: grid (RTX 3050 / Windows) needs 13.869-13.901 bytes per element across its
#: ten rows; the H100 grid fits 12.311, and a real ``trl`` training step
#: measured on that box marginals at 12.0955 (12.0821 once the probe fixture's
#: own per-token cost is removed). The excess the RTX grid requires over that
#: is **1.8031 bytes per element**, fitted across its 3.1x vocab contrast with a
#: flat residual of 21.0 bytes per token — i.e. the excess is vocab-scaled, the
#: size of one bf16 copy of the logits, and not a mis-modelled activation term.
#: A single-variable control reproduces it directly: holding the model
#: output object across ``backward()`` instead of letting it die with the local
#: costs **+2.0000 bytes per element**, at 5 of 6 (vocab, tokens) cells and
#: +2.0480 at the sixth (allocator rounding on a 32k-vocab, 256-token cell).
#:
#: What retains it on that stack is NOT identified. torch, trl and transformers
#: were each swapped to the RTX grid's versions on the H100 box and the real
#: training step measured byte-identical, so none of them is the cause. Until
#: something can OBSERVE the retention at pre-flight — a synthetic probe cannot,
#: it sees only its own reference — this stays charged: dropping it under-
#: predicts 10 of the 10 v0.72.3 rows by up to 10.49%, and on Windows an
#: under-prediction does not raise, it silently spills to host memory.
LOGITS_RETENTION_BYTES_PER_ELEMENT = 2

#: VRAM bytes per logit element at peak: the loss arithmetic plus the retained
#: copy. v0.72.0 charged 6 from first principles, which under-predicts this term
#: by 2.33x — a ~5 GB error on a 152k-vocab model at batch 8, where the logits
#: tensor is 146x the entire buffer pool.
LOGITS_BYTES_PER_ELEMENT = LOGITS_LOSS_BYTES_PER_ELEMENT + LOGITS_RETENTION_BYTES_PER_ELEMENT
#: Forward only, no loss: just the bf16 logits.
LOGITS_BYTES_PER_ELEMENT_NO_LOSS = 2

#: Bytes per trainable adapter parameter at peak: fp32 weight + fp32 grad +
#: Adam ``m`` + ``v``. Charged even under an 8-bit optimizer (which needs 10) —
#: over-charging ~6 bytes across a few million adapter parameters is noise next
#: to the logits term, and it errs in the safe direction.
OPTIMIZER_BYTES_PER_PARAM = 16

#: Constant offset measured across both GATE 2 models (RoPE caches, allocator
#: rounding, small per-run buffers). It came out at ~13.3 MB for a 135M model
#: and a 0.5B model alike, so it is charged as a constant rather than scaled.
STREAM_FIXED_SLACK_BYTES = 13_500_000

#: Fraction of the *measured same-session GEMM ceiling* that real streamed
#: training actually reached on the dev box: 68% (Llama-3.1-8B NF4, 5.26 of
#: 7.74 TFLOPS) up to ~100% (small models, fully pinned store). A forecast
#: quoted as a single number would over-promise by up to 1.5x, so the bracket
#: travels with it.
MEASURED_CEILING_FRACTION = (0.68, 1.0)

#: ``uint8`` is a STORAGE dtype only — NF4 packs two nibbles per byte and (under
#: double quant) stores absmax as uint8 too. It is never a base/compute dtype,
#: which is what ``SUPPORTED_STREAM_DTYPES`` lists.
_DTYPE_BYTES = {"bfloat16": 2, "float16": 2, "float32": 4, "uint8": 1}
SUPPORTED_STREAM_DTYPES = ("bfloat16", "float16", "float32")

# --- NF4 (v0.72.2) --------------------------------------------------------
#: Streamed bytes per parameter under NF4 + double quant: packed ``N/2`` +
#: absmax ``N/64`` + nested absmax ``N/64/256*4`` + a 4-byte offset per weight.
NF4_BYTES_PER_PARAM = 0.5 + 1 / 64 + 4 / (64 * 256)
#: Without double quant the per-block absmax stays float32: ``N/2 + 4*N/64``.
NF4_BYTES_PER_PARAM_SINGLE = 0.5 + 4 / 64

#: CUDA context + allocator fragmentation headroom (plan 4.1).
DEFAULT_WORKSPACE_BYTES = 1_000_000_000

_NVME = "nvme"


def dtype_bytes(name: str) -> int:
    """Bytes per element. THE single dtype-size table for layer streaming —
    the sharder, the planner and the trainer must not each keep their own."""
    try:
        return _DTYPE_BYTES[name]
    except KeyError:
        raise ValueError(
            f"unsupported dtype {name!r} for layer streaming; "
            f"supported: {', '.join(SUPPORTED_STREAM_DTYPES)}"
        ) from None


def resolve_stream_dtype(device: str = "cuda") -> str:
    """The store/compute dtype for a streamed run, chosen from the CARD.

    Until v0.73.x this was the literal ``"bfloat16" if on_cuda else "float32"``
    in ``trainer/stream_setup.py``, which is wrong on every pre-Ampere GPU:
    Colab's free tier is a T4 (sm_75), Kaggle is a T4 or a P100, and V100 /
    GTX 16xx / RTX 20xx all report ``is_bf16_supported() == False``. That is
    most of the hardware this feature exists for, and it could not fail on the
    Ampere card every published measurement was taken on (#385).

    The CUDA capability question is delegated to
    ``utils.gpu.get_compute_dtype`` rather than answered again here — one
    question, one answer, or the two drift. MPS is separate: PyTorch supports
    bfloat16 there from macOS 14, so a one-element allocation probes the exact
    runtime rather than guessing from a version string. CPU stays float32: CPU
    streaming is a test convenience, and half-precision CPU kernels are not
    uniformly available.

    Correctness is unaffected by the choice. Streamed-vs-resident logits were
    measured bit-exact (``0.000000e+00``) in float16 as well as bfloat16, in
    both quantisations, against resident references of matching numerics.
    """
    device_name = str(device).lower()
    if device_name.startswith("mps"):
        from soup_cli.utils.gpu import mps_supports_bf16

        return "bfloat16" if mps_supports_bf16() else "float32"
    if not device_name.startswith("cuda"):
        return "float32"

    from soup_cli.utils.gpu import get_compute_dtype

    name = str(get_compute_dtype()).removeprefix("torch.")
    # get_compute_dtype already returns float32 when no CUDA runtime is present,
    # so a "cuda" string on a CPU-only box does not claim a GPU dtype.
    return name if name in SUPPORTED_STREAM_DTYPES else "float32"


# ==========================================================================
# architecture gate
# ==========================================================================
def stream_arch_of(config: Any) -> str:
    """Return the streaming architecture family, or raise naming the allowlist.

    Mirrors ``utils/shrink.arch_family_of_config`` — an allowlist, not a
    heuristic, because a half-supported architecture streams weights into the
    wrong module and produces silently wrong numbers rather than a crash.
    """
    model_type = getattr(config, "model_type", None)
    text_config = getattr(config, "text_config", None)
    text_model_type = getattr(text_config, "model_type", None)
    raw_family = model_type
    if isinstance(text_model_type, str):
        alias = text_model_type.strip().lower()
        if alias in _STREAM_ARCH_ALIASES:
            raw_family = text_model_type
        elif raw_family is None:
            raw_family = text_model_type
    if not raw_family or not isinstance(raw_family, str):
        raise ValueError(
            "layer streaming needs config.model_type or "
            "config.text_config.model_type to pick an architecture; none was "
            "found on the model config"
        )
    family = _STREAM_ARCH_ALIASES.get(raw_family.strip().lower(), raw_family.strip().lower())
    if family not in SUPPORTED_STREAM_ARCHS:
        raise ValueError(
            f"layer streaming does not support model_type={raw_family!r}. "
            f"Supported: {', '.join(SUPPORTED_STREAM_ARCHS)}. "
            f"More architectures land in v0.72.3."
        )
    return family


# ==========================================================================
# storage tier (plan 5.1)
# ==========================================================================
DISK_KINDS = (_NVME, "ssd", "hdd", "unknown")

#: A media type, or a thunk that determines one. ``choose_tier`` accepts the
#: thunk form so the ~9 s Windows probe is paid only when the tier decision
#: actually depends on the answer.
DiskKind = Union[str, Callable[[], str]]

@dataclass(frozen=True)
class DiskClassification:
    """A disk verdict and, when the verdict was DERIVED from an O_DIRECT read,
    the rate that produced it.

    ``measured_bps`` is set ONLY when ``kind`` came from the measured-throughput
    fallback (the virtio/#365 path). It is ``None`` for NVMe-by-name,
    ``rotational=0``, a non-Linux probe, and — crucially — an explicit
    ``training.stream_disk_kind`` override, whose verdict is the user's, not the
    probe's. Carrying the rate ALONGSIDE the kind (not in module state) means a
    refusal can only ever cite the rate that produced its OWN verdict: the two
    cannot desync across a cache hit or an override.
    """

    kind: str
    measured_bps: Optional[float] = None


_DISK_KIND_CACHE: Dict[str, DiskClassification] = {}

# --- measured-throughput fallback (#365) ----------------------------------
#: Bytes read by the O_DIRECT probe when the rotational flag is untrustworthy.
#: 64 MiB clears a device's small write-back window yet reads in well under a
#: second on anything at or above the tier floor (~0.06 s at 1 GB/s), keeping
#: the fallback bounded (criterion 4).
_MEASURE_READ_BYTES = 64 * 1024 * 1024
#: How many times the O_DIRECT read is repeated; the best (fastest) sample wins.
#: A single read can be slowed by a cold device queue or first-access latency and
#: under-measure a genuinely fast disk, wrongly refusing it — the single-sample
#: threshold weakness the #365 review called out. Three bounded reads of a 64 MiB
#: file stay well under a second total on any tier-eligible device.
_MEASURE_READ_SAMPLES = 3
#: A device must sustain at least this sequential read rate to earn the NVMe
#: disk-overflow tier. The tier is NVMe-class by policy — ``choose_tier``
#: already refuses a SATA SSD (~550 MB/s) — so the floor sits above SATA at
#: 1.0 GB/s: the reported virtio disk (1.5 GB/s read) clears it, a spinning
#: disk (~0.1-0.25 GB/s) does not. Consulted ONLY when the rotational flag is
#: unreliable; where a real media type is readable that route still wins.
NVME_TIER_MIN_BYTES_PER_S = 1_000_000_000


def classify_disk_kind(path: str = ".") -> DiskClassification:
    """Full disk verdict for ``path``: the ``kind`` plus, when the verdict was
    measured, the rate that produced it. Cached per volume. The callable
    ``choose_tier`` holds returns THIS, so a refusal cites the coupled rate and
    never a stale/unrelated figure from module state.
    """
    import os

    resolved = os.path.realpath(os.path.expanduser(path))
    # Key on the volume where there is one (Windows) and on the resolved path
    # otherwise, so two spellings of the same location share a cache entry.
    key = os.path.splitdrive(resolved)[0] or resolved
    if key in _DISK_KIND_CACHE:
        return _DISK_KIND_CACHE[key]
    result = _probe_disk_kind(path)
    _DISK_KIND_CACHE[key] = result
    return result


def detect_disk_kind(path: str = ".") -> str:
    """Media type of the volume holding ``path``: nvme / ssd / hdd / unknown.

    **Costs about 9 s on Windows** (measured), because the only reliable source
    is a PowerShell ``Get-PhysicalDisk`` CIM query. That is why ``choose_tier``
    takes a *callable* and only invokes it when the base does not fit in RAM —
    the answer is irrelevant on the RAM tier, which is the common case. The
    result is cached per process. On Linux this WRITES a small scratch file to
    probe throughput (see ``_measure_seq_read_bytes_per_s``).

    ``unknown`` is returned rather than guessed whenever the platform cannot be
    probed, and ``choose_tier`` refuses it. Refusing is the safe direction: the
    cost of wrongly believing a spinning disk is NVMe is a run that thrashes for
    hours (plan P11 — 80 shards x 2 reads = 160 seeks per step).
    """
    return classify_disk_kind(path).kind


def resolve_disk_kind(
    path: str,
    override: Optional[str] = None,
    *,
    notify: Optional[Callable[[str], None]] = None,
) -> DiskClassification:
    """Detected classification, or an explicit override with a loud notice (#365).

    ``training.stream_disk_kind`` is the escape hatch for the case where even
    the measured fallback is wrong. When set it wins, but detection still runs
    so the notice can report what was overridden and what was detected;
    ``choose_tier`` then sees the override. Detection is wrapped because a
    diagnostic read must not break a run the user has already told us how to
    classify.
    """
    if override is None:
        return classify_disk_kind(path)
    try:
        detected = detect_disk_kind(path)
    except Exception:  # noqa: BLE001 — never let the probe break an overridden run
        detected = "unknown"
    if notify is not None:
        notify(
            f"[yellow]disk kind overridden:[/] using "
            f"training.stream_disk_kind={override!r} (detected {detected!r})"
        )
    # The verdict is the user's override, NOT the probe's — carry no measured
    # rate, so a refusal can never cite a reading the user deliberately overrode.
    return DiskClassification(override)


def _resolve_tool(name: str, *fallbacks: str) -> Optional[str]:
    """Absolute path to a system tool, or None.

    Bare names are deliberately never handed to ``subprocess``: on Windows
    ``CreateProcess`` searches the CURRENT DIRECTORY before ``PATH``, so
    ``["powershell", ...]`` run from a freshly-cloned project would execute an
    attacker-planted ``powershell.exe`` sitting in that checkout (CWE-427). No
    shell metacharacters required. POSIX ``execvp`` searches ``$PATH`` only, so
    this matters most on Windows — but resolving everywhere costs nothing.
    """
    import os
    import shutil

    found = shutil.which(name)
    if found:
        return found
    for candidate in fallbacks:
        if os.path.isfile(candidate):
            return candidate
    return None


_POWERSHELL_FALLBACK = r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"


def _darwin_bsd_whole_disk(identifier: Any) -> Optional[str]:
    """Return the whole-disk BSD name for a validated ``diskNsM`` identifier."""
    import re

    match = re.fullmatch(r"/?(?:dev/)?(disk\d+)(?:s\d+)*", str(identifier).strip())
    return match.group(1) if match is not None else None


def _darwin_apfs_physical_stores(disk_info: dict) -> set[str]:
    """Whole disks backing the APFS volume described by ``diskutil info``."""
    stores = disk_info.get("APFSPhysicalStores")
    if not isinstance(stores, list):
        return set()
    result = set()
    for entry in stores:
        if not isinstance(entry, dict):
            continue
        identifier = entry.get("APFSPhysicalStore") or entry.get("DeviceIdentifier")
        whole_disk = _darwin_bsd_whole_disk(identifier)
        if whole_disk is not None:
            result.add(whole_disk)
    return result


def _darwin_nvme_whole_disks(profile: dict) -> set[str]:
    """Whole disks explicitly listed by ``system_profiler SPNVMeDataType``."""
    sections = profile.get("SPNVMeDataType")
    if not isinstance(sections, list):
        return set()
    result = set()
    pending: list[Any] = list(sections)
    while pending:
        value = pending.pop()
        if isinstance(value, dict):
            whole_disk = _darwin_bsd_whole_disk(value.get("bsd_name"))
            if whole_disk is not None:
                result.add(whole_disk)
            pending.extend(value.values())
        elif isinstance(value, list):
            pending.extend(value)
    return result


def _darwin_disk_kind(path: str, diskutil: str) -> DiskClassification:
    """Classify the exact macOS volume, resolving APFS through its physical store."""
    import json
    import os
    import plistlib
    import subprocess

    resolved = os.path.realpath(os.path.expanduser(path))

    def diskutil_info(target: str) -> Any:
        return subprocess.run(
            [diskutil, "info", "-plist", target],
            capture_output=True,
            timeout=60,
            check=False,
            shell=False,
        )

    info_result = diskutil_info(resolved)
    if info_result.returncode != 0:
        # ``diskutil info`` accepts a device or an exact mount point, not an
        # arbitrary directory within that volume. Resolve the latter with the
        # POSIX ``df -P`` format, validate its device token, then retry.
        df_tool = _resolve_tool("df", "/bin/df")
        if df_tool is None:
            return DiskClassification("unknown")
        df_result = subprocess.run(
            [df_tool, "-P", resolved],
            capture_output=True,
            timeout=60,
            check=False,
            shell=False,
        )
        if df_result.returncode != 0 or not df_result.stdout:
            return DiskClassification("unknown")
        lines = [line for line in df_result.stdout.splitlines() if line.strip()]
        fields = lines[-1].split() if len(lines) >= 2 else []
        device = fields[0].decode("ascii", errors="strict") if fields else ""
        if _darwin_bsd_whole_disk(device) is None:
            return DiskClassification("unknown")
        info_result = diskutil_info(device)
    if info_result.returncode != 0 or not info_result.stdout:
        return DiskClassification("unknown")
    try:
        disk_info = plistlib.loads(info_result.stdout)
    except (plistlib.InvalidFileException, TypeError, ValueError):
        return DiskClassification("unknown")
    if not isinstance(disk_info, dict):
        return DiskClassification("unknown")

    protocol = str(disk_info.get("BusProtocol", "")).strip().lower()
    if "nvme" in protocol or "nvmexpress" in protocol:
        return DiskClassification(_NVME)

    physical_stores = _darwin_apfs_physical_stores(disk_info)
    if physical_stores:
        profiler = _resolve_tool("system_profiler", "/usr/sbin/system_profiler")
        if profiler is not None:
            profile_result = subprocess.run(
                [
                    profiler,
                    "-json",
                    "-detailLevel",
                    "mini",
                    "SPNVMeDataType",
                ],
                capture_output=True,
                timeout=60,
                check=False,
                shell=False,
            )
            if profile_result.returncode == 0 and profile_result.stdout:
                try:
                    profile = json.loads(profile_result.stdout)
                except (UnicodeDecodeError, json.JSONDecodeError, TypeError):
                    profile = None
                if isinstance(profile, dict):
                    nvme_disks = _darwin_nvme_whole_disks(profile)
                    if physical_stores and physical_stores.issubset(nvme_disks):
                        return DiskClassification(_NVME)

    # ``SolidState`` establishes SSD, not NVMe. A failed or unmatched physical
    # lookup must never promote an ordinary SATA SSD into the NVMe-only tier.
    if disk_info.get("SolidState") is True:
        return DiskClassification("ssd")
    media_type = str(disk_info.get("MediaType", "")).strip().lower()
    if "solid state" in media_type or media_type == "ssd":
        return DiskClassification("ssd")
    return DiskClassification("unknown")


def _classify_measured_read(measured_bps: Optional[float]) -> str:
    """Classify by a measured read a device the rotational flag calls spinning.

    virtio and other paravirtual block devices expose no media hint, so the
    guest kernel defaults ``rotational`` to 1 and a genuinely NVMe-backed cloud
    disk is otherwise refused the overflow tier (#365). An actual HDD refusal is
    correct (80 shards x 2 reads = 160 seeks per step, plan P11), so the
    discriminator is throughput, not the flag: at or above
    ``NVME_TIER_MIN_BYTES_PER_S`` the device is NVMe-class and usable; below it —
    or unmeasurable (``None``) — it is treated as ``hdd`` and refused, the safe
    direction.
    """
    if measured_bps is not None and measured_bps >= NVME_TIER_MIN_BYTES_PER_S:
        return _NVME
    return "hdd"


def _measure_seq_read_bytes_per_s(path: str) -> Optional[float]:
    """Sequential ``O_DIRECT`` read throughput of the volume holding ``path``.

    Writes a small scratch file beside ``path``, reopens it with ``O_DIRECT`` to
    bypass the page cache, and times ``_MEASURE_READ_SAMPLES`` page-aligned
    sequential reads, returning the **best** (fastest) one. Repeating the read
    and keeping the best rejects a single cold/slow sample that would otherwise
    under-measure a fast device and wrongly refuse it. Best effort: any failure
    (no ``O_DIRECT`` on this filesystem, no write permission, no monotonic clock)
    returns ``None`` so the caller stays conservative, and the scratch file is
    always removed. Each read is bounded by ``_MEASURE_READ_BYTES`` so the probe
    cannot become the ~9 s cost the Windows probe already carries (criterion 4).
    """
    import mmap
    import os
    import tempfile
    import time

    o_direct = getattr(os, "O_DIRECT", None)
    if o_direct is None:  # non-Linux — the caller only reaches here on Linux
        return None
    # Probe the volume that actually holds the shards. The caller passes a
    # DIRECTORY (shard_dir), so write the scratch file inside it; only fall back
    # to the parent for a file path. Taking dirname of a directory would measure
    # the PARENT filesystem — wrong when the target is its own mount point.
    resolved = os.path.realpath(os.path.expanduser(path))
    directory = resolved if os.path.isdir(resolved) else (os.path.dirname(resolved) or ".")
    fd_w = None
    fd_r = None
    scratch = None
    buf = None
    try:
        fd_w, scratch = tempfile.mkstemp(dir=directory, prefix=".soup-diskprobe-")
        block = b"\0" * (1024 * 1024)
        written = 0
        while written < _MEASURE_READ_BYTES:
            written += os.write(fd_w, block[: _MEASURE_READ_BYTES - written])
        os.fsync(fd_w)
        os.close(fd_w)
        fd_w = None

        buf = mmap.mmap(-1, _MEASURE_READ_BYTES)  # page-aligned for O_DIRECT
        fd_r = os.open(scratch, os.O_RDONLY | o_direct)
        best_bps: Optional[float] = None
        for _ in range(_MEASURE_READ_SAMPLES):
            os.lseek(fd_r, 0, os.SEEK_SET)  # offset 0 stays O_DIRECT-aligned
            start = time.monotonic()
            read_total = os.readv(fd_r, [buf])
            elapsed = time.monotonic() - start
            if elapsed <= 0 or read_total <= 0:
                continue
            bps = read_total / elapsed
            if best_bps is None or bps > best_bps:
                best_bps = bps
        return best_bps
    except (OSError, ValueError):
        return None
    finally:
        for fd in (fd_w, fd_r):
            if fd is not None:
                try:
                    os.close(fd)
                except OSError:
                    pass
        if buf is not None:
            try:
                buf.close()
            except (BufferError, ValueError):
                # Isolated from the unlink below so a mmap-close failure can
                # never leak the 64 MiB scratch file.
                pass
        if scratch is not None:
            try:
                os.unlink(scratch)
            except OSError:
                pass


_LINUX_SYS_DEV_BLOCK = "/sys/dev/block"


def _linux_block_kind(device: str, ancestors: frozenset[str] = frozenset()) -> str:
    """Classify one sysfs device, following partitions and stacked-volume members."""
    import os

    device = os.path.realpath(device)
    if device in ancestors or len(ancestors) >= 32 or not os.path.isdir(device):
        return "unknown"
    ancestors = ancestors | {device}
    if os.path.exists(os.path.join(device, "partition")):
        return _linux_block_kind(os.path.dirname(device), ancestors)
    slaves = os.path.join(device, "slaves")
    if os.path.isdir(slaves):
        members = os.listdir(slaves)
        if members:
            kinds = {_linux_block_kind(os.path.join(slaves, name), ancestors) for name in members}
            return next(kind for kind in ("hdd", "unknown", "ssd", _NVME) if kind in kinds)
    if os.path.basename(device).startswith("nvme"):
        return _NVME
    rotational = os.path.join(device, "queue", "rotational")
    if os.path.isfile(rotational):
        with open(rotational, encoding="utf-8") as handle:
            value = handle.read().strip()
        return {"0": "ssd", "1": "hdd"}.get(value, "unknown")
    return "unknown"


def _linux_path_disk_kind(path: str) -> str:
    """Attribute the volume to sysfs before reading its media type."""
    import os

    try:
        dev = os.stat(os.path.expanduser(path)).st_dev
        device = f"{_LINUX_SYS_DEV_BLOCK}/{os.major(dev)}:{os.minor(dev)}"
        return _linux_block_kind(device)
    except (OSError, ValueError):
        return "unknown"


def _probe_disk_kind(path: str) -> DiskClassification:
    """Classify the volume holding ``path``, carrying the measured rate only when
    the verdict was DERIVED from a read (so a refusal can never cite a rate that
    did not produce its own verdict — see ``DiskClassification``)."""
    import platform
    import subprocess

    system = platform.system()
    try:
        if system == "Linux":
            kind = _linux_path_disk_kind(path)
            if kind in (_NVME, "ssd"):
                return DiskClassification(kind)
            # rotational=1 can describe fast virtio storage (#365). Unknown
            # volumes (network, overlay, missing sysfs) also need a path-based
            # read, never the name of an unrelated physical disk (#1800).
            measured = _measure_seq_read_bytes_per_s(path)
            return DiskClassification(_classify_measured_read(measured), measured)
        if system == "Windows":
            shell = _resolve_tool("powershell", _POWERSHELL_FALLBACK)
            if shell is None:
                return DiskClassification("unknown")
            out = subprocess.run(
                [
                    shell, "-NoProfile", "-NonInteractive", "-Command",
                    "Get-PhysicalDisk | Select-Object MediaType,BusType "
                    "| ConvertTo-Json -Compress",
                ],
                capture_output=True, text=True, timeout=60, check=False,
            )
            if out.returncode != 0 or not out.stdout.strip():
                return DiskClassification("unknown")
            import json

            payload = json.loads(out.stdout)
            disks = payload if isinstance(payload, list) else [payload]
            kinds = {_windows_kind(d) for d in disks if isinstance(d, dict)}
            # Several physical disks and no way to attribute the volume to one:
            # report the WORST, so a machine with a spinning disk is not told it
            # has NVMe.
            for candidate in ("hdd", "unknown", "ssd", _NVME):
                if candidate in kinds:
                    return DiskClassification(candidate)
            return DiskClassification("unknown")
        if system == "Darwin":
            tool = _resolve_tool("diskutil", "/usr/sbin/diskutil")
            if tool is None:
                return DiskClassification("unknown")
            return _darwin_disk_kind(path, tool)
    except (OSError, ValueError, subprocess.SubprocessError):
        return DiskClassification("unknown")
    return DiskClassification("unknown")


def _windows_kind(disk: dict) -> str:
    bus = str(disk.get("BusType", "")).strip().lower()
    media = str(disk.get("MediaType", "")).strip().lower()
    if bus == "nvme":
        return _NVME
    # MSFT_PhysicalDisk.MediaType ValueMap is {0 Unspecified, 3 HDD, 4 SSD,
    # 5 SCM} — verified on this box, whose NVMe reports raw value 4. PowerShell
    # usually renders the friendly string, but ConvertTo-Json emits the integer
    # on some systems, so both forms are handled and neither is guessed.
    if media in ("hdd", "3"):
        return "hdd"
    if media in ("ssd", "4"):
        return "ssd"
    return "unknown"


def choose_tier(
    store_bytes: int,
    free_ram_bytes: int,
    disk_kind: DiskKind,
    *,
    headroom: float = RAM_TIER_HEADROOM,
    resident_bytes: int = 0,
    total_ram_bytes: Optional[int] = None,
) -> str:
    """RAM is Tier 1; disk is overflow only; spinning rust is refused.

    plan 2.3: streaming from NVMe measured 2-11 TFLOPS against multiples of
    that from CPU RAM, so RAM is the primary source and disk is a fallback.
    plan P11: on an HDD, 80 shards x 2 reads = 160 seeks per step.

    ``disk_kind`` may be a **callable**, and normally should be: detecting the
    media type costs ~9 s on Windows and is irrelevant whenever the base fits in
    RAM. Passing ``detect_disk_kind`` here means that cost is paid only by runs
    that are actually about to use the disk tier.
    """
    physical_limit = (
        None
        if total_ram_bytes is None
        else total_ram_bytes * PHYSICAL_RAM_TIER_HEADROOM
    )
    resident_store_bytes = store_bytes + int(resident_bytes)
    fits_available_ram = resident_store_bytes < free_ram_bytes * headroom
    fits_physical_ram = physical_limit is None or resident_store_bytes < physical_limit
    if fits_available_ram and fits_physical_ram:
        return TIER_RAM
    result = disk_kind() if callable(disk_kind) else disk_kind
    # The callable may return a DiskClassification (kind + the rate that produced
    # it) or a bare kind string (tests, explicit callers). Either way the rate is
    # taken FROM the same result, so it can only ever describe THIS verdict.
    if isinstance(result, DiskClassification):
        kind, measured = result.kind, result.measured_bps
    else:
        kind, measured = result, None
    if kind == _NVME:
        return TIER_DISK
    # When the verdict came from a measured read (the virtio/#365 path), cite the
    # rate that earned the refusal so "not NVMe" is not an opaque verdict — the
    # operator can see how far under the floor the disk landed. measured is None
    # for a name/flag/override verdict, so the note only appears when it is true.
    measured_note = (
        f" (measured {measured / 1e9:.2f} GB/s, under the "
        f"{NVME_TIER_MIN_BYTES_PER_S / 1e9:.1f} GB/s NVMe floor)"
        if measured is not None
        else ""
    )
    physical_note = (
        ""
        if physical_limit is None or fits_physical_ram
        else (
            f"; the store plus resident extras need {resident_store_bytes / 1e9:.1f} GB, "
            f"which exceeds {PHYSICAL_RAM_TIER_HEADROOM_PERCENT}% of physical RAM"
        )
    )
    resident_note = (
        ""
        if resident_bytes == 0
        else f" plus {resident_bytes / 1e9:.1f} GB of resident extras"
    )
    raise ValueError(
        f"layer streaming needs NVMe or more RAM: the base needs "
        f"{store_bytes / 1e9:.1f} GB{resident_note}, only "
        f"{free_ram_bytes / 1e9:.1f} GB of RAM "
        f"is free{physical_note}, and the detected disk is {kind!r} "
        f"(not NVMe){measured_note}. "
        f"Free RAM, pick a smaller base, or move the model to an NVMe drive."
    )


@dataclass(frozen=True)
class PinDecision:
    """Whether the RAM store can be page-locked, and why not when it cannot."""

    pinned: bool
    reason: str


def decide_pinning(
    store_bytes: int,
    pinned_limit_bytes: Optional[int],
    *,
    stream_pin: Optional[bool] = None,
) -> PinDecision:
    """Pin the RAM store when the box can actually page-lock it.

    Measured on the dev box (RTX 3050 4 GB / 16.9 GB RAM): the maximum
    page-locked host allocation was 7.12 GB even with 9.1 GB "available", so a
    5.55 GB base plus a CUDA context and a model skeleton did not fit. Falling
    back to a pageable store is correct, but it makes
    ``copy_(non_blocking=True)`` synchronous and therefore costs overlap:
    measured GPU utilisation dropped from 96.8% (pinned) to 79.3% (pageable).
    That cost is stated out loud rather than absorbed silently.

    ``stream_pin`` (``training.stream_pin``) overrides the automatic choice:
    ``None`` keeps the behaviour above; ``False`` forces pageable host memory
    and states its throughput cost; ``True`` forces the page-locked one — on a
    CUDA target the run then refuses (in the runtime) rather than falling back
    if the box cannot page-lock it. The refusal itself lives where the pin is
    actually attempted, which is also why a non-CUDA target does not refuse:
    nothing is attempted there. Here ``True`` only records the intent so the
    pre-flight reflects it.

    BOTH TIERS, since #971. The flag used to describe the RAM store alone,
    because the disk tier held nothing to page-lock; it now decides whether the
    async reader's host STAGING is page-locked, and the runtime honours or
    refuses it there exactly as it does on the RAM tier. The reasons below name
    what each tier actually has rather than assuming a RAM store.
    """
    if stream_pin is False:
        return PinDecision(
            pinned=False,
            reason=(
                "training.stream_pin=false forces PAGEABLE host memory — the "
                "base store on the RAM tier, the async reader's staging on the "
                "disk tier. Host-to-device copies become synchronous, which "
                "costs overlap: measured GPU utilisation drops from ~97% to "
                f"~79%, and page-locking is worth up to "
                f"{PIN_THROUGHPUT_GAIN_REAL:.2f}x measured throughput "
                f"(Qwen2.5-32B NF4), {PIN_THROUGHPUT_GAIN_SYNTHETIC:.2f}x on a "
                "synthetic. Unset stream_pin to let the box page-lock when it "
                "can."
            ),
        )
    if stream_pin is True:
        return PinDecision(
            pinned=True,
            # The tier gate this note used to carry ("on the RAM tier the run
            # refuses") was dropped in #971 because the disk tier now HAS
            # staging to page-lock and does refuse over it. The gate's OTHER
            # reason is answered in the sentence rather than dropped with it:
            # on a non-CUDA target nothing is page-locked at all — setup passes
            # ``pin=plan.pinned and on_cuda`` and gates ``require_pin`` on the
            # same flag — so an unconditional "the run refuses" would print a
            # promise that path does not keep. It announces and proceeds.
            reason=(
                "training.stream_pin=true forces page-locked host memory — the "
                "base store on the RAM tier, the async reader's staging on the "
                "disk tier. On a CUDA target the run refuses, on either tier, "
                "rather than falling back to pageable memory if the box cannot "
                "page-lock it; on a non-CUDA target there is no CUDA pinning to "
                "force, so the request is announced as inapplicable and the run "
                "proceeds with pageable host memory."
            ),
        )
    if pinned_limit_bytes is None:
        return PinDecision(
            pinned=True,
            reason="pinned-host ceiling unknown; attempting a page-locked store",
        )
    if store_bytes <= pinned_limit_bytes:
        return PinDecision(pinned=True, reason="store fits under the page-locked ceiling")
    return PinDecision(
        pinned=False,
        reason=(
            f"base store is {store_bytes / 1e9:.2f} GB but this box can only "
            f"page-lock {pinned_limit_bytes / 1e9:.2f} GB — falling back to a "
            f"pageable store. Host-to-device copies become synchronous, which "
            f"costs overlap: measured GPU utilisation drops from ~97% to ~79%. "
            f"Free RAM or use a smaller base to keep the pinned store."
        ),
    )


# ==========================================================================
# buffers
# ==========================================================================
def validate_stream_buffers(value: Any) -> int:
    """plan 8: 1 buffer is a scheduler bug, not a config."""
    if isinstance(value, bool):
        raise ValueError("training.stream_buffers must be an int, not bool")
    if not isinstance(value, int):
        raise ValueError(f"training.stream_buffers must be an int; got {type(value).__name__}")
    if value < MIN_STREAM_BUFFERS or value > MAX_STREAM_BUFFERS:
        raise ValueError(
            f"training.stream_buffers must be between {MIN_STREAM_BUFFERS} and "
            f"{MAX_STREAM_BUFFERS}; got {value}. A single buffer cannot overlap "
            f"load with compute — that is a scheduler bug, not a configuration."
        )
    return value


# ==========================================================================
# arithmetic (plan 4.1 / 4.3)
# ==========================================================================
def should_enable_hf_gradient_checkpointing(
    gradient_checkpointing: Any, *, stream_layers: bool
) -> bool:
    """Whether HF's own gradient checkpointing should be switched on.

    ``StreamedDecoderLayer`` already wraps every layer in
    ``checkpoint(use_reentrant=False)`` — that is what bounds activation memory
    AND what triggers the second weight read per step. Letting the HF Trainer
    also enable checkpointing double-recomputes every layer: the run still
    converges, it is just silently ~1.5x slower. So streaming always wins.
    """
    return bool(gradient_checkpointing) and not stream_layers


def free_ram_bytes() -> Optional[int]:
    """Available host RAM, or None when psutil is not installed."""
    try:
        import psutil
    except ImportError:
        return None
    try:
        return int(psutil.virtual_memory().available)
    except (AttributeError, OSError, ValueError):
        return None


def total_ram_bytes() -> Optional[int]:
    """Total physical host RAM, or None when psutil cannot report it."""
    try:
        import psutil
    except ImportError:
        return None
    try:
        return int(psutil.virtual_memory().total)
    except (AttributeError, OSError, ValueError):
        return None


def estimate_stream_store_bytes(
    source_bytes: int,
    *,
    dtype: str,
    quant: str = QUANT_NONE,
    double_quant: bool = True,
) -> int:
    """Scale an on-disk checkpoint size to the RAM store streaming will hold.

    The pre-flight probe compares the base against free RAM *before* sharding,
    to avoid spending minutes rewriting a checkpoint that will then be refused.
    Under NF4 the store is ~0.26x the bf16 on-disk size, so measuring the raw
    file size would refuse an 8B run on a 16.9 GB box — precisely the
    configuration this release exists to enable.

    Deliberately coarse: it assumes the source is stored at ``dtype`` and
    charges the NF4 rate to every parameter, while layernorms and (in the shard
    set) the embeddings actually stay unquantised. That errs LOW — the
    authoritative check runs after sharding, on the real shard sizes. It errs far
    low for a fused MoE base, whose stacked routed experts stay at ``dtype``, and
    for an FP8 block source, whose files hold one byte per weight. So the disk
    pre-flight and the early RAM check both size the cache from the source headers
    instead (``layer_shard.estimate_shard_cache_bytes``) and fall back to this
    ratio only when those headers cannot be listed or read.
    """
    if quant not in SUPPORTED_STREAM_QUANTS:
        raise ValueError(
            f"unsupported quant {quant!r} for layer streaming; supported: "
            f"{', '.join(SUPPORTED_STREAM_QUANTS)}"
        )
    if source_bytes < 0:
        raise ValueError(f"source_bytes must be non-negative; got {source_bytes}")
    if quant == QUANT_NONE:
        return int(source_bytes)
    per_param = NF4_BYTES_PER_PARAM if double_quant else NF4_BYTES_PER_PARAM_SINGLE
    return int(source_bytes * per_param / dtype_bytes(dtype))


def estimate_stream_tokens_per_sec(params: int, effective_tflops: float) -> float:
    """tok/s ceiling when compute-bound: TFLOPS_eff / (C * P), C = 6."""
    if params <= 0:
        raise ValueError(f"params must be positive; got {params}")
    if effective_tflops <= 0 or not math.isfinite(effective_tflops):
        raise ValueError(f"effective_tflops must be positive and finite; got {effective_tflops}")
    return (effective_tflops * 1e12) / (FLOPS_PER_PARAM_PER_TOKEN * params)


def estimate_epoch_seconds(tokens: int, tokens_per_sec: float) -> float:
    """Wall-clock for a token budget — the pre-flight forecast (plan 10)."""
    if tokens_per_sec <= 0 or not math.isfinite(tokens_per_sec):
        raise ValueError(f"tokens_per_sec must be positive and finite; got {tokens_per_sec}")
    if tokens < 0:
        raise ValueError(f"tokens must be non-negative; got {tokens}")
    return tokens / tokens_per_sec


def estimate_logits_bytes(
    *,
    vocab_size: int,
    seq_len: int,
    batch_size: int = 1,
    upcast_fp32: bool = True,
    bytes_per_element: Optional[float] = None,
) -> int:
    """plan P5: logits, not weights, OOM you first on a small card.

    Measured (GATE 2), not derived: the loss path holds ``LOGITS_BYTES_PER_ELEMENT``
    bytes per element live at peak, not the 2 + 4 a first-principles reading
    suggests. At Qwen2.5-0.5B (vocab 151936), batch 8, S=512 this single term is
    8.71 GB — 146x the whole buffer pool — so a pre-flight that budgets only
    weights and buffers green-lights a config that cannot run.

    ``bytes_per_element`` accepts a stack measurement from
    :func:`calibrated_logits_bytes_per_element`. It is **floored at the shipped
    constant and can therefore only raise the budget**, which is the only
    direction the evidence supports: the loss arithmetic is measurable at
    pre-flight, the retention (issue #327) is not, so a reading below the
    constant is a probe that could not see the retention rather than a stack
    that does not pay it.
    """
    if vocab_size <= 0 or seq_len <= 0 or batch_size <= 0:
        raise ValueError("vocab_size, seq_len and batch_size must all be positive")
    elements = vocab_size * seq_len * batch_size
    per: float = LOGITS_BYTES_PER_ELEMENT if upcast_fp32 else LOGITS_BYTES_PER_ELEMENT_NO_LOSS
    if bytes_per_element is not None:
        if not math.isfinite(bytes_per_element):
            raise ValueError(f"bytes_per_element must be finite; got {bytes_per_element}")
        per = max(per, bytes_per_element)
    return math.ceil(elements * per)


def measure_logits_loss_bytes_per_element(
    *,
    vocab_size: int = 8192,
    tokens: Tuple[int, int] = (1024, 2048),
    device: Optional[str] = None,
) -> Optional[float]:
    """Measure THIS stack's loss-path cost per logit element, or ``None``.

    Reports the **marginal** slope between two token counts, so the fixed
    overhead of the probe itself cancels rather than being modelled. The source
    logits tensor is allocated *before* the peak is reset, so what comes back is
    the loss arithmetic alone — :data:`LOGITS_LOSS_BYTES_PER_ELEMENT`, not the
    retained copy on top of it.

    Costs one transient allocation of ``14 * vocab_size * max(tokens)`` bytes —
    96 MiB at the defaults. Returns ``None`` rather than raising when there is no
    CUDA device or the probe cannot run: a pre-flight that dies because its own
    instrument failed is worse than one that falls back to the shipped constant.

    Measured 12.000000 with spread 0.00e+00 over 3 repeats at each of
    (8192, 1024->2048), (4096, 1024->2048) and (16384, 512->1024).
    """
    if vocab_size <= 0:
        raise ValueError(f"vocab_size must be positive; got {vocab_size}")
    small, large = tokens
    if small <= 0 or large <= small:
        raise ValueError(f"tokens must be an increasing positive pair; got {tokens!r}")
    try:  # pragma: no cover - exercised only where torch + CUDA exist
        import torch
        from transformers.loss.loss_utils import ForCausalLMLoss
    except Exception:
        return None
    if not torch.cuda.is_available():
        return None
    where = device or "cuda"

    def _peak(count: int) -> int:
        torch.cuda.empty_cache()
        labels = torch.randint(0, vocab_size, (1, count), device=where)
        logits = torch.randn(
            1, count, vocab_size, dtype=torch.bfloat16, device=where, requires_grad=True
        )
        torch.cuda.synchronize()
        torch.cuda.reset_peak_memory_stats(where)
        base = torch.cuda.memory_allocated(where)
        loss = ForCausalLMLoss(logits, labels, vocab_size)
        loss.backward()
        torch.cuda.synchronize()
        peak = torch.cuda.max_memory_allocated(where) - base
        del loss, logits, labels
        torch.cuda.empty_cache()
        return peak

    try:  # pragma: no cover - exercised only where torch + CUDA exist
        low, high = _peak(small), _peak(large)
    except Exception:
        return None
    return (high - low) / (vocab_size * (large - small))


def calibrated_logits_bytes_per_element(**kwargs: Any) -> float:
    """The shipped constant, raised if this stack's loss path costs more.

    Never lowered. The measurable half is the loss arithmetic; the retention
    term is charged on top of whatever is measured, because no synthetic probe
    can observe whether the training loop that will actually run still holds the
    logits when the loss backward peaks (issue #327). A stack whose loss path
    grew a fourth fp32 buffer would otherwise be under-budgeted by 12.5% with no
    guard at all, and under-prediction is the failure that does not raise.
    """
    measured = measure_logits_loss_bytes_per_element(**kwargs)
    if measured is None:
        return float(LOGITS_BYTES_PER_ELEMENT)
    return max(float(LOGITS_BYTES_PER_ELEMENT), measured + LOGITS_RETENTION_BYTES_PER_ELEMENT)


def estimate_activation_bytes(
    *,
    hidden_size: int,
    intermediate_size: int,
    n_layers: int,
    seq_len: int,
    batch_size: int = 1,
    dtype: str = "bfloat16",
) -> int:
    """Activation memory for one streamed step (GATE 2).

    Two terms, and keeping them separate is the whole memory argument for
    streaming:

    * ``2 * n_layers * hidden`` per token — ``checkpoint(use_reentrant=False)``
      saves each layer's *input*, so this one does scale with depth. It is small.
    * ``4 * (hidden + intermediate)`` per token — the tensors live inside the ONE
      layer currently being recomputed. Independent of depth, which is precisely
      why a 32-layer model costs no more here than a 4-layer one.
    """
    for name, value in (
        ("hidden_size", hidden_size),
        ("intermediate_size", intermediate_size),
        ("n_layers", n_layers),
        ("seq_len", seq_len),
        ("batch_size", batch_size),
    ):
        if value <= 0:
            raise ValueError(f"{name} must be positive; got {value}")
    element = dtype_bytes(dtype)
    tokens = seq_len * batch_size
    boundary = element * n_layers * hidden_size
    transient = 2 * element * (hidden_size + intermediate_size)
    return tokens * (boundary + transient)


def estimate_optimizer_bytes(
    adapter_params: int, *, bytes_per_param: int = OPTIMIZER_BYTES_PER_PARAM
) -> int:
    """Adapter weights + gradients + optimizer moments."""
    if adapter_params < 0:
        raise ValueError(f"adapter_params must be non-negative; got {adapter_params}")
    return adapter_params * bytes_per_param


def nf4_scale_cache_bytes(quant_specs: Mapping[str, Any], n_layers: int) -> int:
    """Device bytes ``training.stream_nf4_scale_cache`` holds once every layer ran.

    One fp32 scale per block of every double-quantised decoder weight, in every
    layer. ``quant_specs`` is keyed by the layer-relative name shared across
    layers, so a model whose layers do not all carry a weight is over-counted,
    which is the safe direction for a number that can refuse a run.
    """
    per_layer = sum(
        4 * -(-math.prod(spec.shape) // spec.blocksize)
        for spec in quant_specs.values()
        if spec.nested
    )
    return per_layer * n_layers


#: training.stream_recompute='auto' — the share of the card's total memory the
#: retained activations may grow the step into.
STREAM_RECOMPUTE_HEADROOM = 0.85

#: The three caps of :func:`stream_recompute_budget`, by the name it reports.
RECOMPUTE_CAP_CARD = "card"
RECOMPUTE_CAP_FREE = "free"
RECOMPUTE_CAP_OVERRIDE = "override"


@dataclass(frozen=True)
class RecomputeBudget:
    """The VRAM the retained activations of ``stream_recompute='auto'`` may use."""

    #: The smallest of the caps, in bytes.
    cap_bytes: int
    #: Which cap that was: RECOMPUTE_CAP_CARD / _FREE / _OVERRIDE.
    bound: str
    #: ``cap_bytes`` minus the measured reserved peak; <= 0 means none.
    budget_bytes: int


def stream_recompute_budget(
    *,
    total_bytes: int,
    free_bytes: int,
    reserved_bytes: int,
    reserved_peak_bytes: int,
    override_bytes: Optional[int] = None,
    headroom: float = STREAM_RECOMPUTE_HEADROOM,
) -> RecomputeBudget:
    """``min(headroom x card, free + reserved now, stream_vram_override) - reserved peak``.

    The card term keeps headroom below the physical size; the free term is what
    this process can still reach on a card other processes share
    (``mem_get_info`` free plus what the allocator already holds); the override,
    when set, is the user's own cap for what the driver cannot report. The peak
    is the RESERVED one, not the allocated one: the caching allocator's
    fragmentation is real VRAM.
    """
    if (
        isinstance(headroom, bool)
        or not isinstance(headroom, (int, float))
        or not 0.0 < headroom <= 1.0
    ):
        raise ValueError(f"headroom must be a float in (0, 1]; got {headroom!r}")
    caps = {
        RECOMPUTE_CAP_CARD: int(headroom * total_bytes),
        RECOMPUTE_CAP_FREE: int(free_bytes) + int(reserved_bytes),
    }
    if override_bytes is not None:
        caps[RECOMPUTE_CAP_OVERRIDE] = int(override_bytes)
    bound = min(caps, key=lambda name: caps[name])
    cap = caps[bound]
    return RecomputeBudget(cap_bytes=cap, bound=bound, budget_bytes=cap - int(reserved_peak_bytes))


def retained_layer_charge(
    allocated_bytes: int, peak_reserved_bytes: int, peak_allocated_bytes: int
) -> int:
    """One retained layer's charge: its allocated bytes times the step's
    reserved/allocated ratio when that is above 1 (the allocator's rounding and
    fragmentation grow with what it holds), rounded up."""
    allocated_bytes = int(allocated_bytes)
    if peak_allocated_bytes <= 0 or peak_reserved_bytes <= peak_allocated_bytes:
        return allocated_bytes
    return -(-allocated_bytes * int(peak_reserved_bytes) // int(peak_allocated_bytes))


def plan_retained_layers(budget_bytes: int, per_layer_bytes: int, n_layers: int) -> int:
    """How many of the last streamed layers may skip the checkpoint recompute:
    ``budget_bytes // per_layer_bytes`` clamped to ``[0, n_layers]``. A zero or
    negative budget, or a layer charged at zero bytes or less, plans nothing,
    which is the checkpoint recompute everywhere."""
    if budget_bytes <= 0 or per_layer_bytes <= 0:
        return 0
    return max(0, min(int(n_layers), int(budget_bytes) // int(per_layer_bytes)))


def estimate_loss_chunk_bytes(
    *, hidden_size: int, seq_len: int, batch_size: int = 1, dtype: str = "bfloat16"
) -> int:
    """What ``training.stream_loss_chunk`` keeps that an unchunked step does not (#1693).

    Two ``[batch, seq, hidden]`` tensors in the stream dtype: the gradient on
    the hidden states that the forward fills chunk by chunk, and its product
    with the incoming gradient in the backward. They are not live together with
    a chunk's logits, so charging both next to the logits term errs high.
    """
    if hidden_size <= 0 or seq_len <= 0 or batch_size <= 0:
        raise ValueError("hidden_size, seq_len and batch_size must all be positive")
    return 2 * hidden_size * seq_len * batch_size * dtype_bytes(dtype)


def estimate_stream_peak_vram(
    *,
    layer_bytes: int,
    buffers: int,
    extras_bytes: int,
    adapter_params: int,
    vocab_size: int,
    hidden_size: int,
    intermediate_size: int,
    n_layers: int,
    seq_len: int,
    batch_size: int = 1,
    dtype: str = "bfloat16",
    logits_bytes_per_element: Optional[float] = None,
    large_layer_bytes: int = 0,
    persistent_bytes: int = 0,
    loss_chunk: Optional[int] = None,
) -> int:
    """Predicted ``torch.cuda.max_memory_allocated()`` for a streaming step.

    Assembles the GATE 2 terms. Validated against 10 real runs across two models
    (a 3.1x vocab contrast), batch 1..8 and two sequence lengths: **worst
    absolute error 0.85%, and it never under-predicts** — the only safe direction
    for a number that is allowed to refuse a run. An independent check against
    the published v0.72.2 Llama-3.1-8B NF4 row (untied embeddings, different
    quantisation, different session, nothing fitted to it) brackets it at +7.5%.

    ``extras_bytes`` contains only the genuinely resident non-decoder weights.
    ``large_layer_bytes`` is one reusable slot sized to the larger of
    ``embed_tokens`` and an untied ``lm_head``; the two matrices no longer add
    together at peak (#324). A loss with a second forward per step on an untied
    checkpoint also holds a private copy of the head (#1049); the caller charges
    that as a second slot, so pass the sum.

    Returns allocator-visible bytes only. The CUDA context and driver reservation
    sit outside the caching allocator (0.85 GB on the dev box, which also drives
    a display), so the fit decision compares this against *measured free VRAM*
    rather than against the card's nameplate size.

    ``logits_bytes_per_element`` forwards a stack measurement from
    :func:`calibrated_logits_bytes_per_element`; it is floored at the shipped
    constant, so passing it can only raise the prediction.

    ``persistent_bytes`` is device memory the runtime holds for the whole run on
    top of the pool, such as :func:`nf4_scale_cache_bytes`.

    ``loss_chunk`` is ``training.stream_loss_chunk`` (#1693): the logits term is
    charged on ``min(seq_len, loss_chunk)`` tokens, at the same bytes per
    element, and :func:`estimate_loss_chunk_bytes` is added. None of the runs
    this formula was fitted to used it.
    """
    logits_tokens = seq_len
    chunk_bytes = 0
    if loss_chunk is not None:
        if isinstance(loss_chunk, bool) or not isinstance(loss_chunk, int) or loss_chunk < 1:
            raise ValueError(f"loss_chunk must be a positive integer, got {loss_chunk!r}")
        logits_tokens = min(seq_len, loss_chunk)
        chunk_bytes = estimate_loss_chunk_bytes(
            hidden_size=hidden_size, seq_len=seq_len, batch_size=batch_size, dtype=dtype
        )
    return (
        layer_bytes * buffers
        + large_layer_bytes
        + persistent_bytes
        + extras_bytes
        + estimate_optimizer_bytes(adapter_params)
        + STREAM_FIXED_SLACK_BYTES
        + estimate_activation_bytes(
            hidden_size=hidden_size,
            intermediate_size=intermediate_size,
            n_layers=n_layers,
            seq_len=seq_len,
            batch_size=batch_size,
            dtype=dtype,
        )
        + estimate_logits_bytes(
            vocab_size=vocab_size,
            seq_len=logits_tokens,
            batch_size=batch_size,
            bytes_per_element=logits_bytes_per_element,
        )
        + chunk_bytes
    )


@dataclass(frozen=True)
class VramFit:
    """Whether a streaming step is predicted to fit in the VRAM actually free.

    ``measured_bytes`` is ``None`` for a decision taken from the prediction alone
    and carries the real peak once :func:`decide_measured_fit` has run, so a
    caller can print both numbers and let a divergence be seen rather than
    silently replacing one with the other.
    """

    fits: bool
    predicted_bytes: int
    available_bytes: int
    reason: str
    measured_bytes: Optional[int] = None


def decide_stream_fit(*, predicted_bytes: int, available_bytes: int) -> VramFit:
    """Compare predicted demand against measured free VRAM.

    Refusing rather than warning is deliberate. On Linux an over-budget step is a
    hard OOM. On Windows it is worse: WDDM silently spills to host memory and the
    run merely becomes an order of magnitude slower — measured here as a 9.27 GB
    peak on a 4.29 GB card with no exception raised at all. A user reading that
    as "streaming is slow" would draw exactly the wrong conclusion.
    """
    if predicted_bytes <= available_bytes:
        return VramFit(
            fits=True,
            predicted_bytes=predicted_bytes,
            available_bytes=available_bytes,
            reason=(
                f"predicted peak {predicted_bytes / 1e9:.2f} GB fits in "
                f"{available_bytes / 1e9:.2f} GB of free VRAM"
            ),
        )
    return VramFit(
        fits=False,
        predicted_bytes=predicted_bytes,
        available_bytes=available_bytes,
        reason=(
            f"a streaming step is predicted to need "
            f"{predicted_bytes / 1e9:.2f} GB of VRAM but only "
            f"{available_bytes / 1e9:.2f} GB is free. Streaming bounds the "
            f"WEIGHTS, not the activations or the logits — lower "
            f"training.batch_size or data.max_length, both of which scale this "
            f"linearly."
        ),
    )


def decide_measured_fit(
    *, measured_bytes: int, predicted_bytes: int, available_bytes: int
) -> VramFit:
    """Decide the fit on the REAL peak of one step, with the prediction beside it.

    :func:`decide_stream_fit` compares a formula against free VRAM, and the
    formula's documented contract is that it never under-predicts. Measured on an
    RTX 3050 Laptop against SmolLM2-135M streamed in bf16, that holds to seq 4352
    (1.081x the real peak) and then fails: at seq 5120 the prediction is 0.934x
    the real peak and at seq 6144 it is 0.787x, i.e. it under-predicts by 21% on
    a shape a user can reach by editing one line of YAML. Against the probe the
    same formula reads 0.992x at seq 4096 and 0.830x at 5120, because the probe
    itself runs 12.5-14.3% above the real training step — the direction that
    makes it safe as a gate. The measurement is deterministic (repeats at a fixed
    shape return bit-identical peaks, #395). The mechanism is NOT established —
    an attention ``seq**2`` term is the obvious candidate and does not fit the
    numbers — which is exactly why this takes a measurement rather than another
    coefficient: a formula cannot model a term nobody has identified.

    ``measured_bytes`` is ``torch.cuda.max_memory_allocated``, deliberately not
    ``max_memory_reserved``. Reserved runs 1.08x-1.41x allocated here and
    overshoots what has to fit, because the caching allocator keeps freed blocks
    and hands them back under pressure: the flagship Llama-3.1-8B NF4
    configuration reserved 3.70 GB against 3.45 GB free and runs, with
    ``num_alloc_retries`` at 0 on every shape measured. Gating on reserved would
    refuse the headline config of the feature it protects.
    """
    if measured_bytes <= available_bytes:
        return VramFit(
            fits=True,
            predicted_bytes=predicted_bytes,
            available_bytes=available_bytes,
            measured_bytes=measured_bytes,
            reason=(
                f"measured peak {measured_bytes / 1e9:.2f} GB fits in "
                f"{available_bytes / 1e9:.2f} GB of free VRAM "
                f"(predicted {predicted_bytes / 1e9:.2f} GB)"
            ),
        )
    return VramFit(
        fits=False,
        predicted_bytes=predicted_bytes,
        available_bytes=available_bytes,
        measured_bytes=measured_bytes,
        reason=(
            f"a streaming step MEASURED {measured_bytes / 1e9:.2f} GB of VRAM at "
            f"the configured shape but only {available_bytes / 1e9:.2f} GB is "
            f"free (the formula predicted {predicted_bytes / 1e9:.2f} GB). "
            f"Streaming bounds the WEIGHTS, not the activations or the logits — "
            f"lower training.batch_size or data.max_length, both of which scale "
            f"this. This run was measured, not estimated: it does not fit."
        ),
    )


def resolve_available_vram_bytes(
    *, measured_bytes: int, override_bytes: Optional[int]
) -> int:
    """The free-VRAM figure the pre-flight fit check measures against.

    ``mem_get_info()`` is a device-level driver query, so it cannot see a
    per-process cap (``set_per_process_memory_fraction``, a MIG slice, a card
    another process is also using). ``training.stream_vram_override``, when
    set, REPLACES the driver reading rather than padding it, in either
    direction: raised to let a documented over-prediction through, or lowered
    to enforce a cap the driver itself cannot report.
    """
    return measured_bytes if override_bytes is None else override_bytes


#: Measured (GATE 3) speed-up from reaching an effective batch by raising
#: ``batch_size`` rather than ``gradient_accumulation_steps``: Qwen2.5-0.5B bf16,
#: S=256, pinned store — 1393.4 tok/s at batch 4 / accum 1 against 553.2 at
#: batch 1 / accum 4. One weight read amortised over 4x the tokens.
ACCUM_VS_BATCH_SPEEDUP = 2.5


def accumulation_advice(*, batch_size: int, accum: int) -> Optional[str]:
    """Tell an accumulating user what the measurement says, or stay quiet.

    The intuition most people bring — "accumulation costs streaming I/O linearly"
    — is right per optimizer step and wrong per token, which is the unit that
    decides wall-clock: ``accum=N`` reads the base N times *and* processes N
    times the tokens, so layer reads per 1k tokens do not move at all (measured
    constant at 175.78 across accum 1/2/4).

    What is true is the trade the user is actually making. Raising ``batch_size``
    reaches the same effective batch in ONE read instead of N, and measured
    2.52x faster — but it costs VRAM, while accumulation holds peak flat. So the
    advice is only worth printing when there might be VRAM headroom to spend.
    """
    if batch_size <= 0 or accum <= 0:
        raise ValueError(
            f"batch_size and accum must be positive; got {batch_size}, {accum}"
        )
    if accum == 1:
        return None
    return (
        f"accumulating {accum}x at batch {batch_size}: the base is re-read once "
        f"per micro-batch. Per token that is free, but reaching effective batch "
        f"{batch_size * accum} by raising training.batch_size instead measured "
        f"~{ACCUM_VS_BATCH_SPEEDUP:.1f}x faster. Accumulation holds peak VRAM "
        f"flat, so raise batch_size while the budget above allows, then "
        f"accumulate for the rest."
    )


@dataclass(frozen=True)
class ThroughputForecast:
    """A compute-bound CEILING, plus the observed fraction of it. Never a point
    estimate: real streamed runs landed at 68%-100% of the measured ceiling."""

    effective_tflops: float
    tokens_per_sec_ceiling: float
    tokens_per_sec_low: float
    epoch_seconds_floor: float
    epoch_seconds_high: float
    sm_clock_mhz: Optional[int] = None


def forecast_stream_throughput(
    *,
    params: int,
    effective_tflops: float,
    tokens_per_epoch: int,
    sm_clock_mhz: Optional[int] = None,
) -> ThroughputForecast:
    """Bracket a streaming run's throughput from a *measured* GEMM ceiling.

    The ceiling itself is arithmetic (``TFLOPS / (6 * P)``); the honest part is
    that ``effective_tflops`` must come from a matmul benchmarked on the user's
    own card in the same session — a per-card constant baked into the source
    would be a fabrication, and this box's boost clock alone moved 442..952 MHz
    inside a single measurement run.
    """
    ceiling = estimate_stream_tokens_per_sec(params, effective_tflops)
    low_fraction, high_fraction = MEASURED_CEILING_FRACTION
    low = ceiling * low_fraction
    return ThroughputForecast(
        effective_tflops=effective_tflops,
        tokens_per_sec_ceiling=ceiling * high_fraction,
        tokens_per_sec_low=low,
        epoch_seconds_floor=(
            estimate_epoch_seconds(tokens_per_epoch, ceiling * high_fraction)
            if tokens_per_epoch
            else 0.0
        ),
        epoch_seconds_high=(
            estimate_epoch_seconds(tokens_per_epoch, low) if tokens_per_epoch else 0.0
        ),
        sm_clock_mhz=sm_clock_mhz,
    )


def estimate_stream_vram(
    *,
    layer_bytes: int,
    buffers: int,
    embed_bytes: int,
    adapter_bytes: int = 0,
    activation_bytes: int = 0,
    logits_bytes: int = 0,
    workspace_bytes: int = DEFAULT_WORKSPACE_BYTES,
    large_layer_bytes: int = 0,
) -> int:
    """Peak VRAM for a streaming step (plan 4.1)."""
    return (
        layer_bytes * buffers
        + large_layer_bytes
        + embed_bytes
        + adapter_bytes
        + activation_bytes
        + logits_bytes
        + workspace_bytes
    )


# ==========================================================================
# specs and plans
# ==========================================================================
@dataclass(frozen=True)
class LayerSpec:
    """Shape + dtype of ONE tensor inside a decoder layer."""

    name: str
    shape: Tuple[int, ...]
    dtype: str

    def __post_init__(self) -> None:
        dtype_bytes(self.dtype)  # fail fast on an unsupported dtype

    @property
    def nbytes(self) -> int:
        return math.prod(self.shape) * dtype_bytes(self.dtype)


@dataclass(frozen=True)
class StreamPlan:
    """What a streaming run will do, before it does it."""

    arch: str
    tier: str
    n_layers: int
    layer_bytes: int
    store_bytes: int
    embed_bytes: int
    large_store_bytes: int
    large_buffer_bytes: int
    buffers: int
    buffer_bytes: int
    pinned: bool
    notes: Tuple[str, ...]
    read_ahead: int = DEFAULT_STREAM_READ_AHEAD
    staging_bytes: int = 0


def staging_bytes_for(
    *,
    read_ahead: int,
    n_layers: int,
    layer_bytes: int,
    large_store_bytes: int = 0,
) -> int:
    """Host memory the disk tier's async reader page-locks, before it does.

    THE one formula, so the pre-flight, the panel and the RAM refusal cannot
    each keep their own. The disk tier used to predict zero host residency,
    which was true of the synchronous source it replaced and false of this one:
    ``AsyncDiskSource`` allocates ``min(read_ahead, members) x group_bytes``
    per DISTINCT layer spec and holds it for the whole run.

    ``read_ahead`` does NOT bound the vocabulary weights. The embedding and an
    untied ``lm_head`` are one member each, so each takes a full slot at any
    depth — and they are the largest tensors in the model, left unquantised by
    ``replace_with_bnb_linear``. ``large_store_bytes`` is exactly that pair, so
    it is charged once rather than multiplied. Worked, on the 70B NF4 shape at
    the default depth 2: ~0.9 GB of decoder staging plus ~2.1 GB embed plus
    ~2.1 GB head, i.e. ~5 GB of unswappable host memory on a box that reached
    this tier BECAUSE its RAM could not hold the model.
    """
    depth = max(0, min(int(read_ahead), int(n_layers)))
    return depth * int(layer_bytes) + int(large_store_bytes)


def build_stream_plan(
    *,
    arch: str,
    n_layers: int,
    layer_bytes: int,
    embed_bytes: int,
    available_ram_bytes: int,
    pinned_limit_bytes: Optional[int],
    total_ram_bytes: Optional[int] = None,
    buffers: int = DEFAULT_STREAM_BUFFERS,
    disk_kind: DiskKind = _NVME,
    stream_pin: Optional[bool] = None,
    store_bytes: Optional[int] = None,
    large_store_bytes: int = 0,
    large_buffer_bytes: int = 0,
    read_ahead: int = DEFAULT_STREAM_READ_AHEAD,
) -> StreamPlan:
    """Decide tier + pinning and record every caveat as a visible note."""
    buffers = validate_stream_buffers(buffers)
    if n_layers <= 0:
        raise ValueError(f"n_layers must be positive; got {n_layers}")
    store_bytes = n_layers * layer_bytes if store_bytes is None else int(store_bytes)
    host_store_bytes = store_bytes + int(large_store_bytes)
    resident_bytes = int(embed_bytes)
    model_bytes = host_store_bytes + resident_bytes
    physical_limit = (
        None
        if total_ram_bytes is None
        else total_ram_bytes * PHYSICAL_RAM_TIER_HEADROOM
    )
    available_budget_exceeded = (
        model_bytes >= available_ram_bytes * RAM_TIER_HEADROOM
        and host_store_bytes < available_ram_bytes * RAM_TIER_HEADROOM
    )
    physical_budget_exceeded = (
        physical_limit is not None
        and model_bytes >= physical_limit
        and host_store_bytes < available_ram_bytes * RAM_TIER_HEADROOM
    )
    tier = choose_tier(
        host_store_bytes,
        available_ram_bytes,
        disk_kind,
        resident_bytes=resident_bytes,
        total_ram_bytes=total_ram_bytes,
    )
    notes = []
    if tier == TIER_DISK:
        # Falling back is the point of stream_source='auto', but a silent
        # fallback to a slower path is the failure mode this project keeps
        # calling out elsewhere. Say what happened, and say what it costs:
        # benchmarks/gate-971-async-nvme-source.md measured the gap against a
        # same-day control of the synchronous source, cold on a store larger
        # than RAM and warm with the store fully cached. The honest summary is
        # that the disk tier is slower than RAM either way, which is why this
        # is a fallback and not a choice.
        if physical_budget_exceeded:
            notes.append(
                "base exceeds the physical RAM safety ceiling — streaming from "
                "the NVMe disk tier instead. The RAM tier keeps the base "
                "resident while shard reads pressure the page cache, so Soup "
                f"requires the store plus resident extras to stay under "
                f"{PHYSICAL_RAM_TIER_HEADROOM_PERCENT}% of physical RAM. Set "
                "stream_source='ram' to refuse rather than fall back."
            )
        elif available_budget_exceeded:
            notes.append(
                "base plus resident extras do not fit the free-RAM safety "
                "headroom — streaming from the NVMe disk tier instead. Set "
                "stream_source='ram' to refuse rather than fall back."
            )
        else:
            notes.append(
                "base does not fit in RAM — streaming from the NVMe disk tier "
                "instead. An async reader stages training.stream_read_ahead layers "
                "in host RAM (page-locked where the box allows) rather than "
                "holding the base resident. It is "
                "slower than the RAM tier — measured 1.9-2.3x its step time with the "
                "store fully cached, on one box "
                "(benchmarks/gate-971-async-nvme-source.md); a store larger than "
                "RAM has no RAM-tier comparison, which is what makes this a "
                "fallback. Set stream_source='ram' to refuse rather than fall "
                "back."
            )
    decision = decide_pinning(host_store_bytes, pinned_limit_bytes, stream_pin=stream_pin)
    # #366 review round 3 — "record, never silence". An automatic pinned store is
    # the unremarkable default and stays quiet, but an EXPLICIT stream_pin=true is
    # a user decision, so the pre-flight states it too. Without this the forced-on
    # branch was the one path that decided something and said nothing, which is
    # also what decide_pinning's docstring already promised it did not do.
    #
    # NOT scoped to the RAM tier. It was, because the reason claimed a RAM store
    # the disk tier did not have and the runtime announced the inapplicability
    # instead — but #971 gave the disk tier host staging that stream_pin now
    # honours or refuses, and deleted that announcement. Gating on the tier left
    # the two spellings of reaching disk printing different prose for the same
    # config: `stream_source: disk` on a RAM-sized box planned tier=ram, so this
    # fired and printed a RAM-tier promise under a panel headed `tier disk`,
    # while `auto` on a RAM-poor box planned tier=disk and recorded the explicit
    # request nowhere at all. The reason above now names both tiers, so it is
    # true wherever it prints.
    if not decision.pinned or stream_pin is True:
        notes.append(decision.reason)
    return StreamPlan(
        arch=arch,
        tier=tier,
        n_layers=n_layers,
        layer_bytes=layer_bytes,
        store_bytes=host_store_bytes,
        embed_bytes=embed_bytes,
        large_store_bytes=int(large_store_bytes),
        large_buffer_bytes=int(large_buffer_bytes),
        buffers=buffers,
        buffer_bytes=layer_bytes * buffers + int(large_buffer_bytes),
        pinned=decision.pinned,
        notes=tuple(notes),
        read_ahead=int(read_ahead),
        # Zero on the RAM tier because the reader does not exist there — the
        # whole base is resident and already charged as `store_bytes`.
        staging_bytes=(
            staging_bytes_for(
                read_ahead=read_ahead,
                n_layers=n_layers,
                layer_bytes=layer_bytes,
                large_store_bytes=large_store_bytes,
            )
            if tier == TIER_DISK
            else 0
        ),
    )


def render_stream_panel(plan: StreamPlan, extra_lines: Sequence[str] = ()) -> Panel:
    """Pre-flight summary. plan 10: tell the user the cost BEFORE the run."""
    if plan.tier == TIER_DISK:
        # `plan.store_bytes` is 0 on this tier and reporting it as a pinned
        # store would read as a bug rather than as the design. What the disk
        # tier does hold is the reader's staging, whose size depends on
        # training.stream_read_ahead and is not known until the source is
        # built — the runtime's own ready line prints it. So this says the
        # SHAPE and leaves the number to the line that has it, rather than
        # claiming "nothing held resident", which stopped being true in #971.
        store_line = (
            f"  base         streamed from disk across {plan.n_layers} layers, "
            f"staged by an async reader (no resident copy)"
        )
    else:
        store_line = (
            f"  base store   {plan.store_bytes / 1e9:.2f} GB across "
            f"{plan.n_layers} layers "
            f"({'pinned' if plan.pinned else 'pageable'})"
        )
    lines = [
        f"[bold]Layer streaming[/] [yellow]BETA[/] — arch [cyan]{plan.arch}[/], "
        f"tier [cyan]{plan.tier}[/]",
        store_line,
    ]
    if plan.tier == TIER_DISK:
        # The number the disk tier used to predict as zero. It is host memory,
        # so it does not belong in the VRAM line below, and it is held for the
        # whole run — an operator choosing a depth is choosing this.
        lines.append(
            f"  host staging read_ahead {plan.read_ahead} -> "
            f"{plan.staging_bytes / 1e6:.0f} MB "
            f"(page-locked when the box allows)"
        )
    lines += [
        f"  VRAM buffers {plan.buffers} x {plan.layer_bytes / 1e6:.0f} MB "
        f"+ 1 x {plan.large_buffer_bytes / 1e6:.0f} MB large-layer slot "
        f"= {plan.buffer_bytes / 1e6:.0f} MB",
        f"  resident     {plan.embed_bytes / 1e6:.0f} MB extras + adapters",
    ]
    lines.extend(extra_lines)
    for note in plan.notes:
        lines.append(f"  [yellow]![/] {note}")
    return Panel("\n".join(lines), title="training.stream_layers", border_style="cyan")