Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

96.5 KB
Raw Blame
"""Shared layer-streaming setup for every trainer wrapper that supports it.

v0.72.4 — extracted verbatim from ``trainer/sft.py`` so that SFT and the four
preference losses (DPO / ORPO / SimPO / KTO) cannot drift. There is exactly one
copy of the NF4 pre-flight, the RAM/disk tier decision, the VRAM fit refusal and
the runtime release; a per-wrapper copy would be five places to fix the next
time any of them is wrong.

The move is behaviour-preserving for SFT by design, so v0.72.0-.3's
bit-exactness gates remain valid without being re-run. The ONE addition is
``_STREAM_ROWS_PER_EXAMPLE``: DPO, ORPO and SimPO build their forward through
TRL's ``concatenated_inputs`` + ``torch.cat``, so 2 x ``batch_size`` rows reach
the model in a single tensor. v0.72.3's VRAM estimator was validated on the
property that it NEVER under-predicts, and budgeting those three at 1x rows
would break exactly that — on Windows the consequence is not an exception but a
silent WDDM spill to host memory that makes the run an order of magnitude
slower with no error at all.

NO top-level torch: this module is imported by five trainer modules.
"""

import contextlib
import math
import os
import shutil
from dataclasses import dataclass
from typing import Any, Optional, Sequence

from rich.console import Console
from rich.panel import Panel

from soup_cli.utils.config_bounds import DEFAULT_STREAM_READ_AHEAD, MAX_STREAM_READ_AHEAD
from soup_cli.utils.paths import quote_path
from soup_cli.utils.run_provenance import base_revision_kwargs
from soup_cli.utils.stripe_roots import ReadAheadDecision, effective_read_ahead

console = Console()


#: How far over the predicted budget `training.stream_vram_probe` may still
#: defer to a measurement. The formula's worst measured error is 0.787x (21%
#: under, seq 6144), so anything beyond a small multiple is not the formula
#: being wrong — it is the config being too big, and that is refusable by
#: arithmetic without touching the GPU.
_PROBE_DEFERRAL_CEILING = 4.0


def _stream_source_line(stats: dict) -> str:
    """The source half of the ``Layer streaming ready:`` line, from ``runtime.stats()``.

    On the RAM tier the store's bytes are stated, and — since #901 — the bytes
    the box actually page-locked for it when the source knows them: pinned host
    memory is handed out in power-of-two blocks, so the two differ. A pageable
    store, and a source that does not account for it (``pinned_bytes`` None),
    print no second figure. The disk tier names its reader depth and staging
    instead: not "nothing held resident" (#971).
    """
    from soup_cli.utils.layer_stream import TIER_RAM

    pinned = "pinned" if stats["pinned"] else "pageable"
    if stats["tier"] == TIER_RAM:
        return f"{stats['store_bytes'] / 1e9:.2f} GB {pinned} RAM store" + _page_locked_note(
            stats
        )
    # Since #974 the staging lives in the same power-of-two arenas as the RAM
    # store, so the disk tier can say what was page-locked too.
    drive_count = stats.get("drive_count", 1)
    direct_io = stats.get("direct_io_drives", drive_count)
    media = "an NVMe volume"
    if drive_count > 1:
        media = f"{drive_count} NVMe volumes, direct I/O {direct_io}/{drive_count}"
    return (
        f"streamed from DISK ({stats['disk_bytes'] / 1e9:.2f} GB on {media}) "
        f"by an async reader, read_ahead={stats['read_ahead']}, "
        f"{stats['store_bytes'] / 1e6:.0f} MB {pinned} host staging"
    ) + _page_locked_note(stats)


def _page_locked_note(stats: dict) -> str:
    """The ``(X GB page-locked)`` suffix, when the source accounts for it (#901).

    Pinned host memory is handed out in power-of-two blocks, so the figure
    differs from the store's own bytes; a pageable store, and a source that
    does not account for it (``pinned_bytes`` None), print nothing.
    """
    if stats["pinned"] and stats.get("pinned_bytes"):
        return f" ({stats['pinned_bytes'] / 1e9:.2f} GB page-locked)"
    return ""


def _validate_qwen4_streaming_mode(*, arch: str, task: str, quant: str) -> None:
    """Keep unvalidated Qwen4 training modes outside the streamed path."""
    if arch != "qwen4_exp":
        return
    if task != "sft":
        raise ValueError(
            "Qwen4-Exp layer streaming is initially validated for task='sft' "
            f"only; got task={task!r}. Preference-loss parity is pending."
        )
    if quant != "none":
        raise ValueError(
            "Qwen4-Exp layer streaming currently requires quantization='none'. "
            "Its exact PLE path is validated, but streamed NF4 parity is pending."
        )


def _validate_qwen4_ngram_disk(*, disk_kind: str, weights_dir: str) -> None:
    """Refuse sparse PLE mmap on media outside the measured SSD classes."""
    if disk_kind in ("nvme", "ssd"):
        return
    raise ValueError(
        "training.stream_ngram_source='disk' needs an SSD or NVMe "
        f"checkpoint volume; detected {disk_kind!r} at {weights_dir}. "
        "Move the checkpoint or set an accurate training.stream_disk_kind override."
    )


def _resolve_qwen4_ngram_source(
    *,
    oq_ngram: bool,
    requested: str,
    store_total: int,
    ngram_bytes: int,
    free_ram: int,
    resident_ram: int = 0,
    total_ram: int | None = None,
    stream_source: str,
) -> str:
    """Resolve Qwen4 PLE storage and refuse unsupported oQ materialisation."""
    from soup_cli.utils.layer_stream import (
        PHYSICAL_RAM_TIER_HEADROOM,
        RAM_TIER_HEADROOM,
    )

    if oq_ngram:
        if requested == "ram":
            raise ValueError(
                "oQ PLE embeddings require "
                "training.stream_ngram_source='disk' (or 'auto'): the packed "
                "source stays read-only and only requested rows are dequantized."
            )
        return "disk"
    if requested != "auto":
        return requested
    ram_budget = free_ram * RAM_TIER_HEADROOM
    base_in_ram = (
        store_total if stream_source != "disk" and store_total + resident_ram < ram_budget else 0
    )
    ram_bytes = base_in_ram + ngram_bytes
    fits_available_ram = ram_bytes + resident_ram < ram_budget
    physical_limit = None if total_ram is None else total_ram * PHYSICAL_RAM_TIER_HEADROOM
    fits_physical_ram = physical_limit is None or ram_bytes + resident_ram < physical_limit
    return "ram" if fits_available_ram and fits_physical_ram else "disk"


def _validate_qwen4_ngram_ram_fit(
    *,
    stream_source: str,
    ngram_source: str,
    required_ram: int,
    free_ram: int,
    resident_ram: int = 0,
    total_ram: int | None = None,
) -> None:
    """Refuse a RAM base or PLE before either source allocates its store."""
    from soup_cli.utils.layer_stream import (
        PHYSICAL_RAM_TIER_HEADROOM,
        PHYSICAL_RAM_TIER_HEADROOM_PERCENT,
        RAM_TIER_HEADROOM,
    )

    ram_required = stream_source == "ram" or ngram_source == "ram"
    total_required_ram = required_ram + int(resident_ram)
    if (
        ram_required
        and total_ram is not None
        and total_required_ram >= total_ram * PHYSICAL_RAM_TIER_HEADROOM
    ):
        policy = (
            "training.stream_ngram_source='ram'"
            if ngram_source == "ram"
            else "training.stream_source='ram'"
        )
        fallback = (
            "stream_ngram_source='auto' to use read-only SSD streaming"
            if ngram_source == "ram"
            else "stream_source='auto' to allow the disk tier"
        )
        raise ValueError(
            f"{policy} but the base plus resident extras and selected PLE "
            f"storage needs {total_required_ram / 1e9:.1f} GB, "
            "which exceeds "
            f"{PHYSICAL_RAM_TIER_HEADROOM_PERCENT}% of physical RAM "
            f"({total_ram / 1e9:.1f} GB). Set {fallback}, free RAM, "
            "or pick a smaller base."
        )
    if ram_required and total_required_ram >= free_ram * RAM_TIER_HEADROOM:
        policy = (
            "training.stream_ngram_source='ram'"
            if ngram_source == "ram"
            else "training.stream_source='ram'"
        )
        fallback = (
            "stream_ngram_source='auto' to use read-only SSD streaming"
            if ngram_source == "ram"
            else "stream_source='auto' to allow the disk tier"
        )
        raise ValueError(
            f"{policy} but the base plus resident extras and selected PLE "
            f"storage needs {total_required_ram / 1e9:.1f} GB and only "
            f"{free_ram / 1e9:.1f} GB of RAM is free. Set {fallback}, free RAM, "
            "or pick a smaller base."
        )


def _validate_early_ram_fit(
    *,
    base: str,
    source_bytes: int,
    ram_bytes: int,
    approximate: bool,
    quantised: bool,
    free_ram: Optional[int],
    total_ram: Optional[int],
) -> None:
    """Refuse a ``stream_source: ram`` run BEFORE sharding when the RAM tier cannot hold it.

    ``ram_bytes`` is what the check after sharding (:func:`_validate_qwen4_ngram_ram_fit`)
    charges, sized from the source headers (``ShardCacheEstimate.ram_tier_bytes``): the
    decoder-layer files, the embedding/head shards and the resident extras. The rule is
    that check's rule (70% of free RAM, 55% of physical RAM, either reached refuses), so
    a base the exact check would refuse is refused here, before minutes of disk writes.

    ``approximate`` marks the fallback for source headers that cannot be listed or read:
    the flat NF4 rate on the size on disk, which errs low (far low for a fused MoE base)
    and is labelled as a guess, the way the disk pre-flight panel labels it.

    A figure psutil cannot report (``None``) is not checked. That mirrors the check
    after sharding, which replaces an unknown free RAM by ten times the store and so
    can still refuse only by the physical ceiling, when the total is known.
    """
    from soup_cli.utils.layer_stream import (
        PHYSICAL_RAM_TIER_HEADROOM,
        PHYSICAL_RAM_TIER_HEADROOM_PERCENT,
        RAM_TIER_HEADROOM,
    )

    if total_ram is not None and ram_bytes >= total_ram * PHYSICAL_RAM_TIER_HEADROOM:
        limit = (
            f"more than {PHYSICAL_RAM_TIER_HEADROOM_PERCENT}% of physical RAM "
            f"({total_ram / 1e9:.1f} GB)"
        )
    elif free_ram is not None and ram_bytes >= free_ram * RAM_TIER_HEADROOM:
        limit = f"and only {free_ram / 1e9:.1f} GB of RAM is free"
    else:
        return
    if approximate:
        streamed = "once quantised to NF4" if quantised else "in RAM"
        need = (
            f"is {source_bytes / 1e9:.1f} GB on disk, approximately "
            f"{ram_bytes / 1e9:.1f} GB {streamed} (its safetensors headers could not "
            "be listed or read),"
        )
    else:
        need = (
            f"needs {ram_bytes / 1e9:.1f} GB of RAM once sharded (sized from its "
            f"safetensors headers; {source_bytes / 1e9:.1f} GB on disk),"
        )
    raise ValueError(
        f"training.stream_source='ram' but {base} {need} {limit}. Refusing before "
        "sharding. Set stream_source='auto' to fall back to the NVMe disk tier, free "
        "RAM, or pick a smaller base."
    )


def _validate_stream_staging_ram_fit(
    *,
    staging_bytes: int,
    read_ahead: int,
    free_ram: int,
    resident_ram: int = 0,
    read_ahead_decision: Optional[ReadAheadDecision] = None,
) -> None:
    """Refuse a disk-tier run whose host staging will not fit free RAM.

    The disk tier had no host-RAM check at all: it predicted zero residency,
    which was true of the synchronous source it replaced and false of the async
    one, which holds ``min(read_ahead, members) x group_bytes`` of host RAM per
    distinct layer shape for the whole run — page-locked where the box allows,
    pageable otherwise, and the check is the same either way because the RAM is
    held in both cases. On the 70B NF4 shape at the default depth that is
    ~5 GB — on a box that reached this tier BECAUSE its RAM could not hold the
    model. The RAM tier has had this check since v0.72.0
    (``free_ram_bytes`` against ``choose_tier``'s 0.7 headroom, strict ``<``);
    this is the same rule applied to the same resource.

    Refusing beats clamping ``read_ahead``: a depth the operator set is a
    decision, and silently lowering it would hand back a slower run than the
    one they configured with no line saying why.

    ``read_ahead_decision`` (R4) says how the depth came about. On a striped
    cache the default is raised to drives + 1, so a bare "lower it" loops: the
    operator writes the default back and it is raised again. The advice then
    names a value that is not re-raised.
    """
    from soup_cli.utils.layer_stream import RAM_TIER_HEADROOM

    required = int(staging_bytes) + int(resident_ram)
    budget = free_ram * RAM_TIER_HEADROOM
    if required < budget:
        return
    decision = read_ahead_decision or ReadAheadDecision(depth=read_ahead, configured=read_ahead)
    # At the floor there is no lower depth to suggest, and an impossible remedy
    # is worse than none: it reads as "you did not try hard enough".
    advice = decision.lowering_advice()
    lower = f"{advice}, " if advice else ""
    raise ValueError(
        f"layer streaming's disk tier would hold "
        f"{staging_bytes / 1e9:.2f} GB of host staging (page-locked when the "
        f"box allows) at training.stream_read_ahead={read_ahead}, and with "
        f"{resident_ram / 1e9:.2f} GB of resident extras that needs "
        f"{required / 1e9:.2f} GB — more than the "
        f"{budget / 1e9:.2f} GB safety headroom on "
        f"{free_ram / 1e9:.1f} GB of free RAM. The reader stages whole layers, "
        f"and the embedding and lm_head take one slot each at ANY depth "
        f"because they are one layer each. "
        f"{lower}free RAM, or use a smaller base."
    )


def _warn_if_ngram_source_unused(*, arch: str, requested: str, ngram_bytes: int, notify) -> None:
    """Make a user-supplied PLE policy visible when the checkpoint has no PLE."""
    if arch == "qwen4_exp" and requested != "auto" and not ngram_bytes:
        notify(
            "[yellow]training.stream_ngram_source="
            f"{requested!r} has no effect: this Qwen4 checkpoint has no PLE "
            "N-gram table.[/]"
        )


@dataclass(frozen=True)
class _ProbePlan:
    """What the post-build measured probe (#349) needs from the pre-flight.

    Carried forward rather than recomputed so the shape the probe measures and
    the shape the formula budgeted are the same by construction — two
    independent derivations of ``rows`` would be free to drift, and the whole
    point is to compare the two numbers against each other.
    """

    rows: int
    seq_len: int
    vocab_size: int
    predicted_bytes: int
    available_bytes: int


def _existing_disk_anchor(path: str) -> str:
    """Nearest existing ancestor, for paths whose final cache dir is not made yet."""
    anchor = os.path.realpath(os.path.expanduser(path))
    while not os.path.exists(anchor):
        parent = os.path.dirname(anchor)
        if parent == anchor:
            raise OSError(f"cannot locate an existing filesystem ancestor for {quote_path(path)}")
        anchor = parent
    return anchor


def _disk_volume(path: str) -> tuple[int, int]:
    """Filesystem identity and currently free bytes for a prospective write."""
    anchor = _existing_disk_anchor(path)
    return int(os.stat(anchor).st_dev), int(shutil.disk_usage(anchor).free)


def _say_read_ahead_decision(decision: ReadAheadDecision, console: Any) -> None:
    """Explain a striped disk reader's effective depth."""
    n_roots = decision.n_roots
    if n_roots <= 1:
        return
    configured = decision.configured
    wanted = min(n_roots + 1, MAX_STREAM_READ_AHEAD)
    if decision.raised:
        console.print(
            f"[dim]training.stream_read_ahead {configured} -> {decision.depth}: the layer "
            f"cache spans {n_roots} drives, and each needs a read in flight "
            f"({DEFAULT_STREAM_READ_AHEAD} counts as the default; any other value is kept as "
            f"set).[/]"
        )
    elif configured < wanted:
        console.print(
            f"[yellow]training.stream_read_ahead={configured} with {n_roots} stripe drives "
            f"keeps {wanted - configured} of them idle between layers; {wanted} lets every "
            f"drive read at once (one more layer of host staging each).[/]"
        )


def _effective_read_ahead(
    tcfg: Any, n_roots: int, console: Any, *, announce: bool = True
) -> ReadAheadDecision:
    """The async reader's depth with ``n_roots`` drives (R4), and how it came about.

    Every drive needs a read in flight beside the slot the consumer holds, so the DEFAULT depth
    becomes ``n_roots + 1`` — one more layer of host staging per extra drive. "Default" is read
    the way the schema's own read-ahead check reads it ("a default is not a decision",
    ``config/schema.py``): the configured value equals DEFAULT_STREAM_READ_AHEAD. That also
    holds for a config round-tripped through ``model_dump`` (``train --replay``, ``soup
    sweep``), where every field looks explicitly set. Any other value is the operator's call:
    kept, with a warning naming the drives it leaves idle — a speed setting, not a correctness
    one, so never a refusal. The rule itself is ``stripe_roots.effective_read_ahead``.

    Returns the whole decision, not just the depth: every message that advises lowering the
    depth needs to know whether Soup raised it, or its advice loops (the operator writes the
    default back and it is raised again).
    """
    configured = int(tcfg.stream_read_ahead)
    decision = ReadAheadDecision(
        depth=effective_read_ahead(configured, n_roots), configured=configured, n_roots=n_roots
    )
    if announce:
        _say_read_ahead_decision(decision, console)
    return decision


def _stripe_write_shares(total: int, n_roots: int) -> list[int]:
    """Fallback even split across roots when source safetensors headers cannot be read.

    Used when source headers are unreadable or for architectures like qwen4_exp.
    Evenly divides total across roots with divmod remainder distribution.
    """
    base, extra = divmod(int(total), n_roots)
    return [base + (1 if position < extra else 0) for position in range(n_roots)]


def estimate_stripe_write_shares(
    weights_dir: Optional[str],
    *,
    total_bytes: int,
    n_roots: int,
    dtype: str = "bfloat16",
    quant: str = "none",
    double_quant: bool = True,
    arch: str = "",
    estimate: Any = None,
) -> tuple[list[int], bool]:
    """Calculate per-root shard write shares, charging root 0 non-decoder bytes (#1612).

    Returns ``(shares, approximate)``:
    - When ``estimate`` (a ``layer_shard.ShardCacheEstimate``) is given, its
      per-layer and non-decoder bytes are what each root is charged: the same
      header-derived sizes the pre-flight's total comes from. ``approximate`` is
      False.
    - Otherwise, when source safetensors headers can be parsed without tensor loads:
      calculates the parameter count per decoder layer and for non-decoder weights.
      Root k is charged decoder bytes for layers placed on root k according to
      ``layer_roots_for(n_layers, n_roots)``.
      Root 0 is charged its decoder layers plus all non-decoder weights at the
      streaming dtype rate.
      ``approximate`` is False.
    - When headers cannot be read, or n_roots == 1, or total_bytes <= 0, or arch == "qwen4_exp":
      falls back to ``_stripe_write_shares(total_bytes, n_roots)``, with
      ``approximate`` True (when n_roots > 1 and total_bytes > 0).
    """
    if arch == "qwen4_exp":
        return _stripe_write_shares(total_bytes, n_roots), n_roots > 1
    if n_roots <= 1 or total_bytes <= 0:
        return _stripe_write_shares(total_bytes, n_roots), False
    if estimate is not None:
        return _shares_from_layer_bytes(
            dict(enumerate(estimate.layer_bytes)),
            float(estimate.other_bytes),
            total_bytes=total_bytes,
            n_roots=n_roots,
        )
    if not weights_dir:
        return _stripe_write_shares(total_bytes, n_roots), True

    try:
        resolved_dir = os.path.realpath(os.path.expanduser(str(weights_dir)))
        if not os.path.isdir(resolved_dir):
            return _stripe_write_shares(total_bytes, n_roots), True

        from soup_cli.utils.layer_shard import (
            _LAYER_RE,
            QUANT_NF4,
            _canonical_stream_key,
            _discover_safetensors,
        )
        from soup_cli.utils.layer_stream import (
            NF4_BYTES_PER_PARAM,
            NF4_BYTES_PER_PARAM_SINGLE,
        )
        from soup_cli.utils.safetensors_reader import _ITEMSIZE, read_header

        shards = _discover_safetensors(resolved_dir)
        itemsize = _ITEMSIZE.get(dtype, 2)

        decoder_layer_bytes: dict[int, float] = {}
        non_decoder_bytes: float = 0.0

        for shard_path in shards:
            header = read_header(shard_path)
            for source_key, tensor_range in header.items():
                canonical = _canonical_stream_key(source_key)
                if canonical.endswith((".scales", ".biases")):
                    continue
                match = _LAYER_RE.match(canonical)
                numel = math.prod(tensor_range.shape)
                if match:
                    layer_idx = int(match.group(1))
                    if quant == QUANT_NF4:
                        if len(tensor_range.shape) >= 2:
                            rate = (
                                NF4_BYTES_PER_PARAM
                                if double_quant
                                else NF4_BYTES_PER_PARAM_SINGLE
                            )
                        else:
                            rate = itemsize
                    else:
                        rate = itemsize
                    decoder_layer_bytes[layer_idx] = (
                        decoder_layer_bytes.get(layer_idx, 0.0) + numel * rate
                    )
                else:
                    non_decoder_bytes += numel * itemsize

        return _shares_from_layer_bytes(
            decoder_layer_bytes, non_decoder_bytes, total_bytes=total_bytes, n_roots=n_roots
        )
    except Exception:
        return _stripe_write_shares(total_bytes, n_roots), True


def _shares_from_layer_bytes(
    decoder_layer_bytes: dict,
    non_decoder_bytes: float,
    *,
    total_bytes: int,
    n_roots: int,
) -> tuple[list[int], bool]:
    """Split ``total_bytes`` over the roots in proportion to what each root receives.

    Root ``k`` gets the decoder layers ``layer_roots_for`` places on it; root 0 also
    gets everything outside the decoder layers. The total is raised to the raw sum
    when that is larger. Falls back to the even split when nothing was measured.
    """
    from soup_cli.utils.stripe_roots import layer_roots_for

    if not decoder_layer_bytes:
        return _stripe_write_shares(total_bytes, n_roots), True

    n_layers = max(decoder_layer_bytes.keys()) + 1
    layer_roots = layer_roots_for(n_layers, n_roots)
    raw_root_shares = [0.0] * n_roots
    for layer_idx in range(n_layers):
        raw_root_shares[layer_roots[layer_idx]] += decoder_layer_bytes.get(layer_idx, 0.0)
    raw_root_shares[0] += non_decoder_bytes

    raw_total = sum(raw_root_shares)
    if raw_total <= 0:
        return _stripe_write_shares(total_bytes, n_roots), True

    effective_total = max(int(total_bytes), int(math.ceil(raw_total)))
    raw_scaled = [effective_total * (s / raw_total) for s in raw_root_shares]
    shares = [int(v) for v in raw_scaled]
    remainder = effective_total - sum(shares)
    fractions = sorted(range(n_roots), key=lambda i: raw_scaled[i] - shares[i], reverse=True)
    for i in range(remainder):
        shares[fractions[i]] += 1

    return shares, False


def _planned_source_shards(weights_plan: Any) -> list[str]:
    """The ``*.safetensors`` files the sharder will read, listed before any copy is made.

    Without a copy that is the weight directory itself, listed the way the sharder
    lists it. A symlinked Hugging Face snapshot is copied to a regular-file
    directory first, and that copy does not exist yet when the pre-flight runs, so
    its files are read where the copy step will read them, through the copy step's
    own validation: a link is followed only into the snapshot's blob stores.
    """
    from soup_cli.utils.layer_shard import _discover_safetensors

    if not weights_plan.needs_materialization:
        return _discover_safetensors(weights_plan.weights_dir)
    from soup_cli.utils.spectrum_scan import _snapshot_materialization_entries

    revision = getattr(weights_plan, "source_revision", None)
    if revision is None:
        # The copy step refuses such a snapshot; the pre-flight does not guess around it.
        raise ValueError("cannot read a symlinked Hub snapshot without its resolved revision")
    entries = _snapshot_materialization_entries(
        weights_plan.source_dir, source_revision=revision
    )
    return sorted(
        resolved
        for relative, resolved, _blob in entries
        if relative.endswith(".safetensors") and not os.path.dirname(relative)
    )


@dataclass(frozen=True)
class _ResolvedWeights:
    """A weight directory already resolved to regular files, as a pre-flight plan.

    No copy is pending, so :func:`_planned_source_shards` lists it the way the sharder
    lists it.
    """

    weights_dir: str
    needs_materialization: bool = False


#: What listing the source files can raise (a missing directory, a snapshot the copy
#: step would refuse, a malformed plan); the pre-flight then falls back.
_SOURCE_LISTING_ERRORS = (OSError, ValueError, TypeError, OverflowError)


def _preflight_cache_estimate(weights_plan: Any, *, arch: str, **estimate_kwargs: Any) -> Any:
    """The shard cache the sharder will write for this plan, or ``None``.

    ``None`` for an architecture with its own estimator (``qwen4_exp``, whose
    source is then not even listed), and when the source files cannot be listed
    or a header cannot be read: the caller then charges the coarse NF4-rate
    figure and labels it approximate. Nothing else falls back: a bad request
    (dtype, layout, quant suffixes) and a checkpoint the sharder would refuse
    raise here, before any copy or shard is written.
    """
    from soup_cli.utils.layer_shard import SourceHeaderError, estimate_shard_cache_bytes

    if arch == "qwen4_exp":
        return None
    try:
        source_shards = _planned_source_shards(weights_plan)
    except _SOURCE_LISTING_ERRORS:
        return None
    try:
        return estimate_shard_cache_bytes(source_shards, arch=arch, **estimate_kwargs)
    except SourceHeaderError:
        return None


def _render_stream_disk_preflight(
    *,
    source_bytes: int,
    materialized_copy_bytes: int,
    materialize_bytes: int,
    materialized_path: str,
    shard_bytes: int,
    shard_write_bytes: int,
    shard_path: str,
    stripe_writes: Sequence[tuple[str, int]] = (),
    approximate: bool = False,
) -> None:
    """Print and enforce the complete on-disk cost before either cache writes.

    ``stripe_writes`` are the (folder, bytes) writes bound for each extra stripe drive (R4);
    each is charged to its own volume and refused by name when that drive cannot hold it.
    """
    writes = (
        ("materialized weight copy", materialized_path, materialize_bytes),
        ("layer-shard cache", shard_path, shard_write_bytes),
    ) + tuple(
        (f"layer-shard stripe {position}", path, required)
        for position, (path, required) in enumerate(stripe_writes, start=1)
    )
    required_by_device: dict[int, int] = {}
    free_by_device: dict[int, int] = {}
    labels_by_device: dict[int, list[str]] = {}
    for label, path, required in writes:
        if required <= 0:
            continue
        device, free = _disk_volume(path)
        required_by_device[device] = required_by_device.get(device, 0) + required
        free_by_device[device] = min(free_by_device.get(device, free), free)
        labels_by_device.setdefault(device, []).append(label)

    projected_total = source_bytes + materialized_copy_bytes + shard_bytes
    additional = (
        materialize_bytes + shard_write_bytes + sum(required for _path, required in stripe_writes)
    )
    lines = [
        f"HF/local source: {source_bytes / 1e9:.2f} GB",
        (
            f"Soup materialized copy: {materialized_copy_bytes / 1e9:.2f} GB "
            f"({'write required' if materialize_bytes else 'no write required'})"
        ),
        (
            f"Layer-shard cache: {'approximately ' if approximate else ''}"
            f"{shard_bytes / 1e9:.2f} GB "
            f"({'write required' if shard_write_bytes else 'reusable'})"
        ),
        f"Projected total on disk: {projected_total / 1e9:.2f} GB",
        f"Additional writes before training: {additional / 1e9:.2f} GB",
    ]
    for device in sorted(required_by_device):
        required = required_by_device[device]
        free = free_by_device[device]
        labels = " + ".join(labels_by_device[device])
        lines.append(f"Free on target volume ({labels}): {free / 1e9:.2f} GB")
        if required > free:
            console.print(Panel("\n".join(lines), title="Layer streaming disk pre-flight"))
            from soup_cli.utils.stripe_roots import STRIPE_DIRS_ENV

            # R4: a full stripe drive is fixed through the stripe variable, which the
            # containment policy does not cover — name it when a stripe write is on this volume.
            remedy = "SOUP_SPECTRUM_CACHE_DIR / SOUP_LAYER_STREAM_CACHE_DIR at a larger contained"
            if any(label.startswith("layer-shard stripe") for label in labels_by_device[device]):
                remedy = (
                    "SOUP_SPECTRUM_CACHE_DIR / SOUP_LAYER_STREAM_CACHE_DIR / "
                    f"{STRIPE_DIRS_ENV} at a larger"
                )
            raise ValueError(
                f"layer streaming needs {required / 1e9:.2f} GB of additional "
                f"disk space for {labels}, but only {free / 1e9:.2f} GB is free. "
                f"Refusing before copying or sharding. Free disk space, point "
                f"{remedy} volume, or choose a smaller base."
            )
    console.print(Panel("\n".join(lines), title="Layer streaming disk pre-flight"))


def _distributed_launch() -> bool:
    """True when the process was launched by torchrun / accelerate / deepspeed.

    Those set ``WORLD_SIZE`` and HF then reports ``n_gpu == 1`` per process, so
    ``nn.DataParallel`` is never applied and the guard below must not fire.
    A malformed value is treated as non-distributed: refusing with a clear
    message beats proceeding into a raw torch error.
    """
    try:
        return int(os.environ.get("WORLD_SIZE", "1") or "1") > 1
    except (TypeError, ValueError):
        return False


def refuse_if_data_parallel(device) -> None:
    """Refuse layer streaming when HF Trainer would wrap the model in DataParallel.

    ``TrainingArguments`` sets ``_n_gpu = torch.cuda.device_count()`` for a
    non-distributed run, and ``Trainer._wrap_model`` then does
    ``model = nn.DataParallel(model)`` whenever ``n_gpu > 1``. DataParallel
    replicates by requiring every parameter to live on ``device_ids[0]``, and
    layer streaming keeps the decoder on ``meta`` by design — the two are
    incompatible by construction, not by accident.

    Without this the user gets torch's bare ``module must have its parameters
    and buffers on device cuda:0 ... but found one of them on device: meta``,
    which names nothing they set and points at nothing they can change.

    Refusing rather than silently dropping to one GPU is deliberate and matches
    the rest of this path (the VRAM fit decision refuses too): a run that
    quietly used 1 of 8 visible cards would look like it was using all of them.
    """
    if not str(device).startswith("cuda"):
        return
    import torch

    if not torch.cuda.is_available():
        return
    visible = torch.cuda.device_count()
    if visible <= 1 or _distributed_launch():
        return
    raise ValueError(
        f"training.stream_layers=true, but {visible} CUDA devices are visible. "
        f"transformers wraps the model in nn.DataParallel whenever more than one "
        f"GPU is visible and the run is not distributed, and DataParallel requires "
        f"every parameter on cuda:0 — layer streaming keeps the decoder on 'meta' "
        f"by design, so the two cannot be combined. Layer streaming is a "
        f"single-GPU technique: re-run with one card visible, e.g. "
        f"CUDA_VISIBLE_DEVICES=0, or set stream_layers=false to train resident "
        f"across all {visible}."
    )


class StreamingSetupMixin:
    """Builds a layer-streamed model in place of the resident load.

    Requires the host wrapper to provide ``self.device``,
    ``self._trust_remote_code``, and to accept ``self.model`` / ``self.tokenizer``
    / ``self._stream_runtime`` being set.
    """

    #: Rows that reach the model per dataset example. 1 for a plain causal LM
    #: step; 2 for a loss whose forward concatenates chosen and rejected.
    _STREAM_ROWS_PER_EXAMPLE = 1

    #: True for a loss that runs a second forward before each step's backward
    #: (the reference pass of DPO and KTO). On an untied checkpoint that
    #: forward refills the large slot the output head's autograd view points
    #: into, so the head takes a private copy of its weight and the VRAM
    #: pre-flight charges a second large slot for it (#1049).
    _STREAM_REFILL_BEFORE_BACKWARD = False

    #: Set by :meth:`_setup_streaming_transformers`; absent on a resident run.
    _stream_runtime = None
    # #1693 -- the head-and-loss hook of a run with `stream_loss_chunk` set.
    _stream_loss_chunk_hook = None

    #: ``trust_remote_code`` for the streamed MODEL (config, meta skeleton,
    #: install): always False. Every entry of ``SUPPORTED_STREAM_ARCHS`` passed
    #: its resident-vs-streamed parity gate on transformers' own class, and a hub
    #: repo's remote modelling code lays the same weights out differently
    #: (Moonlight's ``modeling_deepseek.py`` keeps one module per expert, so the
    #: shards would not fit it). The tokenizer still gets the user's
    #: ``trust_remote_code``: that is where such repos need it (Moonlight's
    #: tokenizer exists only as remote code).
    _STREAM_NATIVE_MODEL_CODE = False

    @classmethod
    def _stream_meta_skeleton(cls, base, *, dtype, quant):
        """The setup's meta probe of the streamed model, from native code."""
        from soup_cli.utils.layer_stream_runtime import build_meta_skeleton

        return build_meta_skeleton(
            base, dtype=dtype, quant=quant, trust_remote_code=cls._STREAM_NATIVE_MODEL_CODE
        )

    @staticmethod
    def _stream_shape_config(model_config):
        """Text sub-config for multimodal wrappers; plain config otherwise."""
        text_config = getattr(model_config, "text_config", None)
        return text_config if text_config is not None else model_config

    @staticmethod
    def _stream_intermediate_size(model_config) -> int:
        """Activation width estimate, including Qwen3.5 MoE text configs."""
        direct = int(getattr(model_config, "intermediate_size", 0) or 0)
        if direct:
            return direct
        moe = int(getattr(model_config, "moe_intermediate_size", 0) or 0)
        per_tok = int(getattr(model_config, "num_experts_per_tok", 0) or 0)
        shared = int(getattr(model_config, "shared_expert_intermediate_size", 0) or 0)
        return moe * max(per_tok, 1) + shared

    @staticmethod
    def _stream_total_experts(model_config) -> int:
        """Expert instances per layer when the config describes an MoE model."""
        shape_cfg = StreamingSetupMixin._stream_shape_config(model_config)
        for cfg in (shape_cfg, model_config):
            for key in (
                "num_local_experts",
                "num_experts",
                "n_routed_experts",
                "moe_num_experts",
            ):
                value = getattr(cfg, key, None)
                if isinstance(value, (int, float)) and value > 1:
                    return int(value)
        return 0

    @staticmethod
    def _stream_layer_budget_bytes(layer_specs) -> int:
        """Per-buffer bytes from the same union spec the runtime pool uses."""
        from soup_cli.utils.layer_stream import dtype_bytes
        from soup_cli.utils.layer_stream_runtime import RamSource

        merged = RamSource.merge_layer_specs(layer_specs)
        return sum(math.prod(shape) * dtype_bytes(stored) for shape, stored in merged.values())

    @contextlib.contextmanager
    def _training_context(self, *contexts):
        """The `with` block every trainer runs `trainer.train()` inside.

        Its whole job is ordering: ``_close_stream_runtime`` is registered
        FIRST so it runs LAST, after every other context has unwound, and it
        runs even when training raises. That matters because an OOM mid-run is
        a realistic outcome on exactly the small cards this feature targets,
        and on the disk tier the runtime holds one open shard handle per decoder
        layer — which is the case that leaks across back-to-back runs in one
        process (`soup sweep`, the web UI).

        Yields the stack so a caller can enter further contexts conditionally.
        """
        with contextlib.ExitStack() as stack:
            stack.callback(self._close_stream_runtime)
            for context in contexts:
                stack.enter_context(context)
            yield stack

    def _setup_streaming_transformers(self, cfg, tcfg):
        """v0.72.0 BETA — layer streaming. The resident base load NEVER happens.

        Builds the skeleton on ``meta`` (``accelerate.init_empty_weights``),
        materialises only embeddings / final norm / LoRA, and streams each
        decoder layer from CPU RAM into a small pool of pre-allocated VRAM
        buffers. Peak VRAM becomes the size of ONE layer instead of the model.
        """
        from dataclasses import replace

        from peft import TaskType
        from transformers import AutoTokenizer

        from soup_cli.utils.kimi_k2 import load_stream_model_config

        # BEFORE the tokenizer load, the weight resolve and the shard write:
        # this configuration cannot work, and finding out minutes into disk I/O
        # is worse than finding out now.
        refuse_if_data_parallel(self.device)

        from soup_cli.utils.expert_stream import EXPERT_LAYOUT_PER_EXPERT
        from soup_cli.utils.fp8_block import check_source_quantization, quantization_config_of
        from soup_cli.utils.layer_shard import (
            PER_EXPERT_LAYOUT_ARCHS,
            QUANT_NF4,
            QUANT_NONE,
            checkpoint_source_components,
            estimate_oq_stream_cache_bytes,
            fingerprint_source_files,
            inspect_shard_cache,
            layer_paths,
            resolve_shard_dir,
            shard_cache_disk_bytes,
            shard_checkpoint,
            source_identity_includes_config,
            source_weight_bytes,
            stripe_dirs,
        )
        from soup_cli.utils.layer_stream import (
            RAM_TIER_HEADROOM,
            TIER_DISK,
            build_stream_plan,
            dtype_bytes,
            estimate_stream_store_bytes,
            free_ram_bytes,
            render_stream_panel,
            resolve_disk_kind,
            resolve_stream_dtype,
            staging_bytes_for,
            stream_arch_of,
            total_ram_bytes,
        )
        from soup_cli.utils.layer_stream_runtime import (
            RamSource,
            build_streamed_model,
            expandable_segments_status,
            extras_resident_bytes,
            large_layer_buffer_bytes,
            large_layer_specs,
            large_layer_store_bytes,
            quantised_layer_suffixes,
        )
        from soup_cli.utils.moe import resolve_moe_lora_targets
        from soup_cli.utils.peft_wiring import resolve_lora_target_modules
        from soup_cli.utils.qwen4_ple import external_tensor_bytes
        from soup_cli.utils.spectrum_scan import resolve_model_weights
        from soup_cli.utils.stripe_roots import resolve_stripe_roots
        from soup_cli.utils.terminal import for_terminal

        # The streamed decoder is always transformers' own class (see
        # _STREAM_NATIVE_MODEL_CODE); trust_remote_code reaches the tokenizer only. A Kimi K2
        # config (model_type kimi_k2: custom code to AutoConfig) loads into the native
        # DeepseekV3Config behind a guard that refuses by name what that class would compute
        # differently (utils/kimi_k2.py). Before the tokenizer: a refused K2 is told so first,
        # not sent to install the tokenizer's requirements.
        model_config = load_stream_model_config(
            cfg.base, trust_remote_code=self._STREAM_NATIVE_MODEL_CODE,
            **base_revision_kwargs(cfg.base),
        )

        console.print(f"[dim]Loading tokenizer: {cfg.base}[/]")
        try:
            # config=: without it AutoTokenizer reads the config again, with the user's
            # trust_remote_code, and for a repository with an auto_map (Kimi K2, Moonlight)
            # imports and runs its configuration module (second review of #2107). The
            # native config is what the tokenizer is resolved against; the flag still
            # reaches the tokenizer's own code (K2's tokenization_kimi.py).
            self.tokenizer = AutoTokenizer.from_pretrained(
                cfg.base, config=model_config, trust_remote_code=self._trust_remote_code,
                **base_revision_kwargs(cfg.base),
            )
        except Exception:
            # Kimi K2's tokenizer is repository code that imports tiktoken: say so by name.
            from soup_cli.utils.kimi_k2 import print_kimi_k2_tokenizer_hint

            print_kimi_k2_tokenizer_hint(cfg.base, console, **base_revision_kwargs(cfg.base))
            raise
        if self.tokenizer.pad_token is None:
            self.tokenizer.pad_token = self.tokenizer.eos_token

        # Allowlist, not a heuristic — a half-supported architecture streams
        # weights into the wrong module and mis-trains silently.
        arch = stream_arch_of(model_config)
        # training.stream_expert_layout, in the sharder's spelling ("" = fused).
        # Refused here for an architecture the layout is not gated on: before any
        # weight is resolved, downloaded or sharded.
        expert_layout = (
            EXPERT_LAYOUT_PER_EXPERT
            if getattr(tcfg, "stream_expert_layout", "fused") == "per_expert"
            else ""
        )
        if expert_layout and arch not in PER_EXPERT_LAYOUT_ARCHS:
            raise ValueError(
                "training.stream_expert_layout='per_expert' is gated on resident parity "
                f"per architecture; admitted: {', '.join(PER_EXPERT_LAYOUT_ARCHS)}; this "
                f"base streams as {arch!r}. Use stream_expert_layout='fused'."
            )
        # An FP8 block-quantised base (DeepSeek-V3's published format) is dequantised by
        # the sharder. A spelling it does not dequantise, an FP8 base on another
        # architecture, or any other declared quantiser (compressed-tensors, GPTQ, AWQ,
        # bitsandbytes, ...) is refused here from the config alone: before any weight is
        # resolved or downloaded. The disk pre-flight sizes the cache from the same
        # config (the shards hold dequantised weights, the scales are not stored).
        # The top-level declaration, or a multimodal wrapper's text_config one.
        source_quantization = quantization_config_of(model_config)
        fp8_block = check_source_quantization(source_quantization, arch=arch)

        on_cuda = str(self.device).startswith("cuda")
        # #385 — ASK THE CARD. bf16 needs Ampere, and a T4 (Colab free), a P100
        # (Kaggle), a V100 or a GTX 16xx does not have it. Hardcoding bf16 here
        # made the entire free tier unsupported without saying so, and could not
        # fail on the Ampere card every published measurement came from.
        dtype = resolve_stream_dtype(str(self.device))
        if fp8_block is not None:
            console.print(
                f"[dim]FP8 block-quantised base ({fp8_block[0]}x{fp8_block[1]} blocks): "
                f"dequantised to {dtype} as the layer cache is written[/]"
            )

        # v0.72.2 — NF4. The decoder linears ship as packed nibbles + per-block
        # absmax, so the RAM store is ~0.26x its bf16 size; embeddings, norms and
        # an untied head stay at `dtype`, exactly as replace_with_bnb_linear
        # leaves them.
        quant = QUANT_NF4 if tcfg.quantization == "4bit" else QUANT_NONE
        _validate_qwen4_streaming_mode(arch=arch, task=getattr(cfg, "task", "sft"), quant=quant)
        # #321 — the streamed skeleton and the shards must quantise with the
        # SAME double-quant setting or the streamed-vs-resident bit-exactness
        # claim breaks. Read the flag once here (resolving the tri-state unset to
        # the shipped default) and thread it into both the sharder (its cache
        # already keys on double_quant) and the skeleton.
        double_quant = tcfg.double_quant_on

        shard_dir = resolve_shard_dir(cfg.base)
        # R4: extra NVMe roots, validated once here (the ~9 s disk probe runs per volume only
        # when the variable is set). A bad entry refuses the run by name.
        stripe_roots = resolve_stripe_roots(
            os.path.dirname(shard_dir),
            disk_kind=lambda path: resolve_disk_kind(
                path, tcfg.stream_disk_kind, notify=console.print
            ).kind,
        )
        if stripe_roots:
            console.print(
                "[dim]Layer cache striped over "
                + ", ".join(
                    for_terminal(root) for root in (os.path.dirname(shard_dir), *stripe_roots)
                )
                + "[/]"
            )
        quant_device_kind = str(self.device).split(":", 1)[0] if quant == QUANT_NF4 else ""
        # DeepSeek-V3 / Kimi K2 store their MTP module as layers past the last
        # decoder layer; the sharder drops them against the config's count (and the
        # disk pre-flight does not charge them).
        stream_num_hidden_layers = (
            int(self._stream_shape_config(model_config).num_hidden_layers)
            if arch == "deepseek_v3"
            else None
        )

        # The authoritative list of weights to quantise is whatever
        # replace_with_bnb_linear actually converts, read off a meta skeleton —
        # not a hard-coded name list that would drift per architecture.
        #
        # This builds a second, throwaway skeleton (build_streamed_model makes
        # its own). Deliberate: a meta skeleton allocates NO weight storage, so
        # the cost is module-tree construction only, and threading a pre-built
        # model into build_streamed_model would couple suffix discovery to model
        # construction for no memory saving.
        #
        # It runs BEFORE the weights are resolved: it needs only the config, and
        # the disk pre-flight below sizes the shard cache by these same suffixes.
        quant_suffixes = ()
        # The attention targets (``auto`` resolved, or the user's list), first:
        # they need only the config, and the moe_lora helper below must EXTEND a
        # list target, as it does on the resident trainers since #1590 (a string
        # target -- qwen4_exp's "all-linear", a regex -- it keeps, without the MoE
        # routers, on both paths). Handing it ``None`` and replacing the list with its answer
        # dropped every name the MoE list does not repeat -- MLA's q_a/q_b/kv_a/kv_b
        # projections, Qwen3.5's in_proj_qkv / out_proj -- and an explicit list.
        target_modules = resolve_lora_target_modules(
            model_config, tcfg.lora.target_modules, console
        )
        # #798: the ONE helper that turns moe_lora into targets, and refuses a
        # dropout peft cannot honour on fused experts. This path kept its own
        # copy of the block, which is why it was the one path with no refusal.
        # The probe is a meta skeleton and is the only model available here, so
        # the call happens while it is alive rather than at the attach below.
        # With moe_lora off it returns ``target_modules`` unchanged.
        if quant == QUANT_NF4:
            probe = self._stream_meta_skeleton(cfg.base, dtype=dtype, quant=quant)
            target_modules = resolve_moe_lora_targets(
                probe, tcfg, target_modules, console=console
            )
            quant_suffixes = quantised_layer_suffixes(probe)
            del probe
        elif tcfg.moe_lora:
            probe = self._stream_meta_skeleton(cfg.base, dtype=dtype, quant=quant)
            target_modules = resolve_moe_lora_targets(
                probe, tcfg, target_modules, console=console
            )
            del probe

        # One set of arguments for every header estimate of this run's cache.
        estimate_kwargs = dict(
            dtype=dtype,
            arch=arch,
            quant=quant,
            quant_suffixes=quant_suffixes,
            double_quant=double_quant,
            expert_layout=expert_layout,
            num_hidden_layers=stream_num_hidden_layers,
            quantization_config=source_quantization,
        )
        # The disk pre-flight's estimate, kept for the early RAM check below so the
        # source headers are read once. Present (possibly None) once it has run.
        preflight_estimate: dict = {}

        def _disk_preflight(weights_plan) -> None:
            # What the sharder will write, from the source headers and the sharder's
            # own storage rules: fused routed experts stay at the stream dtype under
            # NF4, which a flat NF4 rate under-counted ~3.6x on Moonlight-16B.
            cache_estimate = _preflight_cache_estimate(weights_plan, **estimate_kwargs)
            preflight_estimate["cache"] = cache_estimate
            if cache_estimate is not None:
                shard_estimate = cache_estimate.total_bytes
            else:
                shard_estimate = estimate_stream_store_bytes(
                    weights_plan.source_bytes,
                    dtype=dtype,
                    quant=quant,
                    double_quant=double_quant,
                )
            oq_shard_estimate = estimate_oq_stream_cache_bytes(
                weights_plan.weights_dir,
                dtype=dtype,
                arch=arch,
            )
            if oq_shard_estimate is not None:
                shard_estimate = oq_shard_estimate
            cached = None
            if not weights_plan.needs_materialization:
                source_components = checkpoint_source_components(
                    weights_plan.weights_dir,
                    weights_plan.source_files,
                    include_config=source_identity_includes_config(arch),
                )
                cached, _reason = inspect_shard_cache(
                    shard_dir,
                    dtype,
                    fingerprint_source_files(source_components),
                    source_components,
                    quant,
                    double_quant,
                    quant_device_kind,
                    "qwen4_ple" if arch == "qwen4_exp" else "",
                    stripe_roots=stripe_roots,
                    expert_layout=expert_layout,
                )
            cached_bytes = None
            if cached is not None:
                cached_bytes = shard_cache_disk_bytes(shard_dir, cached)
                if cached_bytes is not None:
                    shard_estimate = cached_bytes
            write_total = 0 if cached is not None else shard_estimate
            shares, approximate = estimate_stripe_write_shares(
                weights_plan.weights_dir,
                total_bytes=write_total,
                n_roots=1 + len(stripe_roots),
                dtype=dtype,
                quant=quant,
                double_quant=double_quant,
                arch=arch,
                estimate=cache_estimate,
            )
            dirs = stripe_dirs(shard_dir, stripe_roots)
            effective_shard_bytes = max(shard_estimate, sum(shares))
            # The flat-rate fallback is a guess and says so; a measured cache is exact.
            coarse = cache_estimate is None and oq_shard_estimate is None
            _render_stream_disk_preflight(
                source_bytes=weights_plan.source_bytes,
                materialized_copy_bytes=weights_plan.materialized_copy_bytes,
                materialize_bytes=weights_plan.materialize_bytes,
                materialized_path=weights_plan.weights_dir,
                shard_bytes=effective_shard_bytes,
                shard_write_bytes=shares[0],
                shard_path=shard_dir,
                stripe_writes=tuple(zip(dirs[1:], shares[1:])),
                approximate=(approximate or coarse) and cached_bytes is None,
            )

        weights_dir = resolve_model_weights(
            cfg.base,
            before_materialize=_disk_preflight,
        )

        # RAM check BEFORE sharding: re-writing a checkpoint we are about to refuse
        # for not fitting in RAM costs minutes of disk I/O. Only `stream_source: ram`
        # insists on the RAM tier; `auto` falls back to disk after sharding, from the
        # real store. Charged what the check after sharding charges, from the disk
        # pre-flight's header estimate (fused MoE experts at the stream dtype, FP8
        # weights dequantised, embeddings and norms unquantised), not the on-disk
        # size at a flat NF4 rate, which errs low. That rate is the fallback only
        # when the headers cannot be listed or read, and says approximately.
        # Qwen4's source size includes the PLE table, which the sharder leaves
        # external. Its exact RAM/disk decision is made from the safetensors header
        # below; counting it here would reject the very `stream_ngram_source: disk`
        # run this path enables. A RAM figure psutil cannot report is not checked,
        # as after sharding (see `_validate_early_ram_fit`).
        if arch != "qwen4_exp" and tcfg.stream_source == "ram":
            if "cache" in preflight_estimate:
                cache_estimate = preflight_estimate["cache"]
            else:
                # A resolver that did not call the pre-flight back: the resolved
                # directory holds regular files, so it is sized where it lies.
                cache_estimate = _preflight_cache_estimate(
                    _ResolvedWeights(weights_dir), **estimate_kwargs
                )
            source_bytes = source_weight_bytes(weights_dir)
            if cache_estimate is not None:
                ram_bytes = cache_estimate.ram_tier_bytes
            else:
                ram_bytes = estimate_stream_store_bytes(
                    source_bytes, dtype=dtype, quant=quant, double_quant=double_quant
                )
            _validate_early_ram_fit(
                base=cfg.base,
                source_bytes=source_bytes,
                ram_bytes=ram_bytes,
                approximate=cache_estimate is None,
                quantised=quant != QUANT_NONE,
                free_ram=free_ram_bytes(),
                total_ram=total_ram_bytes(),
            )

        console.print(f"[dim]Preparing layer shards -> {shard_dir}[/]")
        index = shard_checkpoint(
            weights_dir,
            shard_dir,
            dtype=dtype,
            arch=arch,
            quant=quant,
            quant_suffixes=quant_suffixes,
            double_quant=double_quant,
            # Quantise on the device that will run the model: CPU and CUDA agree
            # on the packed nibbles but not on every float32 nested statistic.
            quant_device=str(self.device),
            stripe_roots=stripe_roots,
            notify=console.print,
            num_hidden_layers=stream_num_hidden_layers,
            expert_layout=expert_layout,
        )
        if expert_layout:
            # Whatever the tier, per_expert reads every routed expert from the cache
            # volume (about three times per step: forward, recompute, backward); the
            # RAM tier holds only the dense part, so the tier choice never looks at it.
            experts_disk = resolve_disk_kind(
                shard_dir, tcfg.stream_disk_kind, notify=console.print
            )
            if experts_disk.kind != "nvme":
                console.print(
                    "[yellow]training.stream_expert_layout='per_expert' reads every routed "
                    "expert from the cache volume about three times per step, and that "
                    f"volume reads as {for_terminal(repr(str(experts_disk.kind)))}, not NVMe: "
                    "expect steps bound by that disk.[/]"
                )
        # R4: decided once, from the roots the index actually carries (a reused cache keeps its
        # own), so the plan's staging figure, the host pre-flight and the reader all agree. Read
        # with getattr like `external_tensors` below: an index without the field is one root.
        # The whole decision travels to every message that advises lowering the depth.
        read_ahead_decision = _effective_read_ahead(
            tcfg,
            1 + len(getattr(index, "stripe_roots", None) or ()),
            console,
            announce=False,
        )
        read_ahead = read_ahead_decision.depth

        layer_specs = RamSource.layer_specs_from_paths(layer_paths(shard_dir, index))
        # Measured from the shard headers, not derived from `total_params`:
        # under NF4 a layer holds packed uint8 alongside float32 statistics, so
        # element counts no longer convert to bytes at a single rate.
        layer_byte_sizes = [
            sum(math.prod(shape) * dtype_bytes(stored) for shape, stored in per_layer.values())
            for per_layer in layer_specs
        ]
        layer_bytes = self._stream_layer_budget_bytes(layer_specs)
        layer_store_bytes = sum(layer_byte_sizes)
        embed_bytes = extras_resident_bytes(shard_dir)
        large_store_bytes = large_layer_store_bytes(shard_dir, index)
        large_buffer_bytes = large_layer_buffer_bytes(shard_dir, index)
        large_budget_bytes = self._stream_large_budget_bytes(
            large_buffer_bytes, len(large_layer_specs(shard_dir, index))
        )
        ngram_bytes = external_tensor_bytes(getattr(index, "external_tensors", None) or {})

        free_ram = free_ram_bytes()
        total_ram = total_ram_bytes()
        if free_ram is None:
            console.print(
                "[yellow]psutil unavailable — cannot size the RAM tier; "
                "proceeding and letting the allocation fail loudly if it must[/]"
            )
            free_ram = (layer_bytes * index.n_layers + large_store_bytes + embed_bytes) * 10

        store_total = layer_store_bytes + large_store_bytes
        ngram_source = "disk"
        if ngram_bytes:
            requested_ngram = tcfg.stream_ngram_source
            oq_ngram = any(hasattr(spec, "bits") for spec in index.external_tensors.values())
            ngram_source = _resolve_qwen4_ngram_source(
                oq_ngram=oq_ngram,
                requested=requested_ngram,
                store_total=store_total,
                ngram_bytes=ngram_bytes,
                free_ram=free_ram,
                resident_ram=embed_bytes,
                total_ram=total_ram,
                stream_source=tcfg.stream_source,
            )
            storage = "CPU RAM"
            if ngram_source == "disk":
                ngram_disk = resolve_disk_kind(
                    weights_dir, tcfg.stream_disk_kind, notify=console.print
                )
                ngram_disk_kind = ngram_disk.kind
                _validate_qwen4_ngram_disk(disk_kind=ngram_disk_kind, weights_dir=weights_dir)
                storage = f"read-only {ngram_disk_kind.upper()} mmap"
            console.print(
                f"[cyan]Qwen4 PLE:[/] {ngram_bytes / 1e9:.2f} GB via {storage} "
                f"(stream_ngram_source={tcfg.stream_ngram_source!r})"
            )
        _warn_if_ngram_source_unused(
            arch=arch,
            requested=getattr(tcfg, "stream_ngram_source", "auto"),
            ngram_bytes=ngram_bytes,
            notify=console.print,
        )
        # Checked BEFORE build_stream_plan so a `ram`-only run is refused with
        # the message about stream_source rather than choose_tier's generic
        # "needs NVMe or more RAM" — and without paying the ~9 s disk probe for
        # an answer that cannot change the outcome.
        required_ram = store_total + (ngram_bytes if ngram_source == "ram" else 0)
        _validate_qwen4_ngram_ram_fit(
            stream_source=tcfg.stream_source,
            ngram_source=ngram_source,
            required_ram=required_ram,
            free_ram=free_ram,
            resident_ram=embed_bytes,
            total_ram=total_ram,
        )
        plan_free_ram = free_ram
        if ngram_source == "ram":
            plan_free_ram = max(
                0,
                free_ram - math.ceil(ngram_bytes / RAM_TIER_HEADROOM),
            )
        plan = build_stream_plan(
            arch=arch,
            n_layers=index.n_layers,
            layer_bytes=layer_bytes,
            embed_bytes=embed_bytes,
            store_bytes=layer_store_bytes,
            large_store_bytes=large_store_bytes,
            large_buffer_bytes=large_buffer_bytes,
            available_ram_bytes=plan_free_ram,
            total_ram_bytes=total_ram,
            # The page-locked ceiling is a property of the box, not of free RAM;
            # rather than probe it destructively we attempt the pinned store and
            # fall back loudly (see layer_stream_runtime._build_source).
            pinned_limit_bytes=None,
            buffers=tcfg.stream_buffers,
            # v0.72.3: the REAL media type, not a constant. Passed as a callable
            # because probing costs ~9 s on Windows and the answer only matters
            # when the base does not fit in RAM. #365: honour a
            # stream_disk_kind override (with a loud detected-vs-override notice)
            # for a disk the auto-probe still misreads.
            disk_kind=lambda: resolve_disk_kind(
                shard_dir, tcfg.stream_disk_kind, notify=console.print
            ),
            # #366: training.stream_pin (None/False/True) overrides the automatic
            # pinning choice so the pageable escape hatch is reachable from config.
            stream_pin=tcfg.stream_pin,
            # #971: the depth decides how much host memory the async reader
            # page-locks, so the plan has to carry it or the pre-flight is
            # predicting zero residency for a tier that holds GBs of it.
            read_ahead=read_ahead,
        )
        # v0.72.3 — the disk overflow tier is live, so a base that does not fit
        # in RAM is no longer fatal. `stream_source` decides: 'ram' insists,
        # 'disk' forces, 'auto' (the default) takes RAM when it fits and falls
        # back to disk when it does not. build_stream_plan already refused a
        # non-NVMe disk, so reaching here with tier='disk' means NVMe.
        tier = TIER_DISK if tcfg.stream_source == "disk" else plan.tier
        if tier == TIER_DISK:
            _say_read_ahead_decision(read_ahead_decision, console)
        if tier != plan.tier:
            # The panel is rendered from `plan`, so a forced tier has to be
            # reflected there or the pre-flight reports "tier ram" immediately
            # before the runtime announces it is streaming from disk. Every
            # field that describes the RAM store is corrected with it, so no
            # consumer can read a stale value.
            # `pinned` is deliberately NOT zeroed with them. It described the
            # RAM store, but `pin=plan.pinned and on_cuda` below now also
            # decides whether the disk tier's host STAGING is page-locked
            # (#971). Zeroing it here would stage pageable for a run that
            # reached the disk tier via `stream_source: disk` and pinned for one
            # that reached the same tier via `auto` — one tier, two behaviours,
            # chosen by the spelling.
            plan = replace(
                plan,
                tier=tier,
                store_bytes=0,
                large_store_bytes=0,
                # Computed here for the same reason `pinned` is not zeroed
                # above: `build_stream_plan` leaves it 0 on a RAM-tier plan, so
                # carrying that through would report no host staging for a run
                # that reached disk by spelling rather than by RAM pressure —
                # one tier, two numbers, chosen by the spelling. `large_store
                # _bytes` is read off the PRE-replace plan, since the line above
                # has just zeroed it.
                staging_bytes=staging_bytes_for(
                    read_ahead=read_ahead,
                    n_layers=plan.n_layers,
                    layer_bytes=plan.layer_bytes,
                    large_store_bytes=plan.large_store_bytes,
                ),
                notes=plan.notes
                + (
                    "streaming from disk because stream_source='disk' was set, "
                    "not because RAM was short. An async reader stages "
                    "training.stream_read_ahead layers in host RAM (page-locked "
                    "where the box allows) rather than holding the base "
                    "resident, and is slower than the RAM "
                    "tier it is being used instead of — measured 1.9-2.3x its step "
                    "time with the store fully cached, on one box "
                    "(benchmarks/gate-971-async-nvme-source.md).",
                ),
            )
        # #971 — HOST pre-flight, and it has to come after the tier is settled
        # above: the async reader's staging is page-locked for the whole run and
        # nothing was charging it. Before the panel, because a refusal an
        # operator has to scroll past a summary to find reads as an afterthought.
        if plan.tier == TIER_DISK:
            _validate_stream_staging_ram_fit(
                staging_bytes=plan.staging_bytes,
                read_ahead=read_ahead,
                free_ram=free_ram,
                resident_ram=embed_bytes,
                read_ahead_decision=read_ahead_decision,
            )
        # v0.72.3 — VRAM pre-flight. Streaming bounds the WEIGHTS; activations
        # and the logits tensor are untouched by it and both scale with batch x
        # seq. On a large-vocab model the logits term alone dwarfs the buffer
        # pool (measured: 146x at batch 8), so a plan that reports only tier and
        # buffer sizes will happily green-light a config that cannot run.
        forecast_lines, probe_plan = self._stream_budget_lines(
            cfg,
            tcfg,
            model_config=model_config,
            layer_bytes=layer_bytes,
            embed_bytes=embed_bytes,
            large_layer_bytes=large_budget_bytes,
            index=index,
            on_cuda=on_cuda,
            target_modules=target_modules,
        )
        console.print(render_stream_panel(plan, forecast_lines))
        console.print(
            "[yellow]Layer streaming is BETA:[/] slower than resident training, "
            "but this model may not run resident on this card at all."
        )
        if on_cuda:
            enabled, why_not = expandable_segments_status()
            if not enabled:
                console.print(f"[dim]expandable_segments allocator hint not enabled: {why_not}[/]")

        from soup_cli.utils.peft_wiring import build_lora_config

        # ``target_modules`` was settled before the probe was deleted, moe_lora's
        # additions included (already announced by resolve_moe_lora_targets).
        lora_config = build_lora_config(
            tcfg.lora,
            target_modules=target_modules,
            task_type=TaskType.CAUSAL_LM,
        )

        # #366 / #434 — CUDA host pinning is inapplicable on every non-CUDA
        # target. An explicit stream_pin=true is honoured by saying so, not by
        # dropping it silently. On MPS the pageable CPU source is also what keeps
        # the frozen base out of the accelerator allocator.
        if tcfg.stream_pin is True and not on_cuda:
            console.print(
                "[yellow]training.stream_pin=true, but no CUDA device is present: "
                "CUDA host pinning does not apply to this target. Proceeding with "
                "a pageable CPU source.[/]"
            )

        model, runtime = build_streamed_model(
            model_id=cfg.base,
            shard_dir=shard_dir,
            index=index,
            lora_config=lora_config,
            device=self.device,
            dtype=dtype,
            buffers=tcfg.stream_buffers,
            # #971: the depth the async reader stages to on the disk tier.
            # Ignored on the RAM tier, which holds every layer and reads
            # nothing ahead.
            read_ahead=read_ahead,
            # R4: how that depth came about, for the page-lock refusal and fallback advice.
            read_ahead_decision=read_ahead_decision,
            pin=plan.pinned and on_cuda,
            # #366: stream_pin=true refuses rather than silently falling back to
            # pageable memory — on the RAM tier that is the store, and since
            # #971 on the disk tier it is the reader's host staging. On
            # non-CUDA targets the notice above covers it, so require_pin is
            # gated on a real CUDA device.
            require_pin=(tcfg.stream_pin is True) and on_cuda,
            seed=tcfg.seed if getattr(tcfg, "seed", None) is not None else 0,
            trust_remote_code=self._STREAM_NATIVE_MODEL_CODE,
            console=console,
            quant=quant,
            double_quant=double_quant,
            tier=tier,
            weights_dir=weights_dir,
            ngram_source=ngram_source,
            refill_before_backward=self._STREAM_REFILL_BEFORE_BACKWARD,
            nf4_scale_cache=bool(getattr(tcfg, "stream_nf4_scale_cache", False)),
        )
        self.model = model
        self._stream_runtime = runtime
        # #792 — on the materialised adapters, before the probe measures a step.
        from soup_cli.utils.peft_wiring import apply_fast_lora_patches, fast_lora_summary

        try:
            fast_lora_counts = apply_fast_lora_patches(model, tcfg)
        except Exception:
            # Training's cleanup context has not started yet: release the
            # runtime here, as the probe below does on its own failure.
            self._close_stream_runtime()
            raise
        if fast_lora_counts is not None:
            console.print(f"[green]{fast_lora_summary(fast_lora_counts)}[/]")
        loss_chunk = getattr(tcfg, "stream_loss_chunk", None)
        if loss_chunk is not None:
            # #1693 -- before the probe, so that it measures the step that runs.
            from soup_cli.utils.stream_loss_chunk import ChunkedHeadLoss, causal_lm_of

            try:
                self._stream_loss_chunk_hook = ChunkedHeadLoss(causal_lm_of(model), loss_chunk)
            except Exception:
                # As for #792's patch above: training's cleanup context has
                # not started yet, so release the runtime here.
                self._close_stream_runtime()
                raise
        # training.stream_recompute -- before the probe, so that the probe
        # measures the mode that will train. The default ('on') adds nothing to
        # the setup: no attribute is read and nothing is called, so the shape
        # and task arguments below are only evaluated for 'off' / 'auto'.
        if getattr(tcfg, "stream_recompute", "on") != "on":
            try:
                self._configure_stream_recompute(
                    model,
                    tcfg,
                    tier=tier,
                    on_cuda=on_cuda,
                    task=cfg.task,
                    step_shape=self._stream_step_shape(cfg, tcfg, model_config),
                )
            except Exception:
                self._close_stream_runtime()
                raise
        if probe_plan is not None:
            self._run_stream_vram_probe(model, probe_plan)
        stats = runtime.stats()
        source_line = _stream_source_line(stats)
        large_runtime_buffer = stats.get("large_buffer_bytes", 0)
        decoder_buffers = stats["buffer_bytes"] - large_runtime_buffer
        buffer_line = (
            f"{stats['buffers']} x "
            f"{decoder_buffers / stats['buffers'] / 1e6:.0f} MB decoder buffers + "
            f"1 x {large_runtime_buffer / 1e6:.0f} MB large-layer slot"
        )
        console.print(
            f"[green]Layer streaming ready:[/] {stats['n_layers']} layers, "
            f"{source_line}, {buffer_line}"
        )

    def _stream_large_budget_bytes(self, slot_bytes: int, n_large_keys: int) -> int:
        """Large-layer bytes the VRAM pre-flight charges.

        One reusable slot, sized to the larger of ``embed_tokens`` and an untied
        ``lm_head`` (#324). An untied checkpoint (two large keys) under a loss
        with a second forward per step also holds a private copy of the head's
        weight for the whole graph (#1049). It is charged as a second full slot.
        That is the head's size when the head is the larger matrix and an
        over-count otherwise, which keeps ``estimate_stream_peak_vram`` on the
        side it promises never to leave.
        """
        if self._STREAM_REFILL_BEFORE_BACKWARD and n_large_keys > 1:
            return 2 * slot_bytes
        return slot_bytes

    def _configure_stream_recompute(
        self,
        model,
        tcfg,
        *,
        tier: str,
        on_cuda: bool,
        task: str,
        step_shape: tuple = (0, 0, 0),
    ) -> None:
        """``training.stream_recompute``: 'on' leaves the checkpoint recompute;
        'off' switches every streamed layer; 'auto' (RAM tier, CUDA) measures one
        synthetic step at ``step_shape`` (rows, seq_len, vocab_size) and switches
        the last layers that fit, all before training starts."""
        from soup_cli.utils.layer_stream import STREAM_RECOMPUTE_HEADROOM, TIER_DISK

        mode = getattr(tcfg, "stream_recompute", "on")
        if mode == "on":
            return
        if task != "sft":
            # The config refuses this first; repeated here so that no other
            # caller of the setup can switch a loss with two grad-enabled
            # forwards per backward.
            raise ValueError(
                f"training.stream_recompute={mode!r} supports task='sft' only; "
                f"got task={task!r}."
            )
        if mode == "off":
            from soup_cli.utils.layer_stream_runtime import set_stream_recompute

            switched = set_stream_recompute(model, False)
            console.print(
                f"[cyan]training.stream_recompute='off':[/] {switched} streamed layers "
                "keep their activations instead of recomputing them in the backward "
                "(faster on the RAM tier, more VRAM; the formula pre-flight above does "
                "not count them, training.stream_vram_probe measures them)"
            )
            if tier == TIER_DISK:
                console.print(
                    "[yellow]training.stream_recompute='off' on the disk tier:[/] a "
                    "disk-tier step is usually read-bound, and skipping the recompute "
                    "does not skip the backward read of each layer, so it is likely no "
                    "faster and holds more VRAM; the recompute ('on') may be cheaper here."
                )
            return
        if tier == TIER_DISK:
            console.print(
                "[cyan]training.stream_recompute='auto' keeps the recompute on the disk "
                "tier:[/] a disk-tier step is read-bound, and skipping the recompute does "
                "not skip the backward read of each layer."
            )
            return
        if not on_cuda:
            console.print(
                "[cyan]training.stream_recompute='auto' keeps the recompute:[/] no CUDA "
                "device to measure VRAM on."
            )
            return
        from soup_cli.utils.layer_stream_runtime import plan_stream_recompute_auto

        rows, seq_len, vocab_size = step_shape
        console.print(
            f"[cyan]training.stream_recompute='auto':[/] measuring one step at batch {rows} "
            f"x seq {seq_len} to choose how many of the last layers keep their activations "
            f"within {STREAM_RECOMPUTE_HEADROOM:.0%} of the card."
        )
        plan_stream_recompute_auto(
            model,
            rows=rows,
            seq_len=seq_len,
            vocab_size=vocab_size,
            notify=console.print,
            override_bytes=getattr(tcfg, "stream_vram_override", None),
        )

    def _stream_step_shape(self, cfg, tcfg, model_config) -> tuple:
        """``(rows, seq_len, vocab_size)`` of the worst-case step: the shape
        :meth:`_stream_budget_lines` budgets and the VRAM probe measures."""
        shape_cfg = self._stream_shape_config(model_config)
        batch = tcfg.batch_size if isinstance(tcfg.batch_size, int) else 1
        return (
            batch * self._STREAM_ROWS_PER_EXAMPLE,
            int(cfg.data.max_length),
            int(getattr(shape_cfg, "vocab_size", 0) or 0),
        )

    def _attach_streamed_save_guard(self) -> None:
        """Check every ``checkpoint-*`` adapter a streamed run writes (#1011)."""
        if getattr(self, "_stream_runtime", None) is None:
            return
        from soup_cli.utils.layer_stream_runtime import build_streamed_save_guard_callback

        self.trainer.add_callback(build_streamed_save_guard_callback())

    def _assert_streamed_adapter_saved(self, output_dir: str) -> None:
        """Check the final adapter a streamed run wrote (#1011).

        ``save_model`` dispatches no ``on_save``, so the callback above does not
        see this save. Gated on ``args.should_save``, the condition ``save_model``
        gates the write on.
        """
        if getattr(self, "_stream_runtime", None) is None:
            return
        if not getattr(self.trainer.args, "should_save", True):
            return
        from soup_cli.utils.layer_stream_runtime import assert_streamed_adapter_saved

        assert_streamed_adapter_saved(self.trainer.model, output_dir)

    def _close_stream_runtime(self) -> None:
        """Release the streaming weight source, if this run had one."""
        runtime = getattr(self, "_stream_runtime", None)
        if runtime is not None:
            runtime.close()

    def _remove_stream_loss_chunk_hook(self) -> None:
        """Take the chunked head and loss off the model when training ends (#1693).

        A labelled forward later in the same process (an eval gate, ``infer``)
        then gets the full logits again, and the trainer's ``compute_loss``
        takes TRL's own path.
        """
        hook = self._stream_loss_chunk_hook
        if hook is None:
            return
        from soup_cli.utils.stream_loss_chunk import HOOK_ATTR

        hook.remove()
        trainer = getattr(self, "trainer", None)
        if trainer is not None and getattr(trainer, HOOK_ATTR, None) is hook:
            setattr(trainer, HOOK_ATTR, None)
        self._stream_loss_chunk_hook = None

    def _estimate_adapter_params(self, tcfg, model_config, target_modules=None) -> int:
        """Trainable adapter parameters, before the model exists.

        Deliberately coarse and biased HIGH: it assumes every targeted module is
        hidden x hidden. Gate/up/down projections are larger, but the whole
        adapter term is ~0.5% of a streaming step's peak, so precision here buys
        nothing while under-counting would eat into the safety margin.

        ``target_modules`` is what the setup resolved. Under ``moe_lora`` it is
        counted instead of the configured value, because it is what attaches: the
        union of the attention targets and the MoE names, once per layer, plus one
        adapter per routed expert for each fused expert tensor peft is handed as
        ``target_parameters``. Counting the configured value charged
        ``['o_proj']`` one target per layer. Measured against a real attach: 1.80x
        at DeepSeek-V3's dimensions, 1.45x at Qwen3.5-MoE's. Without ``moe_lora``
        the configured value is counted, as before.
        """
        shape_cfg = StreamingSetupMixin._stream_shape_config(model_config)
        hidden = int(getattr(shape_cfg, "hidden_size", 0) or 0)
        layers = int(getattr(shape_cfg, "num_hidden_layers", 0) or 0)
        experts = StreamingSetupMixin._stream_total_experts(model_config)
        moe = bool(getattr(tcfg, "moe_lora", False)) and experts > 1
        targets = tcfg.lora.target_modules
        if moe and target_modules is not None:
            targets = target_modules
        if isinstance(targets, (list, tuple)):
            target_names = [str(name) for name in targets]
            n_targets = len(target_names)
            fused = list(getattr(targets, "target_parameters", None) or ())
            if moe and fused:
                # Fused routed experts (transformers 5): the module names reach the
                # dense / shared-expert projections once per layer, and each fused
                # tensor carries one adapter per routed expert.
                n_targets += experts * len(fused)
            elif moe:
                # One nn.Linear per expert: an expert name matches every expert.
                expert_suffixes = {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
                expert_patterns = {name for name in target_names if name in expert_suffixes}
                n_targets += (experts - 1) * len(expert_patterns)
        elif moe:
            n_targets = 4 + 3 * experts
        else:
            n_targets = 4
        return layers * n_targets * 2 * tcfg.lora.r * hidden

    def _stream_budget_lines(
        self,
        cfg,
        tcfg,
        *,
        model_config,
        layer_bytes,
        embed_bytes,
        index,
        on_cuda,
        large_layer_bytes=0,
        target_modules=None,
    ):
        """Predict peak VRAM + bracket throughput, and REFUSE a run that cannot fit.

        Returns ``(panel_lines, probe_plan)``, where ``probe_plan`` is ``None``
        unless ``training.stream_vram_probe`` asked for the measured gate (#349).
        ``target_modules`` is the resolved LoRA target list the attach will use
        (see :meth:`_estimate_adapter_params`).

        Raises when the step is predicted not to fit: on Linux that would be a
        hard OOM, and on Windows something worse — WDDM spills to host memory
        without raising, so the run silently becomes an order of magnitude
        slower and looks like the feature is merely slow. Under the probe the
        prediction is demoted to advice instead, because refusing here would
        prevent the measurement that exists to overrule it.
        """
        from soup_cli.utils.expert_stream import (
            EXPERT_LAYOUT_PER_EXPERT,
            expert_stream_vram_bytes,
        )
        from soup_cli.utils.layer_stream import (
            LOGITS_BYTES_PER_ELEMENT,
            accumulation_advice,
            calibrated_logits_bytes_per_element,
            decide_stream_fit,
            dtype_bytes,
            estimate_logits_bytes,
            estimate_stream_peak_vram,
            forecast_stream_throughput,
            nf4_scale_cache_bytes,
            resolve_available_vram_bytes,
        )
        from soup_cli.utils.layer_stream_runtime import measure_gemm_tflops

        # Charged whole even though it fills lazily: the cache is complete after
        # the first step and stays resident for the run.
        scale_cache_bytes = (
            nf4_scale_cache_bytes(getattr(index, "quant_specs", None) or {}, index.n_layers)
            if getattr(tcfg, "stream_nf4_scale_cache", False)
            else 0
        )
        # training.stream_expert_layout='per_expert': the layer buffers above hold
        # only the dense part (the layer files no longer carry the experts); one
        # routed expert is in flight at a time on top of them.
        expert_bytes = 0
        expert_shapes = getattr(index, "expert_shapes", None)
        if getattr(index, "expert_layout", "") == EXPERT_LAYOUT_PER_EXPERT and expert_shapes:
            expert_bytes = expert_stream_vram_bytes(
                expert_shapes,
                dtype_bytes=dtype_bytes(index.dtype),
                quantised=bool(getattr(index, "expert_quant_specs", None)),
            )

        shape_cfg = self._stream_shape_config(model_config)
        vocab = int(getattr(shape_cfg, "vocab_size", 0) or 0)
        hidden = int(getattr(shape_cfg, "hidden_size", 0) or 0)
        inter = self._stream_intermediate_size(shape_cfg)
        seq_len = int(cfg.data.max_length)
        # #1693 -- with the head and the loss in chunks, the logits term stops
        # at one chunk. (`getattr`: the stand-in configs of #348 and #349 name
        # only the fields this method read when they were written.)
        loss_chunk = getattr(tcfg, "stream_loss_chunk", None)
        batch = tcfg.batch_size if isinstance(tcfg.batch_size, int) else 1
        # v0.72.4 — a paired loss concatenates chosen and rejected into ONE
        # tensor, so twice the rows reach the model per configured batch. The
        # estimator's contract is that it never under-predicts; budgeting a
        # paired loss at 1x rows would halve the logits term, which is the
        # dominant one (measured 146x the buffer pool at batch 8).
        rows = batch * self._STREAM_ROWS_PER_EXAMPLE
        if not (vocab and hidden and inter):
            # Never silently: skipping the budget also skips the refusal that
            # stops a run from OOMing (or, on Windows, spilling to host memory
            # and running an order of magnitude slower with no error at all).
            console.print(
                "[yellow]Layer streaming could not read vocab_size / hidden_size "
                "/ intermediate_size from the model config, so peak VRAM cannot "
                "be predicted — the pre-flight fit check is SKIPPED for this "
                "run.[/]"
            )
            # No shape to probe either: the probe needs vocab_size to build the
            # synthetic batch, which is one of the fields that could not be read.
            return (), None

        # calibrated_logits_bytes_per_element() is floored at LOGITS_BYTES_PER_ELEMENT,
        # so forwarding it here can only raise the budget, never lower it (issue #348).
        calibrated = calibrated_logits_bytes_per_element()
        predicted = estimate_stream_peak_vram(
            layer_bytes=layer_bytes,
            buffers=tcfg.stream_buffers,
            extras_bytes=embed_bytes,
            adapter_params=self._estimate_adapter_params(
                tcfg, model_config, target_modules=target_modules
            ),
            vocab_size=vocab,
            hidden_size=hidden,
            intermediate_size=inter,
            n_layers=index.n_layers,
            seq_len=seq_len,
            batch_size=rows,
            logits_bytes_per_element=calibrated,
            large_layer_bytes=large_layer_bytes,
            persistent_bytes=scale_cache_bytes + expert_bytes,
            loss_chunk=loss_chunk,
        )
        logits = estimate_logits_bytes(
            vocab_size=vocab,
            seq_len=seq_len if loss_chunk is None else min(seq_len, loss_chunk),
            batch_size=rows,
            bytes_per_element=calibrated,
        )
        paired = (
            "" if rows == batch else f" ({rows} rows — chosen+rejected are one concatenated tensor)"
        )
        chunked = "" if loss_chunk is None else f" in chunks of {loss_chunk} tokens"
        lines = [
            f"  peak VRAM    ~{predicted / 1e9:.2f} GB at batch {batch} x seq "
            f"{seq_len}{paired} (logits {logits / 1e9:.2f} GB{chunked})"
        ]
        if scale_cache_bytes:
            lines.append(
                f"  scale cache  {scale_cache_bytes / 1e9:.2f} GB resident "
                "(training.stream_nf4_scale_cache), included above"
            )
        if expert_bytes:
            lines.append(
                f"  experts      {expert_bytes / 1e9:.3f} GB for the routed expert in flight "
                "(training.stream_expert_layout='per_expert'), included above"
            )
        if calibrated > LOGITS_BYTES_PER_ELEMENT:
            lines.append(
                f"  logits       calibrated {calibrated:.3f} B/element on this stack, "
                f"above the shipped {LOGITS_BYTES_PER_ELEMENT:.0f}: budget raised to match"
            )

        if not on_cuda:
            # Nothing to measure against: the probe reads CUDA peak counters.
            # Say so rather than no-opping, mirroring the unreadable-config skip
            # above — a silently inactive gate reads exactly like an active one.
            if tcfg.stream_vram_probe:
                console.print(
                    "[yellow]training.stream_vram_probe is set but this run is "
                    "not on CUDA, so there is no peak to measure — the pre-flight "
                    "falls back to the predicted budget.[/]"
                )
            return tuple(lines), None

        import torch

        measured_available = int(torch.cuda.mem_get_info()[0])
        available = resolve_available_vram_bytes(
            measured_bytes=measured_available, override_bytes=tcfg.stream_vram_override
        )
        fit = decide_stream_fit(predicted_bytes=predicted, available_bytes=available)
        if not fit.fits:
            if not tcfg.stream_vram_probe:
                raise ValueError(fit.reason)
            if predicted > available * _PROBE_DEFERRAL_CEILING:
                # The probe exists to settle a DISAGREEMENT, and the largest
                # disagreement ever measured is 21% (formula 0.787x the real peak
                # at seq 6144). A config predicted several times over budget is
                # not a disagreement, and deferring it would trade a free
                # arithmetic refusal for minutes of sharding plus a real
                # allocation attempt at that shape — driven by a soup.yaml whose
                # author need not be whoever runs it.
                raise ValueError(
                    f"{fit.reason} training.stream_vram_probe cannot overrule a "
                    f"prediction this far over budget "
                    f"({predicted / available:.1f}x): the probe corrects a "
                    f"margin, not an order of magnitude."
                )
            # #349 — with the probe on, the formula is advisory: refusing here
            # would stop the measurement that exists to overrule it from ever
            # being taken. Say so rather than passing silently, because the
            # build about to happen is minutes of work that may still be refused.
            console.print(
                f"[yellow]Predicted over budget "
                f"({predicted / 1e9:.2f} GB vs {available / 1e9:.2f} GB free), but "
                f"training.stream_vram_probe is on: measuring the real peak "
                f"before deciding.[/]"
            )
        if tcfg.stream_vram_override is None:
            lines.append(f"  free VRAM    {available / 1e9:.2f} GB")
        else:
            lines.append(
                f"  free VRAM    {available / 1e9:.2f} GB (training.stream_vram_override; "
                f"driver reports {measured_available / 1e9:.2f} GB)"
            )

        # A per-card TFLOPS constant baked into the source would be a
        # fabrication; measuring the user's own card in this session is the only
        # honest input, and the result is reported as a bracket because real
        # streamed runs landed at 68%-100% of their measured ceiling.
        ceiling = measure_gemm_tflops(device=str(self.device))
        if ceiling is not None and index.total_params:
            shaped = forecast_stream_throughput(
                params=index.total_params,
                effective_tflops=ceiling.tflops,
                tokens_per_epoch=0,
                sm_clock_mhz=ceiling.sm_clock_mhz,
            )
            clock = f" @ {ceiling.sm_clock_mhz} MHz" if ceiling.sm_clock_mhz else ""
            lines.append(
                f"  forecast     {shaped.tokens_per_sec_low:.0f}-"
                f"{shaped.tokens_per_sec_ceiling:.0f} tok/s — a compute-bound "
                f"bound, not a promise"
            )
            lines.append(
                f"               (from {ceiling.tflops:.2f} TFLOPS measured on "
                f"this card now using {ceiling.dtype}{clock})"
            )
        advice = accumulation_advice(batch_size=batch, accum=tcfg.gradient_accumulation_steps)
        if advice is not None:
            lines.append(f"  [yellow]![/] {advice}")
        plan = None
        if tcfg.stream_vram_probe:
            plan = _ProbePlan(
                rows=rows,
                seq_len=seq_len,
                vocab_size=vocab,
                predicted_bytes=predicted,
                available_bytes=available,
            )
        return tuple(lines), plan

    def _run_stream_vram_probe(self, model, plan: _ProbePlan) -> None:
        """Measure one real step and let THAT decide, not the formula (#349).

        Raises when the measured peak does not fit. The streaming runtime is
        released first: it holds the pinned RAM store, and `setup()` raising is
        outside the ExitStack that `_training_context` installs around training.
        """
        from soup_cli.utils.layer_stream import decide_measured_fit
        from soup_cli.utils.layer_stream_runtime import measure_step_peak_bytes

        try:
            peak = measure_step_peak_bytes(
                model,
                rows=plan.rows,
                seq_len=plan.seq_len,
                vocab_size=plan.vocab_size,
                device=str(self.device),
            )
        except Exception:
            # measure_step_peak_bytes validates its own arguments and raises
            # before its internal handler exists. Unreachable today (the plan is
            # only built from a validated shape), but the docstring promises the
            # runtime is released before anything propagates, and a promise that
            # holds only for the paths written so far is the kind that breaks
            # when a second caller appears.
            self._close_stream_runtime()
            raise
        if peak is None:
            # Instrument failure, not a verdict. Fall back to the formula so the
            # run is never left with no gate at all — but if the formula had
            # already refused, honour that refusal rather than proceeding on
            # the strength of a probe that did not happen.
            console.print(
                "[yellow]The measured VRAM probe could not run; falling back to "
                "the predicted budget.[/]"
            )
            if plan.predicted_bytes > plan.available_bytes:
                self._close_stream_runtime()
                raise ValueError(
                    f"a streaming step is predicted to need "
                    f"{plan.predicted_bytes / 1e9:.2f} GB of VRAM but only "
                    f"{plan.available_bytes / 1e9:.2f} GB is free, and the "
                    f"measured probe that could have overruled that prediction "
                    f"failed to run. Lower training.batch_size or "
                    f"data.max_length."
                )
            return
        if peak.failed:
            # The probe ran a real CUDA op and it raised. The fit is unknown and
            # the context may be unusable, so this refuses rather than falling
            # back to the prediction: "the arithmetic was happy" is not a reason
            # to keep driving a device that just failed.
            self._close_stream_runtime()
            raise ValueError(
                f"the measured VRAM probe raised {peak.error} while running one "
                f"step at batch {plan.rows} x seq {plan.seq_len}. The fit could "
                f"not be established and the CUDA context may no longer be "
                f"usable, so this run is refused rather than continued on the "
                f"predicted budget ({plan.predicted_bytes / 1e9:.2f} GB). Re-run "
                f"without training.stream_vram_probe to use the prediction."
            )
        if peak.oom:
            self._close_stream_runtime()
            raise ValueError(
                f"a streaming step at batch {plan.rows} x seq {plan.seq_len} ran "
                f"out of VRAM while being measured (predicted "
                f"{plan.predicted_bytes / 1e9:.2f} GB, "
                f"{plan.available_bytes / 1e9:.2f} GB free). Lower "
                f"training.batch_size or data.max_length."
            )
        fit = decide_measured_fit(
            measured_bytes=peak.peak_bytes,
            predicted_bytes=plan.predicted_bytes,
            available_bytes=plan.available_bytes,
        )
        console.print(
            f"[dim]measured peak {peak.peak_bytes / 1e9:.2f} GB "
            f"({peak.reserved_bytes / 1e9:.2f} GB reserved) in "
            f"{peak.seconds:.2f} s at batch {plan.rows} x seq {plan.seq_len}; "
            f"predicted {plan.predicted_bytes / 1e9:.2f} GB[/]"
        )
        if not fit.fits:
            self._close_stream_runtime()
            raise ValueError(fit.reason)