Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

51.1 KB
Raw Blame
#!/usr/bin/env python3
"""Layer-streaming throughput probe: what bounds the streamed step on THIS card.

Re-runnable form of the instrumentation behind
``benchmarks/probe-v0.73.0-what-bounds-streaming.md``. Those scripts lived in a
scratchpad and left with the machine (#379); this one ships in the repo so the
next card can be measured the same way, through the SHIPPED streaming path
(``shard_checkpoint`` -> ``build_streamed_model``), never a re-implementation.

Everything runs in ONE process so the GEMM ceiling and the streamed step share
a session and a clock: a fraction-of-ceiling across sessions is meaningless
(the old box spread 13% on boost clock alone).

Modes, combinable::

  --ceiling  the shipped 4096^3 GEMM probe plus shape-matched projection GEMMs
             at M = batch * seq, FLOP-weighted per decoder layer
  --step     warm-up + timed steps uninstrumented, then the same steps with the
             CUDA-event instrumentation on (the instrumentation must be free)
  --sweep    sequence-length sweep at the configured batch
  --ablate   interleaved arms per round: A baseline / B no host-to-device
             copies / C no NF4 dequantisation / D neither. B, C and D are
             TIMING-ONLY and compute garbage (stale buffers, a cached zero
             weight); they run LAST because they corrupt the adapters.

Every point is appended to ``--out`` the moment it exists: a 20-minute sweep
once died with an empty log, and that is how the data was lost. An
out-of-memory error inside a --step or --ablate block is written as a point of
kind ``oom`` (the peak, the error, the traceback's tail) and the run exits 3:
at a larger batch it is a result, not a crash (the R2 batch control).

Typical invocation::

    python benchmarks/harness/stream_probe.py \
        --weights unsloth/mistral-7b-instruct-v0.3 --quant nf4 \
        --seq 512 --batch 1 --steps 8 --warmup 3 \
        --ceiling --step --ablate --rounds 2 --out probe.json

A machine without CUDA is an intentional skip and exits 0.
"""

from __future__ import annotations

import argparse
import inspect
import json
import math
import sys
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

SWEEP_DEFAULT = "16,32,64,128,256,384,512"
ARMS = (
    ("A_baseline", False, False),
    ("B_nocopy", True, False),
    ("C_nodequant", False, True),
    ("D_neither", True, True),
)


def cuda_available() -> bool:
    try:
        import torch

        return torch.cuda.is_available()
    except Exception:
        return False


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    parser.add_argument("--weights", required=True, help="model id or checkpoint path")
    parser.add_argument(
        "--shards",
        default=None,
        help="shard cache dir (default: the same ~/.soup/layer-stream/<slug> soup train uses)",
    )
    # The sharder validates each root's own syntax/existence/symlink/overlap rules; it does
    # NOT re-run the distinct-volume or NVMe-kind checks resolve_stripe_roots does at setup
    # time (that ~9 s probe is deliberately not repeated here). Pointing two --stripe-dir
    # entries at the same physical drive is on the operator running this harness.
    parser.add_argument(
        "--stripe-dir",
        action="append",
        default=[],
        help="extra cache root on another NVMe drive (repeatable); the cache is striped over it",
    )
    parser.add_argument("--quant", choices=("none", "nf4"), default="nf4")
    parser.add_argument("--tier", choices=("ram", "disk"), default="ram")
    parser.add_argument(
        "--no-pin",
        action="store_true",
        help=(
            "pageable host memory on EITHER tier: the RAM tier's store, or the disk "
            "tier's read-ahead staging"
        ),
    )
    parser.add_argument("--buffers", type=int, default=2)
    parser.add_argument(
        "--read-ahead",
        type=int,
        default=2,
        help="layers the async disk source reads ahead (disk tier only)",
    )
    parser.add_argument(
        "--control-sync-source",
        action="store_true",
        help=(
            "CONTROL (disk tier only): measure the SHIPPED synchronous "
            "DiskSource instead of the async reader, in the same session as "
            "the async block. --read-ahead then has no effect"
        ),
    )
    parser.add_argument("--seq", type=int, default=512)
    parser.add_argument("--batch", type=int, default=1)
    parser.add_argument("--steps", type=int, default=8, help="timed steps per point")
    parser.add_argument("--warmup", type=int, default=3, help="untimed steps per point")
    parser.add_argument("--lora-r", type=int, default=8)
    parser.add_argument("--lora-targets", default="q_proj,k_proj,v_proj,o_proj")
    parser.add_argument("--seed", type=int, default=3, help="adapter init seed")
    parser.add_argument("--input-seed", type=int, default=17)
    parser.add_argument("--ceiling", action="store_true")
    parser.add_argument("--step", action="store_true")
    parser.add_argument(
        "--skip-events",
        action="store_true",
        help=(
            "run only the uninstrumented `step_plain` block of --step. Added for the "
            "#974 A/B, where one arm reads 3x faster than the other and the CUDA-event "
            "brackets cost each of them a different fraction of its step: measured free "
            "on the fast arm (16.71 -> 16.84 s/step), not measured to completion on the "
            "slow one. Halves a cold 70B block and keeps the two arms comparable"
        ),
    )
    parser.add_argument("--sweep", nargs="?", const=SWEEP_DEFAULT, default=None)
    parser.add_argument("--ablate", action="store_true")
    parser.add_argument("--rounds", type=int, default=2, help="interleaved ablation rounds")
    parser.add_argument(
        "--arms",
        default=None,
        help=(
            "comma-separated subset of the ablation arms (A_baseline,B_nocopy,C_nodequant,"
            "D_neither); default all. On a cold disk tier a baseline step is minutes, so "
            "B and D alone give the compute floor cheaply"
        ),
    )
    parser.add_argument(
        "--evict-gb",
        type=float,
        default=0.0,
        help=(
            "HEURISTIC page-cache eviction before each timed block on the disk tier: "
            "commit this many GB of pageable RAM and free it"
        ),
    )
    parser.add_argument(
        "--lazy-shard-handles",
        action="store_true",
        help=(
            "OBSOLETE since #926: the shipped sharder now keeps at most two source "
            "handles alive and every read owns its memory. Kept so the 2026-09-12 record "
            "can be re-run as it was: before #926 the sharder held every source shard's "
            "mmap open at once and died with an access violation on Windows past ~100 GB "
            "of live mappings, and this flag replaced safe_open during sharding ONLY with "
            "a wrapper of the same shape, so the runtime and every timed number still "
            "came from the shipped code"
        ),
    )
    parser.add_argument(
        "--recompute",
        choices=("on", "off"),
        default="on",
        help=(
            "training.stream_recompute: off = every streamed layer keeps its activations "
            "instead of the checkpoint recompute"
        ),
    )
    parser.add_argument("--out", required=True, help="JSON results file, rewritten per point")
    parser.add_argument("--label", default="", help="free text stored beside the results")
    return parser.parse_args()


# ==========================================================================
# the same-day control (see --control-sync-source)
# ==========================================================================
#: The shim's class name. Every run prints the source class it actually
#: built and refuses when that disagrees with the flag, so a control block
#: and a measured block can never be confused in the results file.
CONTROL_SOURCE_NAME = "SyncDiskSourceControl"


def install_sync_source_control() -> str:
    """Point ``_build_source``'s lazy import at the SHIPPED synchronous source.

    The pre-#971 "before" for the disk tier was measured on 2026-09-12, in a
    different session on a card whose boost clock varies ~13% between
    sessions. The runtime no longer constructs ``DiskSource`` for
    ``tier='disk'``, and a measurement task must not change ``src/``, so the
    same-day control is made here instead: the ONE name ``_build_source``
    imports is replaced by a subclass of the shipped ``DiskSource`` that
    accepts and ignores the two keyword arguments the async source added.

    Nothing else moves. The buffer pool, the prefetcher and the layer wrapper
    stay the shipped ones — exactly as they were before the release contract
    landed — and ``_release_source`` is duck-typed, so it is a no-op against a
    source that defines no ``release``. The control therefore isolates the
    READ PATH and not a different scheduler.

    ``pinned`` and ``read_ahead`` are set to what the shipped source honestly
    has: no page-locked staging and no reader depth. ``runtime.stats()`` reads
    both off the source, and ``nbytes`` is already 0 by ``DiskSource``'s own
    design, so the preamble labels the block without being told to.
    """
    from soup_cli.utils import async_disk_source
    from soup_cli.utils.layer_stream_runtime import DiskSource

    class SyncDiskSourceControl(DiskSource):  # type: ignore[misc]
        def __init__(
            self, *args: Any, read_ahead: Any = None, pin: Any = False, **kwargs: Any
        ):
            del read_ahead, pin  # the async source's two additions, ignored
            super().__init__(*args, **kwargs)
            self.pinned = False
            self.read_ahead = None

    assert SyncDiskSourceControl.__name__ == CONTROL_SOURCE_NAME
    async_disk_source.AsyncDiskSource = SyncDiskSourceControl
    return CONTROL_SOURCE_NAME


# ==========================================================================
# sharding workaround (see --lazy-shard-handles)
# ==========================================================================
class _LazyHandleLRU:
    """At most ``capacity`` real ``safe_open`` handles alive at once.

    The shipped sharder enters every source shard into one ExitStack and keeps
    the mappings for the whole of pass 2. On this Windows box that pattern dies
    with an access violation once ~100 GB of mappings are live, whatever the
    file count (measured: 62 x 1.71 GB and 14 x 6.85 GB). Opening a file only
    while a layer needs it is the fix shape; this wrapper applies it from the
    outside so the sharder's own logic is exercised unchanged.
    """

    def __init__(self, real_safe_open: Any, capacity: int = 2):
        self.real = real_safe_open
        self.capacity = capacity
        self.order: List[Any] = []

    def touch(self, handle: Any) -> None:
        if handle in self.order:
            self.order.remove(handle)
        self.order.append(handle)
        while len(self.order) > self.capacity:
            victim = self.order.pop(0)
            victim.release()


class _LazyHandle:
    def __init__(self, lru: _LazyHandleLRU, path: str, framework: str, device: str):
        self.lru = lru
        self.path = path
        self.framework = framework
        self.device = device
        self._real: Any = None

    def _handle(self) -> Any:
        if self._real is None:
            self._real = self.lru.real(self.path, framework=self.framework, device=self.device)
            self._real.__enter__()
        self.lru.touch(self)
        return self._real

    def release(self) -> None:
        if self._real is not None:
            self._real.__exit__(None, None, None)
            self._real = None

    def __enter__(self) -> "_LazyHandle":
        return self

    def __exit__(self, *exc: Any) -> None:
        self.release()
        if self in self.lru.order:
            self.lru.order.remove(self)

    def keys(self) -> Any:
        return self._handle().keys()

    def metadata(self) -> Any:
        return self._handle().metadata()

    def get_slice(self, key: str) -> Any:
        return self._handle().get_slice(key)

    def get_tensor(self, key: str) -> Any:
        # A COPY, never the zero-copy view: the sharder keeps non-quantised
        # tensors (norms, embeddings, the head) until it writes them, and a view
        # over a released mapping is exactly the access violation being avoided.
        return self._handle().get_tensor(key).clone()


class lazy_shard_handles:  # noqa: N801 — context manager, used like a function
    """Patch ``safetensors.safe_open`` for the duration of one ``with`` block."""

    def __init__(self, capacity: int = 2):
        self.capacity = capacity
        self._saved: Any = None

    def __enter__(self) -> "lazy_shard_handles":
        import safetensors

        self._saved = safetensors.safe_open
        lru = _LazyHandleLRU(self._saved, self.capacity)

        def patched(path: Any, framework: str = "pt", device: str = "cpu") -> _LazyHandle:
            return _LazyHandle(lru, str(path), framework, device)

        safetensors.safe_open = patched
        return self

    def __exit__(self, *exc: Any) -> None:
        import safetensors

        safetensors.safe_open = self._saved


# ==========================================================================
# model + shapes
# ==========================================================================
def make_lora(args: argparse.Namespace):
    from peft import LoraConfig, TaskType

    targets = [name.strip() for name in args.lora_targets.split(",") if name.strip()]
    return LoraConfig(
        r=args.lora_r,
        lora_alpha=2 * args.lora_r,
        lora_dropout=0.0,
        bias="none",
        target_modules=targets,
        task_type=TaskType.CAUSAL_LM,
    )


def model_shapes(config: Any) -> Dict[str, Any]:
    """Projection shapes and the FLOP-per-token accounting the gates use."""
    hidden = int(config.hidden_size)
    inter = int(config.intermediate_size)
    layers = int(config.num_hidden_layers)
    heads = int(config.num_attention_heads)
    kv_heads = int(getattr(config, "num_key_value_heads", None) or heads)
    head_dim = int(getattr(config, "head_dim", None) or hidden // heads)
    q_out = heads * head_dim
    kv_out = kv_heads * head_dim
    vocab = int(config.vocab_size)
    # q, o ; k, v ; gate, up, down. Biases and norms are noise at this scale.
    per_layer = 2 * hidden * q_out + 2 * hidden * kv_out + 3 * hidden * inter
    decoder = per_layer * layers
    lm_head = vocab * hidden
    # C=6 on the frozen decoder (forward + recompute + dL/dx, no dL/dW); the
    # head is not checkpointed and has no weight grad either: C=4. Embeddings 0.
    flop_per_token = 6 * decoder + 4 * lm_head
    projections = {
        "q_proj": (hidden, q_out, 1),
        "k_proj": (hidden, kv_out, 1),
        "v_proj": (hidden, kv_out, 1),
        "o_proj": (q_out, hidden, 1),
        "gate_proj": (hidden, inter, 1),
        "up_proj": (hidden, inter, 1),
        "down_proj": (inter, hidden, 1),
    }
    return {
        "hidden": hidden,
        "intermediate": inter,
        "layers": layers,
        "kv_out": kv_out,
        "q_out": q_out,
        "vocab": vocab,
        "decoder_params": decoder,
        "lm_head_params": lm_head,
        "flop_per_token": flop_per_token,
        "projections": projections,
        "tied": bool(getattr(config, "tie_word_embeddings", False)),
    }


def build(args: argparse.Namespace, device: str, dtype: str) -> Tuple[Any, ...]:
    from soup_cli.utils.layer_shard import resolve_shard_dir, shard_checkpoint
    from soup_cli.utils.layer_stream_runtime import (
        build_meta_skeleton,
        build_streamed_model,
        quantised_layer_suffixes,
    )
    from soup_cli.utils.spectrum_scan import resolve_model_weights

    weights_dir = resolve_model_weights(args.weights)
    shard_dir = args.shards or resolve_shard_dir(args.weights)
    Path(shard_dir).mkdir(parents=True, exist_ok=True)

    probe = build_meta_skeleton(weights_dir, dtype=dtype, quant=args.quant)
    config = probe.config
    arch = str(config.model_type)
    suffixes = quantised_layer_suffixes(probe) if args.quant == "nf4" else ()
    del probe

    started = time.perf_counter()
    import contextlib

    guard: Any = contextlib.nullcontext()
    if args.lazy_shard_handles:
        print(
            "sharding      WORKAROUND: safe_open patched to hold <= 2 handles and return "
            "copies (the shipped sharder dies past ~100 GB of live mappings on Windows)"
        )
        guard = lazy_shard_handles(capacity=2)
    with guard:
        index = shard_checkpoint(
            weights_dir,
            shard_dir,
            dtype=dtype,
            arch=arch,
            quant=args.quant,
            quant_suffixes=suffixes,
            double_quant=True,
            quant_device=device if args.quant == "nf4" else None,
            notify=print,
            # getattr, not args.stripe_dir: head_prefetch_ab.py, issue901_stream_step_probe.py,
            # issue974_warm_stages.py and layer0_wait.py each hand-build their own bare
            # argparse.Namespace and call this build() directly, none of them spelling a
            # stripe_dir field, and build() reading it unconditionally would raise
            # AttributeError in all four the next time they run.
            stripe_roots=tuple(getattr(args, "stripe_dir", [])),
        )
    shard_seconds = time.perf_counter() - started

    if args.control_sync_source:
        print(
            f"source        CONTROL: {install_sync_source_control()} — the shipped "
            "synchronous DiskSource replaces the async reader for this block"
        )

    started = time.perf_counter()
    model, runtime = build_streamed_model(
        model_id=weights_dir,
        shard_dir=shard_dir,
        index=index,
        lora_config=make_lora(args),
        device=device,
        dtype=dtype,
        buffers=args.buffers,
        read_ahead=args.read_ahead,
        # Both tiers stage through host memory now: the RAM tier's store, the
        # disk tier's read-ahead staging (#971). Forcing pageable on disk here
        # would have measured the fallback path, not the shipped one.
        pin=not args.no_pin,
        seed=args.seed,
        quant=args.quant,
        double_quant=True,
        tier=args.tier,
    )
    build_seconds = time.perf_counter() - started
    return model, runtime, config, index, weights_dir, shard_dir, shard_seconds, build_seconds


# ==========================================================================
# instrumentation: replicas of the two shipped copy paths, plus stall timing
# ==========================================================================
class Instruments:
    """CUDA-event timing on the copy stream and the compute stream, and the
    two timing-only ablation switches. Installed on the live pool objects the
    prefetcher and the layer wrappers already hold, so the shipped scheduler
    is untouched."""

    def __init__(self, runtime: Any, model: Any, quant: str):
        self.pool = runtime.pool
        self.large_pool = runtime.large_pool
        self.events_on = False
        self.nocopy = False
        self.nodequant = False
        self.patched_linears = 0
        self._copy_pairs: List[Tuple[Any, Any]] = []
        self._stall_pairs: List[Tuple[Any, Any]] = []
        self._zero_cache: Dict[Tuple[Tuple[int, ...], Any], Any] = {}
        self._install_pool()
        self._install_large_pool()
        if quant == "nf4":
            self._install_nodequant(model)

    # -- drift guard: the replica below must stay a copy of the shipped body --
    @staticmethod
    def _guard(kind: Any, method: str, needles: Tuple[str, ...]) -> None:
        source = inspect.getsource(getattr(kind, method))
        missing = [needle for needle in needles if needle not in source]
        if missing:
            raise RuntimeError(
                f"shipped {kind.__name__}.{method} no longer contains {missing!r}; "
                "the probe's replica would measure a different scheduler. Update "
                "benchmarks/harness/stream_probe.py to match the shipped code."
            )

    def _install_pool(self) -> None:
        import torch

        # The shipped body tells the source its staging slot is free; a replica
        # that skipped it would be refused by AsyncDiskSource at the second
        # layer (#971) and would measure a different contract on the RAM tier.
        from soup_cli.utils.layer_stream_runtime import _release_source

        pool = self.pool
        self._guard(
            type(pool),
            "load_async",
            (
                "stream.wait_stream(torch.cuda.current_stream())",
                "dst.copy_(source.get(idx, name), non_blocking=True)",
                "self.events[slot].record(stream)",
                "_release_source(source, idx, self.events[slot])",
                "_release_source(source, idx, None)",
                "self.generation[slot] += 1",
            ),
        )
        inst = self

        def load_async(idx: int, source: Any, stream: Any = None) -> int:
            slot = pool.slot_for(idx)
            keys = (
                pool.active_keys_by_layer[0]
                if len(pool.active_keys_by_layer) == 1
                else pool.active_keys_by_layer[idx]
            )
            if pool.is_cuda and stream is not None:
                stream.wait_stream(torch.cuda.current_stream())
                with torch.cuda.stream(stream):
                    start = None
                    if inst.events_on:
                        start = torch.cuda.Event(enable_timing=True)
                        start.record(stream)
                    if not inst.nocopy:
                        for name in keys:
                            dst = pool.buffers[slot][name]
                            dst.copy_(source.get(idx, name), non_blocking=True)
                    if inst.events_on:
                        end = torch.cuda.Event(enable_timing=True)
                        end.record(stream)
                        inst._copy_pairs.append((start, end))
                    pool.events[slot].record(stream)
                _release_source(source, idx, pool.events[slot])
            else:
                for name in keys:
                    pool.buffers[slot][name].copy_(source.get(idx, name))
                _release_source(source, idx, None)
            pool.owner[slot] = idx
            # The no-recompute backward refuses a slot whose load generation
            # moved since its first visit; a replacement load must bump it too.
            pool.generation[slot] += 1
            pool.loads += 1
            return slot

        original_wait = pool.wait

        def wait(idx: int) -> Any:
            if inst.events_on and pool.is_cuda:
                before = torch.cuda.Event(enable_timing=True)
                before.record()
                out = original_wait(idx)
                after = torch.cuda.Event(enable_timing=True)
                after.record()
                inst._stall_pairs.append((before, after))
                return out
            return original_wait(idx)

        pool.load_async = load_async
        pool.wait = wait

    def _install_large_pool(self) -> None:
        import torch

        from soup_cli.utils.layer_stream_runtime import _release_source

        large = self.large_pool
        if large is None:
            return
        self._guard(
            type(large),
            "load_async",
            (
                "if self.owner == key:",
                "stream.wait_stream(torch.cuda.current_stream())",
                "dst.copy_(source.get(source_idx, key), non_blocking=True)",
                "self.event.record(stream)",
                "_release_source(source, source_idx, self.event)",
                "_release_source(source, source_idx, None)",
            ),
        )
        inst = self

        def load_async(key: str, source: Any, stream: Any = None) -> None:
            if large.owner == key:
                return
            if key not in large.specs or key not in large.source_indices:
                raise ValueError(f"large-layer source is missing {key!r}")
            dst = large._view(key)
            source_idx = large.source_indices[key]
            if large.is_cuda and stream is not None:
                stream.wait_stream(torch.cuda.current_stream())
                with torch.cuda.stream(stream):
                    start = None
                    if inst.events_on:
                        start = torch.cuda.Event(enable_timing=True)
                        start.record(stream)
                    if not inst.nocopy:
                        dst.copy_(source.get(source_idx, key), non_blocking=True)
                    if inst.events_on:
                        end = torch.cuda.Event(enable_timing=True)
                        end.record(stream)
                        inst._copy_pairs.append((start, end))
                    large.event.record(stream)
                _release_source(source, source_idx, large.event)
            else:
                dst.copy_(source.get(source_idx, key))
                _release_source(source, source_idx, None)
            large.owner = key
            large.loads += 1

        original_wait = large.wait

        def wait(key: str) -> Any:
            if inst.events_on and large.is_cuda:
                before = torch.cuda.Event(enable_timing=True)
                before.record()
                out = original_wait(key)
                after = torch.cuda.Event(enable_timing=True)
                after.record()
                inst._stall_pairs.append((before, after))
                return out
            return original_wait(key)

        large.load_async = load_async
        large.wait = wait

    def _install_nodequant(self, model: Any) -> None:
        import types

        import bitsandbytes as bnb
        import torch.nn.functional as functional

        inst = self
        count = 0
        for child in model.modules():
            if not isinstance(child, bnb.nn.Linear4bit):
                continue
            original = child.forward  # the #331 dequant-forward, bound

            def forward(self, x, _original=original):
                if not inst.nodequant:
                    return _original(x)
                quant_state = self.weight.quant_state
                inp_dtype = x.dtype
                if getattr(self, "compute_dtype", None) is not None:
                    x = x.to(self.compute_dtype)
                bias = self.bias
                if bias is not None:
                    bias = bias.to(x.dtype)
                weight = inst.zero_weight(tuple(quant_state.shape), x.dtype, x.device)
                return functional.linear(x, weight, bias).to(inp_dtype)

            child.forward = types.MethodType(forward, child)
            count += 1
        self.patched_linears = count

    def zero_weight(self, shape: Tuple[int, ...], dtype: Any, device: Any) -> Any:
        import torch

        key = (shape, dtype)
        weight = self._zero_cache.get(key)
        if weight is None:
            weight = torch.zeros(shape, dtype=dtype, device=device)
            self._zero_cache[key] = weight
        return weight

    def reset(self) -> None:
        self._copy_pairs = []
        self._stall_pairs = []

    def consume(self) -> Tuple[float, float, int]:
        """(copy seconds on the prefetch stream, compute-stream stall seconds,
        number of copy events). Call only after torch.cuda.synchronize()."""
        copy_ms = sum(start.elapsed_time(end) for start, end in self._copy_pairs)
        stall_ms = sum(before.elapsed_time(after) for before, after in self._stall_pairs)
        copies = len(self._copy_pairs)
        self.reset()
        return copy_ms / 1000.0, stall_ms / 1000.0, copies


# ==========================================================================
# measurement
# ==========================================================================
def gpu_facts(device: str) -> Dict[str, Any]:
    import subprocess

    import torch

    import soup_cli
    from soup_cli.utils.layer_stream import _resolve_tool

    facts: Dict[str, Any] = {
        "gpu": torch.cuda.get_device_name(0),
        "torch": torch.__version__,
        "vram_total_gb": round(torch.cuda.mem_get_info()[1] / 1e9, 3),
        # Which source tree this run actually imported. An arm that claims to be
        # "the old code" is only evidence if the JSON says where it came from.
        "soup_cli_file": soup_cli.__file__,
    }
    tool = _resolve_tool("nvidia-smi")
    if tool is not None:
        try:
            out = subprocess.run(
                [
                    tool,
                    "--query-gpu=driver_version,pcie.link.gen.current,pcie.link.width.current",
                    "--format=csv,noheader,nounits",
                ],
                capture_output=True,
                text=True,
                timeout=20,
                check=False,
            )
            driver, gen, width = [part.strip() for part in out.stdout.strip().split(",")]
            facts.update({"driver": driver, "pcie_gen": gen, "pcie_width": width})
        except (OSError, ValueError, subprocess.SubprocessError):
            pass
    return facts


def finite(value: float) -> Optional[float]:
    return value if math.isfinite(value) else None


def evict_page_cache(gigabytes: float) -> None:
    """Heuristic: commit pageable RAM and release it so the standby list is
    pressured. Reported as a heuristic, never as a guarantee of cold reads."""
    import torch

    if gigabytes <= 0:
        return
    buffer = torch.empty(int(gigabytes * 1e9), dtype=torch.uint8, device="cpu")
    buffer.fill_(1)
    del buffer


#: Exit code of a run that stopped on an out-of-memory error after recording it (R2). 2 stays
#: "refused before measuring"; anything else non-zero is a crash the JSON says nothing about.
OOM_EXIT = 3


def is_out_of_memory(exc: BaseException) -> bool:
    """True for a device allocation failure: torch's own class, or the text CUDA and cuBLAS use.

    Added for the R2 batch control, where an out-of-memory error at a larger batch is a RESULT
    to record and not a crash. The text match also catches a stale CUDA error that only SAYS
    out of memory (#901: a failed page-lock surfaced at the next launch); the record quotes the
    message and the traceback, so the two can be told apart after the fact.
    """
    import torch

    if isinstance(exc, torch.OutOfMemoryError):
        return True
    text = str(exc)
    return "out of memory" in text.lower() or "ALLOC_FAILED" in text


def memory_facts() -> Dict[str, Any]:
    """The allocator's counters and the device's free memory, read after a block (R2).

    ``num_alloc_retries`` and ``num_ooms`` are cumulative over the process. ``free_gb`` is
    ``cudaMemGetInfo`` at the end of the block. None of it is a fit verdict on Windows, where an
    over-allocation spills into shared host memory without an error.
    """
    import torch

    stats = torch.cuda.memory_stats()
    free, total = torch.cuda.mem_get_info()
    return {
        "num_alloc_retries": stats.get("num_alloc_retries"),
        "num_ooms": stats.get("num_ooms"),
        "free_gb": free / 1e9,
        "total_gb": total / 1e9,
    }


def oom_record(label: str, exc: BaseException) -> Dict[str, Any]:
    """What an out-of-memory stop leaves in the JSON: the peak, the error and where it came from.

    The peak counters were reset at the block's first timed step, so an error inside a warm-up
    step reports the peak since the previous reset or, in the first block, since process start.
    """
    import traceback

    import torch

    frames = traceback.format_exception(type(exc), exc, exc.__traceback__)
    return {
        "label": label,
        "error_type": type(exc).__name__,
        "error": " ".join(str(exc).split())[:2000],
        "traceback_tail": "".join(frames[-8:])[-4000:],
        "peak_alloc_gb": torch.cuda.max_memory_allocated() / 1e9,
        "peak_reserved_gb": torch.cuda.max_memory_reserved() / 1e9,
        "memory": memory_facts(),
    }


def ids_fingerprint(ids: Any) -> Dict[str, Any]:
    """A SHA-256 per row of the token ids a block trains on, read outside the timed region (R2).

    A later layer-major (L2L) arm that runs k micro-batches of 1 x seq has to train on the same
    rows as the batch-k arm it is compared with; these hashes are how it shows that. Whether
    row i of a (b, seq) draw equals the i-th (1, seq) draw is not assumed either way: the
    hashes record which it is.
    """
    import hashlib

    rows = ids.detach().to("cpu").contiguous()
    return {
        "shape": [int(dim) for dim in rows.shape],
        "dtype": str(rows.dtype).replace("torch.", ""),
        "row_sha256": [hashlib.sha256(row.numpy().tobytes()).hexdigest() for row in rows],
    }


def stop_on_oom(sink: "Sink", runtime: Any, label: str, exc: BaseException) -> int:
    """Record an out-of-memory stop as a point of its own, close the source, return OOM_EXIT."""
    record = oom_record(label, exc)
    sink.add("oom", record)
    print(
        f"OOM           {label}: {record['error_type']}: {record['error'][:300]} "
        f"(peak {record['peak_alloc_gb']:.3f} GB allocated / "
        f"{record['peak_reserved_gb']:.3f} GB reserved)"
    )
    try:
        runtime.close()
    except Exception as close_exc:  # the OOM is the result; a failed close must not hide it
        print(f"OOM           runtime.close() also failed: {type(close_exc).__name__}: {close_exc}")
    return OOM_EXIT


def run_steps(
    model: Any,
    optimizer: Any,
    inst: Instruments,
    ids: Any,
    *,
    steps: int,
    warmup: int,
    label: str,
) -> Dict[str, Any]:
    import torch

    from soup_cli.utils.layer_stream_runtime import sm_clock_mhz

    pool = inst.pool
    large = inst.large_pool
    tokens = int(ids.numel())
    records: List[Dict[str, Any]] = []
    clock_start = sm_clock_mhz()
    for index in range(warmup + steps):
        loads_before = pool.loads
        large_before = large.loads if large is not None else 0
        inst.reset()
        torch.cuda.synchronize()
        if index == warmup:
            torch.cuda.reset_peak_memory_stats()
        # Wall-clock start, read outside the timed region: the sustained two-drive probe lines
        # each step up with its driver's 1-s drive counters. perf_counter stays the step timer.
        started_unix = time.time()
        wall_start = time.perf_counter()
        out = model(input_ids=ids, labels=ids)
        out.loss.backward()
        optimizer.step()
        optimizer.zero_grad(set_to_none=True)
        torch.cuda.synchronize()
        wall = time.perf_counter() - wall_start
        copy_s, stall_s, copies = inst.consume()
        if index >= warmup:
            records.append(
                {
                    "step_s": wall,
                    "started_unix": started_unix,
                    "loss": finite(float(out.loss.detach())),
                    "layer_loads": pool.loads - loads_before,
                    "large_loads": (large.loads - large_before) if large is not None else 0,
                    "copy_s": copy_s,
                    "stall_s": stall_s,
                    "copy_events": copies,
                }
            )
    clock_end = sm_clock_mhz()
    step_mean = sum(rec["step_s"] for rec in records) / len(records)
    per_layer_bytes = pool.nbytes // pool.n
    large_bytes = large.nbytes if large is not None else 0
    loads_mean = sum(rec["layer_loads"] for rec in records) / len(records)
    large_mean = sum(rec["large_loads"] for rec in records) / len(records)
    moved = loads_mean * per_layer_bytes + large_mean * large_bytes
    copy_mean = sum(rec["copy_s"] for rec in records) / len(records)
    stall_mean = sum(rec["stall_s"] for rec in records) / len(records)
    return {
        "label": label,
        "tokens_per_step": tokens,
        "steps": steps,
        "warmup": warmup,
        "events_on": inst.events_on,
        "nocopy": inst.nocopy,
        "nodequant": inst.nodequant,
        "step_s_mean": step_mean,
        "step_s_min": min(rec["step_s"] for rec in records),
        "step_s_max": max(rec["step_s"] for rec in records),
        "tok_per_s": tokens / step_mean,
        "layer_loads_per_step": loads_mean,
        "large_loads_per_step": large_mean,
        "bytes_moved_per_step": moved,
        "implied_h2d_gb_per_s": moved / step_mean / 1e9,
        "copy_s_per_step": copy_mean,
        "copy_stream_gb_per_s": (moved / copy_mean / 1e9) if copy_mean > 0 else None,
        "stall_s_per_step": stall_mean,
        "stall_share": stall_mean / step_mean,
        "peak_alloc_gb": torch.cuda.max_memory_allocated() / 1e9,
        "peak_reserved_gb": torch.cuda.max_memory_reserved() / 1e9,
        "sm_clock_mhz_start": clock_start,
        "sm_clock_mhz_end": clock_end,
        "records": records,
    }


def measure_ceiling(device: str, dtype: str, shapes: Dict[str, Any], tokens: int) -> Dict[str, Any]:
    import torch

    from soup_cli.utils.layer_stream_runtime import measure_gemm_tflops, sm_clock_mhz

    square = measure_gemm_tflops(device)
    torch_dtype = getattr(torch, dtype)
    grouped: Dict[Tuple[int, int], int] = {}
    for _name, (k_dim, n_dim, count) in shapes["projections"].items():
        grouped[(k_dim, n_dim)] = grouped.get((k_dim, n_dim), 0) + count
    per_shape: List[Dict[str, Any]] = []
    weighted_num = 0.0
    weighted_den = 0.0
    for (k_dim, n_dim), count in grouped.items():
        left = torch.randn(tokens, k_dim, device=device, dtype=torch_dtype)
        right = torch.randn(k_dim, n_dim, device=device, dtype=torch_dtype)
        flops = 2.0 * tokens * k_dim * n_dim
        best = 0.0
        for _rep in range(3):
            for _warm in range(3):
                left @ right
            torch.cuda.synchronize()
            start = torch.cuda.Event(enable_timing=True)
            end = torch.cuda.Event(enable_timing=True)
            start.record()
            iters = 20
            for _it in range(iters):
                left @ right
            end.record()
            torch.cuda.synchronize()
            seconds = start.elapsed_time(end) / 1000.0
            best = max(best, flops * iters / seconds / 1e12)
        del left, right
        per_shape.append({"m": tokens, "k": k_dim, "n": n_dim, "count": count, "tflops": best})
        weighted_num += best * flops * count
        weighted_den += flops * count
    torch.cuda.empty_cache()
    square_payload = None
    if square is not None:
        square_payload = {
            "tflops": square.tflops,
            "samples": list(square.samples),
            "sm_clock_mhz": square.sm_clock_mhz,
            "dtype": square.dtype,
        }
    return {
        "square_4096": square_payload,
        "shape_matched": per_shape,
        "shape_matched_weighted_tflops": weighted_num / weighted_den,
        "sm_clock_mhz": sm_clock_mhz(),
        "dtype": dtype,
    }


class Sink:
    """Rewrite the JSON file after every point."""

    def __init__(self, path: str, meta: Dict[str, Any]):
        self.path = Path(path)
        self.payload: Dict[str, Any] = {"meta": meta, "records": []}
        self.flush()

    def add(self, kind: str, record: Dict[str, Any]) -> None:
        self.payload["records"].append({"kind": kind, **record})
        self.flush()

    def flush(self) -> None:
        self.path.parent.mkdir(parents=True, exist_ok=True)
        self.path.write_text(json.dumps(self.payload, indent=1), encoding="utf-8")


def describe(summary: Dict[str, Any], flop_per_token: float, ceiling: Optional[float]) -> str:
    eff = flop_per_token * summary["tokens_per_step"] / summary["step_s_mean"] / 1e12
    parts = [
        f"{summary['label']:<24}",
        f"{summary['tok_per_s']:8.1f} tok/s",
        f"step {summary['step_s_mean']:.3f} s",
        f"peak {summary['peak_alloc_gb']:.3f} GB",
        f"loads {summary['layer_loads_per_step']:.1f}+{summary['large_loads_per_step']:.1f}",
        f"eff {eff:.2f} TFLOPS",
    ]
    if ceiling:
        parts.append(f"= {100.0 * eff / ceiling:.1f}% of ceiling")
    if summary["events_on"]:
        parts.append(
            f"copy {summary['copy_s_per_step']:.3f} s "
            f"({summary['copy_stream_gb_per_s'] or 0:.2f} GB/s) "
            f"stall {summary['stall_s_per_step'] * 1000:.1f} ms "
            f"({100 * summary['stall_share']:.2f}%)"
        )
    if summary["sm_clock_mhz_start"] is not None:
        parts.append(f"clock {summary['sm_clock_mhz_start']}->{summary['sm_clock_mhz_end']} MHz")
    return "  ".join(parts)


def main() -> int:
    args = parse_args()
    if not cuda_available():
        print("SKIP: CUDA is required for stream_probe.py")
        return 0
    if not (args.ceiling or args.step or args.sweep or args.ablate):
        print("ERROR: choose at least one of --ceiling --step --sweep --ablate")
        return 2
    if args.control_sync_source and args.tier != "disk":
        print(
            "ERROR: --control-sync-source is a DISK-tier control; the RAM tier "
            "has no disk source to swap"
        )
        return 2

    import torch

    from soup_cli.utils.layer_stream import resolve_stream_dtype

    device = "cuda"
    dtype = resolve_stream_dtype(device)
    facts = gpu_facts(device)
    print("RUN: layer-streaming throughput probe")
    for key, value in facts.items():
        print(f"{key:<16}{value}")
    print(f"{'dtype':<16}{dtype}")
    print(f"{'quant':<16}{args.quant}")
    print(f"{'tier':<16}{args.tier}{' (pageable)' if args.no_pin else ''}")

    model, runtime, config, index, weights_dir, shard_dir, shard_s, build_s = build(
        args, device, dtype
    )
    shapes = model_shapes(config)
    stats = runtime.stats()
    print(f"{'arch':<16}{config.model_type}  layers {shapes['layers']}")
    print(f"{'shards':<16}{shard_dir}  ({shard_s:.1f} s)")
    print(
        f"{'store':<16}{stats['store_bytes'] / 1e9:.3f} GB "
        f"{'pinned' if stats['pinned'] else 'pageable'} on tier {stats['tier']} "
        f"(disk {stats['disk_bytes'] / 1e9:.3f} GB); build {build_s:.1f} s"
    )
    # Guard: the class the runtime ACTUALLY built, every block, control or
    # not. A shim left installed by accident would otherwise publish the
    # pre-#971 read path as the measured one, which is the single worst
    # mistake this record could make.
    source_class = type(runtime.source).__name__
    is_control = source_class == CONTROL_SOURCE_NAME
    if is_control != bool(args.control_sync_source):
        print(
            f"ERROR: source guard — the runtime built a {source_class} while "
            f"--control-sync-source was "
            f"{'set' if args.control_sync_source else 'not set'}"
        )
        return 2
    print(
        f"{'source':<16}{source_class}  read_ahead {stats['read_ahead']}  "
        f"staging {stats['store_bytes'] / 1e6:.0f} MB "
        f"{'pinned' if stats['pinned'] else 'pageable'}"
        + ("  [CONTROL: the pre-#971 synchronous read path]" if is_control else "")
    )
    # direct_io and _open_of exist only on the disk-tier source; getattr reads as None/{} on
    # the RAM tier. A pinned staging region that is not sector-aligned silently turns direct
    # I/O off for every drive (see async_disk_source.py), so this line is what lets the
    # operator catch that live instead of inferring it from throughput after the fact.
    stripe_roots_seen = list(getattr(index, "stripe_roots", ()) or ())
    layer_roots_seen = list(getattr(index, "layer_roots", ()) or ())
    layer_roots_text = ",".join(str(root) for root in layer_roots_seen[:12])
    if len(layer_roots_seen) > 12:
        layer_roots_text += ",..."
    print(
        f"{'stripe':<16}direct_io {getattr(runtime.source, 'direct_io', None)}  "
        f"stripe roots {stripe_roots_seen}  layer roots {layer_roots_text}"
    )
    per_buffer_mb = stats["buffer_bytes"] / stats["buffers"] / 1e6
    print(f"{'buffers':<16}{stats['buffers']} x {per_buffer_mb:.1f} MB")

    import bitsandbytes as bnb

    trainable = [param for param in model.parameters() if param.requires_grad]
    optimizer = bnb.optim.PagedAdamW8bit(trainable, lr=1e-4)
    inst = Instruments(runtime, model, args.quant)
    if args.quant == "nf4":
        print(f"{'nf4 linears':<16}{inst.patched_linears} patched for the ablation")
    if args.recompute == "off":
        from soup_cli.utils.layer_stream_runtime import set_stream_recompute

        switched = set_stream_recompute(model, False)
        print(f"{'recompute':<16}OFF on {switched} streamed layers (stream_recompute='off')")

    meta = {
        **facts,
        "recompute": args.recompute,
        "label": args.label,
        "weights": args.weights,
        "weights_dir": weights_dir,
        "shard_dir": shard_dir,
        "quant": args.quant,
        "dtype": dtype,
        "tier": stats["tier"],
        "pinned": stats["pinned"],
        "source_class": source_class,
        "control_sync_source": bool(args.control_sync_source),
        "read_ahead": stats["read_ahead"],
        "stripe_dirs": list(args.stripe_dir),
        "stripe_roots": list(getattr(index, "stripe_roots", ()) or ()),
        "layer_roots": list(getattr(index, "layer_roots", ()) or ()),
        "direct_io": getattr(runtime.source, "direct_io", None),
        "open_per_root": {
            str(root): getattr(opener, "__name__", repr(opener))
            for root, opener in getattr(runtime.source, "_open_of", {}).items()
        },
        "store_gb": stats["store_bytes"] / 1e9,
        "disk_gb": stats["disk_bytes"] / 1e9,
        "buffers": stats["buffers"],
        "buffer_bytes": stats["buffer_bytes"],
        "large_buffer_bytes": stats["large_buffer_bytes"],
        "n_layers": stats["n_layers"],
        "total_params": stats["total_params"],
        "shapes": {key: value for key, value in shapes.items() if key != "projections"},
        "lora_r": args.lora_r,
        "lora_targets": args.lora_targets,
        "batch": args.batch,
        "seq": args.seq,
        "steps": args.steps,
        "warmup": args.warmup,
        "shard_seconds": shard_s,
        "build_seconds": build_s,
        "evict_gb": args.evict_gb,
        "started": time.strftime("%Y-%m-%dT%H:%M:%S"),
    }
    sink = Sink(args.out, meta)
    flop_per_token = float(shapes["flop_per_token"])
    ceiling_tflops: Optional[float] = None

    generator = torch.Generator(device=device).manual_seed(args.input_seed)

    def make_ids(seq: int) -> Any:
        return torch.randint(
            0, shapes["vocab"], (args.batch, seq), generator=generator, device=device
        )

    if args.ceiling:
        print("ceiling       measuring (same session, same clock)...")
        ceiling = measure_ceiling(device, dtype, shapes, args.batch * args.seq)
        ceiling_tflops = ceiling["shape_matched_weighted_tflops"]
        sink.add("ceiling", ceiling)
        square = ceiling["square_4096"]
        square_tflops = square["tflops"] if square else float("nan")
        print(
            f"ceiling       square 4096^3 {square_tflops:.2f} TFLOPS; "
            f"shape-matched FLOP-weighted {ceiling_tflops:.2f} TFLOPS "
            f"@ {ceiling['sm_clock_mhz']} MHz"
        )

    if args.step:
        ids = make_ids(args.seq)
        fingerprint = ids_fingerprint(ids)
        evict_page_cache(args.evict_gb)
        inst.events_on = False
        try:
            plain = run_steps(
                model,
                optimizer,
                inst,
                ids,
                steps=args.steps,
                warmup=args.warmup,
                label="step_plain",
            )
        except Exception as exc:  # re-raised unless it is an out-of-memory error
            if not is_out_of_memory(exc):
                raise
            return stop_on_oom(sink, runtime, "step_plain", exc)
        plain["ids"] = fingerprint
        plain["memory"] = memory_facts()
        sink.add("step", plain)
        print(describe(plain, flop_per_token, ceiling_tflops))
        if not args.skip_events:
            inst.events_on = True
            evict_page_cache(args.evict_gb)
            try:
                timed = run_steps(
                    model,
                    optimizer,
                    inst,
                    ids,
                    steps=args.steps,
                    warmup=args.warmup,
                    label="step_events",
                )
            except Exception as exc:  # re-raised unless it is an out-of-memory error
                if not is_out_of_memory(exc):
                    raise
                return stop_on_oom(sink, runtime, "step_events", exc)
            inst.events_on = False
            timed["ids"] = fingerprint
            timed["memory"] = memory_facts()
            sink.add("step", timed)
            print(describe(timed, flop_per_token, ceiling_tflops))

    if args.sweep:
        seqs = [int(part) for part in args.sweep.split(",") if part.strip()]
        inst.events_on = True
        for seq in seqs:
            ids = make_ids(seq)
            evict_page_cache(args.evict_gb)
            point = run_steps(
                model,
                optimizer,
                inst,
                ids,
                steps=args.steps,
                warmup=args.warmup,
                label=f"sweep_seq{seq}",
            )
            point["seq"] = seq
            sink.add("sweep", point)
            print(describe(point, flop_per_token, ceiling_tflops))
        inst.events_on = False

    if args.ablate:
        ids = make_ids(args.seq)
        arms = ARMS if args.quant == "nf4" else ARMS[:2]
        if args.arms:
            wanted = {name.strip() for name in args.arms.split(",") if name.strip()}
            unknown = wanted - {arm[0] for arm in ARMS}
            if unknown:
                print(f"ERROR: unknown ablation arm(s) {sorted(unknown)}")
                return 2
            arms = tuple(arm for arm in arms if arm[0] in wanted)
        print(
            "ablate        arms B/C/D are timing-only and compute garbage; "
            f"{args.rounds} interleaved rounds"
        )
        inst.events_on = False
        for round_index in range(args.rounds):
            for name, nocopy, nodequant in arms:
                inst.nocopy = nocopy
                inst.nodequant = nodequant
                evict_page_cache(args.evict_gb)
                try:
                    point = run_steps(
                        model,
                        optimizer,
                        inst,
                        ids,
                        steps=args.steps,
                        warmup=max(1, args.warmup),
                        label=f"{name}_r{round_index}",
                    )
                except Exception as exc:  # re-raised unless it is an out-of-memory error
                    if not is_out_of_memory(exc):
                        raise
                    return stop_on_oom(sink, runtime, f"{name}_r{round_index}", exc)
                point["arm"] = name
                point["round"] = round_index
                point["memory"] = memory_facts()
                sink.add("ablate", point)
                print(describe(point, flop_per_token, ceiling_tflops))
        inst.nocopy = False
        inst.nodequant = False

    if args.ceiling:
        print("ceiling       re-measuring at the end of the session...")
        again = measure_ceiling(device, dtype, shapes, args.batch * args.seq)
        sink.add("ceiling_end", again)
        print(
            f"ceiling       shape-matched {again['shape_matched_weighted_tflops']:.2f} TFLOPS "
            f"@ {again['sm_clock_mhz']} MHz"
        )

    runtime.close()
    print(f"wrote         {args.out}")
    return 0


if __name__ == "__main__":
    sys.exit(main())