Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#974 — the disk tier reads a layer as sector-aligned byte ranges, through direct I/O,
into ONE staging region per slot.
Measured on the dev box (RTX 5070, Windows 11, Samsung PM9B1 NVMe, 2026-09-15; the
probes and JSON are in the #974 record): every BUFFERED read primitive tops out at
1.2-2.9 GB/s cold — per-tensor ``readinto`` (the shipped ``AsyncDiskSource`` path), one
``readinto`` per byte range, any open flag — because the cache manager, not the request
shape, is the ceiling; and the async source was already there (``gate-971`` §10:
1.46-2.32 GB/s at the source, 84.7% of a cold 70B step inside the read). Unbuffered
I/O (``FILE_FLAG_NO_BUFFERING`` / ``O_DIRECT``) into a 4 KiB-aligned pinned buffer reads
the same layers at 3.5-5.65 GB/s, best at 2-4 parallel ranges. Warm, with the store in
the page cache, the async arm was 0.91-0.97x the synchronous control position-matched:
the "regression" #974 was filed over was a run-order effect (and one 2.4x-slow block
that did not reproduce), not a cost of the design.
So the reader changes in three ways, each pinned here:
* a layer's whole data section (contiguous in a safetensors file: the tensors are
dtype-descending with zero gap bytes) is read as K sector-aligned byte ranges by K
worker threads, each through its own direct-I/O handle, with buffered ``open`` as
the fallback where direct I/O is unavailable (tmpfs, an unsupported platform);
* every staging slot is ONE contiguous region — tensors are views at their file offset
minus the aligned start — packed into power-of-two pinned arenas by the #901 packer,
so the disk tier's staging stops paying the per-tensor rounding the RAM store stopped
paying in #901 (1.7-1.9x measured);
* ``_plan_queue`` / ``_claim_slot`` / ``get`` are untouched; the byte-identity gate
against the shipped ``DiskSource`` is the proof that nothing on that path moved.
"""
from __future__ import annotations
import json
import struct
import threading
import time
from pathlib import Path
import pytest
torch = pytest.importorskip("torch")
pytest.importorskip("safetensors")
from safetensors.torch import save_file # noqa: E402
from soup_cli.utils.async_disk_source import AsyncDiskSource # noqa: E402
from soup_cli.utils.layer_stream_runtime import DiskSource, RamSource # noqa: E402
N_LAYERS = 4
SECTOR = 4096
#: Far beyond any box's RAM, so ``cuMemHostAlloc`` refuses it at once (#901).
_IMPOSSIBLE_PIN_BYTES = 2**40
class _Console:
def __init__(self):
self.printed = []
def print(self, msg):
self.printed.append(str(msg))
def _shards(
tmp_path: Path,
n_layers: int = N_LAYERS,
*,
long_header_on: int | None = None,
extra_tensor_on: int | None = None,
extra_elements: int = 33,
empty_tensor: bool = False,
big: bool = False,
) -> str:
"""NF4-shaped shards (mixed uint8 / float32 / bf16 plus a 0-dim scalar), as in the
#971 suite, with two optional per-layer variations that a range reader must
survive and a per-tensor reader never saw:
* ``long_header_on``: that layer's header carries 5000 bytes of metadata, so its
data section starts PAST the first 4 KiB sector — the only fixture in which
the aligned read start is not 0 and the ``entry.start - aligned_start`` term
of every view offset is actually exercised (a parallel session's mutation
run found the absolute offset survived every other fixture, including the
real 70B store, whose data section starts at byte 2976);
* ``extra_tensor_on``: that layer's shard holds a tensor the spec does not ask
for, sorted INTO the middle of the wanted ones, so the wanted span is not the
whole data section and the wanted tensors sit at different offsets.
* ``extra_elements``: how large that foreign float32 tensor is — small by default,
large to make the wanted span mostly gap;
* ``empty_tensor``: every layer also carries a zero-element float32 tensor;
* ``big``: a 256 KiB packed weight, so a layer spans many sectors and can be
split across several ranges.
"""
from soup_cli.utils.layer_shard import layer_shard_path
out = tmp_path / "shards"
out.mkdir()
torch.manual_seed(927)
rows = 4096 if big else 64
for idx in range(n_layers):
blob = {
"self_attn.q_proj.weight": torch.randint(0, 255, (rows, 64), dtype=torch.uint8),
"self_attn.q_proj.weight::absmax": torch.rand(16, dtype=torch.float32),
"self_attn.q_proj.weight::nested_offset": torch.tensor(
0.125 * (idx + 1), dtype=torch.float32
),
"input_layernorm.weight": torch.rand(64, dtype=torch.bfloat16),
}
if extra_tensor_on == idx:
blob["self_attn.q_proj.weight::absmax_extra"] = torch.rand(
extra_elements, dtype=torch.float32
)
if empty_tensor:
blob["self_attn.q_proj.weight::empty"] = torch.empty(0, dtype=torch.float32)
metadata = {"note": "x" * 5000} if long_header_on == idx else None
save_file(blob, layer_shard_path(str(out), idx), metadata=metadata)
return str(out)
def _spec(shard_dir: str, n_layers: int = N_LAYERS):
"""Layer 0's spec for every layer — what ``_build_source`` hands a decoder group."""
return [RamSource.spec_from_shard(shard_dir, 0)] * n_layers
def _raw_bytes(tensor):
return tensor.reshape(-1).view(torch.uint8)
def _assert_identical_to_disk_source(source, shard_dir, spec, n_layers=N_LAYERS):
shipped = DiskSource(shard_dir, n_layers, spec)
try:
for idx in range(n_layers):
for name in spec[idx]:
theirs = shipped.get(idx, name)
mine = source.get(idx, name)
assert mine.dtype == theirs.dtype, (idx, name)
assert mine.shape == theirs.shape, (idx, name)
assert torch.equal(_raw_bytes(mine), _raw_bytes(theirs)), (idx, name)
# Pinned staging is a borrow: give the slot back before the next layer.
source.release(idx, None)
finally:
shipped.close()
# ==========================================================================
# The read plan: sector-aligned ranges over a layer's data section
# ==========================================================================
class TestRangePlanning:
"""Pure arithmetic. Direct I/O needs offset, length and address on sector
boundaries, and the safetensors data section starts wherever the header ends
(byte 2976 on the 70B store), so the plan covers the aligned superset."""
def test_aligned_span_floors_the_start_and_ceils_the_end(self):
from soup_cli.utils.safetensors_reader import aligned_span
assert aligned_span(2976, 441433020) == (0, 441434112)
assert aligned_span(4096, 8192) == (4096, 8192)
assert aligned_span(4097, 8193) == (4096, 12288)
def test_ranges_are_sector_aligned_contiguous_and_cover_the_span(self):
from soup_cli.utils.safetensors_reader import plan_ranges
lo, hi = 4096, 4096 + SECTOR * 10
ranges = plan_ranges(lo, hi, 4)
assert len(ranges) == 4
assert ranges[0][0] == lo and ranges[-1][1] == hi
for (_a, b), (c, _d) in zip(ranges, ranges[1:]):
assert b == c, "ranges must be contiguous"
assert all(a % SECTOR == 0 and b % SECTOR == 0 and b > a for a, b in ranges)
def test_more_parts_than_sectors_collapses_to_what_the_span_can_carry(self):
from soup_cli.utils.safetensors_reader import plan_ranges
assert plan_ranges(0, SECTOR, 4) == [(0, SECTOR)]
assert len(plan_ranges(0, 2 * SECTOR, 4)) == 2
def test_a_malformed_plan_is_refused_by_name(self):
from soup_cli.utils.safetensors_reader import plan_ranges
with pytest.raises(ValueError, match="parts"):
plan_ranges(0, SECTOR, 0)
with pytest.raises(ValueError, match="aligned"):
plan_ranges(1, SECTOR, 1)
with pytest.raises(ValueError, match="empty"):
plan_ranges(SECTOR, SECTOR, 1)
# ==========================================================================
# One range into the front of a view
# ==========================================================================
class TestReadRangeInto:
@staticmethod
def _file(tmp_path: Path, size: int):
data = bytes(i % 251 for i in range(size))
path = tmp_path / "blob.bin"
path.write_bytes(data)
return str(path), data
def test_fills_exactly_the_requested_bytes_at_the_offset(self, tmp_path):
from soup_cli.utils.safetensors_reader import read_range_into
path, data = self._file(tmp_path, 3 * SECTOR + 100)
view = torch.zeros(2 * SECTOR, dtype=torch.uint8)
with open(path, "rb") as handle:
read_range_into(handle, SECTOR, view, 2 * SECTOR)
assert bytes(view.numpy()) == data[SECTOR : 3 * SECTOR]
def test_a_view_running_past_the_end_of_the_file_gets_the_tail_and_stops(self, tmp_path):
"""The aligned superset's last range runs past EOF; the file's tail arrives
and the read stops at ``expected`` rather than demanding the padding."""
from soup_cli.utils.safetensors_reader import read_range_into
path, data = self._file(tmp_path, 3 * SECTOR + 100)
view = torch.zeros(2 * SECTOR, dtype=torch.uint8)
with open(path, "rb") as handle:
read_range_into(handle, 2 * SECTOR, view, SECTOR + 100)
assert bytes(view[: SECTOR + 100].numpy()) == data[2 * SECTOR :]
def test_a_file_shorter_than_expected_is_a_named_short_read(self, tmp_path):
from soup_cli.utils.safetensors_reader import read_range_into
path, _ = self._file(tmp_path, SECTOR + 10)
view = torch.zeros(2 * SECTOR, dtype=torch.uint8)
with open(path, "rb") as handle:
with pytest.raises(OSError, match="short read"):
read_range_into(handle, 0, view, 2 * SECTOR)
def test_a_view_that_cannot_hold_expected_is_refused(self, tmp_path):
from soup_cli.utils.safetensors_reader import read_range_into
path, _ = self._file(tmp_path, SECTOR)
with open(path, "rb") as handle:
with pytest.raises(ValueError, match="holds"):
read_range_into(handle, 0, torch.zeros(10, dtype=torch.uint8), 100)
def test_a_zero_return_before_expected_is_a_short_read(self):
"""The one signal that ends the loop, driven directly: a handle that has
nothing at all must not let the read return as if it had filled the view."""
from soup_cli.utils.safetensors_reader import read_range_into
class Empty:
def seek(self, pos: int) -> None:
pass
def readinto(self, buffer) -> int:
return 0
with pytest.raises(OSError, match="short read, 0 of 4096"):
read_range_into(Empty(), 0, torch.zeros(SECTOR, dtype=torch.uint8), SECTOR)
def test_a_negative_expected_is_refused(self, tmp_path):
"""A negative count would return without reading and leave the view
undefined while claiming success (security review of #974)."""
from soup_cli.utils.safetensors_reader import read_range_into
path, _ = self._file(tmp_path, SECTOR)
with open(path, "rb") as handle:
with pytest.raises(ValueError, match="expected"):
read_range_into(handle, 0, torch.zeros(SECTOR, dtype=torch.uint8), -1)
def test_a_partial_read_that_is_not_the_end_of_the_file_is_retried(self):
"""``io.FileIO.readinto`` is one syscall, and POSIX lets it return fewer
bytes than asked with more still to come (Linux documents exactly that for
O_DIRECT). Only a ZERO return is the end of the data; a short one means
ask again (python review of #974)."""
from soup_cli.utils.safetensors_reader import read_range_into
data = bytes(i % 251 for i in range(3 * SECTOR))
class Dribbling:
"""Hands back half of every request, never zero until the data is gone."""
def __init__(self) -> None:
self.pos = 0
self.calls = 0
def seek(self, pos: int) -> None:
self.pos = pos
def readinto(self, buffer) -> int:
self.calls += 1
chunk = data[self.pos : self.pos + max(1, len(buffer) // 2)]
buffer[: len(chunk)] = chunk
self.pos += len(chunk)
return len(chunk)
view = torch.zeros(2 * SECTOR, dtype=torch.uint8)
handle = Dribbling()
read_range_into(handle, SECTOR, view, 2 * SECTOR)
assert bytes(view.numpy()) == data[SECTOR : 3 * SECTOR]
assert handle.calls >= 2, "the short return was accepted without asking again"
# ==========================================================================
# Direct I/O, where the platform has it
# ==========================================================================
class TestOpenDirect:
@staticmethod
def _file(tmp_path: Path, size: int):
data = bytes(i % 253 for i in range(size))
path = tmp_path / "direct.bin"
path.write_bytes(data)
return str(path), data
@staticmethod
def _aligned_view(numel: int):
buffer = torch.empty(numel + SECTOR, dtype=torch.uint8)
pad = (-buffer.data_ptr()) % SECTOR
return buffer, buffer[pad : pad + numel]
def _open_or_skip(self, path: str):
from soup_cli.utils.safetensors_reader import open_direct
try:
return open_direct(path)
except OSError as exc:
pytest.skip(f"direct I/O is unavailable on this filesystem/platform: {exc}")
def test_a_direct_handle_reads_the_same_bytes_as_a_buffered_one(self, tmp_path):
from soup_cli.utils.safetensors_reader import read_range_into
path, data = self._file(tmp_path, 5 * SECTOR + 7)
handle = self._open_or_skip(path)
keep, view = self._aligned_view(6 * SECTOR)
with handle:
read_range_into(handle, 0, view, len(data))
assert bytes(view[: len(data)].numpy()) == data
del keep
def test_a_direct_handle_carries_the_files_identity(self, tmp_path):
"""The shard-replaced-under-the-run check (#971) fstat's the handle it
reads through; a direct handle must answer the same way."""
from soup_cli.utils.safetensors_reader import identity_of
path, _ = self._file(tmp_path, SECTOR)
handle = self._open_or_skip(path)
with handle, open(path, "rb") as buffered:
assert identity_of(handle) == identity_of(buffered)
# ==========================================================================
# Staging: one region per slot, tensors as views
# ==========================================================================
class TestStagingIsOneRegionPerSlot:
def test_every_staged_tensor_is_a_view_into_its_slots_region(self, tmp_path):
shard_dir = _shards(tmp_path)
source = AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=2, pin=False)
try:
assert len(source._regions) == len(source._slots) == 2
for region, slot in zip(source._regions, source._slots):
lo = region.data_ptr()
hi = lo + region.numel()
assert slot, "an empty slot stages nothing"
for dst in slot.values():
start = dst.data_ptr()
assert lo <= start < hi, "a staged tensor lives outside its region"
assert start + dst.numel() * dst.element_size() <= hi
finally:
source.close()
def test_regions_start_on_a_sector_boundary_even_when_pageable(self, tmp_path):
shard_dir = _shards(tmp_path)
source = AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=2, pin=False)
try:
assert all(region.data_ptr() % SECTOR == 0 for region in source._regions)
finally:
source.close()
def test_staging_is_accounted_three_ways(self, tmp_path):
"""``nbytes`` stays what it was (the staged tensors' own bytes, the figure
the pre-flight budgets); ``staging_bytes`` is the regions, which cover the
aligned superset; ``pinned_bytes`` is what the box page-locked, 0 here."""
shard_dir = _shards(tmp_path)
source = AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=2, pin=False)
try:
observed = sum(
dst.numel() * dst.element_size()
for slot in source._slots
for dst in slot.values()
)
assert source.nbytes == observed
assert source.staging_bytes == sum(r.numel() for r in source._regions)
assert source.staging_bytes >= source.nbytes
assert source.pinned_bytes == 0
finally:
source.close()
def test_read_ranges_is_validated_by_name(self, tmp_path):
shard_dir = _shards(tmp_path)
spec = _spec(shard_dir)
with pytest.raises(ValueError, match="read_ranges.*not bool"):
AsyncDiskSource(shard_dir, N_LAYERS, spec, pin=False, read_ranges=True)
for bad in (0, -1, 17):
with pytest.raises(ValueError, match="read_ranges"):
AsyncDiskSource(shard_dir, N_LAYERS, spec, pin=False, read_ranges=bad)
def test_the_default_depth_of_ranges_is_the_measured_one(self, tmp_path):
from soup_cli.utils.async_disk_source import DEFAULT_STREAM_READ_RANGES
assert DEFAULT_STREAM_READ_RANGES == 4
shard_dir = _shards(tmp_path)
source = AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), pin=False)
try:
assert source.read_ranges == DEFAULT_STREAM_READ_RANGES
assert isinstance(source.direct_io, bool)
finally:
source.close()
# ==========================================================================
# THE gate: the same bytes as DiskSource through the range reader
# ==========================================================================
class TestByteIdentityThroughTheRangeReader:
@pytest.mark.parametrize("read_ranges", [1, 3, 4])
def test_every_tensor_matches_disk_source(self, tmp_path, read_ranges):
shard_dir = _shards(tmp_path, big=True)
spec = _spec(shard_dir)
source = AsyncDiskSource(
shard_dir, N_LAYERS, spec, read_ahead=2, pin=False, read_ranges=read_ranges
)
try:
_assert_identical_to_disk_source(source, shard_dir, spec)
finally:
source.close()
def test_identity_holds_without_direct_io(self, tmp_path, monkeypatch):
"""tmpfs, a network share, a platform without the flag: the buffered
fallback must read the same bytes, and the source must SAY it fell back."""
import soup_cli.utils.async_disk_source as mod
def unavailable(path):
raise OSError("no direct I/O here")
monkeypatch.setattr(mod, "open_direct", unavailable)
shard_dir = _shards(tmp_path, big=True)
spec = _spec(shard_dir)
source = AsyncDiskSource(shard_dir, N_LAYERS, spec, read_ahead=2, pin=False)
try:
assert source.direct_io is False
_assert_identical_to_disk_source(source, shard_dir, spec)
finally:
source.close()
def test_a_layer_whose_header_is_longer_than_its_siblings_is_placed_correctly(
self, tmp_path
):
"""Views are per LAYER, not per slot: the sector pad differs when the
header length does, and a slot reused for such a layer must re-derive
where each tensor landed."""
shard_dir = _shards(tmp_path, long_header_on=2)
spec = _spec(shard_dir)
source = AsyncDiskSource(shard_dir, N_LAYERS, spec, read_ahead=2, pin=False)
try:
starts = [plan.start for plan in source._plans]
assert starts[2] >= SECTOR and all(s == 0 for i, s in enumerate(starts) if i != 2), (
f"the fixture no longer moves layer 2's data section past the first "
f"sector (aligned starts {starts}); the offset term is not exercised"
)
_assert_identical_to_disk_source(source, shard_dir, spec)
finally:
source.close()
def test_a_shard_carrying_a_tensor_the_spec_does_not_want_is_still_read_correctly(
self, tmp_path
):
shard_dir = _shards(tmp_path, extra_tensor_on=1)
spec = _spec(shard_dir)
source = AsyncDiskSource(shard_dir, N_LAYERS, spec, read_ahead=2, pin=False)
try:
_assert_identical_to_disk_source(source, shard_dir, spec)
with pytest.raises(KeyError):
source.get(1, "self_attn.q_proj.weight::absmax_extra")
finally:
source.close()
def test_a_shard_spreading_the_wanted_tensors_over_a_far_larger_span_is_refused(
self, tmp_path
):
"""Staging is sized to the span between the first and last wanted tensor,
so a foreign tensor BETWEEN them is staged with them. A small one is the
cost of the design; a large one is a page-locked allocation the spec never
asked for (the security review of #974 built a shard whose two 16-byte
tensors cost an 8 MiB region). Refused at construction, by name."""
shard_dir = _shards(tmp_path, extra_tensor_on=1, extra_elements=2_000_000)
with pytest.raises(ValueError, match="span"):
AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=2, pin=False)
def test_a_layer_that_wants_no_tensors_is_refused_by_name(self, tmp_path):
shard_dir = _shards(tmp_path)
spec = [dict(layer) for layer in _spec(shard_dir)]
spec[1] = {}
with pytest.raises(ValueError, match="no tensors"):
AsyncDiskSource(shard_dir, N_LAYERS, spec, read_ahead=2, pin=False)
def test_a_layer_whose_wanted_tensors_hold_no_bytes_is_refused_by_name(self, tmp_path):
"""Otherwise the empty span is refused by ``plan_ranges`` at the first
read, i.e. at the ``get`` that wanted it, instead of at construction."""
shard_dir = _shards(tmp_path, empty_tensor=True)
spec = [{"self_attn.q_proj.weight::empty": ((0,), "float32")}] * N_LAYERS
with pytest.raises(ValueError, match="no bytes"):
AsyncDiskSource(shard_dir, N_LAYERS, spec, read_ahead=2, pin=False)
def test_a_zero_element_tensor_rides_along_intact(self, tmp_path):
"""An empty tensor has no bytes to read and no alignment to check; it must
still come back as an empty tensor of the right dtype from every layer."""
shard_dir = _shards(tmp_path, empty_tensor=True)
spec = _spec(shard_dir)
assert spec[0]["self_attn.q_proj.weight::empty"] == ((0,), "float32")
source = AsyncDiskSource(shard_dir, N_LAYERS, spec, read_ahead=2, pin=False)
try:
_assert_identical_to_disk_source(source, shard_dir, spec)
empty = source.get(0, "self_attn.q_proj.weight::empty")
assert empty.numel() == 0 and empty.dtype == torch.float32
finally:
source.close()
def test_a_tensor_offset_not_aligned_to_its_dtype_is_refused_at_construction(
self, tmp_path
):
"""A view needs its byte offset on the dtype's boundary. safetensors' own
writer orders tensors so that holds; a hand-built shard that breaks it is
refused by name rather than read into a view torch cannot make."""
from soup_cli.utils.layer_shard import layer_shard_path
out = tmp_path / "misaligned"
out.mkdir()
header = {
"a": {"dtype": "U8", "shape": [3], "data_offsets": [0, 3]},
"b": {"dtype": "F32", "shape": [2], "data_offsets": [3, 11]},
}
body = json.dumps(header).encode()
# The refusal is about 'b' starting off a 4-byte boundary from the aligned
# read start (0): if the header length ever makes 8 + len + 3 a multiple of
# four, this fixture stops testing anything — say so rather than pass.
assert (8 + len(body) + 3) % 4 != 0, "fixture no longer misaligns 'b'"
for idx in range(2):
with open(layer_shard_path(str(out), idx), "wb") as handle:
handle.write(struct.pack("<Q", len(body)) + body + bytes(11))
spec = [{"a": ((3,), "uint8"), "b": ((2,), "float32")}] * 2
with pytest.raises(ValueError, match="aligned"):
AsyncDiskSource(str(out), 2, spec, read_ahead=1, pin=False)
# ==========================================================================
# The ranges are read by workers, in parallel, off the calling thread
# ==========================================================================
class TestTheReadIsSplitAcrossWorkers:
def test_ranges_of_one_layer_are_read_concurrently(self, tmp_path, monkeypatch):
import soup_cli.utils.async_disk_source as mod
real = mod.read_range_into
lock = threading.Lock()
live = {"now": 0, "peak": 0}
def counting(handle, start, view, expected):
with lock:
live["now"] += 1
live["peak"] = max(live["peak"], live["now"])
try:
time.sleep(0.02)
return real(handle, start, view, expected)
finally:
with lock:
live["now"] -= 1
monkeypatch.setattr(mod, "read_range_into", counting)
shard_dir = _shards(tmp_path, big=True)
source = AsyncDiskSource(
shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=1, pin=False, read_ranges=4
)
try:
source.get(0, "input_layernorm.weight")
assert live["peak"] >= 2, "the ranges of one layer were read one after another"
finally:
source.close()
def test_the_reads_never_run_on_the_calling_thread(self, tmp_path, monkeypatch):
import soup_cli.utils.async_disk_source as mod
real = mod.read_range_into
seen = []
def recording(handle, start, view, expected):
seen.append(threading.current_thread())
return real(handle, start, view, expected)
monkeypatch.setattr(mod, "read_range_into", recording)
shard_dir = _shards(tmp_path, big=True)
source = AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=2, pin=False)
try:
source.get(0, "input_layernorm.weight")
assert seen, "nothing was read at all"
assert threading.current_thread() not in seen
assert {t.name.rsplit("-", 1)[0] for t in seen} == {"soup-layer-range"}
assert all(t.daemon for t in seen), "a wedged range read must not block exit"
finally:
source.close()
def test_the_workers_are_gone_after_close(self, tmp_path):
"""THIS source's workers, not every ``soup-layer-range`` thread in the
process: in a full test session other sources' workers are still
winding down, and a process-wide sweep blamed them on this one."""
shard_dir = _shards(tmp_path)
source = AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=2, pin=False)
source.get(0, "input_layernorm.weight")
workers = list(source._readers._threads)
assert workers and all(t.is_alive() for t in workers), "no range workers were started"
assert all(t.name.startswith("soup-layer-range") for t in workers)
source.close()
# No join here: ``close()`` returning is the promise that the workers are
# gone, the same promise it makes for the reader thread.
assert not any(t.is_alive() for t in workers)
def test_close_releases_the_staging_buffers(self, tmp_path):
"""The regions and their arenas are gigabytes of page-locked memory on a
real run; a ``close()`` that dropped the slots but kept them referenced
would hold that for the process's lifetime with nothing to say so (a
parallel session's mutation run: no test looked)."""
shard_dir = _shards(tmp_path)
source = AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=2, pin=False)
source.get(0, "input_layernorm.weight")
assert source._regions and source._arenas
source.close()
assert source._slots == [] and source._regions == [] and source._arenas == []
def test_the_reader_reads_each_layer_through_read_layer(self, tmp_path, monkeypatch):
"""The one seam the harness times (``reader_read`` in
``benchmarks/harness/issue974_warm_stages.py``): a layer read is exactly one
``_read_layer`` call, whatever the ranges do inside it."""
calls = []
real = AsyncDiskSource._read_layer
def recording(self, idx, region):
calls.append(idx)
return real(self, idx, region)
monkeypatch.setattr(AsyncDiskSource, "_read_layer", recording)
shard_dir = _shards(tmp_path)
spec = _spec(shard_dir)
source = AsyncDiskSource(shard_dir, N_LAYERS, spec, read_ahead=1, pin=False)
try:
for idx in range(N_LAYERS):
source.get(idx, "input_layernorm.weight")
assert calls == list(range(N_LAYERS))
finally:
source.close()
def test_an_error_in_one_range_surfaces_at_the_get_that_wanted_it(
self, tmp_path, monkeypatch
):
import soup_cli.utils.async_disk_source as mod
real = mod.read_range_into
count = {"n": 0}
def failing(handle, start, view, expected):
count["n"] += 1
if count["n"] == 2:
raise OSError("range two went away")
return real(handle, start, view, expected)
monkeypatch.setattr(mod, "read_range_into", failing)
shard_dir = _shards(tmp_path, big=True)
source = AsyncDiskSource(
shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=1, pin=False, read_ranges=4
)
try:
with pytest.raises(OSError, match="range two went away"):
source.get(0, "input_layernorm.weight")
finally:
source.close()
# ==========================================================================
# The ready line says what the disk tier page-locked
# ==========================================================================
class TestTheReadyLineNamesThePageLockedStaging:
@staticmethod
def _stats(pinned: bool, pinned_bytes):
return {
"tier": "disk",
"pinned": pinned,
"pinned_bytes": pinned_bytes,
"disk_bytes": 36.39e9,
"read_ahead": 2,
"store_bytes": 1.96e9,
}
def test_pinned_staging_prints_the_page_locked_figure(self):
from soup_cli.trainer.stream_setup import _stream_source_line
line = _stream_source_line(self._stats(True, 2**31))
assert "1960 MB pinned host staging" in line
assert "(2.15 GB page-locked)" in line
def test_pageable_staging_prints_no_second_figure(self):
from soup_cli.trainer.stream_setup import _stream_source_line
line = _stream_source_line(self._stats(False, 0))
assert "pageable host staging" in line
assert "page-locked" not in line
# ==========================================================================
# The arena plan on the real 70B staging
# ==========================================================================
class TestTheArenaPlanForARealStore:
def test_the_70b_staging_page_locks_within_ten_percent_of_its_bytes(self):
"""Two decoder slots (441.4 MB aligned spans) plus the embed and the untied
head (536.9 MB each) on the 70B-shaped NF4 store: one 2 GiB arena for
1.96 GB, where per-tensor pinning paid 1.7-1.9x."""
from soup_cli.utils.layer_stream_runtime import plan_pinned_arenas
spans = [441434112, 441434112, 536875008, 536875008]
plan = plan_pinned_arenas(spans, align=SECTOR)
assert plan.pinned_bytes / sum(spans) <= 1.10
assert all(offset % SECTOR == 0 for _arena, offset in plan.placements)
# ==========================================================================
# Real hardware: pinned arenas, and the disk tier's own failed page-lock
# ==========================================================================
@pytest.mark.gpu
class TestOnRealHardware:
def test_every_region_is_a_pinned_sector_aligned_view_of_a_power_of_two_arena(
self, tmp_path
):
shard_dir = _shards(tmp_path, big=True)
source = AsyncDiskSource(shard_dir, N_LAYERS, _spec(shard_dir), read_ahead=2, pin=True)
try:
assert source.pinned is True
assert source._arenas, "pinned staging must live in arenas"
for arena in source._arenas:
assert arena.is_pinned()
assert arena.numel() & (arena.numel() - 1) == 0, "arena is not a power of two"
assert source.pinned_bytes == sum(a.numel() for a in source._arenas)
for region in source._regions:
assert region.is_pinned()
assert region.data_ptr() % SECTOR == 0
staged = [dst for slot in source._slots for dst in slot.values()]
assert all(dst.is_pinned() for dst in staged)
finally:
source.close()
def test_byte_identity_holds_with_pinned_staging(self, tmp_path):
shard_dir = _shards(tmp_path, big=True)
spec = _spec(shard_dir)
source = AsyncDiskSource(shard_dir, N_LAYERS, spec, read_ahead=2, pin=True, read_ranges=4)
try:
_assert_identical_to_disk_source(source, shard_dir, spec)
finally:
source.close()
def test_the_disk_tier_falls_back_to_pageable_staging_after_a_real_failed_page_lock(
self, tmp_path, monkeypatch
):
"""The #901 round trip, on the DISK tier: a genuine driver refusal of the
first arena, the genuine stale error, the shipped fallback — and the first
kernel launch after ``_build_source`` returns must work. The RAM tier had
this test; the disk tier did not (TDD review of #989)."""
import soup_cli.utils.layer_stream_runtime as rt
shard_dir = _shards(tmp_path)
spec = _spec(shard_dir)
real_empty = torch.empty
def _impossible_when_pinned(*args, **kwargs):
if kwargs.get("pin_memory"):
return real_empty(_IMPOSSIBLE_PIN_BYTES, dtype=torch.uint8, pin_memory=True)
return real_empty(*args, **kwargs)
monkeypatch.setattr(torch, "empty", _impossible_when_pinned)
console = _Console()
source, pinned = rt._build_source(
shard_dir, N_LAYERS, spec, True, console, "disk", read_ahead=2
)
monkeypatch.undo()
try:
assert pinned is False
assert source.pinned is False
assert any("PAGEABLE" in msg for msg in console.printed)
# The first kernel launch of the run, where #901 died.
torch.ones(1, device="cuda")
torch.cuda.synchronize()
_assert_identical_to_disk_source(source, shard_dir, spec)
finally:
source.close()