Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Parse a safetensors header without mapping the file.
``safe_open`` memory-maps, and on Windows a mapping charges commit for the
file's whole size (#926, measured: 48.99 GB mapped raised the commit charge
46.17 GB). The streaming disk tier holds one shard per decoder layer for a
whole run, so mapping is what we are avoiding, not an implementation detail.
The format is: 8 bytes little-endian header length, that many bytes of JSON
mapping name -> {dtype, shape, data_offsets}, then the tensor bytes. Offsets in
the JSON are relative to the end of the header; ``TensorRange`` stores them
ABSOLUTE so a caller can seek directly.
NO top-level torch or safetensors: this module is import-light by design.
"""
import io
import json
import logging
import math
import os
import struct
import sys
from dataclasses import dataclass
from typing import IO, TYPE_CHECKING, Dict, List, Tuple
if TYPE_CHECKING: # pragma: no cover — annotations only; the module stays torch-free
import torch
logger = logging.getLogger(__name__)
# A real header is a few hundred KB. The cap turns a corrupt length field into
# a named refusal instead of a 16 EB allocation attempt.
_MAX_HEADER_BYTES = 100_000_000
#: The unit direct I/O demands of a file offset, a request length and a buffer
#: address: a multiple of every sector size in use (512-byte and 4 KiB volumes
#: alike). Measured on the dev box (#974): an unaligned LENGTH is refused with
#: EINVAL; an unaligned address happened to be tolerated and is not relied on.
SECTOR_BYTES = 4096
# Win32 flags for ``open_direct``; ``_winapi`` exports GENERIC_READ and
# OPEN_EXISTING but not these.
_FILE_SHARE_READ = 0x00000001
_FILE_SHARE_WRITE = 0x00000002
_FILE_FLAG_NO_BUFFERING = 0x20000000
_FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000
# Safetensors' spelling -> Soup's, matching layer_stream_runtime._SAFETENSORS_DTYPES.
# The FP8 spellings are here so the disk pre-flight can read a SOURCE checkpoint's
# header (an FP8 block source, which the sharder dequantises, or an FP8 spelling it
# refuses by name); Soup's own cache files never hold FP8, and their readers check
# every dtype against the index.
_DTYPES: Dict[str, str] = {
"BF16": "bfloat16",
"F16": "float16",
"F32": "float32",
"F64": "float64",
"I8": "int8",
"I16": "int16",
"I32": "int32",
"I64": "int64",
"U8": "uint8",
"BOOL": "bool",
"F8_E4M3": "float8_e4m3fn",
"F8_E4M3FNUZ": "float8_e4m3fnuz",
"F8_E5M2": "float8_e5m2",
"F8_E5M2FNUZ": "float8_e5m2fnuz",
}
_ITEMSIZE: Dict[str, int] = {
"bfloat16": 2,
"float16": 2,
"float32": 4,
"float64": 8,
"int8": 1,
"int16": 2,
"int32": 4,
"int64": 8,
"uint8": 1,
"bool": 1,
"float8_e4m3fn": 1,
"float8_e4m3fnuz": 1,
"float8_e5m2": 1,
"float8_e5m2fnuz": 1,
}
#: Soup's spelling -> safetensors', the inverse of ``_DTYPES``.
RAW_DTYPES: Dict[str, str] = {soup: raw for raw, soup in _DTYPES.items()}
@dataclass(frozen=True)
class ShardIdentity:
"""Which FILE a set of byte ranges was read off.
``read_header`` closes the file, so between the parse and every later read
there is no handle, no inode pin and no fingerprint — and ``read_into``
checks how many bytes arrived, never that they came from the same file.
A shard replaced in place by one of the SAME SIZE and a different layout
therefore reads at stale offsets and trains on garbage with no error
anywhere. ``layer_shard`` re-shards into the same directory with
``os.replace`` whenever the base's fingerprint moves, so the window is the
whole run rather than a microsecond.
Four fields because no one of them is sufficient: size alone misses a
same-size rewrite, mtime alone misses a preserved timestamp, and
``(ino, dev)`` alone misses an in-place rewrite that keeps the inode.
This type only REPORTS identity; the policy and the message belong to the
caller that knows what the file is for.
"""
size: int
mtime_ns: int
ino: int
dev: int
def identity_of(handle: "object") -> ShardIdentity:
"""The identity of the file behind an OPEN handle (``os.fstat``, not a path).
Taking it off the handle rather than the path is what makes it atomic with
the bytes read through that handle: a path-based ``stat`` could describe a
different file than the one the caller is about to read.
"""
st = os.fstat(handle.fileno())
return ShardIdentity(
size=st.st_size, mtime_ns=st.st_mtime_ns, ino=st.st_ino, dev=st.st_dev
)
@dataclass(frozen=True)
class TensorRange:
"""One tensor's identity and its ABSOLUTE byte range in the shard."""
name: str
dtype: str
shape: Tuple[int, ...]
start: int
end: int
@property
def nbytes(self) -> int:
return self.end - self.start
class UnsupportedDtypeError(ValueError):
"""A header names a tensor in a dtype this reader does not read (``U32``, ``F4``, ...).
A ``ValueError`` like every other malformed-header refusal here; its own type so the
layer-stream disk pre-flight can refuse that tensor in the sharder's words
(``layer_shard._read_source_tensors``) instead of treating the header as unreadable.
``tensor`` and ``dtype`` are the header's own (untrusted) strings. Both have defaults
so the exception rebuilds from its message alone, as ``pickle`` and ``copy`` rebuild
it (its attributes then come back from its ``__dict__``).
"""
def __init__(self, message: str, *, tensor: str = "", dtype: object = None) -> None:
super().__init__(message)
self.tensor = tensor
self.dtype = dtype
def read_header(path: str) -> Dict[str, TensorRange]:
"""Every tensor in ``path``, with absolute byte ranges. Never maps the file."""
return read_header_with_identity(path)[0]
def read_header_with_identity(path: str) -> Tuple[Dict[str, TensorRange], ShardIdentity]:
"""``read_header``, plus the identity of the file the ranges were read off.
The identity comes from an ``os.fstat`` on the SAME handle the header is
read through, so a caller that keeps the ranges for the rest of a run can
prove, at every later open, that it is still addressing the file it parsed.
"""
with open(path, "rb") as handle:
identity = identity_of(handle)
size = identity.size
raw_length = handle.read(8)
if len(raw_length) != 8:
raise ValueError(f"{path}: header is truncated (not even a length field)")
(length,) = struct.unpack("<Q", raw_length)
if length > _MAX_HEADER_BYTES:
raise ValueError(
f"{path}: header claims {length} bytes, above the "
f"{_MAX_HEADER_BYTES} cap; refusing rather than allocating it"
)
body = handle.read(length)
if len(body) != length:
raise ValueError(f"{path}: header is truncated ({len(body)} of {length} bytes)")
try:
payload = json.loads(body)
except json.JSONDecodeError as exc:
raise ValueError(f"{path}: header is not valid JSON ({exc})") from exc
if not isinstance(payload, dict):
raise ValueError(f"{path}: header is not a JSON object")
base = 8 + length
entries: Dict[str, TensorRange] = {}
for name, meta in payload.items():
if name == "__metadata__":
continue
if not isinstance(meta, dict):
raise ValueError(f"{path}: entry {name!r} is not an object")
raw_dtype = meta.get("dtype")
if raw_dtype not in _DTYPES:
raise UnsupportedDtypeError(
f"{path}: tensor {name!r} has unsupported dtype {raw_dtype!r}; "
f"supported: {', '.join(sorted(_DTYPES))}",
tensor=name,
dtype=raw_dtype,
)
dtype = _DTYPES[raw_dtype]
# No default: ``dict.get(key, default)`` substitutes only when the KEY is
# absent, so a stored ``null`` would slip a default past this check and
# raise a bare TypeError at the comprehension. Mirrors data_offsets below.
raw_shape = meta.get("shape")
if not isinstance(raw_shape, (list, tuple)):
raise ValueError(f"{path}: tensor {name!r} has no valid shape")
shape = tuple(int(dim) for dim in raw_shape)
if any(dim < 0 for dim in shape):
raise ValueError(f"{path}: tensor {name!r} has a negative dimension")
offsets = meta.get("data_offsets")
if not isinstance(offsets, (list, tuple)) or len(offsets) != 2:
raise ValueError(f"{path}: tensor {name!r} has no valid data_offsets")
start, end = base + int(offsets[0]), base + int(offsets[1])
if start < base:
raise ValueError(
f"{path}: tensor {name!r} has data_offsets starting before the "
f"tensor-data region (byte {start}, header ends at {base})"
)
if start > end:
raise ValueError(f"{path}: tensor {name!r} has a reversed byte range")
if end > size:
raise ValueError(
f"{path}: tensor {name!r} ends at byte {end}, past the end of the "
f"file ({size} bytes)"
)
expected = math.prod(shape) * _ITEMSIZE[dtype]
if end - start != expected:
raise ValueError(
f"{path}: tensor {name!r} declares shape {shape} of {dtype} "
f"({expected} bytes) but its byte range holds {end - start}"
)
entries[name] = TensorRange(
name=name, dtype=dtype, shape=shape, start=start, end=end
)
return entries, identity
def read_into(handle: "object", entry: TensorRange, tensor: "object") -> None:
"""Fill ``tensor`` from ``handle`` at ``entry``'s byte range.
``tensor`` is a pre-allocated CPU tensor of the right shape and dtype —
allocating here would defeat the pool. The read goes through a ``uint8``
view of the SAME memory, because ``numpy()`` refuses bfloat16 and a
``frombuffer`` round trip would copy.
The ``reshape(-1)`` comes BEFORE that view, and is not cosmetic: torch
refuses a dtype-``view`` on a 0-dimensional tensor ("self.dim() cannot be 0
to view Float as Byte"), and NF4 double quantisation stores a SCALAR
``::nested_offset`` per quantised weight — 7 of the 30 tensors in a real
decoder-layer shard. Viewing first made every 4-bit layer unreadable
through this path. Reshaping a contiguous tensor returns a view, so the
uint8 flat still aliases the destination's storage and ``readinto`` writes
through to it; the contiguity refusal above is what keeps that true, which
is why it stays ahead of this line rather than being folded into it.
If this raises, ``tensor`` holds undefined contents — a short read leaves
whatever prefix bytes arrived and does not zero or roll back the rest —
and must not be used until a later call fills it successfully.
"""
import torch
if not tensor.is_contiguous():
raise ValueError(
f"tensor {entry.name!r}: destination must be contiguous to be read into"
)
if tensor.device.type != "cpu":
raise ValueError(
f"tensor {entry.name!r}: destination must live on the CPU, "
f"got {tensor.device}"
)
held = tensor.numel() * tensor.element_size()
if held != entry.nbytes:
raise ValueError(
f"tensor {entry.name!r}: shard holds {entry.nbytes} bytes but the "
f"destination holds {held}"
)
flat = tensor.reshape(-1).view(torch.uint8)
handle.seek(entry.start)
got = handle.readinto(memoryview(flat.numpy()))
if got != entry.nbytes:
raise OSError(
f"tensor {entry.name!r}: short read, {got} of {entry.nbytes} bytes "
f"at offset {entry.start}"
)
class DirectIOUnavailableError(OSError):
"""This platform offers no way to read a file past the page cache."""
def aligned_span(lo: int, hi: int, *, sector: int = SECTOR_BYTES) -> Tuple[int, int]:
"""The sector-aligned superset of the byte range ``[lo, hi)``.
A safetensors data section starts wherever its header ends (byte 2976 on
the 70B-shaped store), so a direct read covers a little more than the
tensors and the caller places them at ``start - aligned_lo``.
"""
return lo - lo % sector, -(-hi // sector) * sector
def plan_ranges(
lo: int, hi: int, parts: int, *, sector: int = SECTOR_BYTES
) -> List[Tuple[int, int]]:
"""Split the aligned span ``[lo, hi)`` into at most ``parts`` byte ranges.
Contiguous, non-empty, every boundary on a sector — so each range can be
one direct-I/O request into a sector-aligned slice of the same buffer. A
span shorter than ``parts`` sectors yields fewer ranges rather than empty
ones.
"""
if isinstance(parts, bool) or int(parts) < 1:
raise ValueError(f"parts must be a positive int; got {parts!r}")
if lo % sector or hi % sector:
raise ValueError(f"span [{lo}, {hi}) is not aligned to {sector}-byte sectors")
if hi <= lo:
raise ValueError(f"span [{lo}, {hi}) is empty")
sectors = (hi - lo) // sector
parts = min(int(parts), sectors)
return [
(lo + (sectors * part // parts) * sector, lo + (sectors * (part + 1) // parts) * sector)
for part in range(parts)
]
def read_range_into(
handle: IO[bytes], start: int, view: "torch.Tensor", expected: int
) -> None:
"""Read ``expected`` bytes at ``start`` into the FRONT of ``view``.
``view`` is a contiguous CPU ``uint8`` tensor at least ``expected`` long —
usually longer: it is the sector-aligned slice of a staging region, and the
last range of a layer runs past the end of the file, where the request
returns short and the loop stops at ``expected`` rather than demanding the
padding. A ``readinto`` that returns FEWER bytes than asked is asked again:
``io.FileIO`` is one syscall, and POSIX lets it return short with more still
to come (Linux documents exactly that for ``O_DIRECT``). Only a ZERO return
is the end of the data, and one that comes before ``expected`` is a file
shorter than its header said, refused as a short read. Every request keeps
the aligned length ``view`` has from ``done`` on, which a direct-I/O handle
requires; a short return that leaves ``done`` off a sector boundary makes
the next request one the handle refuses — an ``OSError`` either way, never
a silent partial fill.
If this raises, ``view`` holds undefined contents.
"""
import torch
if expected < 0:
raise ValueError(f"expected byte count must not be negative; got {expected}")
if view.dtype != torch.uint8 or view.device.type != "cpu" or not view.is_contiguous():
raise ValueError("destination must be a contiguous CPU uint8 tensor")
if view.numel() < expected:
raise ValueError(
f"destination holds {view.numel()} bytes but {expected} were expected "
f"at offset {start}"
)
handle.seek(start)
buffer = memoryview(view.numpy())
done = 0
while done < expected:
got = handle.readinto(buffer[done:])
if not got:
raise OSError(f"short read, {done} of {expected} bytes at offset {start}")
done += got
def _owning_fileio(fd: int) -> io.FileIO:
"""Wrap ``fd`` so the descriptor is closed if the wrapper itself cannot be built."""
try:
return io.FileIO(fd, "rb", closefd=True)
except BaseException:
os.close(fd)
raise
def open_direct(path: str) -> io.FileIO:
"""Open ``path`` for reading with the page cache bypassed.
Measured on the dev box (#974, Samsung PM9B1 NVMe, Windows 11): buffered
reads of a store larger than RAM top out at 1.2-2.9 GB/s cold whatever the
request pattern, unbuffered reads of the same layers into a 4 KiB-aligned
buffer reach 3.5-5.65 GB/s. Windows: ``FILE_FLAG_NO_BUFFERING`` on a
handle wrapped as a CRT descriptor; Linux: ``O_DIRECT``; macOS:
``F_NOCACHE``. The caller owes the alignment (:data:`SECTOR_BYTES`) of
every offset, length and address it reads with. Raises ``OSError`` where
the filesystem refuses (tmpfs has no ``O_DIRECT``) and
:class:`DirectIOUnavailableError` where the platform has no such mode at
all; the reader falls back to a buffered ``open`` in both cases.
"""
if sys.platform == "win32":
import _winapi
import msvcrt
try:
handle = _winapi.CreateFile(
path,
_winapi.GENERIC_READ,
_FILE_SHARE_READ | _FILE_SHARE_WRITE,
_winapi.NULL,
_winapi.OPEN_EXISTING,
_FILE_FLAG_NO_BUFFERING | _FILE_FLAG_SEQUENTIAL_SCAN,
_winapi.NULL,
)
except OSError as exc:
# ``_winapi`` raises without the path; a missing shard must name
# itself the way ``open`` does, or the operator gets "[WinError 2]".
raise type(exc)(
exc.errno, exc.strerror, path, getattr(exc, "winerror", None)
) from None
try:
fd = msvcrt.open_osfhandle(handle, os.O_RDONLY | os.O_BINARY)
except OSError:
_winapi.CloseHandle(handle)
raise
return _owning_fileio(fd)
cloexec = getattr(os, "O_CLOEXEC", 0)
direct = getattr(os, "O_DIRECT", None)
if direct is not None:
return _owning_fileio(os.open(path, os.O_RDONLY | direct | cloexec))
if sys.platform == "darwin":
import fcntl
nocache = getattr(fcntl, "F_NOCACHE", None)
if nocache is not None:
fd = os.open(path, os.O_RDONLY | cloexec)
try:
fcntl.fcntl(fd, nocache, 1)
except OSError:
os.close(fd)
raise
return _owning_fileio(fd)
raise DirectIOUnavailableError(f"no direct I/O mode on {sys.platform}")