Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
nvfp4 local pack: read the checkpoint with cached sequential reads, not a process-lifetime np.memmap (#3377, #3378)
Highlander0fbbf068eb7f
diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.pyindex 03f5c8848..a9752e4a0 100644--- a/tools/nvfp4_local_pack.py+++ b/tools/nvfp4_local_pack.py@@ -63,7 +63,19 @@ PLE_ROW_BYTES = 90 # 80 B of E2M1 codes + 10 E4M3 block scales, per 160# --------------------------------------------------------------------------------------------- safetensors accessclass Checkpoint:- """Header-indexed, memory-mapped access to a checkpoint's safetensors shards (raw bytes, any dtype)."""+ """Header-indexed, buffered access to a checkpoint's safetensors shards (raw bytes, any dtype).++ Reads go through a cached file handle into a fresh buffer instead of a process-lifetime np.memmap:+ a memmap sweep of the ~64 GB of expert bytes charges every faulted page to the PROCESS WORKING+ SET, which Windows (and every free-RAM observer reading GlobalMemoryStatusEx) counts as used+ until the OS trims it - the pack step then looks like a ~64 GB RAM demand while its real+ committed need is a few GB, and under memory pressure it is the pack run itself that pays+ (issue: the silent exit-1 mid-pack failure).+ A cached sequential read puts the same pages in the system file cache (standby: counted as+ available). The returned array is a fresh WRITABLE buffer - no consumer writes (a mutator would+ previously have hit the read-only memmap) - and reads share one handle per shard, so raw() is+ not thread-safe, unlike the memmap it replaces. Each shard is read as three ascending streams+ (exceptions, weights, CSF-fixed) walked expert-ascending."""def __init__(self, folder, partial_ok=False):self.dir = pathlib.Path(folder)@@ -76,7 +88,7 @@ class Checkpoint:if not files:raise FileNotFoundError("no .safetensors files in " + str(self.dir))self.meta: dict[str, tuple[str, str, list[int], int, int]] = {}- self._mm: dict[str, np.memmap] = {}+ self._fh: dict[str, "object"] = {}for fn in files:path = self.dir / fnif not path.exists() and partial_ok:@@ -100,9 +112,14 @@ class Checkpoint:def raw(self, name) -> np.ndarray:fn, _, _, a, b = self.meta[name]- if fn not in self._mm:- self._mm[fn] = np.memmap(self.dir / fn, dtype=np.uint8, mode="r")- return self._mm[fn][a:b]+ f = self._fh.get(fn)+ if f is None:+ f = self._fh[fn] = open(self.dir / fn, "rb")+ buf = np.empty(b - a, dtype=np.uint8)+ f.seek(a)+ if f.readinto(buf) != b - a:+ raise EOFError("%s: shard %s ended early at %d" % (name, fn, a))+ return bufdef array(self, name, np_dtype) -> np.ndarray:return self.raw(name).view(np_dtype).reshape(self.shape(name))diff --git a/tools/test_nvfp4_local_pack.py b/tools/test_nvfp4_local_pack.pyindex dd6fbe324..b32c50e16 100644--- a/tools/test_nvfp4_local_pack.py+++ b/tools/test_nvfp4_local_pack.py@@ -1,5 +1,7 @@"""Tests for tools/nvfp4_local_pack.py: the CSF scale decode, the MXFP8 dequantization, the PLE rows."""+import jsonimport pathlib+import structimport sysimport numpy as np@@ -114,3 +116,42 @@ def test_nvfp4_blocks_layout_by_hand():# within a 16-block: element j in the low nibble, element j+8 in the high oneblk0 = out[0, 4:12]assert [int(b & 15) for b in blk0] == list(range(8)) and [int(b >> 4) for b in blk0] == list(range(8, 16))+++def _write_shard(path, tensors):+ """tensors: name -> (dtype, shape, uint8 bytes), written safetensors-style (data after the header)."""+ hdr, data, off = {}, bytearray(), 0+ for name, (dt, shape, buf) in tensors.items():+ hdr[name] = {"dtype": dt, "shape": shape, "data_offsets": [off, off + len(buf)]}+ data += buf+ off += len(buf)+ hj = json.dumps(hdr).encode("utf-8")+ hj += b" " * (-len(hj) % 8)+ with open(path, "wb") as f:+ f.write(struct.pack("<Q", len(hj)) + hj + data)+++def test_checkpoint_reads_the_shard_bytes_exactly(tmp_path):+ a = np.arange(200, dtype=np.uint8)+ _write_shard(tmp_path / "s1.safetensors", {"t.a": ("U8", [100, 2], a.tobytes())})+ d = np.arange(4, dtype=np.uint32)+ c = np.arange(32, dtype=np.uint8)+ _write_shard(tmp_path / "s2.safetensors", {"t.d": ("U32", [4], d.tobytes()), "t.c": ("U8", [8, 4], c.tobytes())})+ (tmp_path / "model.safetensors.index.json").write_text(json.dumps({"weight_map": {+ "t.a": "s1.safetensors", "t.c": "s2.safetensors", "t.d": "s2.safetensors"}}), encoding="utf-8")+ ck = P.Checkpoint(tmp_path)+ assert "t.a" in ck and ck.shape("t.a") == [100, 2] and ck.dtype("t.a") == "U8"+ assert np.array_equal(ck.raw("t.a"), a) # the bytes at the tensor's exact range+ assert np.array_equal(ck.array("t.c", np.uint8), c.reshape(8, 4))+ assert np.array_equal(ck.array("t.d", np.uint32), d) # .view through a fresh buffer+ assert np.array_equal(ck.raw("t.a"), ck.raw("t.a")) # a second read is still correct+++def test_checkpoint_names_a_truncated_shard(tmp_path):+ c = np.arange(32, dtype=np.uint8)+ _write_shard(tmp_path / "s1.safetensors", {"t.c": ("U8", [8, 4], c.tobytes())})+ p = tmp_path / "s1.safetensors"+ p.write_bytes(p.read_bytes()[:-8]) # the last tensor's range now ends past EOF+ ck = P.Checkpoint(tmp_path)+ with pytest.raises(EOFError):+ ck.raw("t.c")