Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

nvfp4 local pack: read the checkpoint with cached sequential reads, not a process-lifetime np.memmap (#3377, #3378)

Highlander0fbbf068eb7f
diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.py
index 03f5c8848..a9752e4a0 100644
--- a/tools/nvfp4_local_pack.py
+++ b/tools/nvfp4_local_pack.py
@@ -63,7 +63,19 @@ PLE_ROW_BYTES = 90 # 80 B of E2M1 codes + 10 E4M3 block scales, per 160
# --------------------------------------------------------------------------------------------- safetensors access
class Checkpoint:
- """Header-indexed, memory-mapped access to a checkpoint's safetensors shards (raw bytes, any dtype)."""
+ """Header-indexed, buffered access to a checkpoint's safetensors shards (raw bytes, any dtype).
+
+ Reads go through a cached file handle into a fresh buffer instead of a process-lifetime np.memmap:
+ a memmap sweep of the ~64 GB of expert bytes charges every faulted page to the PROCESS WORKING
+ SET, which Windows (and every free-RAM observer reading GlobalMemoryStatusEx) counts as used
+ until the OS trims it - the pack step then looks like a ~64 GB RAM demand while its real
+ committed need is a few GB, and under memory pressure it is the pack run itself that pays
+ (issue: the silent exit-1 mid-pack failure).
+ A cached sequential read puts the same pages in the system file cache (standby: counted as
+ available). The returned array is a fresh WRITABLE buffer - no consumer writes (a mutator would
+ previously have hit the read-only memmap) - and reads share one handle per shard, so raw() is
+ not thread-safe, unlike the memmap it replaces. Each shard is read as three ascending streams
+ (exceptions, weights, CSF-fixed) walked expert-ascending."""
def __init__(self, folder, partial_ok=False):
self.dir = pathlib.Path(folder)
@@ -76,7 +88,7 @@ class Checkpoint:
if not files:
raise FileNotFoundError("no .safetensors files in " + str(self.dir))
self.meta: dict[str, tuple[str, str, list[int], int, int]] = {}
- self._mm: dict[str, np.memmap] = {}
+ self._fh: dict[str, "object"] = {}
for fn in files:
path = self.dir / fn
if not path.exists() and partial_ok:
@@ -100,9 +112,14 @@ class Checkpoint:
def raw(self, name) -> np.ndarray:
fn, _, _, a, b = self.meta[name]
- if fn not in self._mm:
- self._mm[fn] = np.memmap(self.dir / fn, dtype=np.uint8, mode="r")
- return self._mm[fn][a:b]
+ f = self._fh.get(fn)
+ if f is None:
+ f = self._fh[fn] = open(self.dir / fn, "rb")
+ buf = np.empty(b - a, dtype=np.uint8)
+ f.seek(a)
+ if f.readinto(buf) != b - a:
+ raise EOFError("%s: shard %s ended early at %d" % (name, fn, a))
+ return buf
def array(self, name, np_dtype) -> np.ndarray:
return self.raw(name).view(np_dtype).reshape(self.shape(name))
diff --git a/tools/test_nvfp4_local_pack.py b/tools/test_nvfp4_local_pack.py
index dd6fbe324..b32c50e16 100644
--- a/tools/test_nvfp4_local_pack.py
+++ b/tools/test_nvfp4_local_pack.py
@@ -1,5 +1,7 @@
"""Tests for tools/nvfp4_local_pack.py: the CSF scale decode, the MXFP8 dequantization, the PLE rows."""
+import json
import pathlib
+import struct
import sys
import numpy as np
@@ -114,3 +116,42 @@ def test_nvfp4_blocks_layout_by_hand():
# within a 16-block: element j in the low nibble, element j+8 in the high one
blk0 = out[0, 4:12]
assert [int(b & 15) for b in blk0] == list(range(8)) and [int(b >> 4) for b in blk0] == list(range(8, 16))
+
+
+def _write_shard(path, tensors):
+ """tensors: name -> (dtype, shape, uint8 bytes), written safetensors-style (data after the header)."""
+ hdr, data, off = {}, bytearray(), 0
+ for name, (dt, shape, buf) in tensors.items():
+ hdr[name] = {"dtype": dt, "shape": shape, "data_offsets": [off, off + len(buf)]}
+ data += buf
+ off += len(buf)
+ hj = json.dumps(hdr).encode("utf-8")
+ hj += b" " * (-len(hj) % 8)
+ with open(path, "wb") as f:
+ f.write(struct.pack("<Q", len(hj)) + hj + data)
+
+
+def test_checkpoint_reads_the_shard_bytes_exactly(tmp_path):
+ a = np.arange(200, dtype=np.uint8)
+ _write_shard(tmp_path / "s1.safetensors", {"t.a": ("U8", [100, 2], a.tobytes())})
+ d = np.arange(4, dtype=np.uint32)
+ c = np.arange(32, dtype=np.uint8)
+ _write_shard(tmp_path / "s2.safetensors", {"t.d": ("U32", [4], d.tobytes()), "t.c": ("U8", [8, 4], c.tobytes())})
+ (tmp_path / "model.safetensors.index.json").write_text(json.dumps({"weight_map": {
+ "t.a": "s1.safetensors", "t.c": "s2.safetensors", "t.d": "s2.safetensors"}}), encoding="utf-8")
+ ck = P.Checkpoint(tmp_path)
+ assert "t.a" in ck and ck.shape("t.a") == [100, 2] and ck.dtype("t.a") == "U8"
+ assert np.array_equal(ck.raw("t.a"), a) # the bytes at the tensor's exact range
+ assert np.array_equal(ck.array("t.c", np.uint8), c.reshape(8, 4))
+ assert np.array_equal(ck.array("t.d", np.uint32), d) # .view through a fresh buffer
+ assert np.array_equal(ck.raw("t.a"), ck.raw("t.a")) # a second read is still correct
+
+
+def test_checkpoint_names_a_truncated_shard(tmp_path):
+ c = np.arange(32, dtype=np.uint8)
+ _write_shard(tmp_path / "s1.safetensors", {"t.c": ("U8", [8, 4], c.tobytes())})
+ p = tmp_path / "s1.safetensors"
+ p.write_bytes(p.read_bytes()[:-8]) # the last tensor's range now ends past EOF
+ ck = P.Checkpoint(tmp_path)
+ with pytest.raises(EOFError):
+ ck.raw("t.c")