Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
# Copyright (c) 2026 the Strata contributors (MIT).
"""tools/dsv41/engram_iq4nl_bench.py - the Phase 1.3 IQ4_NL Engram reader: O_DIRECT row reads + host dequant.
The Engram is a big IQ4_NL table (engram-IQ4_NL.gguf, 110.6 GB in helcig/tmp-oct26). The engine, for one token,
gathers 48 rows of it by hash (tools/dsv41/engram_hash.py gives the row ids), so the reader's job is:
random 2.3 KB row reads straight off the device (O_DIRECT: no page cache, the table does not fit in RAM),
dequantised on the host to f32.
This is the Phase 1.3 skeleton: it does the two things the real run needs to be trusted before it is written into the
engine - it checks the dequant against ggml's own block_IQ4_NL decode, bit for bit, and it measures the two access
patterns (48 rows = one token; a prompt-chunk batch) on whatever file it is pointed at.
The block (ggml's block_iq4_nl, third_party/ggml/ggml-common.h): 32 values, one fp16 scale `d`, 16 packed nibbles:
typedef struct { ggml_half d; uint8_t qs[16]; } block_iq4_nl; // 18 bytes, QK4_NL = 32
y[base + j ] = d * kvalues_iq4nl[qs[j] & 0x0F] (j = 0..15)
y[base + j + 16] = d * kvalues_iq4nl[qs[j] >> 4]
kvalues_iq4nl = {-127,-104,-83,-65,-49,-35,-22,-10, 1, 13, 25, 38, 53, 69, 89, 113}
so a row of n_embd values is (n_embd/32)*18 bytes.
No model is needed to exercise it: `--make-synthetic` writes a valid GGUF with one IQ4_NL tensor (header, one tensor
info, 32-byte alignment, random packed nibbles and small fp16 scales) of the size you ask for, and every mode runs
against that. What the real run needs is listed in REAL RUN below and printed by --help.
# on p1004 (no numpy there: this file is stdlib only)
python3 tools/dsv41/engram_iq4nl_bench.py --make-synthetic /tmp/engram-synth.gguf --size-gb 1.0
python3 tools/dsv41/engram_iq4nl_bench.py --gguf /tmp/engram-synth.gguf --mode all \\
--rows-per-token 48 --batch-tokens 256 --iters 20
REAL RUN (what this needs that p1004 does not have today):
* the file: engram-IQ4_NL.gguf, 110.6 GB. p1004 has 14 GB free on /data and 65 GB on /, so it needs ~50 GB freed
first (P1004.md's /data/launch-test + /data/rel023-amd is 71 GB). It must sit on a real block device:
O_DIRECT works on ext4/xfs/nvme, not on tmpfs or overlayfs.
* the tensor geometry is read from the GGUF header (n_embd, n_rows); nothing is hardcoded here.
* `--ggml-dump`: a 3-line C++ program that calls ggml's dequantize_row_iq4_nl and prints the float bits, linked
against a built libggml-base.a (-I <llama.cpp>/ggml/include -I <llama.cpp>/ggml/src). --build-ggml-dump writes
and compiles it from the source embedded below. Without it the check is the script's scalar reference against
its own fast path, which is still bit-for-bit but not independent of this file.
* numpy: optional. It is not installed on p1004, so dequant here is pure Python (~8k rows/s); the engine's dequant
is the C kernel in ds4_iq.cu / ds4_gg_cpu.cpp. Use --dequant off for the I/O ceiling, --dequant sample (default)
for a measured dequant rate, --dequant all to dequantise the whole batch.
* the real 48-row gather per token is random over 110 GB: IOPS matter as much as MB/s. --mode token is that
pattern; --mode batch is the prompt chunk; --mode seq is the device's sequential ceiling for reference.
"""
from __future__ import annotations
import argparse
import array
import mmap
import os
import random
import struct
import subprocess
import sys
import tempfile
import time
# ---------------------------------------------------------------- ggml IQ4_NL
QK4_NL = 32 # values per block
IQ4NL_ROW_DIV = 32 # n_embd must be a multiple of this
IQ4NL_BLOCK_BYTES = 18 # fp16 d + 16 packed nibbles
GGML_TYPE_IQ4_NL = 20 # ggml.h: IQ2_XXS 16, IQ2_XS 17, IQ3_XXS 18, IQ1_S 19, IQ4_NL 20, IQ3_S 21, IQ2_S 22
KVALUES_IQ4NL = (-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113)
def iq4nl_row_bytes(n_embd: int) -> int:
if n_embd % IQ4NL_ROW_DIV:
raise ValueError(f"n_embd {n_embd} is not a multiple of {IQ4NL_ROW_DIV}")
return (n_embd // QK4_NL) * IQ4NL_BLOCK_BYTES
def dequant_row_iq4nl_ref(row: bytes, n_embd: int) -> array.array:
"""ggml's dequantize_row_iq4_nl, literal: one fp16 scale per block, 16 low nibbles then 16 high ones.
Python floats are doubles; a float32 * a small integer is exact in a double (24 + 7 bits) and array('f') rounds
it once on store, so this reproduces ggml's float32 product bit for bit.
"""
out = array.array("f")
nb = n_embd // QK4_NL
for b in range(nb):
base = b * IQ4NL_BLOCK_BYTES
d = struct.unpack_from("<e", row, base)[0]
qs = row[base + 2:base + IQ4NL_BLOCK_BYTES]
out.extend(d * KVALUES_IQ4NL[q & 0x0F] for q in qs)
out.extend(d * KVALUES_IQ4NL[q >> 4] for q in qs)
return out
def dequant_row_iq4nl_fast(row: bytes, n_embd: int) -> array.array:
"""The same result, with the block's 16 possible values built once: the reader's hot path."""
out = array.array("f")
nb = n_embd // QK4_NL
unpack = struct.unpack_from
kv = KVALUES_IQ4NL
extend = out.extend
for b in range(nb):
base = b * IQ4NL_BLOCK_BYTES
d = unpack("<e", row, base)[0]
tab = (d * kv[0], d * kv[1], d * kv[2], d * kv[3], d * kv[4], d * kv[5], d * kv[6], d * kv[7],
d * kv[8], d * kv[9], d * kv[10], d * kv[11], d * kv[12], d * kv[13], d * kv[14], d * kv[15])
qs = row[base + 2:base + IQ4NL_BLOCK_BYTES]
extend([tab[q & 0x0F] for q in qs])
extend([tab[q >> 4] for q in qs])
return out
def f32_bits(vals) -> bytes:
return array.array("f", vals).tobytes()
# ---------------------------------------------------------------- GGUF reader
class Gguf:
"""Just enough GGUF for one tensor: magic, version, the kv block, the tensor infos, the aligned data start."""
def __init__(self, path: str):
self.path = path
self.size = os.path.getsize(path)
self.tensors: list[dict] = []
self.meta: dict = {}
self.data_start = 0
self.alignment = 32
with open(path, "rb") as f:
magic = f.read(4)
if magic != b"GGUF":
raise ValueError(f"{path}: not a GGUF (magic {magic!r})")
version = struct.unpack("<I", f.read(4))[0]
if version not in (2, 3):
raise ValueError(f"{path}: GGUF v{version}, this reads v2 and v3")
self.version = version
n_tensors, n_kv = struct.unpack("<QQ", f.read(16))
for _ in range(n_kv):
k, v = self._kv(f)
self.meta[k] = v
self.alignment = int(self.meta.get("general.alignment", 32)) or 32
for _ in range(n_tensors):
name = self._str(f)
n_dims = struct.unpack("<I", f.read(4))[0]
dims = list(struct.unpack(f"<{n_dims}Q", f.read(8 * n_dims)))
ttype = struct.unpack("<I", f.read(4))[0]
off = struct.unpack("<Q", f.read(8))[0]
self.tensors.append({"name": name, "dims": dims, "type": ttype, "offset": off})
pos = f.tell()
self.data_start = (pos + self.alignment - 1) // self.alignment * self.alignment
@staticmethod
def _str(f) -> str:
n = struct.unpack("<Q", f.read(8))[0]
return f.read(n).decode("utf-8", "replace")
_SCALAR = {0: "<B", 1: "<b", 2: "<H", 3: "<h", 4: "<I", 5: "<i", 6: "<f", 7: "<B", 10: "<Q", 11: "<q", 12: "<d"}
def _kv(self, f):
key = self._str(f)
vtype = struct.unpack("<I", f.read(4))[0]
if vtype == 8: # string
return key, self._str(f)
if vtype == 9: # array
atype = struct.unpack("<I", f.read(4))[0]
n = struct.unpack("<Q", f.read(8))[0]
if atype == 8:
return key, [self._str(f) for _ in range(n)]
fmt = self._SCALAR[atype]
return key, list(struct.unpack(f"<{n}{fmt[1]}", f.read(struct.calcsize(fmt) * n)))
fmt = self._SCALAR[vtype]
return key, struct.unpack(fmt, f.read(struct.calcsize(fmt)))[0]
def tensor(self, name: str | None = None) -> dict:
if name is not None:
for t in self.tensors:
if t["name"] == name:
return t
raise KeyError(f"{self.path}: no tensor {name!r}; have {[t['name'] for t in self.tensors]}")
for t in self.tensors:
if t["type"] == GGML_TYPE_IQ4_NL:
return t
raise KeyError(f"{self.path}: no IQ4_NL tensor")
def geometry(self, name: str | None = None) -> tuple[int, int, int]:
"""(n_embd, n_rows, row_bytes) for an IQ4_NL table; dims[0] is the row length."""
t = self.tensor(name)
if t["type"] != GGML_TYPE_IQ4_NL:
raise ValueError(f"tensor {t['name']!r} is ggml type {t['type']}, not {GGML_TYPE_IQ4_NL} (IQ4_NL)")
n_embd = int(t["dims"][0])
n_rows = 1
for d in t["dims"][1:]:
n_rows *= int(d)
return n_embd, n_rows, iq4nl_row_bytes(n_embd)
def row_offset(self, t: dict, row: int) -> int:
"""Absolute file offset of a row: the tensor's offset is relative to the data section."""
_, _, rb = self.geometry(t["name"])
return self.data_start + t["offset"] + row * rb
# ---------------------------------------------------------------- O_DIRECT rows
class RowReader:
"""Aligned, unbuffered reads. A row is 18*(n_embd/32) bytes (2304 for n_embd 4096), which is not a multiple of
the device's 512-byte block, so a row read is a 512-aligned window that is then sliced: the device moves up to
511 extra bytes per row and MB/s is reported both ways (payload and what the device was asked for).
"""
def __init__(self, path: str, align: int = 512, odirect: bool = True):
self.path = path
self.align = align
self.size = os.path.getsize(path)
flags = os.O_RDONLY | (getattr(os, "O_DIRECT", 0) if odirect else 0)
self.odirect = bool(odirect and getattr(os, "O_DIRECT", 0))
try:
self.fd = os.open(path, flags)
except OSError as e:
if not self.odirect:
raise
print(f" (O_DIRECT unavailable on {path}: {e}; falling back to buffered reads)", file=sys.stderr)
self.odirect = False
self.fd = os.open(path, os.O_RDONLY)
self._buf = None
self._buf_len = 0
self.device_bytes = 0
def _buffer(self, n: int):
if self._buf is None or self._buf_len < n:
self._buf = mmap.mmap(-1, n) # page aligned, which is a multiple of any 512/4096 sector
self._buf_len = n
return self._buf
def read(self, off: int, length: int) -> bytes:
"""The `length` bytes at `off`, in one aligned pread."""
if self.odirect:
a = self.align
start = off // a * a
end = min((off + length + a - 1) // a * a, self.size)
n = end - start
if n <= 0:
raise ValueError(f"read past end: off {off} len {length} size {self.size}")
buf = self._buffer(n)
got = os.preadv(self.fd, [memoryview(buf)[:n]], start)
self.device_bytes += n
if got != n:
raise OSError(f"short O_DIRECT read at {start}: {got} of {n}")
return bytes(memoryview(buf)[off - start:off - start + length])
data = os.pread(self.fd, length, off)
self.device_bytes += length
if len(data) != length:
raise OSError(f"short read at {off}: {len(data)} of {length}")
return data
def close(self):
if self._buf is not None:
self._buf.close()
self._buf = None
os.close(self.fd)
# ---------------------------------------------------------------- synthetic file
def make_synthetic(path: str, size_gb: float, n_embd: int = 4096, seed: int = 1, verbose: bool = True) -> tuple[int, int]:
"""A valid GGUF v3 with one IQ4_NL tensor: header, one tensor info, 32-byte alignment, then the data.
The blocks are real blocks: fp16 scales in 0.002 .. 0.017 and random packed nibbles, so the dequant is exercised
on the file the benchmarks read. Returns (n_rows, file bytes).
"""
rb = iq4nl_row_bytes(n_embd)
n_rows = max(1, int(size_gb * 1e9) // rb)
rng = random.Random(seed)
scales = [0.002 + 0.001 * i for i in range(16)]
# 8 block templates per scale cycle, each 18 bytes: the fp16 d and 16 random nibble bytes
blobs = []
for s in scales:
parts = []
for _ in range(8):
parts.append(struct.pack("<e", s) + bytes(rng.getrandbits(8) for _ in range(16)))
blobs.append(b"".join(parts)) # 144 bytes = 8 whole blocks, so a repeat keeps block alignment
header = bytearray()
header += b"GGUF"
header += struct.pack("<I", 3)
header += struct.pack("<QQ", 1, 3)
for k, v in (("general.architecture", "engram"), ("general.name", "p1004-synthetic-iq4nl")):
header += struct.pack("<Q", len(k)) + k.encode()
header += struct.pack("<I", 8)
header += struct.pack("<Q", len(v)) + v.encode()
header += struct.pack("<Q", len("general.alignment")) + b"general.alignment"
header += struct.pack("<I", 4) + struct.pack("<I", 32)
name = b"engram.iq4nl"
header += struct.pack("<Q", len(name)) + name
header += struct.pack("<I", 2) + struct.pack("<QQ", n_embd, n_rows)
header += struct.pack("<I", GGML_TYPE_IQ4_NL) + struct.pack("<Q", 0)
header += b"\x00" * ((-len(header)) % 32)
if verbose:
print(f"synthetic: n_embd {n_embd} row_bytes {rb} n_rows {n_rows} data {(n_rows * rb) / 1e9:.3f} GB -> {path}")
chunk_rows = 64
written = 0
with open(path, "wb", buffering=1024 * 1024) as f:
f.write(header)
k = 0
t0 = time.time()
while written < n_rows:
rows = min(chunk_rows, n_rows - written)
blob = blobs[k % len(blobs)]
k += 1
buf = (blob * (rows * rb // len(blob) + 1))[:rows * rb]
f.write(buf)
written += rows
if verbose and written % (chunk_rows * 512) == 0:
print(f"\r {written * rb / 1e9:.2f} GB", end="", file=sys.stderr)
total = os.path.getsize(path)
if verbose:
print(f"\r wrote {total / 1e9:.3f} GB in {time.time() - t0:.1f} s" + " " * 20, file=sys.stderr)
return n_rows, total
# ---------------------------------------------------------------- the ggml cross-check
_GGML_DUMP_SRC = r"""
// Read one IQ4_NL row from stdin, print ggml's own dequantise: one float per line, 8 hex digits of its bits.
#include <cstdio>
#include <cstdlib>
#include <vector>
extern "C" void dequantize_row_iq4_nl(const void * x, float * y, int64_t k);
int main(int argc, char ** argv) {
if (argc < 2) { std::fprintf(stderr, "usage: iq4nl_dump N_EMBD < row.bin\n"); return 2; }
const int n = std::atoi(argv[1]);
const size_t rb = (size_t) (n / 32) * 18;
std::vector<unsigned char> row(rb);
if (std::fread(row.data(), 1, rb, stdin) != rb) { std::fprintf(stderr, "short read\n"); return 2; }
std::vector<float> y((size_t) n);
dequantize_row_iq4_nl(row.data(), y.data(), n);
for (int i = 0; i < n; ++i) {
unsigned int b; __builtin_memcpy(&b, &y[i], 4);
std::printf("%08x\n", b);
}
return 0;
}
"""
def build_ggml_dump(build_dir: str, out: str | None = None) -> str:
"""Compile the dumper against a built llama.cpp tree (ggml headers + libggml-base.a)."""
src_root = os.path.join(build_dir, "_deps", "strata_llamacpp-src", "ggml")
lib = os.path.join(build_dir, "ggml", "src", "libggml-base.a")
for p in (src_root, lib):
if not os.path.exists(p):
raise FileNotFoundError(f"{p} not found; pass --ggml-build-dir with a tree that built ggml")
out = out or os.path.join(tempfile.gettempdir(), "iq4nl_dump")
with open(out + ".cpp", "w") as f:
f.write(_GGML_DUMP_SRC)
cmd = ["g++", "-O2", "-std=c++17", "-I", os.path.join(src_root, "include"), "-I", os.path.join(src_root, "src"),
out + ".cpp", lib, "-o", out, "-lpthread", "-lm"]
subprocess.run(cmd, check=True)
return out
def ggml_dequant_bits(dumper: str, row: bytes, n_embd: int) -> bytes:
"""ggml's dequant of one row, as raw float32 bytes (the dumper prints the bits as %08x, so repack little-endian)."""
p = subprocess.run([dumper, str(n_embd)], input=row, stdout=subprocess.PIPE, check=True)
hx = b"".join(p.stdout.split()).decode()
if len(hx) != 8 * n_embd:
raise RuntimeError(f"the dumper printed {len(hx)} hex digits for {n_embd} values; check its n_embd argument")
return b"".join(struct.pack("<I", int(hx[i:i + 8], 16)) for i in range(0, len(hx), 8))
# ---------------------------------------------------------------- benchmarks
def _fmt(x, unit=""):
return f"{x:,.1f}{unit}" if isinstance(x, float) else f"{x:,}{unit}"
def _dequant_rate(rows, n_embd: int, limit: int) -> tuple[float, int]:
"""(seconds, rows) for the host dequant of rows already in memory: the CPU side, apart from the I/O."""
sel = rows[:limit]
if not sel:
return 0.0, 0
t0 = time.perf_counter()
for row in sel:
dequant_row_iq4nl_fast(row, n_embd)
return time.perf_counter() - t0, len(sel)
def bench_token(gg: Gguf, t: dict, n_embd: int, n_rows: int, rb: int, args) -> dict:
"""48 random rows per token, one aligned pread each: the decode-time pattern. The reads are timed alone and
the dequant is timed afterwards on the rows already in RAM, so neither number hides the other."""
rnd = random.Random(args.seed)
tok_rows = [rnd.sample(range(n_rows), args.rows_per_token) for _ in range(args.iters)]
rd = RowReader(gg.path, align=args.align, odirect=args.odirect)
blobs = []
try:
rd.read(gg.row_offset(t, tok_rows[0][0]), rb) # warm the path, not the cache (O_DIRECT)
t0 = time.perf_counter()
for rows in tok_rows:
for r in rows:
blobs.append(rd.read(gg.row_offset(t, r), rb))
dt = time.perf_counter() - t0
finally:
dev = rd.device_bytes
rd.close()
n = len(blobs)
out = {
"mode": f"token x{args.iters} ({args.rows_per_token} rows each)",
"seconds": dt,
"tokens_s": args.iters / dt,
"rows_s": n / dt,
"payload_mb_s": n * rb / dt / 1e6,
"device_mb_s": dev / dt / 1e6,
"reads": n,
"read_us": dt / n * 1e6,
"dequant": args.dequant,
}
if args.dequant != "off":
lim = n if args.dequant == "all" else args.rows_per_token * min(args.iters, 2)
ddt, dn = _dequant_rate(blobs, n_embd, lim)
out["dequant_rows"], out["dequant_rows_s"] = dn, dn / ddt if ddt else 0.0
return out
def bench_batch(gg: Gguf, t: dict, n_embd: int, n_rows: int, rb: int, args) -> dict:
"""A prompt chunk: batch_tokens x 48 rows, gathered at random like the engine will, plus the same rows read as
contiguous runs of the table (the best case: a chunk's hashes are unrelated, so the real pattern is the random
one). Reads and dequant are timed apart, as in bench_token."""
rnd = random.Random(args.seed + 1)
n = args.batch_tokens * args.rows_per_token
rows = [rnd.randrange(n_rows) for _ in range(n)]
rd = RowReader(gg.path, align=args.align, odirect=args.odirect)
blobs = []
try:
t0 = time.perf_counter()
for r in rows:
blobs.append(rd.read(gg.row_offset(t, r), rb))
dt = time.perf_counter() - t0
finally:
dev = rd.device_bytes
rd.close()
out = {
"mode": f"batch gathered {args.batch_tokens} tokens = {n} rows",
"seconds": dt,
"tokens_s": args.batch_tokens / dt,
"rows_s": n / dt,
"payload_mb_s": n * rb / dt / 1e6,
"device_mb_s": dev / dt / 1e6,
"reads": n,
"read_us": dt / n * 1e6,
"dequant": args.dequant,
}
if args.dequant != "off":
lim = n if args.dequant == "all" else args.rows_per_token * 3
ddt, dn = _dequant_rate(blobs, n_embd, lim)
out["dequant_rows"], out["dequant_rows_s"] = dn, dn / ddt if ddt else 0.0
del blobs
# the same rows, contiguous: merge the sorted ids into runs and read each run with one pread
srows = sorted(rows)
runs = []
lo = prev = srows[0]
for r in srows[1:]:
if r == prev + 1:
prev = r
continue
runs.append((lo, prev))
lo = prev = r
runs.append((lo, prev))
rd = RowReader(gg.path, align=args.align, odirect=args.odirect)
try:
t0 = time.perf_counter()
got = 0
for lo, hi in runs:
got += len(rd.read(gg.row_offset(t, lo), (hi - lo + 1) * rb))
dt2 = time.perf_counter() - t0
finally:
dev2 = rd.device_bytes
rd.close()
out["contig"] = {
"mode": f"batch contiguous {args.batch_tokens} tokens = {len(runs)} runs",
"seconds": dt2,
"tokens_s": args.batch_tokens / dt2,
"rows_s": got / rb / dt2,
"payload_mb_s": got / dt2 / 1e6,
"device_mb_s": dev2 / dt2 / 1e6,
"reads": len(runs),
"read_us": dt2 / max(1, len(runs)) * 1e6,
"dequant": "off",
}
return out
def bench_seq(gg: Gguf, rb: int, args) -> dict:
"""The device's sequential ceiling through the same O_DIRECT path (rows/s is that ceiling in 2.3 KB rows)."""
mb = min(args.seq_mb, gg.size // (1024 * 1024))
rd = RowReader(gg.path, align=args.align, odirect=args.odirect)
chunk = args.align * 1024 # 512 KiB
try:
rd.read(0, chunk)
t0 = time.perf_counter()
off = 0
total = 0
while total < mb * 1024 * 1024:
n = min(chunk, gg.size - off)
if n <= 0:
break
rd.read(off, n)
off += n
total += n
dt = time.perf_counter() - t0
finally:
dev = rd.device_bytes
rd.close()
return {"mode": f"sequential {total / 1e6:.0f} MB, {chunk // 1024} KiB reads", "seconds": dt,
"rows_s": total / rb / dt, "payload_mb_s": total / dt / 1e6, "device_mb_s": dev / dt / 1e6,
"reads": total // chunk, "read_us": dt / max(1, total // chunk) * 1e6, "dequant": "off"}
# ---------------------------------------------------------------- main
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--gguf", help="the Engram GGUF (or a synthetic one)")
ap.add_argument("--tensor", default=None, help="tensor name (default: the first IQ4_NL tensor)")
ap.add_argument("--mode", default="all", choices=["header", "token", "batch", "seq", "all"])
ap.add_argument("--rows-per-token", type=int, default=48)
ap.add_argument("--batch-tokens", type=int, default=256)
ap.add_argument("--iters", type=int, default=20)
ap.add_argument("--seq-mb", type=int, default=512)
ap.add_argument("--align", type=int, default=512, help="O_DIRECT alignment (the device's logical block size)")
ap.add_argument("--odirect", action="store_true", default=True)
ap.add_argument("--no-odirect", dest="odirect", action="store_false")
ap.add_argument("--dequant", default="sample", choices=["off", "sample", "all"])
ap.add_argument("--seed", type=int, default=1)
ap.add_argument("--verify-rows", type=int, default=8, help="rows checked bit-exactly against ggml")
ap.add_argument("--ggml-dump", help="a compiled ggml IQ4_NL dequant dumper (see --build-ggml-dump)")
ap.add_argument("--build-ggml-dump", action="store_true", help="compile the dumper from the source embedded here")
ap.add_argument("--ggml-build-dir", default="/data/dsv41-build", help="a Strata build tree with ggml built")
ap.add_argument("--make-synthetic", metavar="PATH", help="write a synthetic IQ4_NL GGUF and exit (unless --gguf)")
ap.add_argument("--size-gb", type=float, default=1.0)
ap.add_argument("--n-embd", type=int, default=4096)
args = ap.parse_args(argv)
if args.make_synthetic:
make_synthetic(args.make_synthetic, args.size_gb, args.n_embd, args.seed)
if not args.gguf:
args.gguf = args.make_synthetic
if args.mode == "all":
args.mode = "header"
if not args.gguf:
ap.error("--gguf or --make-synthetic is required")
gg = Gguf(args.gguf)
t = gg.tensor(args.tensor)
n_embd, n_rows, rb = gg.geometry(args.tensor)
print(f"{gg.path}: {gg.size / 1e9:.3f} GB, GGUF v{gg.meta.get('general.version', 3)}, {len(gg.tensors)} tensor(s), "
f"alignment {gg.alignment}, data at {gg.data_start}")
print(f" tensor {t['name']!r}: type {t['type']} (IQ4_NL), dims {t['dims']}, n_embd {n_embd}, n_rows {n_rows}, "
f"row_bytes {rb}, table {(n_rows * rb) / 1e9:.3f} GB")
if t["type"] != GGML_TYPE_IQ4_NL:
print(f" not an IQ4_NL tensor: nothing to read", file=sys.stderr)
return 2
# the bit-exact check, always: the scalar reference against the fast path, then against ggml itself
rnd = random.Random(args.seed + 7)
dumper = args.ggml_dump
if args.build_ggml_dump:
dumper = build_ggml_dump(args.ggml_build_dir, dumper)
print(f" built the ggml dumper: {dumper}")
rd = RowReader(gg.path, align=args.align, odirect=args.odirect)
bad = 0
checked = 0
try:
for _ in range(args.verify_rows):
r = rnd.randrange(n_rows)
row = rd.read(gg.row_offset(t, r), rb)
ref = dequant_row_iq4nl_ref(row, n_embd)
fast = dequant_row_iq4nl_fast(row, n_embd)
if f32_bits(ref) != f32_bits(fast):
bad += 1
print(f" row {r}: the scalar reference and the fast path differ", file=sys.stderr)
if dumper:
gw = ggml_dequant_bits(dumper, row, n_embd)
if gw != f32_bits(ref):
bad += 1
print(f" row {r}: ggml's dequant differs from the reference", file=sys.stderr)
checked += 1
finally:
rd.close()
print(f" bit-exact: {checked - bad}/{checked} rows" + (" against ggml's dequantize_row_iq4_nl" if dumper else
" (reference vs fast path; no --ggml-dump given)")
+ (" FAIL" if bad else " OK"))
if bad:
return 1
if args.mode == "header":
return 0
results = []
if args.mode in ("seq", "all"):
results.append(bench_seq(gg, rb, args))
if args.mode in ("token", "all"):
results.append(bench_token(gg, t, n_embd, n_rows, rb, args))
if args.mode in ("batch", "all"):
b = bench_batch(gg, t, n_embd, n_rows, rb, args)
results.append(b["contig"])
results.append(b)
results.sort(key=lambda x: -x["payload_mb_s"])
print()
print(f" {'pattern':44s} {'s':>8s} {'rows/s':>11s} {'tokens/s':>9s} {'MB/s payload':>12s} {'MB/s device':>11s} "
f"{'us/read':>8s} {'dequant rows/s':>14s}")
for r in results:
tok = f"{r['tokens_s']:.1f}" if "tokens_s" in r else "-"
dq = f"{r['dequant_rows_s']:,.0f} ({r['dequant_rows']} rows)" if r.get("dequant_rows_s") else r["dequant"]
print(f" {r['mode']:44s} {r['seconds']:8.3f} {r['rows_s']:11,.0f} {tok:>9s} {r['payload_mb_s']:12,.1f} "
f"{r['device_mb_s']:11,.1f} {r['read_us']:8,.1f} {dq:>14s}")
print(f"\n note: O_DIRECT {'on' if args.odirect else 'off'}, alignment {args.align} B; "
f"payload MB/s counts row bytes only, device MB/s what the reads actually asked for")
print(" the dequant rate is measured apart from the reads (rows/s of the host dequant); the engine's dequant is "
"the C kernel, not this Python")
return 0
if __name__ == "__main__":
sys.exit(main())