Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: device kernels for IQ1_S, IQ4_NL and IQ1_M (the three IST-DAS types that had none)
Niko12215c523d55f7d3
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex 9112befe5..93c1ef26c 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -34,8 +34,8 @@ template <> struct Ty<kIQ3_XXS> { static constexpr int B = 98, TAB = 256, MIN =template <> struct Ty<kIQ3_S> { static constexpr int B = 110, TAB = 512, MIN = 0; };template <> struct Ty<kIQ4_XS> { static constexpr int B = 136, TAB = 4, MIN = 0; };template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; };-// The types with a known layout and no device kernel (the CPU lane's ggml dots take them): their row bytes are needed to-// lay a pack out, nothing else. Ty<> gives the block; the kernels are never instantiated for them.+// The 1.5- and 1.75-bit types (IQ1_S, IQ1_M: an 11-bit index into iq1s_grid_gpu, whose 8 values are 4-bit codes) and+// IQ4_NL (32 values a block: its codebook's 16 int8 values are the first 4 words of its table)template <> struct Ty<kIQ1_S> { static constexpr int B = 50, TAB = 2048, MIN = 0; };template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; };template <> struct Ty<kIQ4_NL> { static constexpr int B = 18, TAB = 16, MIN = 0; };@@ -48,11 +48,12 @@ template <> struct Ty<kQ6_K> { static constexpr int B = 210, TAB = 0, MIN = 0; }// sets take it tooconstexpr int kMXFP4 = 39;template <> struct Ty<kMXFP4> { static constexpr int B = 17, TAB = 0, MIN = 0; };-template <int T> constexpr int kQK = T == kMXFP4 ? 32 : 256; // values a block+template <int T> constexpr int kQK = T == kMXFP4 || T == kIQ4_NL ? 32 : 256; // values a blockstatic_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_xs) == 74 && sizeof(block_iq2_s) == 82 &&sizeof(block_iq3_xxs) == 98 &&sizeof(block_iq3_s) == 110 && sizeof(block_iq4_xs) == 136 && sizeof(block_q4_K) == 144 &&- sizeof(block_q2_K) == 84 && sizeof(block_q3_K) == 110,+ sizeof(block_q2_K) == 84 && sizeof(block_q3_K) == 110 && sizeof(block_iq1_s) == 50 &&+ sizeof(block_iq1_m) == 56 && sizeof(block_iq4_nl) == 18,"ggml's block sizes");template <int T>@@ -61,7 +62,7 @@ __host__ __device__ __forceinline__ size_t row_bytes_t(int k) { return (size_t)// the type's codebook into shared memory (u32 words; IQ4_XS: its 16 values, from bytes - an int8 table's alignment)template <int T>__device__ __forceinline__ void load_tab(uint32_t* tab) {- if constexpr (T == kIQ4_XS) {+ if constexpr (T == kIQ4_XS || T == kIQ4_NL) {if (threadIdx.x < 4) {const uint8_t* kv = (const uint8_t*) kvalues_iq4nl + 4 * threadIdx.x;tab[threadIdx.x] = (uint32_t) kv[0] | (uint32_t) kv[1] << 8 | (uint32_t) kv[2] << 16 | (uint32_t) kv[3] << 24;@@ -69,8 +70,9 @@ __device__ __forceinline__ void load_tab(uint32_t* tab) {} else if constexpr (Ty<T>::TAB > 0) {const uint32_t* src = T == kIQ2_XXS ? (const uint32_t*) iq2xxs_grid: T == kIQ2_XS ? (const uint32_t*) iq2xs_grid- : T == kIQ2_S ? (const uint32_t*) iq2s_grid+ : T == kIQ2_S ? (const uint32_t*) iq2s_grid: T == kIQ3_XXS ? (const uint32_t*) iq3xxs_grid+ : T == kIQ1_S || T == kIQ1_M ? (const uint32_t*) iq1s_grid_gpu: (const uint32_t*) iq3s_grid;for (int i = threadIdx.x; i < Ty<T>::TAB; i += blockDim.x) tab[i] = src[i];}@@ -105,6 +107,19 @@ __device__ __forceinline__ int2 tab16(uint32_t q4, const uint32_t* t) { // vec}return make_int2((int) __byte_perm(tmp[0], tmp[1], 0x6420), (int) __byte_perm(tmp[0], tmp[1], 0x7531));}+// iq1s_grid_gpu's 8 values in 4 bytes, 4 bits each (low nibbles: values 0..3, high: 4..7; 0 is -1, 2 is +1) -> the 4+// values as int8 codes 8 (n - 1) + 1, or - 1 when the group's delta is negative: ggml's weight dl ((n - 1) +- 0.125), whose+// dl (2 sc + 1) / 8 is this kernel's per-sub-block scale, is then d * (s0 lo + s1 hi) exactly+__device__ __forceinline__ uint32_t iq1_codes(uint32_t n, bool neg) {+ const uint32_t c = (uint32_t) __vsub4((int) (n * 8u), 0x07070707); // (n * 8 - 7) in each byte: no carry (n <= 15)+ return neg ? (uint32_t) __vsub4((int) c, 0x02020202) : c;+}+// IQ1_M has no d field: its F16 scale is the top nibbles of its four u16 scales (ggml's dequantize_row_iq1_m)+__device__ __forceinline__ uint16_t iq1m_d(const uint8_t* b) {+ const uint32_t s0 = (uint32_t) b[48] | (uint32_t) b[49] << 8, s1 = (uint32_t) b[50] | (uint32_t) b[51] << 8;+ const uint32_t s2 = (uint32_t) b[52] | (uint32_t) b[53] << 8, s3 = (uint32_t) b[54] | (uint32_t) b[55] << 8;+ return (uint16_t) ((s0 >> 12) | ((s1 >> 8) & 0x00f0u) | ((s2 >> 4) & 0x0f00u) | (s3 & 0xf000u));+}__device__ __forceinline__ int q3k_scale(const uint8_t* sc, int is) {const int lo = is < 8 ? sc[is] & 15 : sc[is - 8] >> 4;const int hi = (sc[8 + (is & 3)] >> (2 * (is >> 2))) & 3;@@ -223,6 +238,45 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,}s.s0 = s.s1 = ls - 32;s.d = h2f(b);+ } else if constexpr (T == kIQ1_S) { // d, qs[32] (4 a 32-value sub-block), qh[8] u16: 11-bit grid, 3-bit scale, delta+ const uint8_t* qs = b + 2;+ const uint16_t qh = ((const uint16_t*) (b + 34))[ib];+ const bool neg = (qh & 0x8000u) != 0;+#pragma unroll+ for (int l = 0; l < 4; ++l) {+ const uint32_t g = tab[qs[4 * ib + l] | (((qh >> (3 * l)) & 7u) << 8)];+ s.q[2 * l] = (int) iq1_codes(g & 0x0F0F0F0Fu, neg);+ s.q[2 * l + 1] = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);+ }+ s.s0 = s.s1 = 2 * (int) ((qh >> 12) & 7u) + 1;+ s.d = h2f(b) * 0.125f;+ } else if constexpr (T == kIQ1_M) { // qs[32], qh[16], scales[8]: no d field; 11-bit grid, two 3-bit scales a 32+ const uint8_t* qs = b + 4 * ib;+ const uint8_t qh0 = b[32 + 2 * ib], qh1 = b[33 + 2 * ib];+ const uint16_t sc = ((const uint16_t*) (b + 48))[ib >> 1];+ const int sh = 6 * (ib & 1);+ const uint32_t ii[4] = {qs[0] | ((uint32_t) (qh0 << 8) & 0x700u), qs[1] | ((uint32_t) (qh0 << 4) & 0x700u),+ qs[2] | ((uint32_t) (qh1 << 8) & 0x700u), qs[3] | ((uint32_t) (qh1 << 4) & 0x700u)};+ const bool neg[4] = {(qh0 & 0x08u) != 0, (qh0 & 0x80u) != 0, (qh1 & 0x08u) != 0, (qh1 & 0x80u) != 0};+#pragma unroll+ for (int l = 0; l < 4; ++l) {+ const uint32_t g = tab[ii[l]];+ s.q[2 * l] = (int) iq1_codes(g & 0x0F0F0F0Fu, neg[l]);+ s.q[2 * l + 1] = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg[l]);+ }+ s.s0 = 2 * (int) ((sc >> sh) & 7u) + 1;+ s.s1 = 2 * (int) ((sc >> (sh + 3)) & 7u) + 1;+ s.d = __half2float(__ushort_as_half(iq1m_d(b))) * 0.125f;+ } else if constexpr (T == kIQ4_NL) { // d, qs[16]: a block of 32 values, its codebook's int8 a nibble+ const uint8_t* qs = b + 2; // (18-byte blocks: even, so ld2's 2-byte alignment - not ld4's 4)+#pragma unroll+ for (int t = 0; t < 4; ++t) {+ const int2 v = tab16(ld2(qs + 4 * t), tab);+ s.q[t] = v.x;+ s.q[4 + t] = v.y;+ }+ s.s0 = s.s1 = 1;+ s.d = h2f(b);} else if constexpr (T == kQ4_K) {int sc, m;q4k_scale(b + 4, ib, sc, m);@@ -316,7 +370,7 @@ __device__ __forceinline__ float sub_dot(const Sub& s, const int4& xa, const inttemplate <int T>__device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, int p, const uint32_t* tab, float* w) {const int ib = kb & 7;- const uint8_t* b = row + (size_t) (kb >> 3) * Ty<T>::B;+ const uint8_t* b = row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B;if constexpr (T == kIQ2_XXS) {const uint8_t* q = b + 2 + 8 * ib;const uint32_t a1 = ld2(q + 4);@@ -378,6 +432,35 @@ __device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, inconst float dl = h2f(b) * (float) (ls - 32);const uint8_t* qs = b + 8 + 16 * ib + 8 * (p & 1);const int8_t* kv = (const int8_t*) tab;+#pragma unroll+ for (int j = 0; j < 8; ++j) w[j] = dl * (float) kv[p < 2 ? qs[j] & 15 : qs[j] >> 4];+ } else if constexpr (T == kIQ1_S) {+ const uint16_t qh = ((const uint16_t*) (b + 34))[ib];+ const uint32_t g = tab[b[2 + 4 * ib + p] | (((qh >> (3 * p)) & 7u) << 8)];+ const uint32_t c0 = iq1_codes(g & 0x0F0F0F0Fu, (qh & 0x8000u) != 0);+ const uint32_t c1 = iq1_codes((g >> 4) & 0x0F0F0F0Fu, (qh & 0x8000u) != 0);+ const float dl = h2f(b) * (float) (2 * (int) ((qh >> 12) & 7u) + 1) * 0.125f;+#pragma unroll+ for (int j = 0; j < 4; ++j) w[j] = dl * (float) (int8_t) (c0 >> (8 * j));+#pragma unroll+ for (int j = 0; j < 4; ++j) w[4 + j] = dl * (float) (int8_t) (c1 >> (8 * j));+ } else if constexpr (T == kIQ1_M) {+ const uint8_t* qs = b + 4 * ib;+ const uint8_t qh0 = b[32 + 2 * ib], qh1 = b[33 + 2 * ib], qh = p < 2 ? qh0 : qh1;+ const uint16_t sc = ((const uint16_t*) (b + 48))[ib >> 1];+ const uint32_t g = tab[qs[p] | ((uint32_t) (qh << (p & 1 ? 4 : 8)) & 0x700u)];+ const bool neg = (p & 1 ? qh & 0x80u : qh & 0x08u) != 0;+ const uint32_t c0 = iq1_codes(g & 0x0F0F0F0Fu, neg), c1 = iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);+ const int sh = (int) ((sc >> (6 * (ib & 1) + (p < 2 ? 0 : 3))) & 7u);+ const float dl = __half2float(__ushort_as_half(iq1m_d(b))) * (float) (2 * sh + 1) * 0.125f;+#pragma unroll+ for (int j = 0; j < 4; ++j) w[j] = dl * (float) (int8_t) (c0 >> (8 * j));+#pragma unroll+ for (int j = 0; j < 4; ++j) w[4 + j] = dl * (float) (int8_t) (c1 >> (8 * j));+ } else if constexpr (T == kIQ4_NL) {+ const float dl = h2f(b);+ const uint8_t* qs = b + 2 + 8 * (p & 1);+ const int8_t* kv = (const int8_t*) tab;#pragma unrollfor (int j = 0; j < 8; ++j) w[j] = dl * (float) kv[p < 2 ? qs[j] & 15 : qs[j] >> 4];} else if constexpr (T == kQ4_K) {@@ -858,6 +941,9 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keepscase kIQ3_XXS: F(kIQ3_XXS); break; \case kIQ3_S: F(kIQ3_S); break; \case kIQ4_XS: F(kIQ4_XS); break; \+ case kIQ1_S: F(kIQ1_S); break; \+ case kIQ1_M: F(kIQ1_M); break; \+ case kIQ4_NL: F(kIQ4_NL); break; \case kQ4_K: F(kQ4_K); break; \case kQ2_K: F(kQ2_K); break; \case kQ3_K: F(kQ3_K); break; \@@ -976,6 +1062,7 @@ bool supported(int t) {// the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)switch (t) {case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS:+ case kIQ1_S: case kIQ1_M: case kIQ4_NL:case kQ4_K: case kQ5_K: case kQ6_K: case kQ2_K: case kQ3_K:return true;default:@@ -1004,7 +1091,7 @@ int block_bytes(int t) {case kQ6_K: return Ty<kQ6_K>::B;case kQ2_K: return Ty<kQ2_K>::B;case kQ3_K: return Ty<kQ3_K>::B;- case kIQ1_S: return Ty<kIQ1_S>::B; // (a known layout, no device kernel: the CPU lane's)+ case kIQ1_S: return Ty<kIQ1_S>::B;case kIQ1_M: return Ty<kIQ1_M>::B;case kIQ4_NL: return Ty<kIQ4_NL>::B;default: return 0;diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 0cf49c957..ff88accba 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -34,8 +34,8 @@ constexpr bool is_tq(int type) { return tq_kbits(type) > 0; }/// Whether the device kernels take this ggml type.bool supported(int type);-/// Whether the type's block layout is known (its row bytes can be computed) - a superset of supported(): a type the CPU-/// lane's ggml dots take but no device kernel does (IQ1_S, IQ1_M, IQ4_NL today) is known and must stay out of VRAM.+/// Whether the type's block layout is known (its row bytes can be computed). It covered more than supported() while+/// IQ1_S, IQ1_M and IQ4_NL had no device kernel; they have one now, so both take the same types.bool known(int type);/// Values a block of the type holds (256; the trellis types 128; 0: not one of these).int block_values(int type);diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cppindex a7bb1894e..8804d30d2 100644--- a/tests/dsv41/iq_expert_test.cpp+++ b/tests/dsv41/iq_expert_test.cpp@@ -90,9 +90,17 @@ static Mat make_mat(std::mt19937_64& rng, int type, int rows, int k) {const size_t nblk = (size_t) rows * (k / bv);m.bytes.resize(nblk * (size_t) bb);for (auto& b : m.bytes) b = (uint8_t) rng();- for (size_t b = 0; b < nblk; ++b) { // a sane F16 scale in the first two bytes (every type here has d at offset 0)+ for (size_t b = 0; b < nblk; ++b) { // a sane F16 scale: at offset 0 (every type here but IQ1_M has d there)const uint16_t d = f2h(0.02f + 0.0002f * (float) (rng() % 600));- std::memcpy(&m.bytes[b * (size_t) bb], &d, 2);+ uint8_t* p = &m.bytes[b * (size_t) bb];+ if (type == 29) { // block_iq1_m has no d field: its F16 scale is the top nibbles of its four u16 scales+ p[49] = (uint8_t) ((p[49] & 0x0f) | ((d & 0x000f) << 4));+ p[51] = (uint8_t) ((p[51] & 0x0f) | (d & 0x00f0));+ p[53] = (uint8_t) ((p[53] & 0x0f) | ((d & 0x0f00) >> 4));+ p[55] = (uint8_t) ((p[55] & 0x0f) | ((d & 0xf000) >> 8));+ } else {+ std::memcpy(p, &d, 2);+ }}m.w.resize((size_t) rows * k);const ggml_type_traits* tr = ggml_get_type_traits((ggml_type) type);diff --git a/tools/dsv41/engram_iq4nl_bench.py b/tools/dsv41/engram_iq4nl_bench.pynew file mode 100644index 000000000..88eedf76d--- /dev/null+++ b/tools/dsv41/engram_iq4nl_bench.py@@ -0,0 +1,623 @@+# Copyright (c) 2026 the Strata contributors (MIT).+"""tools/dsv41/engram_iq4nl_bench.py - the Phase 1.3 IQ4_NL Engram reader: O_DIRECT row reads + host dequant.++The Engram is a big IQ4_NL table (engram-IQ4_NL.gguf, 110.6 GB in helcig/tmp-oct26). The engine, for one token,+gathers 48 rows of it by hash (tools/dsv41/engram_hash.py gives the row ids), so the reader's job is:++ random 2.3 KB row reads straight off the device (O_DIRECT: no page cache, the table does not fit in RAM),+ dequantised on the host to f32.++This is the Phase 1.3 skeleton: it does the two things the real run needs to be trusted before it is written into the+engine - it checks the dequant against ggml's own block_IQ4_NL decode, bit for bit, and it measures the two access+patterns (48 rows = one token; a prompt-chunk batch) on whatever file it is pointed at.++The block (ggml's block_iq4_nl, third_party/ggml/ggml-common.h): 32 values, one fp16 scale `d`, 16 packed nibbles:++ typedef struct { ggml_half d; uint8_t qs[16]; } block_iq4_nl; // 18 bytes, QK4_NL = 32++ y[base + j ] = d * kvalues_iq4nl[qs[j] & 0x0F] (j = 0..15)+ y[base + j + 16] = d * kvalues_iq4nl[qs[j] >> 4]+ kvalues_iq4nl = {-127,-104,-83,-65,-49,-35,-22,-10, 1, 13, 25, 38, 53, 69, 89, 113}++so a row of n_embd values is (n_embd/32)*18 bytes.++No model is needed to exercise it: `--make-synthetic` writes a valid GGUF with one IQ4_NL tensor (header, one tensor+info, 32-byte alignment, random packed nibbles and small fp16 scales) of the size you ask for, and every mode runs+against that. What the real run needs is listed in REAL RUN below and printed by --help.++ # on p1004 (no numpy there: this file is stdlib only)+ python3 tools/dsv41/engram_iq4nl_bench.py --make-synthetic /tmp/engram-synth.gguf --size-gb 1.0+ python3 tools/dsv41/engram_iq4nl_bench.py --gguf /tmp/engram-synth.gguf --mode all \\+ --rows-per-token 48 --batch-tokens 256 --iters 20++REAL RUN (what this needs that p1004 does not have today):+ * the file: engram-IQ4_NL.gguf, 110.6 GB. p1004 has 14 GB free on /data and 65 GB on /, so it needs ~50 GB freed+ first (P1004.md's /data/launch-test + /data/rel023-amd is 71 GB). It must sit on a real block device:+ O_DIRECT works on ext4/xfs/nvme, not on tmpfs or overlayfs.+ * the tensor geometry is read from the GGUF header (n_embd, n_rows); nothing is hardcoded here.+ * `--ggml-dump`: a 3-line C++ program that calls ggml's dequantize_row_iq4_nl and prints the float bits, linked+ against a built libggml-base.a (-I <llama.cpp>/ggml/include -I <llama.cpp>/ggml/src). --build-ggml-dump writes+ and compiles it from the source embedded below. Without it the check is the script's scalar reference against+ its own fast path, which is still bit-for-bit but not independent of this file.+ * numpy: optional. It is not installed on p1004, so dequant here is pure Python (~8k rows/s); the engine's dequant+ is the C kernel in ds4_iq.cu / ds4_gg_cpu.cpp. Use --dequant off for the I/O ceiling, --dequant sample (default)+ for a measured dequant rate, --dequant all to dequantise the whole batch.+ * the real 48-row gather per token is random over 110 GB: IOPS matter as much as MB/s. --mode token is that+ pattern; --mode batch is the prompt chunk; --mode seq is the device's sequential ceiling for reference.+"""+from __future__ import annotations++import argparse+import array+import mmap+import os+import random+import struct+import subprocess+import sys+import tempfile+import time++# ---------------------------------------------------------------- ggml IQ4_NL++QK4_NL = 32 # values per block+IQ4NL_ROW_DIV = 32 # n_embd must be a multiple of this+IQ4NL_BLOCK_BYTES = 18 # fp16 d + 16 packed nibbles+GGML_TYPE_IQ4_NL = 20 # ggml.h: IQ2_XXS 16, IQ2_XS 17, IQ3_XXS 18, IQ1_S 19, IQ4_NL 20, IQ3_S 21, IQ2_S 22+KVALUES_IQ4NL = (-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113)+++def iq4nl_row_bytes(n_embd: int) -> int:+ if n_embd % IQ4NL_ROW_DIV:+ raise ValueError(f"n_embd {n_embd} is not a multiple of {IQ4NL_ROW_DIV}")+ return (n_embd // QK4_NL) * IQ4NL_BLOCK_BYTES+++def dequant_row_iq4nl_ref(row: bytes, n_embd: int) -> array.array:+ """ggml's dequantize_row_iq4_nl, literal: one fp16 scale per block, 16 low nibbles then 16 high ones.++ Python floats are doubles; a float32 * a small integer is exact in a double (24 + 7 bits) and array('f') rounds+ it once on store, so this reproduces ggml's float32 product bit for bit.+ """+ out = array.array("f")+ nb = n_embd // QK4_NL+ for b in range(nb):+ base = b * IQ4NL_BLOCK_BYTES+ d = struct.unpack_from("<e", row, base)[0]+ qs = row[base + 2:base + IQ4NL_BLOCK_BYTES]+ out.extend(d * KVALUES_IQ4NL[q & 0x0F] for q in qs)+ out.extend(d * KVALUES_IQ4NL[q >> 4] for q in qs)+ return out+++def dequant_row_iq4nl_fast(row: bytes, n_embd: int) -> array.array:+ """The same result, with the block's 16 possible values built once: the reader's hot path."""+ out = array.array("f")+ nb = n_embd // QK4_NL+ unpack = struct.unpack_from+ kv = KVALUES_IQ4NL+ extend = out.extend+ for b in range(nb):+ base = b * IQ4NL_BLOCK_BYTES+ d = unpack("<e", row, base)[0]+ tab = (d * kv[0], d * kv[1], d * kv[2], d * kv[3], d * kv[4], d * kv[5], d * kv[6], d * kv[7],+ d * kv[8], d * kv[9], d * kv[10], d * kv[11], d * kv[12], d * kv[13], d * kv[14], d * kv[15])+ qs = row[base + 2:base + IQ4NL_BLOCK_BYTES]+ extend([tab[q & 0x0F] for q in qs])+ extend([tab[q >> 4] for q in qs])+ return out+++def f32_bits(vals) -> bytes:+ return array.array("f", vals).tobytes()+++# ---------------------------------------------------------------- GGUF reader++class Gguf:+ """Just enough GGUF for one tensor: magic, version, the kv block, the tensor infos, the aligned data start."""++ def __init__(self, path: str):+ self.path = path+ self.size = os.path.getsize(path)+ self.tensors: list[dict] = []+ self.meta: dict = {}+ self.data_start = 0+ self.alignment = 32+ with open(path, "rb") as f:+ magic = f.read(4)+ if magic != b"GGUF":+ raise ValueError(f"{path}: not a GGUF (magic {magic!r})")+ version = struct.unpack("<I", f.read(4))[0]+ if version not in (2, 3):+ raise ValueError(f"{path}: GGUF v{version}, this reads v2 and v3")+ self.version = version+ n_tensors, n_kv = struct.unpack("<QQ", f.read(16))+ for _ in range(n_kv):+ k, v = self._kv(f)+ self.meta[k] = v+ self.alignment = int(self.meta.get("general.alignment", 32)) or 32+ for _ in range(n_tensors):+ name = self._str(f)+ n_dims = struct.unpack("<I", f.read(4))[0]+ dims = list(struct.unpack(f"<{n_dims}Q", f.read(8 * n_dims)))+ ttype = struct.unpack("<I", f.read(4))[0]+ off = struct.unpack("<Q", f.read(8))[0]+ self.tensors.append({"name": name, "dims": dims, "type": ttype, "offset": off})+ pos = f.tell()+ self.data_start = (pos + self.alignment - 1) // self.alignment * self.alignment++ @staticmethod+ def _str(f) -> str:+ n = struct.unpack("<Q", f.read(8))[0]+ return f.read(n).decode("utf-8", "replace")++ _SCALAR = {0: "<B", 1: "<b", 2: "<H", 3: "<h", 4: "<I", 5: "<i", 6: "<f", 7: "<B", 10: "<Q", 11: "<q", 12: "<d"}++ def _kv(self, f):+ key = self._str(f)+ vtype = struct.unpack("<I", f.read(4))[0]+ if vtype == 8: # string+ return key, self._str(f)+ if vtype == 9: # array+ atype = struct.unpack("<I", f.read(4))[0]+ n = struct.unpack("<Q", f.read(8))[0]+ if atype == 8:+ return key, [self._str(f) for _ in range(n)]+ fmt = self._SCALAR[atype]+ return key, list(struct.unpack(f"<{n}{fmt[1]}", f.read(struct.calcsize(fmt) * n)))+ fmt = self._SCALAR[vtype]+ return key, struct.unpack(fmt, f.read(struct.calcsize(fmt)))[0]++ def tensor(self, name: str | None = None) -> dict:+ if name is not None:+ for t in self.tensors:+ if t["name"] == name:+ return t+ raise KeyError(f"{self.path}: no tensor {name!r}; have {[t['name'] for t in self.tensors]}")+ for t in self.tensors:+ if t["type"] == GGML_TYPE_IQ4_NL:+ return t+ raise KeyError(f"{self.path}: no IQ4_NL tensor")++ def geometry(self, name: str | None = None) -> tuple[int, int, int]:+ """(n_embd, n_rows, row_bytes) for an IQ4_NL table; dims[0] is the row length."""+ t = self.tensor(name)+ if t["type"] != GGML_TYPE_IQ4_NL:+ raise ValueError(f"tensor {t['name']!r} is ggml type {t['type']}, not {GGML_TYPE_IQ4_NL} (IQ4_NL)")+ n_embd = int(t["dims"][0])+ n_rows = 1+ for d in t["dims"][1:]:+ n_rows *= int(d)+ return n_embd, n_rows, iq4nl_row_bytes(n_embd)++ def row_offset(self, t: dict, row: int) -> int:+ """Absolute file offset of a row: the tensor's offset is relative to the data section."""+ _, _, rb = self.geometry(t["name"])+ return self.data_start + t["offset"] + row * rb+++# ---------------------------------------------------------------- O_DIRECT rows++class RowReader:+ """Aligned, unbuffered reads. A row is 18*(n_embd/32) bytes (2304 for n_embd 4096), which is not a multiple of+ the device's 512-byte block, so a row read is a 512-aligned window that is then sliced: the device moves up to+ 511 extra bytes per row and MB/s is reported both ways (payload and what the device was asked for).+ """++ def __init__(self, path: str, align: int = 512, odirect: bool = True):+ self.path = path+ self.align = align+ self.size = os.path.getsize(path)+ flags = os.O_RDONLY | (getattr(os, "O_DIRECT", 0) if odirect else 0)+ self.odirect = bool(odirect and getattr(os, "O_DIRECT", 0))+ try:+ self.fd = os.open(path, flags)+ except OSError as e:+ if not self.odirect:+ raise+ print(f" (O_DIRECT unavailable on {path}: {e}; falling back to buffered reads)", file=sys.stderr)+ self.odirect = False+ self.fd = os.open(path, os.O_RDONLY)+ self._buf = None+ self._buf_len = 0+ self.device_bytes = 0++ def _buffer(self, n: int):+ if self._buf is None or self._buf_len < n:+ self._buf = mmap.mmap(-1, n) # page aligned, which is a multiple of any 512/4096 sector+ self._buf_len = n+ return self._buf++ def read(self, off: int, length: int) -> bytes:+ """The `length` bytes at `off`, in one aligned pread."""+ if self.odirect:+ a = self.align+ start = off // a * a+ end = min((off + length + a - 1) // a * a, self.size)+ n = end - start+ if n <= 0:+ raise ValueError(f"read past end: off {off} len {length} size {self.size}")+ buf = self._buffer(n)+ got = os.preadv(self.fd, [memoryview(buf)[:n]], start)+ self.device_bytes += n+ if got != n:+ raise OSError(f"short O_DIRECT read at {start}: {got} of {n}")+ return bytes(memoryview(buf)[off - start:off - start + length])+ data = os.pread(self.fd, length, off)+ self.device_bytes += length+ if len(data) != length:+ raise OSError(f"short read at {off}: {len(data)} of {length}")+ return data++ def close(self):+ if self._buf is not None:+ self._buf.close()+ self._buf = None+ os.close(self.fd)+++# ---------------------------------------------------------------- synthetic file++def make_synthetic(path: str, size_gb: float, n_embd: int = 4096, seed: int = 1, verbose: bool = True) -> tuple[int, int]:+ """A valid GGUF v3 with one IQ4_NL tensor: header, one tensor info, 32-byte alignment, then the data.++ The blocks are real blocks: fp16 scales in 0.002 .. 0.017 and random packed nibbles, so the dequant is exercised+ on the file the benchmarks read. Returns (n_rows, file bytes).+ """+ rb = iq4nl_row_bytes(n_embd)+ n_rows = max(1, int(size_gb * 1e9) // rb)+ rng = random.Random(seed)+ scales = [0.002 + 0.001 * i for i in range(16)]+ # 8 block templates per scale cycle, each 18 bytes: the fp16 d and 16 random nibble bytes+ blobs = []+ for s in scales:+ parts = []+ for _ in range(8):+ parts.append(struct.pack("<e", s) + bytes(rng.getrandbits(8) for _ in range(16)))+ blobs.append(b"".join(parts)) # 144 bytes = 8 whole blocks, so a repeat keeps block alignment++ header = bytearray()+ header += b"GGUF"+ header += struct.pack("<I", 3)+ header += struct.pack("<QQ", 1, 3)+ for k, v in (("general.architecture", "engram"), ("general.name", "p1004-synthetic-iq4nl")):+ header += struct.pack("<Q", len(k)) + k.encode()+ header += struct.pack("<I", 8)+ header += struct.pack("<Q", len(v)) + v.encode()+ header += struct.pack("<Q", len("general.alignment")) + b"general.alignment"+ header += struct.pack("<I", 4) + struct.pack("<I", 32)+ name = b"engram.iq4nl"+ header += struct.pack("<Q", len(name)) + name+ header += struct.pack("<I", 2) + struct.pack("<QQ", n_embd, n_rows)+ header += struct.pack("<I", GGML_TYPE_IQ4_NL) + struct.pack("<Q", 0)+ header += b"\x00" * ((-len(header)) % 32)++ if verbose:+ print(f"synthetic: n_embd {n_embd} row_bytes {rb} n_rows {n_rows} data {(n_rows * rb) / 1e9:.3f} GB -> {path}")+ chunk_rows = 64+ written = 0+ with open(path, "wb", buffering=1024 * 1024) as f:+ f.write(header)+ k = 0+ t0 = time.time()+ while written < n_rows:+ rows = min(chunk_rows, n_rows - written)+ blob = blobs[k % len(blobs)]+ k += 1+ buf = (blob * (rows * rb // len(blob) + 1))[:rows * rb]+ f.write(buf)+ written += rows+ if verbose and written % (chunk_rows * 512) == 0:+ print(f"\r {written * rb / 1e9:.2f} GB", end="", file=sys.stderr)+ total = os.path.getsize(path)+ if verbose:+ print(f"\r wrote {total / 1e9:.3f} GB in {time.time() - t0:.1f} s" + " " * 20, file=sys.stderr)+ return n_rows, total+++# ---------------------------------------------------------------- the ggml cross-check++_GGML_DUMP_SRC = r"""+// Read one IQ4_NL row from stdin, print ggml's own dequantise: one float per line, 8 hex digits of its bits.+#include <cstdio>+#include <cstdlib>+#include <vector>+extern "C" void dequantize_row_iq4_nl(const void * x, float * y, int64_t k);+int main(int argc, char ** argv) {+ if (argc < 2) { std::fprintf(stderr, "usage: iq4nl_dump N_EMBD < row.bin\n"); return 2; }+ const int n = std::atoi(argv[1]);+ const size_t rb = (size_t) (n / 32) * 18;+ std::vector<unsigned char> row(rb);+ if (std::fread(row.data(), 1, rb, stdin) != rb) { std::fprintf(stderr, "short read\n"); return 2; }+ std::vector<float> y((size_t) n);+ dequantize_row_iq4_nl(row.data(), y.data(), n);+ for (int i = 0; i < n; ++i) {+ unsigned int b; __builtin_memcpy(&b, &y[i], 4);+ std::printf("%08x\n", b);+ }+ return 0;+}+"""+++def build_ggml_dump(build_dir: str, out: str | None = None) -> str:+ """Compile the dumper against a built llama.cpp tree (ggml headers + libggml-base.a)."""+ src_root = os.path.join(build_dir, "_deps", "strata_llamacpp-src", "ggml")+ lib = os.path.join(build_dir, "ggml", "src", "libggml-base.a")+ for p in (src_root, lib):+ if not os.path.exists(p):+ raise FileNotFoundError(f"{p} not found; pass --ggml-build-dir with a tree that built ggml")+ out = out or os.path.join(tempfile.gettempdir(), "iq4nl_dump")+ with open(out + ".cpp", "w") as f:+ f.write(_GGML_DUMP_SRC)+ cmd = ["g++", "-O2", "-std=c++17", "-I", os.path.join(src_root, "include"), "-I", os.path.join(src_root, "src"),+ out + ".cpp", lib, "-o", out, "-lpthread", "-lm"]+ subprocess.run(cmd, check=True)+ return out+++def ggml_dequant_bits(dumper: str, row: bytes, n_embd: int) -> bytes:+ """ggml's dequant of one row, as raw float32 bytes (the dumper prints the bits as %08x, so repack little-endian)."""+ p = subprocess.run([dumper, str(n_embd)], input=row, stdout=subprocess.PIPE, check=True)+ hx = b"".join(p.stdout.split()).decode()+ if len(hx) != 8 * n_embd:+ raise RuntimeError(f"the dumper printed {len(hx)} hex digits for {n_embd} values; check its n_embd argument")+ return b"".join(struct.pack("<I", int(hx[i:i + 8], 16)) for i in range(0, len(hx), 8))+++# ---------------------------------------------------------------- benchmarks++def _fmt(x, unit=""):+ return f"{x:,.1f}{unit}" if isinstance(x, float) else f"{x:,}{unit}"+++def _dequant_rate(rows, n_embd: int, limit: int) -> tuple[float, int]:+ """(seconds, rows) for the host dequant of rows already in memory: the CPU side, apart from the I/O."""+ sel = rows[:limit]+ if not sel:+ return 0.0, 0+ t0 = time.perf_counter()+ for row in sel:+ dequant_row_iq4nl_fast(row, n_embd)+ return time.perf_counter() - t0, len(sel)+++def bench_token(gg: Gguf, t: dict, n_embd: int, n_rows: int, rb: int, args) -> dict:+ """48 random rows per token, one aligned pread each: the decode-time pattern. The reads are timed alone and+ the dequant is timed afterwards on the rows already in RAM, so neither number hides the other."""+ rnd = random.Random(args.seed)+ tok_rows = [rnd.sample(range(n_rows), args.rows_per_token) for _ in range(args.iters)]+ rd = RowReader(gg.path, align=args.align, odirect=args.odirect)+ blobs = []+ try:+ rd.read(gg.row_offset(t, tok_rows[0][0]), rb) # warm the path, not the cache (O_DIRECT)+ t0 = time.perf_counter()+ for rows in tok_rows:+ for r in rows:+ blobs.append(rd.read(gg.row_offset(t, r), rb))+ dt = time.perf_counter() - t0+ finally:+ dev = rd.device_bytes+ rd.close()+ n = len(blobs)+ out = {+ "mode": f"token x{args.iters} ({args.rows_per_token} rows each)",+ "seconds": dt,+ "tokens_s": args.iters / dt,+ "rows_s": n / dt,+ "payload_mb_s": n * rb / dt / 1e6,+ "device_mb_s": dev / dt / 1e6,+ "reads": n,+ "read_us": dt / n * 1e6,+ "dequant": args.dequant,+ }+ if args.dequant != "off":+ lim = n if args.dequant == "all" else args.rows_per_token * min(args.iters, 2)+ ddt, dn = _dequant_rate(blobs, n_embd, lim)+ out["dequant_rows"], out["dequant_rows_s"] = dn, dn / ddt if ddt else 0.0+ return out+++def bench_batch(gg: Gguf, t: dict, n_embd: int, n_rows: int, rb: int, args) -> dict:+ """A prompt chunk: batch_tokens x 48 rows, gathered at random like the engine will, plus the same rows read as+ contiguous runs of the table (the best case: a chunk's hashes are unrelated, so the real pattern is the random+ one). Reads and dequant are timed apart, as in bench_token."""+ rnd = random.Random(args.seed + 1)+ n = args.batch_tokens * args.rows_per_token+ rows = [rnd.randrange(n_rows) for _ in range(n)]+ rd = RowReader(gg.path, align=args.align, odirect=args.odirect)+ blobs = []+ try:+ t0 = time.perf_counter()+ for r in rows:+ blobs.append(rd.read(gg.row_offset(t, r), rb))+ dt = time.perf_counter() - t0+ finally:+ dev = rd.device_bytes+ rd.close()+ out = {+ "mode": f"batch gathered {args.batch_tokens} tokens = {n} rows",+ "seconds": dt,+ "tokens_s": args.batch_tokens / dt,+ "rows_s": n / dt,+ "payload_mb_s": n * rb / dt / 1e6,+ "device_mb_s": dev / dt / 1e6,+ "reads": n,+ "read_us": dt / n * 1e6,+ "dequant": args.dequant,+ }+ if args.dequant != "off":+ lim = n if args.dequant == "all" else args.rows_per_token * 3+ ddt, dn = _dequant_rate(blobs, n_embd, lim)+ out["dequant_rows"], out["dequant_rows_s"] = dn, dn / ddt if ddt else 0.0+ del blobs+ # the same rows, contiguous: merge the sorted ids into runs and read each run with one pread+ srows = sorted(rows)+ runs = []+ lo = prev = srows[0]+ for r in srows[1:]:+ if r == prev + 1:+ prev = r+ continue+ runs.append((lo, prev))+ lo = prev = r+ runs.append((lo, prev))+ rd = RowReader(gg.path, align=args.align, odirect=args.odirect)+ try:+ t0 = time.perf_counter()+ got = 0+ for lo, hi in runs:+ got += len(rd.read(gg.row_offset(t, lo), (hi - lo + 1) * rb))+ dt2 = time.perf_counter() - t0+ finally:+ dev2 = rd.device_bytes+ rd.close()+ out["contig"] = {+ "mode": f"batch contiguous {args.batch_tokens} tokens = {len(runs)} runs",+ "seconds": dt2,+ "tokens_s": args.batch_tokens / dt2,+ "rows_s": got / rb / dt2,+ "payload_mb_s": got / dt2 / 1e6,+ "device_mb_s": dev2 / dt2 / 1e6,+ "reads": len(runs),+ "read_us": dt2 / max(1, len(runs)) * 1e6,+ "dequant": "off",+ }+ return out+++def bench_seq(gg: Gguf, rb: int, args) -> dict:+ """The device's sequential ceiling through the same O_DIRECT path (rows/s is that ceiling in 2.3 KB rows)."""+ mb = min(args.seq_mb, gg.size // (1024 * 1024))+ rd = RowReader(gg.path, align=args.align, odirect=args.odirect)+ chunk = args.align * 1024 # 512 KiB+ try:+ rd.read(0, chunk)+ t0 = time.perf_counter()+ off = 0+ total = 0+ while total < mb * 1024 * 1024:+ n = min(chunk, gg.size - off)+ if n <= 0:+ break+ rd.read(off, n)+ off += n+ total += n+ dt = time.perf_counter() - t0+ finally:+ dev = rd.device_bytes+ rd.close()+ return {"mode": f"sequential {total / 1e6:.0f} MB, {chunk // 1024} KiB reads", "seconds": dt,+ "rows_s": total / rb / dt, "payload_mb_s": total / dt / 1e6, "device_mb_s": dev / dt / 1e6,+ "reads": total // chunk, "read_us": dt / max(1, total // chunk) * 1e6, "dequant": "off"}+++# ---------------------------------------------------------------- main++def main(argv=None) -> int:+ ap = argparse.ArgumentParser(description=__doc__,+ formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--gguf", help="the Engram GGUF (or a synthetic one)")+ ap.add_argument("--tensor", default=None, help="tensor name (default: the first IQ4_NL tensor)")+ ap.add_argument("--mode", default="all", choices=["header", "token", "batch", "seq", "all"])+ ap.add_argument("--rows-per-token", type=int, default=48)+ ap.add_argument("--batch-tokens", type=int, default=256)+ ap.add_argument("--iters", type=int, default=20)+ ap.add_argument("--seq-mb", type=int, default=512)+ ap.add_argument("--align", type=int, default=512, help="O_DIRECT alignment (the device's logical block size)")+ ap.add_argument("--odirect", action="store_true", default=True)+ ap.add_argument("--no-odirect", dest="odirect", action="store_false")+ ap.add_argument("--dequant", default="sample", choices=["off", "sample", "all"])+ ap.add_argument("--seed", type=int, default=1)+ ap.add_argument("--verify-rows", type=int, default=8, help="rows checked bit-exactly against ggml")+ ap.add_argument("--ggml-dump", help="a compiled ggml IQ4_NL dequant dumper (see --build-ggml-dump)")+ ap.add_argument("--build-ggml-dump", action="store_true", help="compile the dumper from the source embedded here")+ ap.add_argument("--ggml-build-dir", default="/data/dsv41-build", help="a Strata build tree with ggml built")+ ap.add_argument("--make-synthetic", metavar="PATH", help="write a synthetic IQ4_NL GGUF and exit (unless --gguf)")+ ap.add_argument("--size-gb", type=float, default=1.0)+ ap.add_argument("--n-embd", type=int, default=4096)+ args = ap.parse_args(argv)++ if args.make_synthetic:+ make_synthetic(args.make_synthetic, args.size_gb, args.n_embd, args.seed)+ if not args.gguf:+ args.gguf = args.make_synthetic+ if args.mode == "all":+ args.mode = "header"+ if not args.gguf:+ ap.error("--gguf or --make-synthetic is required")++ gg = Gguf(args.gguf)+ t = gg.tensor(args.tensor)+ n_embd, n_rows, rb = gg.geometry(args.tensor)+ print(f"{gg.path}: {gg.size / 1e9:.3f} GB, GGUF v{gg.meta.get('general.version', 3)}, {len(gg.tensors)} tensor(s), "+ f"alignment {gg.alignment}, data at {gg.data_start}")+ print(f" tensor {t['name']!r}: type {t['type']} (IQ4_NL), dims {t['dims']}, n_embd {n_embd}, n_rows {n_rows}, "+ f"row_bytes {rb}, table {(n_rows * rb) / 1e9:.3f} GB")+ if t["type"] != GGML_TYPE_IQ4_NL:+ print(f" not an IQ4_NL tensor: nothing to read", file=sys.stderr)+ return 2++ # the bit-exact check, always: the scalar reference against the fast path, then against ggml itself+ rnd = random.Random(args.seed + 7)+ dumper = args.ggml_dump+ if args.build_ggml_dump:+ dumper = build_ggml_dump(args.ggml_build_dir, dumper)+ print(f" built the ggml dumper: {dumper}")+ rd = RowReader(gg.path, align=args.align, odirect=args.odirect)+ bad = 0+ checked = 0+ try:+ for _ in range(args.verify_rows):+ r = rnd.randrange(n_rows)+ row = rd.read(gg.row_offset(t, r), rb)+ ref = dequant_row_iq4nl_ref(row, n_embd)+ fast = dequant_row_iq4nl_fast(row, n_embd)+ if f32_bits(ref) != f32_bits(fast):+ bad += 1+ print(f" row {r}: the scalar reference and the fast path differ", file=sys.stderr)+ if dumper:+ gw = ggml_dequant_bits(dumper, row, n_embd)+ if gw != f32_bits(ref):+ bad += 1+ print(f" row {r}: ggml's dequant differs from the reference", file=sys.stderr)+ checked += 1+ finally:+ rd.close()+ print(f" bit-exact: {checked - bad}/{checked} rows" + (" against ggml's dequantize_row_iq4_nl" if dumper else+ " (reference vs fast path; no --ggml-dump given)")+ + (" FAIL" if bad else " OK"))+ if bad:+ return 1++ if args.mode == "header":+ return 0++ results = []+ if args.mode in ("seq", "all"):+ results.append(bench_seq(gg, rb, args))+ if args.mode in ("token", "all"):+ results.append(bench_token(gg, t, n_embd, n_rows, rb, args))+ if args.mode in ("batch", "all"):+ b = bench_batch(gg, t, n_embd, n_rows, rb, args)+ results.append(b["contig"])+ results.append(b)+ results.sort(key=lambda x: -x["payload_mb_s"])+ print()+ print(f" {'pattern':44s} {'s':>8s} {'rows/s':>11s} {'tokens/s':>9s} {'MB/s payload':>12s} {'MB/s device':>11s} "+ f"{'us/read':>8s} {'dequant rows/s':>14s}")+ for r in results:+ tok = f"{r['tokens_s']:.1f}" if "tokens_s" in r else "-"+ dq = f"{r['dequant_rows_s']:,.0f} ({r['dequant_rows']} rows)" if r.get("dequant_rows_s") else r["dequant"]+ print(f" {r['mode']:44s} {r['seconds']:8.3f} {r['rows_s']:11,.0f} {tok:>9s} {r['payload_mb_s']:12,.1f} "+ f"{r['device_mb_s']:11,.1f} {r['read_us']:8,.1f} {dq:>14s}")+ print(f"\n note: O_DIRECT {'on' if args.odirect else 'off'}, alignment {args.align} B; "+ f"payload MB/s counts row bytes only, device MB/s what the reads actually asked for")+ print(" the dequant rate is measured apart from the reads (rows/s of the host dequant); the engine's dequant is "+ "the C kernel, not this Python")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/ista_intake.py b/tools/dsv41/ista_intake.pyindex 6626e8aa1..5308d2365 100644--- a/tools/dsv41/ista_intake.py+++ b/tools/dsv41/ista_intake.py@@ -40,10 +40,17 @@ PY = "/data/dsv41/hfvenv/bin/python"HF = "/data/dsv41/hfvenv/bin/hf"TOOLS = PORT / "src/tools/dsv41"# the substitutions the published sets need today (HANDOVER.md §6; Phase 1 removes them)-ENGRAM_DIR = pathlib.Path("/home/niko/engram-ista/2.00bpw")-DSPARK = pathlib.Path("/data/dsv41/source/dspark-DeepSeek-V4.1-Flash-Q8_0.gguf")-DENSE_WOA = pathlib.Path("/home/niko/dsv41-dense/wo_a_q5k")-DENSE_HEAD = pathlib.Path("/home/niko/dsv41-dense/head_q6k")+# the Engram every live pack uses: the reference MXFP8 tables (shards 8/9 of the UD set, 203 GB). The IST-DAS IQ4_NL+# Engram (110.6 GB) is a separate Phase 1.3 experiment, so the intake does not depend on it.+ENGRAM_DIR = pathlib.Path("/home/niko/dsv41-c0/t2/Engram")+DSPARK = pathlib.Path("/data/dsv41/source/dspark-DeepSeek-V4.1-Flash-1017-Q8_0.gguf")+DENSE_WOA = pathlib.Path("/home/niko/dsv41-dense/woaQ5_K")+DENSE_HEAD = pathlib.Path("/home/niko/dsv41-dense/headQ6_K")+# the UD set's first shard carries the Engram hash parameters as ple.* metadata and the vision tower carries the mmproj+ENGRAM_META = pathlib.Path("/data/dsv41/source/UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00001-of-00009.gguf")+MMPROJ = pathlib.Path("/data/dsv41/source/mmproj-BF16.gguf")+DSPARK_NOISE = "128799" # dspark_noise_token_id for this model (see the h225 pack)+DSPARK_RANK = "256" # dspark_markov_rank# ggml type ids the device kernels take (src/dsv41/ds4_iq.cu's supported() plus MXFP4); kept in sync with --pack-infoDEVICE_TYPES = {0, 1, 2, 3, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 39}TYPE_NAMES = {0: "F32", 1: "F16", 2: "Q4_0", 3: "Q4_1", 6: "Q5_0", 7: "Q5_1", 8: "Q8_0", 9: "Q8_1",@@ -126,16 +133,25 @@ def pack_set(commit: str, set_name: str, root: pathlib.Path, report: list[str])if pack.exists():report.append("- pack already exists: %s" % pack)return pack+ # the exact recipe the hand-built h225 pack used (HANDOVER.md §6 and h4_225.sh): no requant, our MXFP8 Engram, and the+ # three workarounds Phase 1 removes - the interleaved 3-D attn_output_a, the engram_(k|q|wkv) tensors, and the Engram+ # hash parameters that live in the UD set's metadata (--engram-meta), without which pack_gguf sees no Engram layers.cmd = [PY, str(TOOLS / "pack_gguf.py"), "--model", str(setdir), "--out", str(pack)]if ENGRAM_DIR.exists():- cmd += ["--engram", str(ENGRAM_DIR.parent)]+ cmd += ["--engram", str(ENGRAM_DIR)]+ if ENGRAM_META.exists():+ cmd += ["--engram-meta", str(ENGRAM_META)]+ if MMPROJ.exists():+ cmd += ["--mmproj", str(MMPROJ)]if DENSE_WOA.exists():cmd += ["--dense-override", str(DENSE_WOA)]if DENSE_HEAD.exists():cmd += ["--dense-override", str(DENSE_HEAD)]- cmd += ["--skip-tensors", r"blk\.\d+\.attn_output_a\.weight"]+ cmd += ["--skip-tensors", r"attn_output_a|engram_(k|q|wkv)\."]if DSPARK.exists():- cmd += ["--dspark-gguf", str(DSPARK), "--dspark-noise-token", "128799", "--dspark-markov-rank", "256"]+ # pack_gguf fills the block/MTP geometry from the drafter but not the noise token or the markov rank: without them+ # the pack reports dspark_noise_token_id 0 and DSpark drafts one token a round (the 2026-10-22 bug).+ cmd += ["--dspark-gguf", str(DSPARK), "--dspark-noise-token", DSPARK_NOISE, "--dspark-markov-rank", DSPARK_RANK]log(" pack: %s" % " ".join(cmd[2:]))r = run(cmd, cwd=str(PORT))(PORT / "runs").mkdir(exist_ok=True)