Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: the Engram reader takes IST-DAS's IQ4_NL table, and a pack with an unknown Engram type stops cleanly instead of segfaulting

Niko1221b2d2cf22eb98
diff --git a/src/dsv41/ds4_engram_io.hpp b/src/dsv41/ds4_engram_io.hpp
index eb190f43c..2402ec1af 100644
--- a/src/dsv41/ds4_engram_io.hpp
+++ b/src/dsv41/ds4_engram_io.hpp
@@ -89,9 +89,9 @@ private:
std::vector<std::unique_ptr<uint8_t[]>> blocks_;
};
-/// One row to fetch: its codes to dst, its scales to scales. mx8: n bytes at off of a file, [scale, 32 codes] blocks
-/// (a GGUF's MXFP8 row); else n code bytes at off and n2 scale bytes at off2 of file2 (two reads; n2 0: none - a
-/// GGUF's Q6_K row, its scales inside its n bytes).
+/// One row to fetch: its bytes to dst, its scales to scales. mx8: n bytes at off of a file, [scale, 32 codes] blocks
+/// (a GGUF's MXFP8 row); else n row bytes at off and n2 scale bytes at off2 of file2 (two reads; n2 0: none - a GGUF's
+/// Q6_K or IQ4_NL row, their scales inside their n bytes: 210 a 256 values, 18 a 32).
struct EgRead {
int file = -1;
uint64_t off = 0;
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index 0cfc3bcc7..789390f78 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1272,14 +1272,30 @@ __global__ void __launch_bounds__(1024) confidence_kernel(const float* __restric
}
// ---------------------------------------------------------------- engram
+// ggml's IQ4_NL: a block of 32 values is 18 bytes, an fp16 d then 16 packed nibble bytes; the low nibble of qs[j] is
+// value j and its high nibble value j + 16, each an index into this codebook (ggml-common.h's kvalues_iq4nl,
+// dequantize_row_iq4_nl). A row is hd / 32 of these blocks, the d of every one inside it - no separate scales.
+__device__ __constant__ int8_t kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10,
+ 1, 13, 25, 38, 53, 69, 89, 113};
+__device__ __forceinline__ float iq4nl_value(const uint8_t* b, int e) {
+ const uint8_t q = b[2 + (e & 15)];
+ const int v = (e & 16) ? (q >> 4) : (q & 15);
+ return __half2float(__ushort_as_half(*(const uint16_t*) b)) * (float) kIq4nl[v];
+}
+
__global__ void engram_dequant_kernel(const uint8_t* __restrict__ rows, const uint8_t* __restrict__ scales, int hd,
- float* __restrict__ out, bool q6k) {
+ float* __restrict__ out, int fmt) {
const int r = blockIdx.x;
- if (q6k) { // (ggml Q6_K superblocks at the row's start)
+ if (fmt == 1) { // (ggml Q6_K superblocks at the row's start)
for (int e = threadIdx.x; e < hd; e += blockDim.x)
out[(size_t) r * hd + e] = q6k_value(rows + (size_t) r * hd + (size_t) (e / 256) * kQ6kBytes, e % 256);
return;
}
+ if (fmt == 2) { // (ggml IQ4_NL blocks at the row's start, their scales inside)
+ for (int e = threadIdx.x; e < hd; e += blockDim.x)
+ out[(size_t) r * hd + e] = iq4nl_value(rows + (size_t) r * hd + (size_t) (e / 32) * 18, e % 32);
+ return;
+ }
for (int e = threadIdx.x; e < hd; e += blockDim.x) {
const float hv = e4m3_value(rows[(size_t) r * hd + e]);
const uint8_t sb = scales[(size_t) r * (hd / 32) + e / 32];
@@ -1685,15 +1701,20 @@ __global__ void __launch_bounds__(kThreads) hc_mix_scale_kernel(const float* __r
}
__global__ void engram_dequant_tok_kernel(const uint8_t* __restrict__ src, size_t per, int cols, int hd,
- float* __restrict__ out, bool q6k) {
+ float* __restrict__ out, int fmt) {
const int r = blockIdx.x, t = r / cols, c = r - t * cols;
const uint8_t* rows = src + (size_t) t * per + (size_t) c * hd;
const uint8_t* scales = src + (size_t) t * per + (size_t) cols * hd + (size_t) c * (hd / 32);
- if (q6k) { // (ggml Q6_K superblocks at the row's start)
+ if (fmt == 1) { // (ggml Q6_K superblocks at the row's start)
for (int e = threadIdx.x; e < hd; e += blockDim.x)
out[(size_t) r * hd + e] = q6k_value(rows + (size_t) (e / 256) * kQ6kBytes, e % 256);
return;
}
+ if (fmt == 2) { // (ggml IQ4_NL blocks at the row's start, their scales inside)
+ for (int e = threadIdx.x; e < hd; e += blockDim.x)
+ out[(size_t) r * hd + e] = iq4nl_value(rows + (size_t) (e / 32) * 18, e % 32);
+ return;
+ }
for (int e = threadIdx.x; e < hd; e += blockDim.x) {
const float hv = e4m3_value(rows[e]);
const uint8_t sb = scales[e / 32];
@@ -1856,8 +1877,8 @@ void hc_post_inplace(const float* y, float* h, const float* post, const float* c
void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {
hc_mix_scale_kernel<<<T, kThreads, 0, s>>>(h, hcdim, n_mix, eps, mixes);
}
-void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, bool q6k) {
- if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, q6k);
+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, int fmt) {
+ if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, fmt);
}
void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,
cudaStream_t s) {
@@ -2053,8 +2074,8 @@ void confidence(const float* x, const uint16_t* E, const int* tok, const float*
confidence_kernel<<<T, 1024, 0, s>>>(x, E, tok, w, dim, R, out);
}
-void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k) {
- engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, q6k);
+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int fmt) {
+ engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, fmt);
}
void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s) {
diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuh
index a2fefcfef..798bca1d9 100644
--- a/src/dsv41/ds4_kernels.cuh
+++ b/src/dsv41/ds4_kernels.cuh
@@ -174,14 +174,15 @@ struct RingBatch {
};
void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s);
-/// Engram. rows: n FP8 E4M3 rows of hd values, scales n x (hd / 32) E8M0 -> out F32 [n * hd]. q6k: the rows are
-/// ggml Q6_K superblocks instead (hd / 256 of them, 210 bytes each, at the start of each row's hd bytes; no scales),
-/// decoded as ggml's dequantize_row_q6_K.
-void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k = false);
+/// Engram. rows: n rows of hd values -> out F32 [n * hd]. fmt, the pack's row layout: 0 FP8 E4M3 values with E8M0
+/// scales (scales: n x (hd / 32)); 1 ggml Q6_K superblocks (hd / 256 of 210 bytes at the start of each row's hd bytes;
+/// no scales), decoded as ggml's dequantize_row_q6_K; 2 ggml IQ4_NL blocks (hd / 32 of 18 bytes, an fp16 d and 16
+/// nibble bytes at the start of each row's hd bytes; no scales), decoded as ggml's dequantize_row_iq4_nl.
+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int fmt = 0);
/// The same over the host's per-token layout (a chunk): token t's cols rows, then their scales, `per` bytes a token;
/// T tokens -> out F32 [T * cols][hd].
void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s,
- bool q6k = false);
+ int fmt = 0);
/// h[t][k] += gate[t][k] * value[t] with the gate from the normalized dot of h[t][k] * w[k] * key[t][k]
/// (kv[t] = [key hc x dim | value dim]).
void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s);
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index cd4a5222c..fe4e457c5 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -884,7 +884,10 @@ struct Model::Impl {
EgRowCache eg_cache;
std::mutex eg_cache_mu;
std::vector<std::unique_ptr<EgBatch>> eg_win;
- std::vector<uint8_t> eg_q6k; // Engram layer li's table in ggml Q6_K (a row one superblock a 256 values), else FP8
+ // Engram layer li's embed rows, as the pack stores them: 0 MXFP8 ([scale, 32 codes] blocks, 33 bytes a 32), 1
+ // ggml Q6_K (210 bytes a 256), 2 ggml IQ4_NL (18 bytes a 32: an fp16 d and 16 nibble bytes), 3 an old pack's FP8
+ // codes with their E8M0 scales in a tensor of their own (see eg_issue)
+ std::vector<uint8_t> eg_fmt;
std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};
uint16_t* h_emb = nullptr; // the window's embedding rows
std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;
@@ -2341,14 +2344,41 @@ struct Model::Impl {
eg_io.start(paths, nth);
for (int li = 0; li < pk.eg_layers; ++li) {
eg_win.emplace_back(new EgBatch());
- const Tensor* tw = pk.get("layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.weight");
- const bool q6k = tw != nullptr && tw->dtype == "Q6_K";
- if (q6k && pk.eg_hd % 256 != 0) {
- err = "Engram layer " + std::to_string(g.eg_layers[(size_t) li]) + ": Q6_K rows of " +
- std::to_string(pk.eg_hd) + " values (not whole superblocks of 256)";
+ const std::string id = std::to_string(g.eg_layers[(size_t) li]);
+ const Tensor* tw = pk.get("layers." + id + ".engram.embed.weight");
+ // The row layout the reader and the dequant take. A type not here (or no tensor at all) would be read
+ // as some other format's bytes - the old code then read the scale tensor of a pack that has none and
+ // crashed in eg_issue - so refuse the pack once, with the type's name, before any token is read.
+ int fmt;
+ if (tw == nullptr) {
+ std::fprintf(stderr, "ds4: Engram layer %s: engram.embed.weight is missing from the pack\n", id.c_str());
+ std::exit(2);
+ } else if (tw->dtype == "MXFP8") {
+ fmt = 0;
+ } else if (tw->dtype == "Q6_K") {
+ fmt = 1;
+ } else if (tw->dtype == "IQ4_NL") {
+ fmt = 2;
+ } else if (pk.get("layers." + id + ".engram.embed.scale") != nullptr) {
+ fmt = 3; // (an old pack: FP8 codes in one tensor, their E8M0 scales in another)
+ } else {
+ std::fprintf(stderr,
+ "ds4: Engram layer %s: engram.embed.weight type %s is not one the Engram reader handles "
+ "(MXFP8, Q6_K, IQ4_NL)\n",
+ id.c_str(), tw->dtype.c_str());
+ std::exit(2);
+ }
+ if (fmt == 1 && pk.eg_hd % 256 != 0) {
+ err = "Engram layer " + id + ": Q6_K rows of " + std::to_string(pk.eg_hd) +
+ " values (not whole superblocks of 256)";
return false;
}
- eg_q6k.push_back(q6k ? 1 : 0);
+ if (fmt == 2 && pk.eg_hd % 32 != 0) {
+ err = "Engram layer " + id + ": IQ4_NL rows of " + std::to_string(pk.eg_hd) +
+ " values (not whole blocks of 32)";
+ return false;
+ }
+ eg_fmt.push_back((uint8_t) fmt);
}
if (o.verbose)
std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "
@@ -4431,8 +4461,10 @@ struct Model::Impl {
const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";
const Tensor* tw = pk.get(P + "weight");
const Tensor* ts = pk.get(P + "scale");
- const bool mx8 = tw->dtype == "MXFP8"; // (a GGUF: the scales between the codes, one per 32)
- const bool q6k = eg_q6k[(size_t) li] != 0; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
+ const int fmt = eg_fmt[(size_t) li];
+ const bool mx8 = fmt == 0; // (a GGUF: the scales between the codes, one per 32)
+ const bool q6k = fmt == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
+ const bool iq4 = fmt == 2; // (a GGUF: 18-byte blocks a 32 values, d and 16 nibble bytes, no scales)
std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read
int64_t hits = 0;
{
@@ -4485,6 +4517,9 @@ struct Model::Impl {
} else if (q6k) { // (into the row's code bytes; its scale bytes unused)
r.n = (uint32_t) (hd / 256 * 210);
r.off = tw->off + (uint64_t) id * r.n;
+ } else if (iq4) { // (the same: one contiguous row, its blocks' d inside them, no scales)
+ r.n = (uint32_t) (hd / 32 * 18);
+ r.off = tw->off + (uint64_t) id * r.n;
} else {
r.n = (uint32_t) hd;
r.off = tw->off + (uint64_t) id * hd;
@@ -5795,7 +5830,7 @@ struct Model::Impl {
CK(cudaEventRecord(c.eg_ev[slot], s));
for (int s0 = 0; s0 < T; s0 += 256) {
const int ns = std::min(256, T - s0);
- engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, eg_q6k[(size_t) l.eg] != 0);
+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);
if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;
engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);
@@ -7320,7 +7355,7 @@ struct Model::Impl {
for (int t = 0; t < T; ++t)
CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
(size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
- engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, eg_q6k[(size_t) l.eg] != 0);
+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
}
diff --git a/tools/dsv41/ista_intake.py b/tools/dsv41/ista_intake.py
index 5308d2365..b9e555711 100644
--- a/tools/dsv41/ista_intake.py
+++ b/tools/dsv41/ista_intake.py
@@ -227,7 +227,17 @@ def main() -> int:
report: list[str] = ["# IST-DAS intake, commit %s" % commit, "", "run %s" % time.strftime("%Y-%m-%dT%H:%M:%S%z"), ""]
log("listing %s at %s" % (REPO, commit[:7]))
- tree = repo_tree(commit)
+ try:
+ tree = repo_tree(commit)
+ except Exception as e: # noqa: BLE001 (HF's API rate-limits unauthenticated boxes; the files are already local)
+ if not a.no_download:
+ raise
+ log(" the Hub API is not answering (%r); building the file list from disk" % (e,))
+ tree = []
+ for set_name in [s.strip() for s in a.sets.split(",") if s.strip()]:
+ for p in sorted((root / set_name).glob("*.gguf")):
+ tree.append(("%s/%s" % (set_name, p.name), p.stat().st_size))
+ tree.sort()
(rep_dir / "tree.txt").write_text("\n".join("%s %d" % (p, s) for p, s in tree))
report += ["## Files at this commit", "", "| path | GB |", "|---|---|"]
report += ["| %s | %.2f |" % (p, s / 1e9) for p, s in tree]
diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.py
index d14a33620..f8c2293d0 100644
--- a/tools/dsv41/pack_gguf.py
+++ b/tools/dsv41/pack_gguf.py
@@ -41,6 +41,11 @@ TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 30: ("BF16", 1
10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110), 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176),
14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98),
20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136),
+ # the two 1.x-bit types the IST-DAS 2.00 set uses: device kernels since 2026-10-23 (ds4_iq.cu), so packs with
+ # them load and run entirely on the card. (The integer metadata tensors - engram_token_map and the like - stay
+ # OUT of this table on purpose: the pack tool skips a tensor whose type it does not know, which is exactly right
+ # for them, and adding them here made them "unmapped" instead.)
+ 19: ("IQ1_S", 256, 50), 29: ("IQ1_M", 256, 56),
# the trellis types (tools/trellis/tqenc.hpp; 145 / 146 are the agentionai fork's ggml ids, 224-227 Strata's own)
145: ("TQK6", 128, 26), 146: ("TQK7", 128, 30), 224: ("TQK5X", 128, 22), 225: ("TQK4X", 128, 18),
226: ("TQK5T", 128, 22), 227: ("TQK4T", 128, 18)}