Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: the Engram reader takes IST-DAS's IQ4_NL table, and a pack with an unknown Engram type stops cleanly instead of segfaulting
Niko1221b2d2cf22eb98
diff --git a/src/dsv41/ds4_engram_io.hpp b/src/dsv41/ds4_engram_io.hppindex eb190f43c..2402ec1af 100644--- a/src/dsv41/ds4_engram_io.hpp+++ b/src/dsv41/ds4_engram_io.hpp@@ -89,9 +89,9 @@ private:std::vector<std::unique_ptr<uint8_t[]>> blocks_;};-/// One row to fetch: its codes to dst, its scales to scales. mx8: n bytes at off of a file, [scale, 32 codes] blocks-/// (a GGUF's MXFP8 row); else n code bytes at off and n2 scale bytes at off2 of file2 (two reads; n2 0: none - a-/// GGUF's Q6_K row, its scales inside its n bytes).+/// One row to fetch: its bytes to dst, its scales to scales. mx8: n bytes at off of a file, [scale, 32 codes] blocks+/// (a GGUF's MXFP8 row); else n row bytes at off and n2 scale bytes at off2 of file2 (two reads; n2 0: none - a GGUF's+/// Q6_K or IQ4_NL row, their scales inside their n bytes: 210 a 256 values, 18 a 32).struct EgRead {int file = -1;uint64_t off = 0;diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 0cfc3bcc7..789390f78 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1272,14 +1272,30 @@ __global__ void __launch_bounds__(1024) confidence_kernel(const float* __restric}// ---------------------------------------------------------------- engram+// ggml's IQ4_NL: a block of 32 values is 18 bytes, an fp16 d then 16 packed nibble bytes; the low nibble of qs[j] is+// value j and its high nibble value j + 16, each an index into this codebook (ggml-common.h's kvalues_iq4nl,+// dequantize_row_iq4_nl). A row is hd / 32 of these blocks, the d of every one inside it - no separate scales.+__device__ __constant__ int8_t kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10,+ 1, 13, 25, 38, 53, 69, 89, 113};+__device__ __forceinline__ float iq4nl_value(const uint8_t* b, int e) {+ const uint8_t q = b[2 + (e & 15)];+ const int v = (e & 16) ? (q >> 4) : (q & 15);+ return __half2float(__ushort_as_half(*(const uint16_t*) b)) * (float) kIq4nl[v];+}+__global__ void engram_dequant_kernel(const uint8_t* __restrict__ rows, const uint8_t* __restrict__ scales, int hd,- float* __restrict__ out, bool q6k) {+ float* __restrict__ out, int fmt) {const int r = blockIdx.x;- if (q6k) { // (ggml Q6_K superblocks at the row's start)+ if (fmt == 1) { // (ggml Q6_K superblocks at the row's start)for (int e = threadIdx.x; e < hd; e += blockDim.x)out[(size_t) r * hd + e] = q6k_value(rows + (size_t) r * hd + (size_t) (e / 256) * kQ6kBytes, e % 256);return;}+ if (fmt == 2) { // (ggml IQ4_NL blocks at the row's start, their scales inside)+ for (int e = threadIdx.x; e < hd; e += blockDim.x)+ out[(size_t) r * hd + e] = iq4nl_value(rows + (size_t) r * hd + (size_t) (e / 32) * 18, e % 32);+ return;+ }for (int e = threadIdx.x; e < hd; e += blockDim.x) {const float hv = e4m3_value(rows[(size_t) r * hd + e]);const uint8_t sb = scales[(size_t) r * (hd / 32) + e / 32];@@ -1685,15 +1701,20 @@ __global__ void __launch_bounds__(kThreads) hc_mix_scale_kernel(const float* __r}__global__ void engram_dequant_tok_kernel(const uint8_t* __restrict__ src, size_t per, int cols, int hd,- float* __restrict__ out, bool q6k) {+ float* __restrict__ out, int fmt) {const int r = blockIdx.x, t = r / cols, c = r - t * cols;const uint8_t* rows = src + (size_t) t * per + (size_t) c * hd;const uint8_t* scales = src + (size_t) t * per + (size_t) cols * hd + (size_t) c * (hd / 32);- if (q6k) { // (ggml Q6_K superblocks at the row's start)+ if (fmt == 1) { // (ggml Q6_K superblocks at the row's start)for (int e = threadIdx.x; e < hd; e += blockDim.x)out[(size_t) r * hd + e] = q6k_value(rows + (size_t) (e / 256) * kQ6kBytes, e % 256);return;}+ if (fmt == 2) { // (ggml IQ4_NL blocks at the row's start, their scales inside)+ for (int e = threadIdx.x; e < hd; e += blockDim.x)+ out[(size_t) r * hd + e] = iq4nl_value(rows + (size_t) (e / 32) * 18, e % 32);+ return;+ }for (int e = threadIdx.x; e < hd; e += blockDim.x) {const float hv = e4m3_value(rows[e]);const uint8_t sb = scales[e / 32];@@ -1856,8 +1877,8 @@ void hc_post_inplace(const float* y, float* h, const float* post, const float* cvoid hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {hc_mix_scale_kernel<<<T, kThreads, 0, s>>>(h, hcdim, n_mix, eps, mixes);}-void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, bool q6k) {- if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, q6k);+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, int fmt) {+ if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, fmt);}void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,cudaStream_t s) {@@ -2053,8 +2074,8 @@ void confidence(const float* x, const uint16_t* E, const int* tok, const float*confidence_kernel<<<T, 1024, 0, s>>>(x, E, tok, w, dim, R, out);}-void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k) {- engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, q6k);+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int fmt) {+ engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, fmt);}void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s) {diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuhindex a2fefcfef..798bca1d9 100644--- a/src/dsv41/ds4_kernels.cuh+++ b/src/dsv41/ds4_kernels.cuh@@ -174,14 +174,15 @@ struct RingBatch {};void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s);-/// Engram. rows: n FP8 E4M3 rows of hd values, scales n x (hd / 32) E8M0 -> out F32 [n * hd]. q6k: the rows are-/// ggml Q6_K superblocks instead (hd / 256 of them, 210 bytes each, at the start of each row's hd bytes; no scales),-/// decoded as ggml's dequantize_row_q6_K.-void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k = false);+/// Engram. rows: n rows of hd values -> out F32 [n * hd]. fmt, the pack's row layout: 0 FP8 E4M3 values with E8M0+/// scales (scales: n x (hd / 32)); 1 ggml Q6_K superblocks (hd / 256 of 210 bytes at the start of each row's hd bytes;+/// no scales), decoded as ggml's dequantize_row_q6_K; 2 ggml IQ4_NL blocks (hd / 32 of 18 bytes, an fp16 d and 16+/// nibble bytes at the start of each row's hd bytes; no scales), decoded as ggml's dequantize_row_iq4_nl.+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int fmt = 0);/// The same over the host's per-token layout (a chunk): token t's cols rows, then their scales, `per` bytes a token;/// T tokens -> out F32 [T * cols][hd].void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s,- bool q6k = false);+ int fmt = 0);/// h[t][k] += gate[t][k] * value[t] with the gate from the normalized dot of h[t][k] * w[k] * key[t][k]/// (kv[t] = [key hc x dim | value dim]).void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s);diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex cd4a5222c..fe4e457c5 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -884,7 +884,10 @@ struct Model::Impl {EgRowCache eg_cache;std::mutex eg_cache_mu;std::vector<std::unique_ptr<EgBatch>> eg_win;- std::vector<uint8_t> eg_q6k; // Engram layer li's table in ggml Q6_K (a row one superblock a 256 values), else FP8+ // Engram layer li's embed rows, as the pack stores them: 0 MXFP8 ([scale, 32 codes] blocks, 33 bytes a 32), 1+ // ggml Q6_K (210 bytes a 256), 2 ggml IQ4_NL (18 bytes a 32: an fp16 d and 16 nibble bytes), 3 an old pack's FP8+ // codes with their E8M0 scales in a tensor of their own (see eg_issue)+ std::vector<uint8_t> eg_fmt;std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};uint16_t* h_emb = nullptr; // the window's embedding rowsstd::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;@@ -2341,14 +2344,41 @@ struct Model::Impl {eg_io.start(paths, nth);for (int li = 0; li < pk.eg_layers; ++li) {eg_win.emplace_back(new EgBatch());- const Tensor* tw = pk.get("layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.weight");- const bool q6k = tw != nullptr && tw->dtype == "Q6_K";- if (q6k && pk.eg_hd % 256 != 0) {- err = "Engram layer " + std::to_string(g.eg_layers[(size_t) li]) + ": Q6_K rows of " +- std::to_string(pk.eg_hd) + " values (not whole superblocks of 256)";+ const std::string id = std::to_string(g.eg_layers[(size_t) li]);+ const Tensor* tw = pk.get("layers." + id + ".engram.embed.weight");+ // The row layout the reader and the dequant take. A type not here (or no tensor at all) would be read+ // as some other format's bytes - the old code then read the scale tensor of a pack that has none and+ // crashed in eg_issue - so refuse the pack once, with the type's name, before any token is read.+ int fmt;+ if (tw == nullptr) {+ std::fprintf(stderr, "ds4: Engram layer %s: engram.embed.weight is missing from the pack\n", id.c_str());+ std::exit(2);+ } else if (tw->dtype == "MXFP8") {+ fmt = 0;+ } else if (tw->dtype == "Q6_K") {+ fmt = 1;+ } else if (tw->dtype == "IQ4_NL") {+ fmt = 2;+ } else if (pk.get("layers." + id + ".engram.embed.scale") != nullptr) {+ fmt = 3; // (an old pack: FP8 codes in one tensor, their E8M0 scales in another)+ } else {+ std::fprintf(stderr,+ "ds4: Engram layer %s: engram.embed.weight type %s is not one the Engram reader handles "+ "(MXFP8, Q6_K, IQ4_NL)\n",+ id.c_str(), tw->dtype.c_str());+ std::exit(2);+ }+ if (fmt == 1 && pk.eg_hd % 256 != 0) {+ err = "Engram layer " + id + ": Q6_K rows of " + std::to_string(pk.eg_hd) ++ " values (not whole superblocks of 256)";return false;}- eg_q6k.push_back(q6k ? 1 : 0);+ if (fmt == 2 && pk.eg_hd % 32 != 0) {+ err = "Engram layer " + id + ": IQ4_NL rows of " + std::to_string(pk.eg_hd) ++ " values (not whole blocks of 32)";+ return false;+ }+ eg_fmt.push_back((uint8_t) fmt);}if (o.verbose)std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "@@ -4431,8 +4461,10 @@ struct Model::Impl {const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";const Tensor* tw = pk.get(P + "weight");const Tensor* ts = pk.get(P + "scale");- const bool mx8 = tw->dtype == "MXFP8"; // (a GGUF: the scales between the codes, one per 32)- const bool q6k = eg_q6k[(size_t) li] != 0; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)+ const int fmt = eg_fmt[(size_t) li];+ const bool mx8 = fmt == 0; // (a GGUF: the scales between the codes, one per 32)+ const bool q6k = fmt == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)+ const bool iq4 = fmt == 2; // (a GGUF: 18-byte blocks a 32 values, d and 16 nibble bytes, no scales)std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its readint64_t hits = 0;{@@ -4485,6 +4517,9 @@ struct Model::Impl {} else if (q6k) { // (into the row's code bytes; its scale bytes unused)r.n = (uint32_t) (hd / 256 * 210);r.off = tw->off + (uint64_t) id * r.n;+ } else if (iq4) { // (the same: one contiguous row, its blocks' d inside them, no scales)+ r.n = (uint32_t) (hd / 32 * 18);+ r.off = tw->off + (uint64_t) id * r.n;} else {r.n = (uint32_t) hd;r.off = tw->off + (uint64_t) id * hd;@@ -5795,7 +5830,7 @@ struct Model::Impl {CK(cudaEventRecord(c.eg_ev[slot], s));for (int s0 = 0; s0 < T; s0 += 256) {const int ns = std::min(256, T - s0);- engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, eg_q6k[(size_t) l.eg] != 0);+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_fmt[(size_t) l.eg]);x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);@@ -7320,7 +7355,7 @@ struct Model::Impl {for (int t = 0; t < T; ++t)CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,(size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));- engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, eg_q6k[(size_t) l.eg] != 0);+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);}diff --git a/tools/dsv41/ista_intake.py b/tools/dsv41/ista_intake.pyindex 5308d2365..b9e555711 100644--- a/tools/dsv41/ista_intake.py+++ b/tools/dsv41/ista_intake.py@@ -227,7 +227,17 @@ def main() -> int:report: list[str] = ["# IST-DAS intake, commit %s" % commit, "", "run %s" % time.strftime("%Y-%m-%dT%H:%M:%S%z"), ""]log("listing %s at %s" % (REPO, commit[:7]))- tree = repo_tree(commit)+ try:+ tree = repo_tree(commit)+ except Exception as e: # noqa: BLE001 (HF's API rate-limits unauthenticated boxes; the files are already local)+ if not a.no_download:+ raise+ log(" the Hub API is not answering (%r); building the file list from disk" % (e,))+ tree = []+ for set_name in [s.strip() for s in a.sets.split(",") if s.strip()]:+ for p in sorted((root / set_name).glob("*.gguf")):+ tree.append(("%s/%s" % (set_name, p.name), p.stat().st_size))+ tree.sort()(rep_dir / "tree.txt").write_text("\n".join("%s %d" % (p, s) for p, s in tree))report += ["## Files at this commit", "", "| path | GB |", "|---|---|"]report += ["| %s | %.2f |" % (p, s / 1e9) for p, s in tree]diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.pyindex d14a33620..f8c2293d0 100644--- a/tools/dsv41/pack_gguf.py+++ b/tools/dsv41/pack_gguf.py@@ -41,6 +41,11 @@ TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 30: ("BF16", 110: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110), 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176),14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98),20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136),+ # the two 1.x-bit types the IST-DAS 2.00 set uses: device kernels since 2026-10-23 (ds4_iq.cu), so packs with+ # them load and run entirely on the card. (The integer metadata tensors - engram_token_map and the like - stay+ # OUT of this table on purpose: the pack tool skips a tensor whose type it does not know, which is exactly right+ # for them, and adding them here made them "unmapped" instead.)+ 19: ("IQ1_S", 256, 50), 29: ("IQ1_M", 256, 56),# the trellis types (tools/trellis/tqenc.hpp; 145 / 146 are the agentionai fork's ggml ids, 224-227 Strata's own)145: ("TQK6", 128, 26), 146: ("TQK7", 128, 30), 224: ("TQK5X", 128, 22), 225: ("TQK4X", 128, 18),226: ("TQK5T", 128, 22), 227: ("TQK4T", 128, 18)}