Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 on Windows+CUDA: IQ1_M device kernels (bit-exact vs ggml), IQ4_NL Engram rows, CUDA build fixes (strtok_r, shfl_xor, test language), IST-DAS pack names, hardware bench
Niko1221c42f4f2969b1
diff --git a/CMakeLists.txt b/CMakeLists.txtindex e07c2462f..b60719e02 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -795,9 +795,13 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_link_libraries(ds4_gg_bench PRIVATE strata_kernels_cpu Threads::Threads)target_compile_definitions(ds4_gg_bench PRIVATE STRATA_DSV41_TQ_CPU=1)endif()+ set(_strata_dsv41_test_lang CUDA)+ if(STRATA_ENABLE_HIP)+ set(_strata_dsv41_test_lang HIP)+ endif()if(STRATA_NATIVE_EXPERTS)# the trellis experts (device kernels ds4_tq.cu, CPU lane ds4_tq_cpu.cpp) against a double reference; needs a GPU, no model- set_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE HIP)+ set_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE ${_strata_dsv41_test_lang})add_executable(ds4_tq_test tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cppsrc/dsv41/ds4_iq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp tools/trellis/tqenc.cpp)target_include_directories(ds4_tq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}@@ -819,7 +823,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)endif()# the i-quant experts (ds4_iq.cu) against ggml's own decode of the same blocks + a double reference, device and CPU# lane, for every type the device takes: the gate a new type (IQ2_XS ..) has to pass before the engine uses it- set_source_files_properties(tests/dsv41/iq_expert_test.cpp PROPERTIES LANGUAGE HIP)+ set_source_files_properties(tests/dsv41/iq_expert_test.cpp PROPERTIES LANGUAGE ${_strata_dsv41_test_lang})add_executable(ds4_iq_test tests/dsv41/iq_expert_test.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cppsrc/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)target_include_directories(ds4_iq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex 9112befe5..a49e2cffb 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -37,7 +37,7 @@ template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; }// The types with a known layout and no device kernel (the CPU lane's ggml dots take them): their row bytes are needed to// lay a pack out, nothing else. Ty<> gives the block; the kernels are never instantiated for them.template <> struct Ty<kIQ1_S> { static constexpr int B = 50, TAB = 2048, MIN = 0; };-template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; };+template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; }; // (the device kernels: ggml's iq1s_grid_gpu, a u32 of eight 4-bit grid+1 values)template <> struct Ty<kIQ4_NL> { static constexpr int B = 18, TAB = 16, MIN = 0; };template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };@@ -71,6 +71,7 @@ __device__ __forceinline__ void load_tab(uint32_t* tab) {: T == kIQ2_XS ? (const uint32_t*) iq2xs_grid: T == kIQ2_S ? (const uint32_t*) iq2s_grid: T == kIQ3_XXS ? (const uint32_t*) iq3xxs_grid+ : T == kIQ1_M ? (const uint32_t*) iq1s_grid_gpu: (const uint32_t*) iq3s_grid;for (int i = threadIdx.x; i < Ty<T>::TAB; i += blockDim.x) tab[i] = src[i];}@@ -197,6 +198,27 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,}s.s0 = s.s1 = 2 * (int) (aux >> 28) + 1;s.d = h2f(b) * 0.25f;+ } else if constexpr (T == kIQ1_M) { // qs[32], qh[16], scales u16[4] (four 3-bit scales each; their top nibbles are d)+ // weight = d (2 sc + 1) (grid + delta), grid in {-1, 0, 1}, delta = +-1/8: with 8 (grid + delta) = 8 v - 8 +- 1 (v =+ // grid + 1, the table's nibbles: values 0..3 are nibbles 0,2,4,6 and 4..7 nibbles 1,3,5,7) as an int8 every 8-value+ // group is an exact integer code, and d / 8 is the scale+ const uint16_t* scp = (const uint16_t*) (b + 48);+ const uint32_t sc = (uint32_t) scp[ib >> 1] >> (6 * (ib & 1));+ const uint8_t* qs = b + 4 * ib;+ const uint8_t* qh = b + 32 + 2 * ib;+#pragma unroll+ for (int l = 0; l < 4; ++l) {+ const uint32_t h = (uint32_t) qh[l >> 1] >> (4 * (l & 1));+ const uint32_t e = tab[(uint32_t) qs[l] | ((h & 7u) << 8)];+ const uint32_t dl = (h & 8u) ? 0xf7f7f7f7u : 0xf9f9f9f9u; // -8 -+ 1 in each byte+ s.q[2 * l] = (int) __vadd4((e & 0x0f0f0f0fu) << 3, dl);+ s.q[2 * l + 1] = (int) __vadd4(((e >> 4) & 0x0f0f0f0fu) << 3, dl);+ }+ s.s0 = 2 * (int) (sc & 7u) + 1;+ s.s1 = 2 * (int) ((sc >> 3) & 7u) + 1;+ const uint32_t du = ((uint32_t) scp[0] >> 12) | (((uint32_t) scp[1] >> 8) & 0x00f0u) | (((uint32_t) scp[2] >> 4) & 0x0f00u) |+ ((uint32_t) scp[3] & 0xf000u);+ s.d = __half2float(__ushort_as_half((unsigned short) du)) * 0.125f;} else if constexpr (T == kIQ3_S) {const uint8_t* qs = b + 2;const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), h = b[66 + ib], s4 = ld2(b + 74 + 4 * ib);@@ -350,6 +372,20 @@ __device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, inconst float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;}+ } else if constexpr (T == kIQ1_M) {+ const uint16_t* scp = (const uint16_t*) (b + 48);+ const uint32_t sc = (uint32_t) scp[ib >> 1] >> (6 * (ib & 1));+ const uint32_t h = (uint32_t) b[32 + 2 * ib + (p >> 1)] >> (4 * (p & 1));+ const uint32_t e = tab[(uint32_t) b[4 * ib + p] | ((h & 7u) << 8)];+ const uint32_t du = ((uint32_t) scp[0] >> 12) | (((uint32_t) scp[1] >> 8) & 0x00f0u) | (((uint32_t) scp[2] >> 4) & 0x0f00u) |+ ((uint32_t) scp[3] & 0xf000u);+ const float dl = __half2float(__ushort_as_half((unsigned short) du)) * (float) (2 * (int) ((sc >> (p < 2 ? 0 : 3)) & 7u) + 1);+ const float delta = (h & 8u) ? -0.125f : 0.125f;+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const int gv = (int) (((j < 4 ? e : e >> 4) >> (8 * (j & 3))) & 15u) - 1;+ w[j] = dl * ((float) gv + delta);+ }} else if constexpr (T == kIQ3_XXS || T == kIQ3_S) {const uint8_t* qs = b + 2;uint32_t g1, g2, sg;@@ -854,6 +890,7 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keepsswitch (t) { \case kIQ2_XXS: F(kIQ2_XXS); break; \case kIQ2_XS: F(kIQ2_XS); break; \+ case kIQ1_M: F(kIQ1_M); break; \case kIQ2_S: F(kIQ2_S); break; \case kIQ3_XXS: F(kIQ3_XXS); break; \case kIQ3_S: F(kIQ3_S); break; \@@ -975,7 +1012,7 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {bool supported(int t) {// the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)switch (t) {- case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS:+ case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS: case kIQ1_M:case kQ4_K: case kQ5_K: case kQ6_K: case kQ2_K: case kQ3_K:return true;default:diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 0cfc3bcc7..9f9a1d484 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1272,9 +1272,24 @@ __global__ void __launch_bounds__(1024) confidence_kernel(const float* __restric}// ---------------------------------------------------------------- engram+// ggml's kvalues_iq4nl (the Engram tables of the IST-DAS sets are IQ4_NL: blocks of 32 values, F16 d + 16 bytes of nibbles)+__device__ __forceinline__ float iq4nl_value(const uint8_t* __restrict__ row, int e) {+ static constexpr int8_t kv[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};+ const uint8_t* b = row + (size_t) (e >> 5) * 18;+ const int j = e & 31;+ const int nib = j < 16 ? (b[2 + j] & 0xf) : (b[2 + j - 16] >> 4);+ const float d = __half2float(__ushort_as_half((unsigned short) (b[0] | (b[1] << 8))));+ return __fmul_rn(d, (float) kv[nib]);+}+__global__ void engram_dequant_kernel(const uint8_t* __restrict__ rows, const uint8_t* __restrict__ scales, int hd,- float* __restrict__ out, bool q6k) {+ float* __restrict__ out, int fmt) {const int r = blockIdx.x;+ const bool q6k = fmt == 1;+ if (fmt == 2) {+ for (int e = threadIdx.x; e < hd; e += blockDim.x) out[(size_t) r * hd + e] = iq4nl_value(rows + (size_t) r * hd, e);+ return;+ }if (q6k) { // (ggml Q6_K superblocks at the row's start)for (int e = threadIdx.x; e < hd; e += blockDim.x)out[(size_t) r * hd + e] = q6k_value(rows + (size_t) r * hd + (size_t) (e / 256) * kQ6kBytes, e % 256);@@ -1685,10 +1700,15 @@ __global__ void __launch_bounds__(kThreads) hc_mix_scale_kernel(const float* __r}__global__ void engram_dequant_tok_kernel(const uint8_t* __restrict__ src, size_t per, int cols, int hd,- float* __restrict__ out, bool q6k) {+ float* __restrict__ out, int fmt) {const int r = blockIdx.x, t = r / cols, c = r - t * cols;const uint8_t* rows = src + (size_t) t * per + (size_t) c * hd;const uint8_t* scales = src + (size_t) t * per + (size_t) cols * hd + (size_t) c * (hd / 32);+ const bool q6k = fmt == 1;+ if (fmt == 2) {+ for (int e = threadIdx.x; e < hd; e += blockDim.x) out[(size_t) r * hd + e] = iq4nl_value(rows, e);+ return;+ }if (q6k) { // (ggml Q6_K superblocks at the row's start)for (int e = threadIdx.x; e < hd; e += blockDim.x)out[(size_t) r * hd + e] = q6k_value(rows + (size_t) (e / 256) * kQ6kBytes, e % 256);@@ -1856,7 +1876,7 @@ void hc_post_inplace(const float* y, float* h, const float* post, const float* cvoid hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {hc_mix_scale_kernel<<<T, kThreads, 0, s>>>(h, hcdim, n_mix, eps, mixes);}-void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, bool q6k) {+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, int q6k) {if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, q6k);}void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,@@ -2053,7 +2073,7 @@ void confidence(const float* x, const uint16_t* E, const int* tok, const float*confidence_kernel<<<T, 1024, 0, s>>>(x, E, tok, w, dim, R, out);}-void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k) {+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int q6k) {engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, q6k);}diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuhindex a2fefcfef..94a13997e 100644--- a/src/dsv41/ds4_kernels.cuh+++ b/src/dsv41/ds4_kernels.cuh@@ -177,11 +177,11 @@ void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T,/// Engram. rows: n FP8 E4M3 rows of hd values, scales n x (hd / 32) E8M0 -> out F32 [n * hd]. q6k: the rows are/// ggml Q6_K superblocks instead (hd / 256 of them, 210 bytes each, at the start of each row's hd bytes; no scales),/// decoded as ggml's dequantize_row_q6_K.-void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k = false);+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int q6k = 0);/// The same over the host's per-token layout (a chunk): token t's cols rows, then their scales, `per` bytes a token;/// T tokens -> out F32 [T * cols][hd].void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s,- bool q6k = false);+ int q6k = 0);/// h[t][k] += gate[t][k] * value[t] with the gate from the normalized dot of h[t][k] * w[k] * key[t][k]/// (kv[t] = [key hc x dim | value dim]).void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s);diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex cd4a5222c..0562e5b94 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -2343,12 +2343,18 @@ struct Model::Impl {eg_win.emplace_back(new EgBatch());const Tensor* tw = pk.get("layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.weight");const bool q6k = tw != nullptr && tw->dtype == "Q6_K";+ const bool iq4 = tw != nullptr && tw->dtype == "IQ4_NL"; // (the IST-DAS sets' tables: 32-value blocks of 18 bytes)+ if (iq4 && pk.eg_hd % 32 != 0) {+ err = "Engram layer " + std::to_string(g.eg_layers[(size_t) li]) + ": IQ4_NL rows of " ++ std::to_string(pk.eg_hd) + " values (not whole blocks of 32)";+ return false;+ }if (q6k && pk.eg_hd % 256 != 0) {err = "Engram layer " + std::to_string(g.eg_layers[(size_t) li]) + ": Q6_K rows of " +std::to_string(pk.eg_hd) + " values (not whole superblocks of 256)";return false;}- eg_q6k.push_back(q6k ? 1 : 0);+ eg_q6k.push_back(q6k ? 1 : iq4 ? 2 : 0);}if (o.verbose)std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "@@ -4432,7 +4438,8 @@ struct Model::Impl {const Tensor* tw = pk.get(P + "weight");const Tensor* ts = pk.get(P + "scale");const bool mx8 = tw->dtype == "MXFP8"; // (a GGUF: the scales between the codes, one per 32)- const bool q6k = eg_q6k[(size_t) li] != 0; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)+ const bool q6k = eg_q6k[(size_t) li] == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)+ const bool iq4 = eg_q6k[(size_t) li] == 2; // (a GGUF: IQ4_NL blocks of 32 values: F16 d, 16 nibble bytes)std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its readint64_t hits = 0;{@@ -4482,8 +4489,8 @@ struct Model::Impl {r.mx8 = true;r.n = (uint32_t) (hd / 32 * 33);r.off = tw->off + (uint64_t) id * r.n;- } else if (q6k) { // (into the row's code bytes; its scale bytes unused)- r.n = (uint32_t) (hd / 256 * 210);+ } else if (q6k || iq4) { // (into the row's code bytes; its scale bytes unused)+ r.n = (uint32_t) (q6k ? hd / 256 * 210 : hd / 32 * 18);r.off = tw->off + (uint64_t) id * r.n;} else {r.n = (uint32_t) hd;@@ -5795,7 +5802,7 @@ struct Model::Impl {CK(cudaEventRecord(c.eg_ev[slot], s));for (int s0 = 0; s0 < T; s0 += 256) {const int ns = std::min(256, T - s0);- engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, eg_q6k[(size_t) l.eg] != 0);+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_q6k[(size_t) l.eg]);x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);@@ -7320,7 +7327,7 @@ struct Model::Impl {for (int t = 0; t < T; ++t)CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,(size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));- engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, eg_q6k[(size_t) l.eg] != 0);+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_q6k[(size_t) l.eg]);if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);}diff --git a/src/dsv41/ds4_packinfo.cpp b/src/dsv41/ds4_packinfo.cppindex ee4783970..c71a264eb 100644--- a/src/dsv41/ds4_packinfo.cpp+++ b/src/dsv41/ds4_packinfo.cpp@@ -14,6 +14,10 @@#include <string>#include <vector>+#if defined(_MSC_VER)+#include <cstring>+#define strtok_r strtok_s+#endifnamespace strata::ds4 {namespace {diff --git a/src/dsv41/ds4_tq.cu b/src/dsv41/ds4_tq.cuindex 04b994d40..9c8e76fe3 100644--- a/src/dsv41/ds4_tq.cu+++ b/src/dsv41/ds4_tq.cu@@ -25,6 +25,9 @@#define DS4_TQ_WMMA 1#endif+#if !defined(STRATA_USE_HIP) && defined(__CUDACC__) // (HIP's non-sync shuffle name, for the portable lane on CUDA)+#define __shfl_xor(v, m) __shfl_xor_sync(0xffffffffu, (v), (m))+#endifnamespace strata::ds4::gguf::iq::tq {namespace {diff --git a/tests/dsv41/hip_as_cuda.h b/tests/dsv41/hip_as_cuda.hnew file mode 100644index 000000000..e4ad41923--- /dev/null+++ b/tests/dsv41/hip_as_cuda.h@@ -0,0 +1,17 @@+// Strata. The few HIP names the DeepSeek-V4.1 kernel tests use, on CUDA (the tests are written against HIP).+#pragma once+#include <cuda_runtime.h>+#define hipError_t cudaError_t+#define hipSuccess cudaSuccess+#define hipGetErrorString cudaGetErrorString+#define hipMalloc cudaMalloc+#define hipFree cudaFree+#define hipMemcpy cudaMemcpy+#define hipMemcpyHostToDevice cudaMemcpyHostToDevice+#define hipMemcpyDeviceToHost cudaMemcpyDeviceToHost+#define hipDeviceSynchronize cudaDeviceSynchronize+#define hipEvent_t cudaEvent_t+#define hipEventCreate cudaEventCreate+#define hipEventRecord(e) cudaEventRecord(e, 0)+#define hipEventSynchronize cudaEventSynchronize+#define hipEventElapsedTime cudaEventElapsedTimediff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cppindex a7bb1894e..a14750f7b 100644--- a/tests/dsv41/iq_expert_test.cpp+++ b/tests/dsv41/iq_expert_test.cpp@@ -3,11 +3,15 @@// (grouped entries), the prompt sets (gate/up and down), and the CPU lane on the same experts. Random legal blocks: any// bit pattern with a sane F16 scale is one. This is the H2 gate for a new type (IQ2_XS first): the kernel's integer sums// must equal ggml's weights times the same activations.-// iq_expert_test [--type 16,17,18,21,22,23] [--bench] [--iq256 0|1]+// iq_expert_test [--type 16,17,18,21,22,23,29] [--bench] [--iq256 0|1]#include "../../src/dsv41/ds4_gguf_cpu.hpp"#include "../../src/dsv41/ds4_iq.cuh"+#if defined(__CUDACC__) || defined(STRATA_DSV41_TEST_CUDA)+#include "hip_as_cuda.h"+#else#include <hip/hip_runtime.h>+#endif#include "ggml.h" // ggml_get_type_traits(type)->to_float: ggml's decode of the same blocks@@ -92,6 +96,11 @@ static Mat make_mat(std::mt19937_64& rng, int type, int rows, int k) {for (auto& b : m.bytes) b = (uint8_t) rng();for (size_t b = 0; b < nblk; ++b) { // a sane F16 scale in the first two bytes (every type here has d at offset 0)const uint16_t d = f2h(0.02f + 0.0002f * (float) (rng() % 600));+ if (type == 29) { // IQ1_M has no d field: F16 d is spread over the top nibbles of its four u16 scale words (bytes 48..55)+ uint8_t* sc = &m.bytes[b * (size_t) bb + 48];+ for (int i = 0; i < 4; ++i) sc[2 * i + 1] = (uint8_t) ((sc[2 * i + 1] & 0x0f) | (((d >> (4 * i)) & 15) << 4));+ continue;+ }std::memcpy(&m.bytes[b * (size_t) bb], &d, 2);}m.w.resize((size_t) rows * k);@@ -137,7 +146,7 @@ static void bench(int type) {}int main(int argc, char** argv) {- std::vector<int> types = {16, 17, 22, 18, 21, 23};+ std::vector<int> types = {16, 17, 22, 18, 21, 23, 29};for (int i = 1; i < argc; ++i) {if (std::string(argv[i]) == "--bench") {std::vector<int> bt;diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.pyindex d14a33620..4fe27ec94 100644--- a/tools/dsv41/pack_gguf.py+++ b/tools/dsv41/pack_gguf.py@@ -133,6 +133,10 @@ LAYER = {"engram_embd.weight": "engram.embed.weight","engram_kv.weight": "engram.wkv.weight","engram_gate.weight": "engram.gate.weight", # q_weight * k_weight, folded by the converter+ # the IST-DAS sets keep q and k apart (the engine multiplies them at load) and the projection as engram_wkv+ "engram_wkv.weight": "engram.wkv.weight",+ "engram_q.weight": "engram.q_weight",+ "engram_k.weight": "engram.k_weight",}GLOBAL = {"token_embd.weight": "embed.weight", "output_norm.weight": "norm.weight", "output.weight": "head.weight"}@@ -244,6 +248,30 @@ def ple_to_engram(kv: dict):kv["%s.engram.%s" % (a, k)] = v+def int_tensors_to_engram(gs, kv: dict):+ """The IST-DAS sets carry the Engram's hash parameters as integer tensors of the first shard (engram_token_map I32,+ engram_primes / engram_offsets / engram_multipliers I64, one row a layer) instead of metadata: read them into the+ engram.* keys this pack reads (when the metadata does not have them already)."""+ a = kv.get("general.architecture")+ want = {"engram_token_map.weight": ("token_map", "i", 4), "engram_primes.weight": ("primes", "q", 8),+ "engram_offsets.weight": ("offsets", "q", 8), "engram_multipliers.weight": ("multipliers", "q", 8)}+ for g in gs:+ for name, typ, dims, off in g.tensors:+ if name not in want:+ continue+ key, fmt, sz = want[name]+ if "%s.engram.%s" % (a, key) in kv:+ continue+ n = 1+ for d in dims:+ n *= d+ with g.path.open("rb") as f:+ f.seek(g.data_start + off)+ kv["%s.engram.%s" % (a, key)] = list(struct.unpack("<%d%s" % (n, fmt), f.read(n * sz)))+ if "%s.engram.pad_token_id" % a not in kv and "%s.engram_pad_token_id" % a in kv:+ kv["%s.engram.pad_token_id" % a] = kv["%s.engram_pad_token_id" % a]++def split_ple(g: Gguf, kv: dict):"""llama.cpp's one Engram tensor (per_layer_token_embd: every layer's table, one after the other) -> each layer'sown blk.N.engram_embd as the earlier GGUFs had it: the same bytes, read in place."""@@ -286,7 +314,7 @@ def engram_set(model: pathlib.Path, explicit: str | None):dirs = ([pathlib.Path(explicit).expanduser().resolve()] if explicit else[d for n in ENGRAM_SETS for d in (model / n, model.parent / n)])for d in dirs:- paths = sorted(d.glob("*.gguf")) if d.is_dir() else []+ paths = [d] if d.is_file() else sorted(d.glob("*.gguf")) if d.is_dir() else [] # (--engram: a folder, or the one file)if not paths:if explicit:raise SystemExit("%s: no .gguf files (--engram: the folder of the Engram tables' GGUF set)" % d)@@ -417,7 +445,17 @@ def main() -> int:have = {t[0] for g in gs for t in g.tensors}arch = kv["general.architecture"]layers = kv.get("%s.engram.layers" % arch, [])- lacking = not layers or any("blk.%d.%s.weight" % (il, part) not in have+ int_tensors_to_engram(gs, kv)++ def has_part(il, part, names):+ if "blk.%d.%s.weight" % (il, part) in names:+ return True+ if part == "engram_kv":+ return "blk.%d.engram_wkv.weight" % il in names+ if part == "engram_gate":+ return "blk.%d.engram_q.weight" % il in names and "blk.%d.engram_k.weight" % il in names+ return False+ lacking = not layers or any(not has_part(il, part, have)for il in layers for part in ("engram_embd", "engram_kv", "engram_gate"))eg_dir, egs = engram_set(model, a.engram) if lacking or a.engram else (None, [])eg_from = {}@@ -432,13 +470,16 @@ def main() -> int:split_ple(g, kv)for p, g in zip(eshards, egs):picked = [t for t in g.tensors if re.match(r"blk\.\d+\.engram_", t[0]) and t[0] not in have]+ picked = [t for t in picked if t[0].endswith("engram_embd.weight") or not has_part(+ int(t[0].split(".")[1]), "engram_kv" if t[0].endswith("engram_kv.weight") else "engram_gate", have)]+if picked:g.tensors = pickedeg_from[p] = gfor il in kv.get("%s.engram.layers" % arch, []):for part in ("engram_embd", "engram_kv", "engram_gate"):nm = "blk.%d.%s.weight" % (il, part)- if nm not in have and not any(t[0] == nm for g in eg_from.values() for t in g.tensors):+ if not has_part(il, part, have) and not any(t[0] == nm for g in eg_from.values() for t in g.tensors):raise SystemExit("%s: missing - not in the quant's shards nor in an Engram set (--engram, or ""<model>/../Engram/: unsloth's UD-Q8_K_XL shard of the Engram tables)" % nm)rel = lambda p: os.path.relpath(p, model).replace(os.sep, "/") # noqa: E731@@ -477,6 +518,8 @@ def main() -> int:if ck.endswith(".attn.wo_a.slice"):# [o_groups * o_lora][in] -> o_groups entries of [o_lora][in] (group g: rows g * o_lora ..)n = int(kv["%s.attention.output_group_count" % kv["general.architecture"]])+ if len(shape) == 3 and shape[0] == n: # (the IST-DAS sets: [groups][o_lora][in] in one tensor)+ shape = [shape[0] * shape[1], shape[2]]if len(shape) != 2 or shape[0] % n:raise SystemExit("%s: %s %s does not split in %d groups" % (g.path.name, name, shape, n))part = nbytes // ndiff --git a/tools/dsv41/win/hwbench.cu b/tools/dsv41/win/hwbench.cunew file mode 100644index 000000000..6903ffc0d--- /dev/null+++ b/tools/dsv41/win/hwbench.cu@@ -0,0 +1,104 @@+// hwbench: host RAM bandwidth, PCIe host->device, and unbuffered overlapped NVMe reads (Windows).+// hwbench ram [threads] [MiB]+// hwbench pcie+// hwbench nvme <file> <block_KiB> <qd> <seconds> [seq|rand]+#include <cuda_runtime.h>+#include <windows.h>+#include <algorithm>+#include <chrono>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <string>+#include <thread>+#include <vector>++static double now() { return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count(); }++static void bench_ram(int threads, size_t mib) {+ size_t n = mib << 20;+ std::vector<char*> src(threads), dst(threads);+ for (int t = 0; t < threads; t++) {+ src[t] = (char*)_aligned_malloc(n, 4096); dst[t] = (char*)_aligned_malloc(n, 4096);+ memset(src[t], t + 1, n); memset(dst[t], 0, n);+ }+ for (int mode = 0; mode < 2; mode++) {+ double best = 0;+ for (int rep = 0; rep < 4; rep++) {+ std::vector<std::thread> th; volatile uint64_t sink = 0;+ double t0 = now();+ for (int t = 0; t < threads; t++)+ th.emplace_back([&, t] {+ if (mode == 0) memcpy(dst[t], src[t], n);+ else { uint64_t s = 0; const uint64_t* p = (const uint64_t*)src[t]; for (size_t i = 0; i < n / 8; i += 1) s += p[i]; sink = sink + s; }+ });+ for (auto& x : th) x.join();+ double dt = now() - t0; double gb = (mode == 0 ? 2.0 : 1.0) * n * threads / 1e9 / dt; // copy counts read+write+ if (gb > best) best = gb;+ }+ printf("ram %s threads=%d size/thread=%zu MiB best=%.1f GB/s (%s)\n", mode == 0 ? "copy" : "read", threads, mib, best, mode == 0 ? "read+write counted" : "read only");+ }+}++static void bench_pcie() {+ for (size_t mib : {1, 4, 16, 64, 256}) {+ size_t n = mib << 20; void *h, *d; cudaStream_t s; cudaStreamCreate(&s);+ cudaMallocHost(&h, n); cudaMalloc(&d, n); memset(h, 1, n);+ void* hp = _aligned_malloc(n, 4096); memset(hp, 1, n);+ for (int pinned = 1; pinned >= 0; pinned--) {+ void* src = pinned ? h : hp;+ for (int w = 0; w < 3; w++) cudaMemcpyAsync(d, src, n, cudaMemcpyHostToDevice, s);+ cudaStreamSynchronize(s);+ int reps = (int)std::max<size_t>(8, (1024u / mib)); double t0 = now();+ for (int i = 0; i < reps; i++) cudaMemcpyAsync(d, src, n, cudaMemcpyHostToDevice, s);+ cudaStreamSynchronize(s); double dt = now() - t0;+ printf("pcie H2D %s %4zu MiB: %.2f GB/s\n", pinned ? "pinned " : "pageable", mib, (double)n * reps / 1e9 / dt);+ }+ cudaFreeHost(h); cudaFree(d); _aligned_free(hp); cudaStreamDestroy(s);+ }+}++static void bench_nvme(const char* path, size_t block, int qd, double secs, bool seq) {+ HANDLE f = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);+ if (f == INVALID_HANDLE_VALUE) { printf("open failed %lu\n", GetLastError()); return; }+ LARGE_INTEGER sz; GetFileSizeEx(f, &sz);+ HANDLE port = CreateIoCompletionPort(f, nullptr, 0, 1);+ uint64_t nblk = (uint64_t)sz.QuadPart / block;+ struct Slot { OVERLAPPED ov; char* buf; };+ std::vector<Slot> slots(qd);+ std::mt19937_64 rng(12345);+ uint64_t seqpos = 0, bytes = 0, ops = 0;+ std::vector<double> lat; std::vector<double> t_issue(qd);+ auto issue = [&](int i) {+ uint64_t b = seq ? (seqpos++ % nblk) : (rng() % nblk);+ uint64_t off = b * block;+ memset(&slots[i].ov, 0, sizeof(OVERLAPPED));+ slots[i].ov.Offset = (DWORD)(off & 0xffffffff); slots[i].ov.OffsetHigh = (DWORD)(off >> 32);+ t_issue[i] = now();+ if (!ReadFile(f, slots[i].buf, (DWORD)block, nullptr, &slots[i].ov) && GetLastError() != ERROR_IO_PENDING) { printf("ReadFile err %lu\n", GetLastError()); exit(1); }+ };+ for (int i = 0; i < qd; i++) { slots[i].buf = (char*)VirtualAlloc(nullptr, block, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); issue(i); }+ double t0 = now(), tend = t0 + secs;+ while (true) {+ DWORD nb; ULONG_PTR key; OVERLAPPED* ov;+ if (!GetQueuedCompletionStatus(port, &nb, &key, &ov, 10000) && !ov) { printf("wait err\n"); break; }+ int i = (int)(((Slot*)((char*)ov - offsetof(Slot, ov))) - slots.data());+ bytes += nb; ops++; lat.push_back(now() - t_issue[i]);+ if (now() < tend) issue(i); else if (--qd == 0) break;+ }+ double dt = now() - t0;+ std::sort(lat.begin(), lat.end());+ printf("nvme %s block=%zuKiB qd=%zu: %.2f GB/s, %.0f IOPS, lat p50=%.2f ms p99=%.2f ms\n", seq ? "seq " : "rand", block >> 10, slots.size(),+ bytes / 1e9 / dt, ops / dt, lat[lat.size() / 2] * 1e3, lat[lat.size() * 99 / 100] * 1e3);+ CloseHandle(f);+}++int main(int argc, char** argv) {+ std::string m = argc > 1 ? argv[1] : "";+ if (m == "ram") bench_ram(argc > 2 ? atoi(argv[2]) : 6, argc > 3 ? atoi(argv[3]) : 512);+ else if (m == "pcie") bench_pcie();+ else if (m == "nvme" && argc > 5) bench_nvme(argv[2], (size_t)atoi(argv[3]) << 10, atoi(argv[4]), atof(argv[5]), argc > 6 && std::string(argv[6]) == "seq");+ else printf("usage: hwbench ram|pcie|nvme ...\n");+}