Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 on Windows+CUDA: IQ1_M device kernels (bit-exact vs ggml), IQ4_NL Engram rows, CUDA build fixes (strtok_r, shfl_xor, test language), IST-DAS pack names, hardware bench

Niko1221c42f4f2969b1
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e07c2462f..b60719e02 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -795,9 +795,13 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_link_libraries(ds4_gg_bench PRIVATE strata_kernels_cpu Threads::Threads)
target_compile_definitions(ds4_gg_bench PRIVATE STRATA_DSV41_TQ_CPU=1)
endif()
+ set(_strata_dsv41_test_lang CUDA)
+ if(STRATA_ENABLE_HIP)
+ set(_strata_dsv41_test_lang HIP)
+ endif()
if(STRATA_NATIVE_EXPERTS)
# the trellis experts (device kernels ds4_tq.cu, CPU lane ds4_tq_cpu.cpp) against a double reference; needs a GPU, no model
- set_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE HIP)
+ set_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE ${_strata_dsv41_test_lang})
add_executable(ds4_tq_test tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp
src/dsv41/ds4_iq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp tools/trellis/tqenc.cpp)
target_include_directories(ds4_tq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
@@ -819,7 +823,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
endif()
# the i-quant experts (ds4_iq.cu) against ggml's own decode of the same blocks + a double reference, device and CPU
# lane, for every type the device takes: the gate a new type (IQ2_XS ..) has to pass before the engine uses it
- set_source_files_properties(tests/dsv41/iq_expert_test.cpp PROPERTIES LANGUAGE HIP)
+ set_source_files_properties(tests/dsv41/iq_expert_test.cpp PROPERTIES LANGUAGE ${_strata_dsv41_test_lang})
add_executable(ds4_iq_test tests/dsv41/iq_expert_test.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp
src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)
target_include_directories(ds4_iq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index 9112befe5..a49e2cffb 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -37,7 +37,7 @@ template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; }
// The types with a known layout and no device kernel (the CPU lane's ggml dots take them): their row bytes are needed to
// lay a pack out, nothing else. Ty<> gives the block; the kernels are never instantiated for them.
template <> struct Ty<kIQ1_S> { static constexpr int B = 50, TAB = 2048, MIN = 0; };
-template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; };
+template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; }; // (the device kernels: ggml's iq1s_grid_gpu, a u32 of eight 4-bit grid+1 values)
template <> struct Ty<kIQ4_NL> { static constexpr int B = 18, TAB = 16, MIN = 0; };
template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };
template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };
@@ -71,6 +71,7 @@ __device__ __forceinline__ void load_tab(uint32_t* tab) {
: T == kIQ2_XS ? (const uint32_t*) iq2xs_grid
: T == kIQ2_S ? (const uint32_t*) iq2s_grid
: T == kIQ3_XXS ? (const uint32_t*) iq3xxs_grid
+ : T == kIQ1_M ? (const uint32_t*) iq1s_grid_gpu
: (const uint32_t*) iq3s_grid;
for (int i = threadIdx.x; i < Ty<T>::TAB; i += blockDim.x) tab[i] = src[i];
}
@@ -197,6 +198,27 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,
}
s.s0 = s.s1 = 2 * (int) (aux >> 28) + 1;
s.d = h2f(b) * 0.25f;
+ } else if constexpr (T == kIQ1_M) { // qs[32], qh[16], scales u16[4] (four 3-bit scales each; their top nibbles are d)
+ // weight = d (2 sc + 1) (grid + delta), grid in {-1, 0, 1}, delta = +-1/8: with 8 (grid + delta) = 8 v - 8 +- 1 (v =
+ // grid + 1, the table's nibbles: values 0..3 are nibbles 0,2,4,6 and 4..7 nibbles 1,3,5,7) as an int8 every 8-value
+ // group is an exact integer code, and d / 8 is the scale
+ const uint16_t* scp = (const uint16_t*) (b + 48);
+ const uint32_t sc = (uint32_t) scp[ib >> 1] >> (6 * (ib & 1));
+ const uint8_t* qs = b + 4 * ib;
+ const uint8_t* qh = b + 32 + 2 * ib;
+#pragma unroll
+ for (int l = 0; l < 4; ++l) {
+ const uint32_t h = (uint32_t) qh[l >> 1] >> (4 * (l & 1));
+ const uint32_t e = tab[(uint32_t) qs[l] | ((h & 7u) << 8)];
+ const uint32_t dl = (h & 8u) ? 0xf7f7f7f7u : 0xf9f9f9f9u; // -8 -+ 1 in each byte
+ s.q[2 * l] = (int) __vadd4((e & 0x0f0f0f0fu) << 3, dl);
+ s.q[2 * l + 1] = (int) __vadd4(((e >> 4) & 0x0f0f0f0fu) << 3, dl);
+ }
+ s.s0 = 2 * (int) (sc & 7u) + 1;
+ s.s1 = 2 * (int) ((sc >> 3) & 7u) + 1;
+ const uint32_t du = ((uint32_t) scp[0] >> 12) | (((uint32_t) scp[1] >> 8) & 0x00f0u) | (((uint32_t) scp[2] >> 4) & 0x0f00u) |
+ ((uint32_t) scp[3] & 0xf000u);
+ s.d = __half2float(__ushort_as_half((unsigned short) du)) * 0.125f;
} else if constexpr (T == kIQ3_S) {
const uint8_t* qs = b + 2;
const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), h = b[66 + ib], s4 = ld2(b + 74 + 4 * ib);
@@ -350,6 +372,20 @@ __device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, in
const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
}
+ } else if constexpr (T == kIQ1_M) {
+ const uint16_t* scp = (const uint16_t*) (b + 48);
+ const uint32_t sc = (uint32_t) scp[ib >> 1] >> (6 * (ib & 1));
+ const uint32_t h = (uint32_t) b[32 + 2 * ib + (p >> 1)] >> (4 * (p & 1));
+ const uint32_t e = tab[(uint32_t) b[4 * ib + p] | ((h & 7u) << 8)];
+ const uint32_t du = ((uint32_t) scp[0] >> 12) | (((uint32_t) scp[1] >> 8) & 0x00f0u) | (((uint32_t) scp[2] >> 4) & 0x0f00u) |
+ ((uint32_t) scp[3] & 0xf000u);
+ const float dl = __half2float(__ushort_as_half((unsigned short) du)) * (float) (2 * (int) ((sc >> (p < 2 ? 0 : 3)) & 7u) + 1);
+ const float delta = (h & 8u) ? -0.125f : 0.125f;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const int gv = (int) (((j < 4 ? e : e >> 4) >> (8 * (j & 3))) & 15u) - 1;
+ w[j] = dl * ((float) gv + delta);
+ }
} else if constexpr (T == kIQ3_XXS || T == kIQ3_S) {
const uint8_t* qs = b + 2;
uint32_t g1, g2, sg;
@@ -854,6 +890,7 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps
switch (t) { \
case kIQ2_XXS: F(kIQ2_XXS); break; \
case kIQ2_XS: F(kIQ2_XS); break; \
+ case kIQ1_M: F(kIQ1_M); break; \
case kIQ2_S: F(kIQ2_S); break; \
case kIQ3_XXS: F(kIQ3_XXS); break; \
case kIQ3_S: F(kIQ3_S); break; \
@@ -975,7 +1012,7 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {
bool supported(int t) {
// the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)
switch (t) {
- case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS:
+ case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS: case kIQ1_M:
case kQ4_K: case kQ5_K: case kQ6_K: case kQ2_K: case kQ3_K:
return true;
default:
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index 0cfc3bcc7..9f9a1d484 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1272,9 +1272,24 @@ __global__ void __launch_bounds__(1024) confidence_kernel(const float* __restric
}
// ---------------------------------------------------------------- engram
+// ggml's kvalues_iq4nl (the Engram tables of the IST-DAS sets are IQ4_NL: blocks of 32 values, F16 d + 16 bytes of nibbles)
+__device__ __forceinline__ float iq4nl_value(const uint8_t* __restrict__ row, int e) {
+ static constexpr int8_t kv[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
+ const uint8_t* b = row + (size_t) (e >> 5) * 18;
+ const int j = e & 31;
+ const int nib = j < 16 ? (b[2 + j] & 0xf) : (b[2 + j - 16] >> 4);
+ const float d = __half2float(__ushort_as_half((unsigned short) (b[0] | (b[1] << 8))));
+ return __fmul_rn(d, (float) kv[nib]);
+}
+
__global__ void engram_dequant_kernel(const uint8_t* __restrict__ rows, const uint8_t* __restrict__ scales, int hd,
- float* __restrict__ out, bool q6k) {
+ float* __restrict__ out, int fmt) {
const int r = blockIdx.x;
+ const bool q6k = fmt == 1;
+ if (fmt == 2) {
+ for (int e = threadIdx.x; e < hd; e += blockDim.x) out[(size_t) r * hd + e] = iq4nl_value(rows + (size_t) r * hd, e);
+ return;
+ }
if (q6k) { // (ggml Q6_K superblocks at the row's start)
for (int e = threadIdx.x; e < hd; e += blockDim.x)
out[(size_t) r * hd + e] = q6k_value(rows + (size_t) r * hd + (size_t) (e / 256) * kQ6kBytes, e % 256);
@@ -1685,10 +1700,15 @@ __global__ void __launch_bounds__(kThreads) hc_mix_scale_kernel(const float* __r
}
__global__ void engram_dequant_tok_kernel(const uint8_t* __restrict__ src, size_t per, int cols, int hd,
- float* __restrict__ out, bool q6k) {
+ float* __restrict__ out, int fmt) {
const int r = blockIdx.x, t = r / cols, c = r - t * cols;
const uint8_t* rows = src + (size_t) t * per + (size_t) c * hd;
const uint8_t* scales = src + (size_t) t * per + (size_t) cols * hd + (size_t) c * (hd / 32);
+ const bool q6k = fmt == 1;
+ if (fmt == 2) {
+ for (int e = threadIdx.x; e < hd; e += blockDim.x) out[(size_t) r * hd + e] = iq4nl_value(rows, e);
+ return;
+ }
if (q6k) { // (ggml Q6_K superblocks at the row's start)
for (int e = threadIdx.x; e < hd; e += blockDim.x)
out[(size_t) r * hd + e] = q6k_value(rows + (size_t) (e / 256) * kQ6kBytes, e % 256);
@@ -1856,7 +1876,7 @@ void hc_post_inplace(const float* y, float* h, const float* post, const float* c
void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {
hc_mix_scale_kernel<<<T, kThreads, 0, s>>>(h, hcdim, n_mix, eps, mixes);
}
-void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, bool q6k) {
+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, int q6k) {
if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, q6k);
}
void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,
@@ -2053,7 +2073,7 @@ void confidence(const float* x, const uint16_t* E, const int* tok, const float*
confidence_kernel<<<T, 1024, 0, s>>>(x, E, tok, w, dim, R, out);
}
-void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k) {
+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int q6k) {
engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, q6k);
}
diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuh
index a2fefcfef..94a13997e 100644
--- a/src/dsv41/ds4_kernels.cuh
+++ b/src/dsv41/ds4_kernels.cuh
@@ -177,11 +177,11 @@ void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T,
/// Engram. rows: n FP8 E4M3 rows of hd values, scales n x (hd / 32) E8M0 -> out F32 [n * hd]. q6k: the rows are
/// ggml Q6_K superblocks instead (hd / 256 of them, 210 bytes each, at the start of each row's hd bytes; no scales),
/// decoded as ggml's dequantize_row_q6_K.
-void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k = false);
+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int q6k = 0);
/// The same over the host's per-token layout (a chunk): token t's cols rows, then their scales, `per` bytes a token;
/// T tokens -> out F32 [T * cols][hd].
void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s,
- bool q6k = false);
+ int q6k = 0);
/// h[t][k] += gate[t][k] * value[t] with the gate from the normalized dot of h[t][k] * w[k] * key[t][k]
/// (kv[t] = [key hc x dim | value dim]).
void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s);
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index cd4a5222c..0562e5b94 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -2343,12 +2343,18 @@ struct Model::Impl {
eg_win.emplace_back(new EgBatch());
const Tensor* tw = pk.get("layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.weight");
const bool q6k = tw != nullptr && tw->dtype == "Q6_K";
+ const bool iq4 = tw != nullptr && tw->dtype == "IQ4_NL"; // (the IST-DAS sets' tables: 32-value blocks of 18 bytes)
+ if (iq4 && pk.eg_hd % 32 != 0) {
+ err = "Engram layer " + std::to_string(g.eg_layers[(size_t) li]) + ": IQ4_NL rows of " +
+ std::to_string(pk.eg_hd) + " values (not whole blocks of 32)";
+ return false;
+ }
if (q6k && pk.eg_hd % 256 != 0) {
err = "Engram layer " + std::to_string(g.eg_layers[(size_t) li]) + ": Q6_K rows of " +
std::to_string(pk.eg_hd) + " values (not whole superblocks of 256)";
return false;
}
- eg_q6k.push_back(q6k ? 1 : 0);
+ eg_q6k.push_back(q6k ? 1 : iq4 ? 2 : 0);
}
if (o.verbose)
std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "
@@ -4432,7 +4438,8 @@ struct Model::Impl {
const Tensor* tw = pk.get(P + "weight");
const Tensor* ts = pk.get(P + "scale");
const bool mx8 = tw->dtype == "MXFP8"; // (a GGUF: the scales between the codes, one per 32)
- const bool q6k = eg_q6k[(size_t) li] != 0; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
+ const bool q6k = eg_q6k[(size_t) li] == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
+ const bool iq4 = eg_q6k[(size_t) li] == 2; // (a GGUF: IQ4_NL blocks of 32 values: F16 d, 16 nibble bytes)
std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read
int64_t hits = 0;
{
@@ -4482,8 +4489,8 @@ struct Model::Impl {
r.mx8 = true;
r.n = (uint32_t) (hd / 32 * 33);
r.off = tw->off + (uint64_t) id * r.n;
- } else if (q6k) { // (into the row's code bytes; its scale bytes unused)
- r.n = (uint32_t) (hd / 256 * 210);
+ } else if (q6k || iq4) { // (into the row's code bytes; its scale bytes unused)
+ r.n = (uint32_t) (q6k ? hd / 256 * 210 : hd / 32 * 18);
r.off = tw->off + (uint64_t) id * r.n;
} else {
r.n = (uint32_t) hd;
@@ -5795,7 +5802,7 @@ struct Model::Impl {
CK(cudaEventRecord(c.eg_ev[slot], s));
for (int s0 = 0; s0 < T; s0 += 256) {
const int ns = std::min(256, T - s0);
- engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, eg_q6k[(size_t) l.eg] != 0);
+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_q6k[(size_t) l.eg]);
x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);
if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;
engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);
@@ -7320,7 +7327,7 @@ struct Model::Impl {
for (int t = 0; t < T; ++t)
CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
(size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
- engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, eg_q6k[(size_t) l.eg] != 0);
+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_q6k[(size_t) l.eg]);
if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
}
diff --git a/src/dsv41/ds4_packinfo.cpp b/src/dsv41/ds4_packinfo.cpp
index ee4783970..c71a264eb 100644
--- a/src/dsv41/ds4_packinfo.cpp
+++ b/src/dsv41/ds4_packinfo.cpp
@@ -14,6 +14,10 @@
#include <string>
#include <vector>
+#if defined(_MSC_VER)
+#include <cstring>
+#define strtok_r strtok_s
+#endif
namespace strata::ds4 {
namespace {
diff --git a/src/dsv41/ds4_tq.cu b/src/dsv41/ds4_tq.cu
index 04b994d40..9c8e76fe3 100644
--- a/src/dsv41/ds4_tq.cu
+++ b/src/dsv41/ds4_tq.cu
@@ -25,6 +25,9 @@
#define DS4_TQ_WMMA 1
#endif
+#if !defined(STRATA_USE_HIP) && defined(__CUDACC__) // (HIP's non-sync shuffle name, for the portable lane on CUDA)
+#define __shfl_xor(v, m) __shfl_xor_sync(0xffffffffu, (v), (m))
+#endif
namespace strata::ds4::gguf::iq::tq {
namespace {
diff --git a/tests/dsv41/hip_as_cuda.h b/tests/dsv41/hip_as_cuda.h
new file mode 100644
index 000000000..e4ad41923
--- /dev/null
+++ b/tests/dsv41/hip_as_cuda.h
@@ -0,0 +1,17 @@
+// Strata. The few HIP names the DeepSeek-V4.1 kernel tests use, on CUDA (the tests are written against HIP).
+#pragma once
+#include <cuda_runtime.h>
+#define hipError_t cudaError_t
+#define hipSuccess cudaSuccess
+#define hipGetErrorString cudaGetErrorString
+#define hipMalloc cudaMalloc
+#define hipFree cudaFree
+#define hipMemcpy cudaMemcpy
+#define hipMemcpyHostToDevice cudaMemcpyHostToDevice
+#define hipMemcpyDeviceToHost cudaMemcpyDeviceToHost
+#define hipDeviceSynchronize cudaDeviceSynchronize
+#define hipEvent_t cudaEvent_t
+#define hipEventCreate cudaEventCreate
+#define hipEventRecord(e) cudaEventRecord(e, 0)
+#define hipEventSynchronize cudaEventSynchronize
+#define hipEventElapsedTime cudaEventElapsedTime
diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cpp
index a7bb1894e..a14750f7b 100644
--- a/tests/dsv41/iq_expert_test.cpp
+++ b/tests/dsv41/iq_expert_test.cpp
@@ -3,11 +3,15 @@
// (grouped entries), the prompt sets (gate/up and down), and the CPU lane on the same experts. Random legal blocks: any
// bit pattern with a sane F16 scale is one. This is the H2 gate for a new type (IQ2_XS first): the kernel's integer sums
// must equal ggml's weights times the same activations.
-// iq_expert_test [--type 16,17,18,21,22,23] [--bench] [--iq256 0|1]
+// iq_expert_test [--type 16,17,18,21,22,23,29] [--bench] [--iq256 0|1]
#include "../../src/dsv41/ds4_gguf_cpu.hpp"
#include "../../src/dsv41/ds4_iq.cuh"
+#if defined(__CUDACC__) || defined(STRATA_DSV41_TEST_CUDA)
+#include "hip_as_cuda.h"
+#else
#include <hip/hip_runtime.h>
+#endif
#include "ggml.h" // ggml_get_type_traits(type)->to_float: ggml's decode of the same blocks
@@ -92,6 +96,11 @@ static Mat make_mat(std::mt19937_64& rng, int type, int rows, int k) {
for (auto& b : m.bytes) b = (uint8_t) rng();
for (size_t b = 0; b < nblk; ++b) { // a sane F16 scale in the first two bytes (every type here has d at offset 0)
const uint16_t d = f2h(0.02f + 0.0002f * (float) (rng() % 600));
+ if (type == 29) { // IQ1_M has no d field: F16 d is spread over the top nibbles of its four u16 scale words (bytes 48..55)
+ uint8_t* sc = &m.bytes[b * (size_t) bb + 48];
+ for (int i = 0; i < 4; ++i) sc[2 * i + 1] = (uint8_t) ((sc[2 * i + 1] & 0x0f) | (((d >> (4 * i)) & 15) << 4));
+ continue;
+ }
std::memcpy(&m.bytes[b * (size_t) bb], &d, 2);
}
m.w.resize((size_t) rows * k);
@@ -137,7 +146,7 @@ static void bench(int type) {
}
int main(int argc, char** argv) {
- std::vector<int> types = {16, 17, 22, 18, 21, 23};
+ std::vector<int> types = {16, 17, 22, 18, 21, 23, 29};
for (int i = 1; i < argc; ++i) {
if (std::string(argv[i]) == "--bench") {
std::vector<int> bt;
diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.py
index d14a33620..4fe27ec94 100644
--- a/tools/dsv41/pack_gguf.py
+++ b/tools/dsv41/pack_gguf.py
@@ -133,6 +133,10 @@ LAYER = {
"engram_embd.weight": "engram.embed.weight",
"engram_kv.weight": "engram.wkv.weight",
"engram_gate.weight": "engram.gate.weight", # q_weight * k_weight, folded by the converter
+ # the IST-DAS sets keep q and k apart (the engine multiplies them at load) and the projection as engram_wkv
+ "engram_wkv.weight": "engram.wkv.weight",
+ "engram_q.weight": "engram.q_weight",
+ "engram_k.weight": "engram.k_weight",
}
GLOBAL = {"token_embd.weight": "embed.weight", "output_norm.weight": "norm.weight", "output.weight": "head.weight"}
@@ -244,6 +248,30 @@ def ple_to_engram(kv: dict):
kv["%s.engram.%s" % (a, k)] = v
+def int_tensors_to_engram(gs, kv: dict):
+ """The IST-DAS sets carry the Engram's hash parameters as integer tensors of the first shard (engram_token_map I32,
+ engram_primes / engram_offsets / engram_multipliers I64, one row a layer) instead of metadata: read them into the
+ engram.* keys this pack reads (when the metadata does not have them already)."""
+ a = kv.get("general.architecture")
+ want = {"engram_token_map.weight": ("token_map", "i", 4), "engram_primes.weight": ("primes", "q", 8),
+ "engram_offsets.weight": ("offsets", "q", 8), "engram_multipliers.weight": ("multipliers", "q", 8)}
+ for g in gs:
+ for name, typ, dims, off in g.tensors:
+ if name not in want:
+ continue
+ key, fmt, sz = want[name]
+ if "%s.engram.%s" % (a, key) in kv:
+ continue
+ n = 1
+ for d in dims:
+ n *= d
+ with g.path.open("rb") as f:
+ f.seek(g.data_start + off)
+ kv["%s.engram.%s" % (a, key)] = list(struct.unpack("<%d%s" % (n, fmt), f.read(n * sz)))
+ if "%s.engram.pad_token_id" % a not in kv and "%s.engram_pad_token_id" % a in kv:
+ kv["%s.engram.pad_token_id" % a] = kv["%s.engram_pad_token_id" % a]
+
+
def split_ple(g: Gguf, kv: dict):
"""llama.cpp's one Engram tensor (per_layer_token_embd: every layer's table, one after the other) -> each layer's
own blk.N.engram_embd as the earlier GGUFs had it: the same bytes, read in place."""
@@ -286,7 +314,7 @@ def engram_set(model: pathlib.Path, explicit: str | None):
dirs = ([pathlib.Path(explicit).expanduser().resolve()] if explicit else
[d for n in ENGRAM_SETS for d in (model / n, model.parent / n)])
for d in dirs:
- paths = sorted(d.glob("*.gguf")) if d.is_dir() else []
+ paths = [d] if d.is_file() else sorted(d.glob("*.gguf")) if d.is_dir() else [] # (--engram: a folder, or the one file)
if not paths:
if explicit:
raise SystemExit("%s: no .gguf files (--engram: the folder of the Engram tables' GGUF set)" % d)
@@ -417,7 +445,17 @@ def main() -> int:
have = {t[0] for g in gs for t in g.tensors}
arch = kv["general.architecture"]
layers = kv.get("%s.engram.layers" % arch, [])
- lacking = not layers or any("blk.%d.%s.weight" % (il, part) not in have
+ int_tensors_to_engram(gs, kv)
+
+ def has_part(il, part, names):
+ if "blk.%d.%s.weight" % (il, part) in names:
+ return True
+ if part == "engram_kv":
+ return "blk.%d.engram_wkv.weight" % il in names
+ if part == "engram_gate":
+ return "blk.%d.engram_q.weight" % il in names and "blk.%d.engram_k.weight" % il in names
+ return False
+ lacking = not layers or any(not has_part(il, part, have)
for il in layers for part in ("engram_embd", "engram_kv", "engram_gate"))
eg_dir, egs = engram_set(model, a.engram) if lacking or a.engram else (None, [])
eg_from = {}
@@ -432,13 +470,16 @@ def main() -> int:
split_ple(g, kv)
for p, g in zip(eshards, egs):
picked = [t for t in g.tensors if re.match(r"blk\.\d+\.engram_", t[0]) and t[0] not in have]
+ picked = [t for t in picked if t[0].endswith("engram_embd.weight") or not has_part(
+ int(t[0].split(".")[1]), "engram_kv" if t[0].endswith("engram_kv.weight") else "engram_gate", have)]
+
if picked:
g.tensors = picked
eg_from[p] = g
for il in kv.get("%s.engram.layers" % arch, []):
for part in ("engram_embd", "engram_kv", "engram_gate"):
nm = "blk.%d.%s.weight" % (il, part)
- if nm not in have and not any(t[0] == nm for g in eg_from.values() for t in g.tensors):
+ if not has_part(il, part, have) and not any(t[0] == nm for g in eg_from.values() for t in g.tensors):
raise SystemExit("%s: missing - not in the quant's shards nor in an Engram set (--engram, or "
"<model>/../Engram/: unsloth's UD-Q8_K_XL shard of the Engram tables)" % nm)
rel = lambda p: os.path.relpath(p, model).replace(os.sep, "/") # noqa: E731
@@ -477,6 +518,8 @@ def main() -> int:
if ck.endswith(".attn.wo_a.slice"):
# [o_groups * o_lora][in] -> o_groups entries of [o_lora][in] (group g: rows g * o_lora ..)
n = int(kv["%s.attention.output_group_count" % kv["general.architecture"]])
+ if len(shape) == 3 and shape[0] == n: # (the IST-DAS sets: [groups][o_lora][in] in one tensor)
+ shape = [shape[0] * shape[1], shape[2]]
if len(shape) != 2 or shape[0] % n:
raise SystemExit("%s: %s %s does not split in %d groups" % (g.path.name, name, shape, n))
part = nbytes // n
diff --git a/tools/dsv41/win/hwbench.cu b/tools/dsv41/win/hwbench.cu
new file mode 100644
index 000000000..6903ffc0d
--- /dev/null
+++ b/tools/dsv41/win/hwbench.cu
@@ -0,0 +1,104 @@
+// hwbench: host RAM bandwidth, PCIe host->device, and unbuffered overlapped NVMe reads (Windows).
+// hwbench ram [threads] [MiB]
+// hwbench pcie
+// hwbench nvme <file> <block_KiB> <qd> <seconds> [seq|rand]
+#include <cuda_runtime.h>
+#include <windows.h>
+#include <algorithm>
+#include <chrono>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <string>
+#include <thread>
+#include <vector>
+
+static double now() { return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
+
+static void bench_ram(int threads, size_t mib) {
+ size_t n = mib << 20;
+ std::vector<char*> src(threads), dst(threads);
+ for (int t = 0; t < threads; t++) {
+ src[t] = (char*)_aligned_malloc(n, 4096); dst[t] = (char*)_aligned_malloc(n, 4096);
+ memset(src[t], t + 1, n); memset(dst[t], 0, n);
+ }
+ for (int mode = 0; mode < 2; mode++) {
+ double best = 0;
+ for (int rep = 0; rep < 4; rep++) {
+ std::vector<std::thread> th; volatile uint64_t sink = 0;
+ double t0 = now();
+ for (int t = 0; t < threads; t++)
+ th.emplace_back([&, t] {
+ if (mode == 0) memcpy(dst[t], src[t], n);
+ else { uint64_t s = 0; const uint64_t* p = (const uint64_t*)src[t]; for (size_t i = 0; i < n / 8; i += 1) s += p[i]; sink = sink + s; }
+ });
+ for (auto& x : th) x.join();
+ double dt = now() - t0; double gb = (mode == 0 ? 2.0 : 1.0) * n * threads / 1e9 / dt; // copy counts read+write
+ if (gb > best) best = gb;
+ }
+ printf("ram %s threads=%d size/thread=%zu MiB best=%.1f GB/s (%s)\n", mode == 0 ? "copy" : "read", threads, mib, best, mode == 0 ? "read+write counted" : "read only");
+ }
+}
+
+static void bench_pcie() {
+ for (size_t mib : {1, 4, 16, 64, 256}) {
+ size_t n = mib << 20; void *h, *d; cudaStream_t s; cudaStreamCreate(&s);
+ cudaMallocHost(&h, n); cudaMalloc(&d, n); memset(h, 1, n);
+ void* hp = _aligned_malloc(n, 4096); memset(hp, 1, n);
+ for (int pinned = 1; pinned >= 0; pinned--) {
+ void* src = pinned ? h : hp;
+ for (int w = 0; w < 3; w++) cudaMemcpyAsync(d, src, n, cudaMemcpyHostToDevice, s);
+ cudaStreamSynchronize(s);
+ int reps = (int)std::max<size_t>(8, (1024u / mib)); double t0 = now();
+ for (int i = 0; i < reps; i++) cudaMemcpyAsync(d, src, n, cudaMemcpyHostToDevice, s);
+ cudaStreamSynchronize(s); double dt = now() - t0;
+ printf("pcie H2D %s %4zu MiB: %.2f GB/s\n", pinned ? "pinned " : "pageable", mib, (double)n * reps / 1e9 / dt);
+ }
+ cudaFreeHost(h); cudaFree(d); _aligned_free(hp); cudaStreamDestroy(s);
+ }
+}
+
+static void bench_nvme(const char* path, size_t block, int qd, double secs, bool seq) {
+ HANDLE f = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);
+ if (f == INVALID_HANDLE_VALUE) { printf("open failed %lu\n", GetLastError()); return; }
+ LARGE_INTEGER sz; GetFileSizeEx(f, &sz);
+ HANDLE port = CreateIoCompletionPort(f, nullptr, 0, 1);
+ uint64_t nblk = (uint64_t)sz.QuadPart / block;
+ struct Slot { OVERLAPPED ov; char* buf; };
+ std::vector<Slot> slots(qd);
+ std::mt19937_64 rng(12345);
+ uint64_t seqpos = 0, bytes = 0, ops = 0;
+ std::vector<double> lat; std::vector<double> t_issue(qd);
+ auto issue = [&](int i) {
+ uint64_t b = seq ? (seqpos++ % nblk) : (rng() % nblk);
+ uint64_t off = b * block;
+ memset(&slots[i].ov, 0, sizeof(OVERLAPPED));
+ slots[i].ov.Offset = (DWORD)(off & 0xffffffff); slots[i].ov.OffsetHigh = (DWORD)(off >> 32);
+ t_issue[i] = now();
+ if (!ReadFile(f, slots[i].buf, (DWORD)block, nullptr, &slots[i].ov) && GetLastError() != ERROR_IO_PENDING) { printf("ReadFile err %lu\n", GetLastError()); exit(1); }
+ };
+ for (int i = 0; i < qd; i++) { slots[i].buf = (char*)VirtualAlloc(nullptr, block, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); issue(i); }
+ double t0 = now(), tend = t0 + secs;
+ while (true) {
+ DWORD nb; ULONG_PTR key; OVERLAPPED* ov;
+ if (!GetQueuedCompletionStatus(port, &nb, &key, &ov, 10000) && !ov) { printf("wait err\n"); break; }
+ int i = (int)(((Slot*)((char*)ov - offsetof(Slot, ov))) - slots.data());
+ bytes += nb; ops++; lat.push_back(now() - t_issue[i]);
+ if (now() < tend) issue(i); else if (--qd == 0) break;
+ }
+ double dt = now() - t0;
+ std::sort(lat.begin(), lat.end());
+ printf("nvme %s block=%zuKiB qd=%zu: %.2f GB/s, %.0f IOPS, lat p50=%.2f ms p99=%.2f ms\n", seq ? "seq " : "rand", block >> 10, slots.size(),
+ bytes / 1e9 / dt, ops / dt, lat[lat.size() / 2] * 1e3, lat[lat.size() * 99 / 100] * 1e3);
+ CloseHandle(f);
+}
+
+int main(int argc, char** argv) {
+ std::string m = argc > 1 ? argv[1] : "";
+ if (m == "ram") bench_ram(argc > 2 ? atoi(argv[2]) : 6, argc > 3 ? atoi(argv[3]) : 512);
+ else if (m == "pcie") bench_pcie();
+ else if (m == "nvme" && argc > 5) bench_nvme(argv[2], (size_t)atoi(argv[3]) << 10, atoi(argv[4]), atof(argv[5]), argc > 6 && std::string(argv[6]) == "seq");
+ else printf("usage: hwbench ram|pcie|nvme ...\n");
+}