Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
markov bias in Q8: the drafter's head is quantised to Q8_0 blocks as it loads (66 MB -> 33 MB, the pack untouched) and read by a Q8 variant of markov_kernel - the markov segment 1.80 -> 1.34 ms, the drafter's round 13.46 -> 12.95, the drafts' acceptance 90-95% (the same band)
Niko1221a266bdfe1d78
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex f803016b8..394dd7f73 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1218,7 +1218,34 @@ __global__ void __launch_bounds__(kThreads) markov_kernel(float* __restrict__ loif (lane == 0) logits[v] += a;}-// ---------------------------------------------------------------- the markov bias over a head group (markov_share)+// the same bias with the head quantised to Q8_0 blocks of 32 (34 B: an F16 d and 32 int8): half the bytes read for the same sums.+// A lane's 8 values sit inside one block (8 divides 32), so a lane reads one d and 8 codes; the sums are the same terms, in the+// block's own order, with the quantisation's error the only difference.+__global__ void __launch_bounds__(kThreads) markov_kernel_q8(float* __restrict__ logits, const uint16_t* __restrict__ E,+ const uint8_t* __restrict__ Hq, const int* __restrict__ tok,+ int V, int R) {+ extern __shared__ float er[];+ const uint16_t* erow = E + (size_t) (*tok) * R;+ for (int i = threadIdx.x; i < R; i += blockDim.x) er[i] = bf16f(erow[i]);+ __syncthreads();+ const int lane = threadIdx.x & 31;+ const int v = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);+ if (v >= V) return;+ const uint8_t* hr = Hq + (size_t) v * (size_t) (R / 32) * 34;+ float a = 0.0f;+ for (int c = lane * 8; c < R; c += 32 * 8) {+ const uint8_t* blk = hr + (size_t) (c >> 5) * 34;+ const float d = __half2float(*(const __half*) blk);+ const int8_t* q = (const int8_t*) (blk + 2 + (c & 31));+ float t = 0.0f;+#pragma unroll+ for (int i = 0; i < 8; ++i) t += (float) q[i] * er[c + i];+ a += d * t;+ }+ a = warp_sum(a);+ if (lane == 0) logits[v] += a;+}+// a member's rows [v0, v0 + vn) of the bias (markov_kernel's sums, the prev token's E row already on the device, BF16)__global__ void __launch_bounds__(kThreads) markov_share_kernel(float* __restrict__ logits,const uint16_t* __restrict__ H,@@ -2135,6 +2162,9 @@ void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { iota_rows_kvoid markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {markov_kernel<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, H, tok, V, R);}+void markov_add_q8(float* logits, const uint16_t* E, const uint8_t* Hq, const int* tok, int V, int R, cudaStream_t s) {+ markov_kernel_q8<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, Hq, tok, V, R);+}void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s) {markov_share_kernel<<<cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, H, erow, vn, R);}diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuhindex ae33e9c83..08075fa00 100644--- a/src/dsv41/ds4_kernels.cuh+++ b/src/dsv41/ds4_kernels.cuh@@ -261,6 +261,8 @@ void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_svoid iota_rows(int* out, int T, int n, int stride, cudaStream_t s);/// The DSpark markov head: logits[v] += E[*tok] . H[v] for v < V (E BF16 [V][R], H F16 [V][R]).void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s);+/// The same bias with the head quantised to Q8_0 blocks of 32 (34 B a block: F16 d and 32 int8, R a multiple of 32).+void markov_add_q8(float* logits, const uint16_t* E, const uint8_t* Hq, const int* tok, int V, int R, cudaStream_t s);/// The markov bias over a head group: a member's rows [v0, v0 + vn) of it added to its share of a logits row (erow:/// the previous token's E row, BF16, on the device) - markov_add's sums; the share's argmax (the greedy sampler's rule)/// into a host-mapped slot {value, index, seq}; every member's slot of the step -> the global argmax into *tok and itsdiff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex a8b558997..9e983c6fb 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -885,6 +885,7 @@ struct Model::Impl {x3::Mat d_main_proj;float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]+ uint8_t* d_mk_head_q8 = nullptr; // the markov head as Q8_0 blocks (the same bias, half the bytes a step)// a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its// share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole// rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq@@ -1619,14 +1620,59 @@ struct Model::Impl {return upload(d, *t, p, err);}+ // The markov bias's head as Q8_0 blocks of 32 (34 B: F16 d + 32 int8) instead of F16 values: the same bias sums with the+ // quantisation's error, half the bytes the drafter's five markov steps read (66 MB -> 33 MB) and half the memory. The+ // pack is untouched: the F32 / F16 / BF16 tensor is quantised as it is loaded.+ bool mk_head_q8(Dev& d, const std::string& name, uint8_t*& p, std::string& err, size_t* acct) {+ const Tensor* t = pk.get(name);+ if (t == nullptr || (t->dtype != "F32" && t->dtype != "F16" && t->dtype != "BF16")) {+ err = name + ": not an F32 / F16 / BF16 tensor";+ return false;+ }+ const size_t n = t->bytes / (t->dtype == "F32" ? 4 : 2);+ if (n == 0 || n % 32 != 0) {+ err = name + ": not a whole number of 32-value blocks";+ return false;+ }+ std::vector<uint8_t> raw(t->bytes);+ if (!pk.read(*t, raw.data())) {+ err = name + ": read failed";+ return false;+ }+ std::vector<float> v(n);+ for (size_t i = 0; i < n; ++i) {+ if (t->dtype == "F32") std::memcpy(&v[i], raw.data() + i * 4, 4);+ else v[i] = t->dtype == "F16" ? f16_to_f32(((const uint16_t*) raw.data())[i])+ : bf16_to_f32(((const uint16_t*) raw.data())[i]);+ }+ const size_t nb = n / 32;+ std::vector<uint8_t> q(nb * 34);+ for (size_t b = 0; b < nb; ++b) {+ const float* s = v.data() + b * 32;+ float amax = 0.0f;+ for (int i = 0; i < 32; ++i) amax = std::max(amax, std::fabs(s[i]));+ const float dd = amax > 0.0f ? amax / 127.0f : 0.0f;+ const uint16_t dh = strata::kernels::f16_from_f32(dd);+ std::memcpy(q.data() + b * 34, &dh, 2);+ int8_t* o = (int8_t*) (q.data() + b * 34 + 2);+ for (int i = 0; i < 32; ++i) {+ const float x = dd > 0.0f ? s[i] / dd : 0.0f;+ o[i] = (int8_t) std::max(-127.0f, std::min(127.0f, std::round(x)));+ }+ }+ if (!dalloc(d, p, nb * 34, err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(p, q.data(), nb * 34, cudaMemcpyHostToDevice));+ return true;+ }+size_t mat_bytes(const std::string& pre) const {if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);size_t b = 0;for (const char* s : {".trellis", ".suh", ".svh"})if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);return b;- }- size_t tensor_bytes(const std::string& name) const {+ } size_t tensor_bytes(const std::string& name) const {const Tensor* t = pk.get(name);return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;}@@ -3790,7 +3836,7 @@ struct Model::Impl {return false;}if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||- !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))+ !mk_head_q8(d, Z + "markov_head.head.weight", d_mk_head_q8, err, acct))return false;}}@@ -6845,7 +6891,7 @@ struct Model::Impl {sp.seed = dq_seed;}for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft- markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);+ markov_add_q8(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head_q8, d_ints + i, V, R, hd.st);if (drafts_q) {sp.counter = dq_counter + (uint64_t) i;strata::dsv41k::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);