Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

markov bias in Q8: the drafter's head is quantised to Q8_0 blocks as it loads (66 MB -> 33 MB, the pack untouched) and read by a Q8 variant of markov_kernel - the markov segment 1.80 -> 1.34 ms, the drafter's round 13.46 -> 12.95, the drafts' acceptance 90-95% (the same band)

Niko1221a266bdfe1d78
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index f803016b8..394dd7f73 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1218,7 +1218,34 @@ __global__ void __launch_bounds__(kThreads) markov_kernel(float* __restrict__ lo
if (lane == 0) logits[v] += a;
}
-// ---------------------------------------------------------------- the markov bias over a head group (markov_share)
+// the same bias with the head quantised to Q8_0 blocks of 32 (34 B: an F16 d and 32 int8): half the bytes read for the same sums.
+// A lane's 8 values sit inside one block (8 divides 32), so a lane reads one d and 8 codes; the sums are the same terms, in the
+// block's own order, with the quantisation's error the only difference.
+__global__ void __launch_bounds__(kThreads) markov_kernel_q8(float* __restrict__ logits, const uint16_t* __restrict__ E,
+ const uint8_t* __restrict__ Hq, const int* __restrict__ tok,
+ int V, int R) {
+ extern __shared__ float er[];
+ const uint16_t* erow = E + (size_t) (*tok) * R;
+ for (int i = threadIdx.x; i < R; i += blockDim.x) er[i] = bf16f(erow[i]);
+ __syncthreads();
+ const int lane = threadIdx.x & 31;
+ const int v = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);
+ if (v >= V) return;
+ const uint8_t* hr = Hq + (size_t) v * (size_t) (R / 32) * 34;
+ float a = 0.0f;
+ for (int c = lane * 8; c < R; c += 32 * 8) {
+ const uint8_t* blk = hr + (size_t) (c >> 5) * 34;
+ const float d = __half2float(*(const __half*) blk);
+ const int8_t* q = (const int8_t*) (blk + 2 + (c & 31));
+ float t = 0.0f;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) t += (float) q[i] * er[c + i];
+ a += d * t;
+ }
+ a = warp_sum(a);
+ if (lane == 0) logits[v] += a;
+}
+
// a member's rows [v0, v0 + vn) of the bias (markov_kernel's sums, the prev token's E row already on the device, BF16)
__global__ void __launch_bounds__(kThreads) markov_share_kernel(float* __restrict__ logits,
const uint16_t* __restrict__ H,
@@ -2135,6 +2162,9 @@ void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { iota_rows_k
void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {
markov_kernel<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, H, tok, V, R);
}
+void markov_add_q8(float* logits, const uint16_t* E, const uint8_t* Hq, const int* tok, int V, int R, cudaStream_t s) {
+ markov_kernel_q8<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, Hq, tok, V, R);
+}
void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s) {
markov_share_kernel<<<cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, H, erow, vn, R);
}
diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuh
index ae33e9c83..08075fa00 100644
--- a/src/dsv41/ds4_kernels.cuh
+++ b/src/dsv41/ds4_kernels.cuh
@@ -261,6 +261,8 @@ void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_s
void iota_rows(int* out, int T, int n, int stride, cudaStream_t s);
/// The DSpark markov head: logits[v] += E[*tok] . H[v] for v < V (E BF16 [V][R], H F16 [V][R]).
void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s);
+/// The same bias with the head quantised to Q8_0 blocks of 32 (34 B a block: F16 d and 32 int8, R a multiple of 32).
+void markov_add_q8(float* logits, const uint16_t* E, const uint8_t* Hq, const int* tok, int V, int R, cudaStream_t s);
/// The markov bias over a head group: a member's rows [v0, v0 + vn) of it added to its share of a logits row (erow:
/// the previous token's E row, BF16, on the device) - markov_add's sums; the share's argmax (the greedy sampler's rule)
/// into a host-mapped slot {value, index, seq}; every member's slot of the step -> the global argmax into *tok and its
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index a8b558997..9e983c6fb 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -885,6 +885,7 @@ struct Model::Impl {
x3::Mat d_main_proj;
float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;
uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]
+ uint8_t* d_mk_head_q8 = nullptr; // the markov head as Q8_0 blocks (the same bias, half the bytes a step)
// a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its
// share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole
// rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq
@@ -1619,14 +1620,59 @@ struct Model::Impl {
return upload(d, *t, p, err);
}
+ // The markov bias's head as Q8_0 blocks of 32 (34 B: F16 d + 32 int8) instead of F16 values: the same bias sums with the
+ // quantisation's error, half the bytes the drafter's five markov steps read (66 MB -> 33 MB) and half the memory. The
+ // pack is untouched: the F32 / F16 / BF16 tensor is quantised as it is loaded.
+ bool mk_head_q8(Dev& d, const std::string& name, uint8_t*& p, std::string& err, size_t* acct) {
+ const Tensor* t = pk.get(name);
+ if (t == nullptr || (t->dtype != "F32" && t->dtype != "F16" && t->dtype != "BF16")) {
+ err = name + ": not an F32 / F16 / BF16 tensor";
+ return false;
+ }
+ const size_t n = t->bytes / (t->dtype == "F32" ? 4 : 2);
+ if (n == 0 || n % 32 != 0) {
+ err = name + ": not a whole number of 32-value blocks";
+ return false;
+ }
+ std::vector<uint8_t> raw(t->bytes);
+ if (!pk.read(*t, raw.data())) {
+ err = name + ": read failed";
+ return false;
+ }
+ std::vector<float> v(n);
+ for (size_t i = 0; i < n; ++i) {
+ if (t->dtype == "F32") std::memcpy(&v[i], raw.data() + i * 4, 4);
+ else v[i] = t->dtype == "F16" ? f16_to_f32(((const uint16_t*) raw.data())[i])
+ : bf16_to_f32(((const uint16_t*) raw.data())[i]);
+ }
+ const size_t nb = n / 32;
+ std::vector<uint8_t> q(nb * 34);
+ for (size_t b = 0; b < nb; ++b) {
+ const float* s = v.data() + b * 32;
+ float amax = 0.0f;
+ for (int i = 0; i < 32; ++i) amax = std::max(amax, std::fabs(s[i]));
+ const float dd = amax > 0.0f ? amax / 127.0f : 0.0f;
+ const uint16_t dh = strata::kernels::f16_from_f32(dd);
+ std::memcpy(q.data() + b * 34, &dh, 2);
+ int8_t* o = (int8_t*) (q.data() + b * 34 + 2);
+ for (int i = 0; i < 32; ++i) {
+ const float x = dd > 0.0f ? s[i] / dd : 0.0f;
+ o[i] = (int8_t) std::max(-127.0f, std::min(127.0f, std::round(x)));
+ }
+ }
+ if (!dalloc(d, p, nb * 34, err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(p, q.data(), nb * 34, cudaMemcpyHostToDevice));
+ return true;
+ }
+
size_t mat_bytes(const std::string& pre) const {
if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);
size_t b = 0;
for (const char* s : {".trellis", ".suh", ".svh"})
if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);
return b;
- }
- size_t tensor_bytes(const std::string& name) const {
+ } size_t tensor_bytes(const std::string& name) const {
const Tensor* t = pk.get(name);
return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;
}
@@ -3790,7 +3836,7 @@ struct Model::Impl {
return false;
}
if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||
- !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))
+ !mk_head_q8(d, Z + "markov_head.head.weight", d_mk_head_q8, err, acct))
return false;
}
}
@@ -6845,7 +6891,7 @@ struct Model::Impl {
sp.seed = dq_seed;
}
for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft
- markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);
+ markov_add_q8(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head_q8, d_ints + i, V, R, hd.st);
if (drafts_q) {
sp.counter = dq_counter + (uint64_t) i;
strata::dsv41k::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);