Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/qwen35/q36ref.cu - NEW-MODELS-PLAN.md phase T1: the reference path for Qwen3.6-35B-A3B (GGUF architecture
// qwen35moe), exactness first. One GPU, the whole model resident in VRAM (UD-Q4_K_XL: 21.3 GiB on a 32 GB card),
// one token at a time (the prompt too), fp32 activations, every weight read in its GGUF form. Two kernel sets:
// --kernels plain, this file's own plain kernels (the reference), and --kernels strata (default), Strata's native
// MMVQ for the quantized matmuls (dense and routed experts) and its fused GDN step with the SiLU-gated norm - the
// engine's kernels on the new family's shapes. No expert cache, no MTP, no graphs: checked against llama.cpp
// (per-layer hidden states, greedy tokens, teacher-forced KL: tools/qwen35/).
//
// The forward pass, as llama.cpp's src/models/qwen35moe.cpp builds it:
// x = token_embd[tok]
// per layer l < 40: h = rms(x) * attn_norm
// GDN (l % 4 != 3): qkv = W_qkv h (8192), z = W_gate h (4096), b = sigmoid(W_beta h), a = W_alpha h (32 each)
// c = silu(causal conv4(qkv)); q, k = l2norm(c[0:2048]), l2norm(c[2048:4096]) (16 heads), v = c[4096:] (32)
// g = softplus(a + dt_bias) * ssm_a; per V head h (K head h % 16): S = S * exp(g_h);
// d = (v - S k) * b_h; S += d k^T; o = S q / sqrt(128); o = rms(o) * ssm_norm * silu(z_h)
// y = W_ssm_out o
// attention (l % 4 == 3): attn_q -> per head [q 256 | gate 256]; q, k = rms(.) * q_norm/k_norm; NEOX RoPE over the
// first 64 dims (base 1e7; text positions: the M-RoPE sections all carry the same position); GQA 16 / 2;
// y = W_o (softmax(q k^T / 16) v * sigmoid(gate))
// x += y; h2 = rms(x) * post_attention_norm
// MoE: p = softmax(W_router h2) over 256, top-8, renormalised; sum_e p_e W_down_e (silu(W_gate_e h2) * W_up_e h2)
// + sigmoid(w_shexp_gate . h2) * W_down_s (silu(W_gate_s h2) * W_up_s h2); x += moe
// logits = W_output (rms(x) * output_norm)
//
// Usage: strata-q36ref --model Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf --tokens 248045,846,198,... [--n 64] [--ctx 4096]
// [--dump-layers FILE] [--dump-logits FILE] [--act q8|f32] [--kernels strata|plain]
// [--force IDS] [--dump-full-logits F]
// prints the greedy ids ("gen: id id ..."); --dump-layers writes every processed position's 40 layer outputs (f32,
// [pos][layer][2048]); --dump-logits writes each greedy step's top-2 (id, logit) pairs for the near-tie report;
// --force feeds the given continuation instead of the greedy ids and --dump-full-logits writes every step's
// logits (f32, [step][vocab]) for the teacher-forced KL check (tools/qwen35/kl.py).
#include "strata/artifact/gguf_reader.hpp"
#include "strata/core/family_layout.hpp"
#include "strata/kernels/fused_gdn.hpp"
#include "strata/kernels/native_mmvq.hpp"
#include "strata/kernels/native_router.hpp"
#include <cuda_runtime.h>
#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
#include <string>
#include <vector>
namespace {
#define CK(x) \
do { \
cudaError_t e_ = (x); \
if (e_ != cudaSuccess) { \
std::fprintf(stderr, "q36ref: %s failed: %s (%s:%d)\n", #x, cudaGetErrorString(e_), __FILE__, __LINE__); \
std::exit(1); \
} \
} while (0)
// ------------------------------------------------------------------------------------------------ dequantisation
// ggml's block formats, 32 values at a time ("chunk" c of a row). The formulas are ggml-quants.c's dequantize_row_*.
enum : int { T_F32 = 0, T_F16 = 1, T_Q8_0 = 8, T_Q4_K = 12, T_Q5_K = 13, T_Q6_K = 14, T_BF16 = 30 };
__device__ __forceinline__ float h2f(uint16_t h) {
const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;
if (e == 0) {
const float v = ldexpf((float) m, -24);
return s ? -v : v;
}
const uint32_t f = e == 31 ? (s | 0x7f800000u | (m << 13)) : (s | ((e + 112) << 23) | (m << 13));
return __uint_as_float(f);
}
__device__ __forceinline__ uint16_t ld16(const uint8_t* p) { return (uint16_t) (p[0] | (p[1] << 8)); }
__device__ __forceinline__ void scale_min_k4(int j, const uint8_t* q, int& d, int& m) {
if (j < 4) {
d = q[j] & 63;
m = q[j + 4] & 63;
} else {
d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4);
}
}
/// The 32 values of chunk `c` of one row (`row` = the row's first byte).
template <int T> __device__ void deq32(const uint8_t* row, int c, float* out) {
if constexpr (T == T_F32) {
const float* r = (const float*) row + 32 * c;
for (int l = 0; l < 32; ++l) out[l] = r[l];
} else if constexpr (T == T_F16) {
const uint8_t* r = row + 64 * c;
for (int l = 0; l < 32; ++l) out[l] = h2f(ld16(r + 2 * l));
} else if constexpr (T == T_BF16) {
const uint8_t* r = row + 64 * c;
for (int l = 0; l < 32; ++l) out[l] = __uint_as_float((uint32_t) ld16(r + 2 * l) << 16);
} else if constexpr (T == T_Q8_0) {
const uint8_t* b = row + 34 * c;
const float d = h2f(ld16(b));
const int8_t* q = (const int8_t*) (b + 2);
for (int l = 0; l < 32; ++l) out[l] = d * (float) q[l];
} else if constexpr (T == T_Q4_K) {
const uint8_t* b = row + 144 * (c >> 3);
const int j = c & 7;
const float d = h2f(ld16(b)), dmin = h2f(ld16(b + 2));
int sc, m;
scale_min_k4(j, b + 4, sc, m);
const float d1 = d * (float) sc, m1 = dmin * (float) m;
const uint8_t* qs = b + 16 + 32 * (j >> 1);
for (int l = 0; l < 32; ++l) out[l] = d1 * (float) ((j & 1) ? (qs[l] >> 4) : (qs[l] & 0xF)) - m1;
} else if constexpr (T == T_Q5_K) {
const uint8_t* b = row + 176 * (c >> 3);
const int j = c & 7;
const float d = h2f(ld16(b)), dmin = h2f(ld16(b + 2));
int sc, m;
scale_min_k4(j, b + 4, sc, m);
const float d1 = d * (float) sc, m1 = dmin * (float) m;
const uint8_t* qh = b + 16;
const uint8_t* qs = b + 48 + 32 * (j >> 1);
for (int l = 0; l < 32; ++l) {
const int q = ((j & 1) ? (qs[l] >> 4) : (qs[l] & 0xF)) + ((qh[l] >> j) & 1 ? 16 : 0);
out[l] = d1 * (float) q - m1;
}
} else if constexpr (T == T_Q6_K) {
const uint8_t* b = row + 210 * (c >> 3);
const int j = c & 7, n = j >> 2, r = j & 3;
const uint8_t* ql = b + 64 * n;
const uint8_t* qh = b + 128 + 32 * n;
const int8_t* sc = (const int8_t*) (b + 192) + 8 * n;
const float d = h2f(ld16(b + 208));
for (int l = 0; l < 32; ++l) {
const int is = l / 16;
int q;
if (r == 0) q = (ql[l] & 0xF) | (((qh[l] >> 0) & 3) << 4);
else if (r == 1) q = (ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4);
else if (r == 2) q = (ql[l] >> 4) | (((qh[l] >> 4) & 3) << 4);
else q = (ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4);
out[l] = d * (float) sc[is + 2 * r] * (float) (q - 32);
}
}
}
__device__ float block_sum(float v, float* sh) {
sh[threadIdx.x] = v;
__syncthreads();
for (int s = blockDim.x / 2; s > 0; s >>= 1) {
if ((int) threadIdx.x < s) sh[threadIdx.x] += sh[threadIdx.x + s];
__syncthreads();
}
const float r = sh[0];
__syncthreads();
return r;
}
constexpr int kThreads = 128;
/// y[sel][o] = W_e[o] . x[sel] for row o (grid.x), selection `sel` (grid.y): expert ids[sel] (or 0 without ids),
/// x of selection sel at x + sel * x_stride. `beta` != 0 adds into y.
template <int T>
__global__ void k_gemv(const uint8_t* W, size_t row_bytes, size_t expert_bytes, const int* ids, int n_in, const float* x,
size_t x_stride, float* y, int n_out) {
__shared__ float sh[kThreads];
const int o = blockIdx.x, sel = blockIdx.y;
const int e = ids ? ids[sel] : 0;
const uint8_t* row = W + (size_t) e * expert_bytes + (size_t) o * row_bytes;
const float* xs = x + (size_t) sel * x_stride;
float acc = 0.f, v[32];
for (int c = threadIdx.x; c < n_in / 32; c += blockDim.x) {
deq32<T>(row, c, v);
const float* xc = xs + 32 * c;
for (int l = 0; l < 32; ++l) acc += v[l] * xc[l];
}
acc = block_sum(acc, sh);
if (threadIdx.x == 0) y[(size_t) sel * n_out + o] = acc;
}
template <int T> __global__ void k_row(const uint8_t* W, size_t row_bytes, int row, int n, float* out) {
const uint8_t* r = W + (size_t) row * row_bytes;
for (int c = blockIdx.x * blockDim.x + threadIdx.x; c < n / 32; c += gridDim.x * blockDim.x) deq32<T>(r, c, out + 32 * c);
}
/// y = x / sqrt(mean(x^2) + eps) * w, per group of n (grid = groups); w (length n) may be null.
__global__ void k_rms(const float* x, const float* w, float* y, int n, float eps) {
__shared__ float sh[kThreads];
const float* xg = x + (size_t) blockIdx.x * n;
float* yg = y + (size_t) blockIdx.x * n;
float s = 0.f;
for (int i = threadIdx.x; i < n; i += blockDim.x) s += xg[i] * xg[i];
s = block_sum(s, sh);
const float r = 1.0f / sqrtf(s / (float) n + eps);
for (int i = threadIdx.x; i < n; i += blockDim.x) yg[i] = xg[i] * r * (w ? w[i] : 1.0f);
}
__device__ __forceinline__ float silu(float v) { return v / (1.0f + expf(-v)); }
__device__ __forceinline__ float sigm(float v) { return 1.0f / (1.0f + expf(-v)); }
__device__ __forceinline__ float softplus(float v) { return v > 20.f ? v : log1pf(expf(v)); }
/// The causal 4-tap conv over the 8192 qkv channels with its 3-input state, then SiLU; the state moves by one.
__global__ void k_conv(const float* qkv, float* state, const float* w, float* out, int C, int K) {
const int c = blockIdx.x * blockDim.x + threadIdx.x;
if (c >= C) return;
float acc = 0.f;
for (int k = 0; k < K - 1; ++k) acc += state[(size_t) k * C + c] * w[(size_t) c * K + k];
acc += qkv[c] * w[(size_t) c * K + K - 1];
for (int k = 0; k < K - 2; ++k) state[(size_t) k * C + c] = state[(size_t) (k + 1) * C + c];
state[(size_t) (K - 2) * C + c] = qkv[c];
out[c] = silu(acc);
}
/// One gated delta-rule step per V head (grid = HV, block = D = 128): S[i][j] row i owned by thread i.
__global__ void k_gdn(const float* conv, const float* alpha, const float* dt_bias, const float* ssm_a,
const float* beta_raw, float* S, float* out, int HK, int /*HV: the grid*/, int D, float eps) {
extern __shared__ float sm[];
float* q = sm;
float* k = sm + D;
float* red = sm + 2 * D;
const int h = blockIdx.x, kh = h % HK, i = threadIdx.x;
const float qi = conv[kh * D + i], ki = conv[HK * D + kh * D + i], vi = conv[2 * HK * D + h * D + i];
const float qn = block_sum(qi * qi, red), kn = block_sum(ki * ki, red);
q[i] = qi / sqrtf(qn + eps) / sqrtf((float) D);
k[i] = ki / sqrtf(kn + eps);
__syncthreads();
const float g = expf(softplus(alpha[h] + dt_bias[h]) * ssm_a[h]);
const float b = sigm(beta_raw[h]);
float* Si = S + ((size_t) h * D + i) * D;
float sk = 0.f;
for (int j = 0; j < D; ++j) {
Si[j] *= g;
sk += Si[j] * k[j];
}
const float d = (vi - sk) * b;
float o = 0.f;
for (int j = 0; j < D; ++j) {
Si[j] += d * k[j];
o += Si[j] * q[j];
}
out[h * D + i] = o;
}
/// The fused path's gate and beta, in place: alpha -> softplus(alpha + dt) * ssm_a (exp'd by the step), beta -> sigmoid.
__global__ void k_gdn_gb(float* alpha, const float* dt_bias, const float* ssm_a, float* beta, int HV) {
const int h = threadIdx.x;
if (h >= HV) return;
alpha[h] = softplus(alpha[h] + dt_bias[h]) * ssm_a[h];
beta[h] = sigm(beta[h]);
}
/// o = rms(o) * w * silu(z), per V head (grid = HV, block = D).
__global__ void k_gated_norm(float* o, const float* w, const float* z, int D, float eps) {
__shared__ float sh[kThreads];
float* oh = o + (size_t) blockIdx.x * D;
const float v = oh[threadIdx.x];
const float s = block_sum(v * v, sh);
oh[threadIdx.x] = v / sqrtf(s / (float) D + eps) * w[threadIdx.x] * silu(z[blockIdx.x * D + threadIdx.x]);
}
/// The attention layer's per-token preparation: q (from the [q | gate] pairs) and k RMS-normed per head with their
/// weights, NEOX RoPE over the first n_rot dims, k and v appended to the cache at `pos`, the gate kept.
__global__ void k_attn_prep(const float* qg, const float* kx, const float* vx, const float* qw, const float* kw,
float* q, float* gate, float* kc, float* vc, int H, int /*HKV: the grid*/, int D, int n_rot, int pos,
int ctx, float base, float eps) {
__shared__ float sh[kThreads];
__shared__ float buf[512];
const int head = blockIdx.x; // 0..H-1: query heads, H..H+HKV-1: key heads
const bool isq = head < H;
const float* src = isq ? qg + (size_t) head * 2 * D : kx + (size_t) (head - H) * D;
float s = 0.f;
for (int i = threadIdx.x; i < D; i += blockDim.x) s += src[i] * src[i];
s = block_sum(s, sh);
const float r = 1.0f / sqrtf(s / (float) D + eps);
for (int i = threadIdx.x; i < D; i += blockDim.x) buf[i] = src[i] * r * (isq ? qw[i] : kw[i]);
__syncthreads();
float* dst = isq ? q + (size_t) head * D : kc + ((size_t) (head - H) * ctx + pos) * D;
const int half = n_rot / 2;
for (int i = threadIdx.x; i < D; i += blockDim.x) {
float v = buf[i];
if (i < n_rot) {
const int p = i < half ? i : i - half;
const float theta = (float) pos * powf(base, -2.0f * (float) p / (float) n_rot);
const float cs = cosf(theta), sn = sinf(theta);
v = i < half ? buf[i] * cs - buf[i + half] * sn : buf[i - half] * sn + buf[i] * cs;
}
dst[i] = v;
}
if (isq)
for (int i = threadIdx.x; i < D; i += blockDim.x) gate[(size_t) head * D + i] = qg[(size_t) head * 2 * D + D + i];
else
for (int i = threadIdx.x; i < D; i += blockDim.x)
vc[((size_t) (head - H) * ctx + pos) * D + i] = vx[(size_t) (head - H) * D + i];
}
/// softmax(q k^T * scale) v for one query head per block (GQA: KV head = head / (H / HKV)), times sigmoid(gate).
__global__ void k_attn(const float* q, const float* kc, const float* vc, const float* gate, float* out, int H, int HKV,
int D, int n_pos, int ctx, float scale) {
extern __shared__ float sc[];
__shared__ float sh[kThreads];
const int h = blockIdx.x, kvh = h / (H / HKV);
const float* qh = q + (size_t) h * D;
for (int p = threadIdx.x; p < n_pos; p += blockDim.x) {
const float* kp = kc + ((size_t) kvh * ctx + p) * D;
float s = 0.f;
for (int i = 0; i < D; ++i) s += qh[i] * kp[i];
sc[p] = s * scale;
}
__syncthreads();
float mx = -INFINITY;
for (int p = threadIdx.x; p < n_pos; p += blockDim.x) mx = fmaxf(mx, sc[p]);
sh[threadIdx.x] = mx;
__syncthreads();
for (int s = blockDim.x / 2; s > 0; s >>= 1) {
if ((int) threadIdx.x < s) sh[threadIdx.x] = fmaxf(sh[threadIdx.x], sh[threadIdx.x + s]);
__syncthreads();
}
mx = sh[0];
__syncthreads();
float sum = 0.f;
for (int p = threadIdx.x; p < n_pos; p += blockDim.x) {
sc[p] = expf(sc[p] - mx);
sum += sc[p];
}
sum = block_sum(sum, sh);
for (int i = threadIdx.x; i < D; i += blockDim.x) {
float acc = 0.f;
for (int p = 0; p < n_pos; ++p) acc += sc[p] * vc[((size_t) kvh * ctx + p) * D + i];
out[(size_t) h * D + i] = acc / sum * sigm(gate[(size_t) h * D + i]);
}
}
__global__ void k_swiglu(const float* g, const float* u, float* h, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n) h[i] = silu(g[i]) * u[i];
}
__global__ void k_add(float* x, const float* y, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n) x[i] += y[i];
}
/// x += sum_s w[s] * y[s] (the routed experts), + sigmoid(gs) * ys (the shared expert)
__global__ void k_moe_sum(float* x, const float* y, const float* w, int k, const float* ys, const float* gs, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n) return;
float acc = 0.f;
for (int s = 0; s < k; ++s) acc += w[s] * y[(size_t) s * n + i];
x[i] += acc + sigm(gs[0]) * ys[i];
}
// ------------------------------------------------------------------------------------------------ the model
struct Tensor {
const uint8_t* d = nullptr; // device
int type = -1;
std::vector<int64_t> shape;
size_t row_bytes = 0;
};
size_t row_bytes_of(int type, int64_t n) {
int be = 0, bb = 0;
if (!strata::block_geometry((uint32_t) type, be, bb)) return 0;
return (size_t) (n / be) * (size_t) bb;
}
template <int T>
void gemv_t(const Tensor& w, size_t expert_bytes, const int* ids, int n_sel, const float* x, size_t x_stride, float* y) {
const int n_in = (int) w.shape[0], n_out = (int) w.shape[1];
k_gemv<T><<<dim3(n_out, n_sel), kThreads>>>(w.d, w.row_bytes, expert_bytes, ids, n_in, x, x_stride, y, n_out);
}
/// llama.cpp's GPU matmuls on quantized weights (MMVQ for one token, MMQ for a batch) read the activations as q8_1:
/// per 32 values d = amax / 127, q = round(x / d). --act q8 (the default) gives this path the same rounding, so
/// its numbers follow llama.cpp's instead of drifting from them by the activation rounding; --act f32 reads them as
/// they are. `x` (n_sel vectors of n, `stride` apart) -> `out` (packed, n apart).
__global__ void k_act_q8(const float* x, size_t stride, int n, float* out) {
const int sel = blockIdx.y, blk = blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32, l = threadIdx.x % 32;
if (blk * 32 >= n) return;
const float v = x[(size_t) sel * stride + blk * 32 + l];
__shared__ float am[kThreads];
am[threadIdx.x] = fabsf(v);
__syncthreads();
const int base = threadIdx.x - l;
float amax = 0.f;
for (int i = 0; i < 32; ++i) amax = fmaxf(amax, am[base + i]);
const float d = amax / 127.0f;
const float q = amax == 0.0f ? 0.0f : roundf(v / d);
out[(size_t) sel * n + blk * 32 + l] = q * d;
}
bool g_act_q8 = true;
float* g_act_buf = nullptr;
constexpr size_t kActBuf = 1 << 16;
// --kernels strata (the default): the quantized matmuls run on Strata's native MMVQ (llama.cpp's pinned MMVQ: q8_1
// activations, one launch per routed expert) and the GDN step on Strata's fused kernels (conv + SiLU + L2, then the
// step with the SiLU-gated norm); --kernels plain keeps every op on this file's own kernels (the reference). The
// Strata kernels run on a blocking stream, so they and the null-stream work here stay in order.
bool g_strata = true;
cudaStream_t g_st = nullptr;
void* g_q8 = nullptr;
std::vector<int> g_ids_host; // the routed experts of the current layer (the native expert path launches per id)
void gemv(const Tensor& w, const float* x, float* y, const int* ids = nullptr, int n_sel = 1, size_t x_stride = 0) {
const size_t eb = w.shape.size() == 3 ? w.row_bytes * (size_t) w.shape[1] : 0;
const bool quant = w.type == T_Q8_0 || w.type == T_Q4_K || w.type == T_Q5_K || w.type == T_Q6_K;
if (quant && g_strata) {
const int n_in = (int) w.shape[0], n_out = (int) w.shape[1], cols = x_stride ? n_sel : 1;
if (x_stride && x_stride != (size_t) n_in) {
std::fprintf(stderr, "q36ref: the native path wants contiguous input columns\n");
std::exit(1);
}
strata::kernels::native_quantize_q8_1(x, g_q8, n_in, cols, g_st);
const size_t qb = strata::kernels::native_q8_1_bytes(n_in, 1);
for (int sel = 0; sel < n_sel; ++sel) {
const uint8_t* we = w.d + (ids ? (size_t) g_ids_host[(size_t) sel] * eb : 0);
strata::kernels::native_mmvq(w.type, we, (const char*) g_q8 + (x_stride ? (size_t) sel * qb : 0),
y + (size_t) sel * n_out, n_in, n_out, 1, g_st);
}
CK(cudaGetLastError());
return;
}
if (quant && g_act_q8) {
const int n = (int) w.shape[0], sel = x_stride ? n_sel : 1;
if ((size_t) n * sel > kActBuf) {
std::fprintf(stderr, "q36ref: activation scratch too small\n");
std::exit(1);
}
k_act_q8<<<dim3((n / 32 + kThreads / 32 - 1) / (kThreads / 32), sel), kThreads>>>(x, x_stride, n, g_act_buf);
x = g_act_buf;
x_stride = x_stride ? (size_t) n : 0;
}
switch (w.type) {
case T_F32: gemv_t<T_F32>(w, eb, ids, n_sel, x, x_stride, y); break;
case T_F16: gemv_t<T_F16>(w, eb, ids, n_sel, x, x_stride, y); break;
case T_BF16: gemv_t<T_BF16>(w, eb, ids, n_sel, x, x_stride, y); break;
case T_Q8_0: gemv_t<T_Q8_0>(w, eb, ids, n_sel, x, x_stride, y); break;
case T_Q4_K: gemv_t<T_Q4_K>(w, eb, ids, n_sel, x, x_stride, y); break;
case T_Q5_K: gemv_t<T_Q5_K>(w, eb, ids, n_sel, x, x_stride, y); break;
case T_Q6_K: gemv_t<T_Q6_K>(w, eb, ids, n_sel, x, x_stride, y); break;
default:
std::fprintf(stderr, "q36ref: no kernel for ggml type %d (%s)\n", w.type, strata::ggml_type_name((uint32_t) w.type));
std::exit(1);
}
CK(cudaGetLastError());
}
/// A 1-D tensor as f32 on the device (F32 as is, BF16 widened at load).
struct Model {
strata::core::ModelGeometry g;
int64_t top_k = 8;
float eps = 1e-6f, rope_base = 1e7f;
int n_rot = 64;
std::map<std::string, Tensor> t;
std::map<std::string, float*> f32; // the small vectors, f32
const Tensor& W(const std::string& n) const {
auto it = t.find(n);
if (it == t.end()) {
std::fprintf(stderr, "q36ref: missing tensor %s\n", n.c_str());
std::exit(1);
}
return it->second;
}
const float* F(const std::string& n) const {
auto it = f32.find(n);
if (it == f32.end()) {
std::fprintf(stderr, "q36ref: missing vector %s\n", n.c_str());
std::exit(1);
}
return it->second;
}
};
float h2f_host(uint16_t h) {
const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;
if (e == 0) return (s ? -1.f : 1.f) * std::ldexp((float) m, -24);
const uint32_t f = e == 31 ? (s | 0x7f800000u | (m << 13)) : (s | ((e + 112) << 23) | (m << 13));
float r;
std::memcpy(&r, &f, 4);
return r;
}
bool load(const std::string& path, Model& m, std::string& err) {
const auto t0 = std::chrono::steady_clock::now();
const strata::GgufModel gm = strata::GgufModel::open(path);
if (!strata::core::geometry_from_gguf(gm.meta(), m.g, m.top_k, err)) return false;
if (m.g.family != strata::ModelFamily::Qwen35Moe) {
err = "this reference path is for Qwen3.6-35B-A3B (qwen35moe) only";
return false;
}
size_t checked = 0;
if (!strata::core::check_gguf_layout(gm, m.g, err, &checked)) return false;
const std::string p = "qwen35moe.";
if (const strata::MetaValue* v = gm.meta().get(p + "attention.layer_norm_rms_epsilon")) m.eps = (float) v->num();
if (const strata::MetaValue* v = gm.meta().get(p + "rope.freq_base")) m.rope_base = (float) v->num();
if (const strata::MetaValue* v = gm.meta().get(p + "rope.dimension_count")) m.n_rot = (int) v->u;
uint64_t bytes = 0;
for (size_t s = 0; s < gm.size(); ++s) {
const strata::GgufFile& f = gm.shard(s);
for (const strata::TensorInfo& ti : f.tensors()) {
const uint64_t nb = strata::tensor_payload_bytes(ti);
const uint8_t* src = f.tensor_data(ti);
// the MTP block (blk.40) is not run by this path: not loaded
if (ti.name.rfind("blk." + std::to_string(m.g.n_layers) + ".", 0) == 0) continue;
const bool conv = ti.name.size() > 17 && ti.name.compare(ti.name.size() - 17, 17, "ssm_conv1d.weight") == 0;
if (ti.shape.size() == 1 || conv) {
// the small vectors (and the conv taps): f32 on the device
std::vector<float> h(ti.elements());
if (ti.type == T_F32) std::memcpy(h.data(), src, h.size() * 4);
else if (ti.type == T_BF16)
for (size_t i = 0; i < h.size(); ++i) {
const uint32_t u = (uint32_t) (src[2 * i] | (src[2 * i + 1] << 8)) << 16;
std::memcpy(&h[i], &u, 4);
}
else if (ti.type == T_F16)
for (size_t i = 0; i < h.size(); ++i) h[i] = h2f_host((uint16_t) (src[2 * i] | (src[2 * i + 1] << 8)));
else {
err = ti.name + ": a 1-D tensor of type " + ti.type_name();
return false;
}
float* d = nullptr;
CK(cudaMalloc(&d, h.size() * 4));
CK(cudaMemcpy(d, h.data(), h.size() * 4, cudaMemcpyHostToDevice));
m.f32[ti.name] = d;
bytes += h.size() * 4;
continue;
}
Tensor w;
w.type = (int) ti.type;
for (uint64_t d : ti.shape) w.shape.push_back((int64_t) d);
w.row_bytes = row_bytes_of(w.type, w.shape[0]);
uint8_t* d = nullptr;
CK(cudaMalloc(&d, nb));
CK(cudaMemcpy(d, src, nb, cudaMemcpyHostToDevice));
w.d = d;
m.t[ti.name] = w;
bytes += nb;
}
}
std::fprintf(stderr, "q36ref: %s: %zu tensors (layout checked), %.2f GiB on the GPU in %.1f s\n", path.c_str(),
checked, (double) bytes / (1 << 30),
std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count());
return true;
}
struct State {
int ctx = 4096, pos = 0;
// scratch
float *x, *h, *y, *qkv, *z, *alpha, *beta, *conv_out, *gdn_out, *qg, *k, *v, *q, *gate, *att, *router, *eg, *eu, *eh,
*ey, *sg, *su, *sh, *sy, *sgate, *wts, *logits;
int* ids;
std::vector<float*> conv_state, S, kc, vc; // per layer (GDN: conv + S; attention: K/V cache)
};
float* dalloc(size_t n) {
float* p = nullptr;
CK(cudaMalloc(&p, n * 4));
CK(cudaMemset(p, 0, n * 4));
return p;
}
void init_state(const Model& m, State& s) {
const auto& g = m.g;
const int N = (int) g.n_embd, FF = (int) g.n_ff, K = (int) m.top_k;
s.x = dalloc(N); s.h = dalloc(N); s.y = dalloc(N);
s.qkv = dalloc(g.ssm_conv_channels); s.z = dalloc(g.ssm_value_dim); s.alpha = dalloc(g.ssm_v_heads);
s.beta = dalloc(g.ssm_v_heads); s.conv_out = dalloc(g.ssm_conv_channels); s.gdn_out = dalloc(g.ssm_value_dim);
s.qg = dalloc(2 * g.n_head * g.head_dim); s.k = dalloc(g.n_head_kv * g.head_dim); s.v = dalloc(g.n_head_kv * g.head_dim);
s.q = dalloc(g.n_head * g.head_dim); s.gate = dalloc(g.n_head * g.head_dim); s.att = dalloc(g.n_head * g.head_dim);
s.router = dalloc(g.n_expert);
s.eg = dalloc((size_t) K * FF); s.eu = dalloc((size_t) K * FF); s.eh = dalloc((size_t) K * FF); s.ey = dalloc((size_t) K * N);
s.sg = dalloc(g.n_ff_shexp); s.su = dalloc(g.n_ff_shexp); s.sh = dalloc(g.n_ff_shexp); s.sy = dalloc(N); s.sgate = dalloc(1);
s.wts = dalloc(K);
s.logits = dalloc(g.n_vocab);
CK(cudaMalloc(&s.ids, K * sizeof(int)));
g_act_buf = dalloc(kActBuf);
CK(cudaStreamCreate(&g_st)); // blocking: ordered with this file's null-stream launches and copies
CK(cudaMalloc(&g_q8, strata::kernels::native_q8_1_bytes((int) kActBuf, 1)));
for (int64_t l = 0; l < g.n_layers; ++l) {
if (strata::core::is_qsa_layer(g, l)) {
s.conv_state.push_back(nullptr);
s.S.push_back(nullptr);
s.kc.push_back(dalloc((size_t) g.n_head_kv * s.ctx * g.head_dim));
s.vc.push_back(dalloc((size_t) g.n_head_kv * s.ctx * g.head_dim));
} else {
s.conv_state.push_back(dalloc((size_t) (g.ssm_d_conv - 1) * g.ssm_conv_channels));
s.S.push_back(dalloc((size_t) g.ssm_v_heads * g.ssm_state_size * g.ssm_state_size));
s.kc.push_back(nullptr);
s.vc.push_back(nullptr);
}
}
}
std::string L(int64_t l, const char* s) { return "blk." + std::to_string(l) + "." + s; }
void embed(const Model& m, State& s, int tok) {
const Tensor& e = m.W("token_embd.weight");
const int N = (int) m.g.n_embd;
switch (e.type) {
case T_Q8_0: k_row<T_Q8_0><<<4, 128>>>(e.d, e.row_bytes, tok, N, s.x); break;
case T_Q4_K: k_row<T_Q4_K><<<4, 128>>>(e.d, e.row_bytes, tok, N, s.x); break;
case T_Q6_K: k_row<T_Q6_K><<<4, 128>>>(e.d, e.row_bytes, tok, N, s.x); break;
case T_BF16: k_row<T_BF16><<<4, 128>>>(e.d, e.row_bytes, tok, N, s.x); break;
case T_F16: k_row<T_F16><<<4, 128>>>(e.d, e.row_bytes, tok, N, s.x); break;
default:
std::fprintf(stderr, "q36ref: token_embd type %d has no row reader\n", e.type);
std::exit(1);
}
CK(cudaGetLastError());
}
/// One token through the model at position s.pos; the layer outputs into `layers` (host, n_layers x N) when given.
void forward(const Model& m, State& s, int tok, float* layers) {
const auto& g = m.g;
const int N = (int) g.n_embd, K = (int) m.top_k, FF = (int) g.n_ff, FS = (int) g.n_ff_shexp;
const int B = 256;
embed(m, s, tok);
for (int64_t l = 0; l < g.n_layers; ++l) {
k_rms<<<1, kThreads>>>(s.x, m.F(L(l, "attn_norm.weight")), s.h, N, m.eps);
if (!strata::core::is_qsa_layer(g, l)) {
const int HK = (int) g.ssm_k_heads, HV = (int) g.ssm_v_heads, D = (int) g.ssm_state_size;
const int C = (int) g.ssm_conv_channels;
gemv(m.W(L(l, "attn_qkv.weight")), s.h, s.qkv);
gemv(m.W(L(l, "attn_gate.weight")), s.h, s.z);
gemv(m.W(L(l, "ssm_beta.weight")), s.h, s.beta);
gemv(m.W(L(l, "ssm_alpha.weight")), s.h, s.alpha);
if (g_strata && D == 128 && g.ssm_d_conv == 4) {
// Strata's fused GDN (its own state layouts: conv history [channel][3], S [row][head][col])
strata::kernels::fused_gdn_conv_l2(s.conv_state[l], s.qkv, m.F(L(l, "ssm_conv1d.weight")), s.conv_out,
C, 2 * HK, m.eps, g_st);
k_gdn_gb<<<1, 64, 0, g_st>>>(s.alpha, m.F(L(l, "ssm_dt.bias")), m.F(L(l, "ssm_a")), s.beta, HV);
strata::kernels::fused_gdn_step_norm_silu(s.S[l], s.conv_out, s.conv_out + HK * D,
s.conv_out + 2 * HK * D, s.alpha, s.beta, s.z,
m.F(L(l, "ssm_norm.weight")), m.eps, s.gdn_out, HK, HV, g_st);
} else {
k_conv<<<(C + B - 1) / B, B>>>(s.qkv, s.conv_state[l], m.F(L(l, "ssm_conv1d.weight")), s.conv_out, C,
(int) g.ssm_d_conv);
k_gdn<<<HV, D, (2 * D + D) * sizeof(float)>>>(s.conv_out, s.alpha, m.F(L(l, "ssm_dt.bias")),
m.F(L(l, "ssm_a")), s.beta, s.S[l], s.gdn_out, HK, HV, D,
m.eps);
k_gated_norm<<<HV, D>>>(s.gdn_out, m.F(L(l, "ssm_norm.weight")), s.z, D, m.eps);
}
gemv(m.W(L(l, "ssm_out.weight")), s.gdn_out, s.y);
} else {
const int H = (int) g.n_head, HKV = (int) g.n_head_kv, D = (int) g.head_dim;
gemv(m.W(L(l, "attn_q.weight")), s.h, s.qg);
gemv(m.W(L(l, "attn_k.weight")), s.h, s.k);
gemv(m.W(L(l, "attn_v.weight")), s.h, s.v);
k_attn_prep<<<H + HKV, kThreads>>>(s.qg, s.k, s.v, m.F(L(l, "attn_q_norm.weight")),
m.F(L(l, "attn_k_norm.weight")), s.q, s.gate, s.kc[l], s.vc[l], H, HKV, D,
m.n_rot, s.pos, s.ctx, m.rope_base, m.eps);
k_attn<<<H, kThreads, (size_t) (s.pos + 1) * sizeof(float)>>>(s.q, s.kc[l], s.vc[l], s.gate, s.att, H, HKV, D,
s.pos + 1, s.ctx, 1.0f / sqrtf((float) D));
gemv(m.W(L(l, "attn_output.weight")), s.att, s.y);
}
CK(cudaGetLastError());
k_add<<<(N + B - 1) / B, B>>>(s.x, s.y, N);
k_rms<<<1, kThreads>>>(s.x, m.F(L(l, "post_attention_norm.weight")), s.h, N, m.eps);
// router: softmax over every expert, top-k, renormalised (host side: exact and simple)
gemv(m.W(L(l, "ffn_gate_inp.weight")), s.h, s.router);
if (g_strata && strata::kernels::native_router_topk_supported((int) g.n_expert, K)) {
// Strata's router (llama.cpp's topk-moe contract) for 256 / 8; the ids come back for the per-expert launches
strata::kernels::native_router_topk(s.router, s.ids, s.wts, (int) g.n_expert, K, 1, g_st);
g_ids_host.resize((size_t) K);
CK(cudaMemcpy(g_ids_host.data(), s.ids, K * sizeof(int), cudaMemcpyDeviceToHost));
} else {
std::vector<float> r((size_t) g.n_expert);
CK(cudaMemcpy(r.data(), s.router, r.size() * 4, cudaMemcpyDeviceToHost));
const float mx = *std::max_element(r.begin(), r.end());
double sum = 0;
for (float& v : r) sum += (v = std::exp(v - mx));
for (float& v : r) v = (float) (v / sum);
std::vector<int> idx((size_t) g.n_expert);
for (int i = 0; i < (int) idx.size(); ++i) idx[i] = i;
std::partial_sort(idx.begin(), idx.begin() + K, idx.end(),
[&](int a, int b) { return r[a] > r[b] || (r[a] == r[b] && a < b); });
std::vector<float> w(K);
float ws = 0;
for (int i = 0; i < K; ++i) ws += (w[i] = r[idx[i]]);
for (float& v : w) v /= ws;
g_ids_host.assign(idx.begin(), idx.begin() + K);
CK(cudaMemcpy(s.ids, idx.data(), K * sizeof(int), cudaMemcpyHostToDevice));
CK(cudaMemcpy(s.wts, w.data(), K * 4, cudaMemcpyHostToDevice));
}
gemv(m.W(L(l, "ffn_gate_exps.weight")), s.h, s.eg, s.ids, K, 0);
gemv(m.W(L(l, "ffn_up_exps.weight")), s.h, s.eu, s.ids, K, 0);
k_swiglu<<<(K * FF + B - 1) / B, B>>>(s.eg, s.eu, s.eh, K * FF);
gemv(m.W(L(l, "ffn_down_exps.weight")), s.eh, s.ey, s.ids, K, FF);
// shared expert with its sigmoid scalar gate
gemv(m.W(L(l, "ffn_gate_shexp.weight")), s.h, s.sg);
gemv(m.W(L(l, "ffn_up_shexp.weight")), s.h, s.su);
k_swiglu<<<(FS + B - 1) / B, B>>>(s.sg, s.su, s.sh, FS);
gemv(m.W(L(l, "ffn_down_shexp.weight")), s.sh, s.sy);
{
Tensor gs;
gs.d = (const uint8_t*) m.F(L(l, "ffn_gate_inp_shexp.weight"));
gs.type = T_F32;
gs.shape = {N, 1};
gs.row_bytes = (size_t) N * 4;
gemv(gs, s.h, s.sgate);
}
k_moe_sum<<<(N + B - 1) / B, B>>>(s.x, s.ey, s.wts, K, s.sy, s.sgate, N);
CK(cudaGetLastError());
if (layers) CK(cudaMemcpy(layers + (size_t) l * N, s.x, (size_t) N * 4, cudaMemcpyDeviceToHost));
}
k_rms<<<1, kThreads>>>(s.x, m.F("output_norm.weight"), s.h, N, m.eps);
gemv(m.W("output.weight"), s.h, s.logits);
CK(cudaDeviceSynchronize());
++s.pos;
}
std::vector<int> parse_ids(const std::string& s) {
std::vector<int> v;
size_t i = 0;
while (i < s.size()) {
while (i < s.size() && (s[i] == ',' || s[i] == ' ' || s[i] == '\n' || s[i] == '\r')) ++i;
if (i >= s.size()) break;
v.push_back(std::atoi(s.c_str() + i));
while (i < s.size() && s[i] != ',' && s[i] != ' ' && s[i] != '\n') ++i;
}
return v;
}
} // namespace
int main(int argc, char** argv) {
std::string model, tokens, dump_layers, dump_logits, force, dump_full;
int n_gen = 64, ctx = 4096;
for (int i = 1; i < argc; ++i) {
const std::string a = argv[i];
auto next = [&]() -> std::string {
if (i + 1 >= argc) {
std::fprintf(stderr, "q36ref: %s needs a value\n", a.c_str());
std::exit(2);
}
return argv[++i];
};
if (a == "--model") model = next();
else if (a == "--tokens") tokens = next();
else if (a == "--tokens-file") {
FILE* f = std::fopen(next().c_str(), "rb");
if (!f) { std::fprintf(stderr, "q36ref: cannot read the tokens file\n"); return 2; }
char buf[4096];
size_t n;
while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) tokens.append(buf, n);
std::fclose(f);
} else if (a == "--n") n_gen = std::atoi(next().c_str());
else if (a == "--ctx") ctx = std::atoi(next().c_str());
else if (a == "--dump-layers") dump_layers = next();
else if (a == "--dump-logits") dump_logits = next();
else if (a == "--force") force = next();
else if (a == "--kernels") {
const std::string v = next();
if (v != "strata" && v != "plain") { std::fprintf(stderr, "q36ref: --kernels strata or plain\n"); return 2; }
g_strata = v == "strata";
}
else if (a == "--dump-full-logits") dump_full = next();
else if (a == "--act") {
const std::string v = next();
if (v != "q8" && v != "f32") { std::fprintf(stderr, "q36ref: --act q8 or f32\n"); return 2; }
g_act_q8 = v == "q8";
}
else {
std::fprintf(stderr, "usage: strata-q36ref --model GGUF --tokens ID,ID,... | --tokens-file F [--n 64] [--ctx 4096] "
"[--dump-layers FILE] [--dump-logits FILE] [--act q8|f32] [--kernels strata|plain] "
"[--force ID,ID,...] [--dump-full-logits FILE]\n");
return 2;
}
}
const std::vector<int> prompt = parse_ids(tokens);
// --force: the continuation is these tokens (teacher forcing: the KL check reads both engines on one sequence);
// the greedy ids are still printed
const std::vector<int> forced = parse_ids(force);
if (!forced.empty()) n_gen = (int) forced.size() + 1;
if (model.empty() || prompt.empty()) {
std::fprintf(stderr, "q36ref: --model and --tokens are required\n");
return 2;
}
Model m;
std::string err;
try {
if (!load(model, m, err)) {
std::fprintf(stderr, "q36ref: %s\n", err.c_str());
return 1;
}
} catch (const std::exception& e) {
std::fprintf(stderr, "q36ref: %s\n", e.what());
return 1;
}
State s;
s.ctx = ctx;
init_state(m, s);
const int N = (int) m.g.n_embd, NL = (int) m.g.n_layers, V = (int) m.g.n_vocab;
FILE* fl = dump_layers.empty() ? nullptr : std::fopen(dump_layers.c_str(), "wb");
FILE* fg = dump_logits.empty() ? nullptr : std::fopen(dump_logits.c_str(), "wb");
FILE* ff = dump_full.empty() ? nullptr : std::fopen(dump_full.c_str(), "wb");
std::vector<float> layers((size_t) NL * N), logits((size_t) V);
const auto t0 = std::chrono::steady_clock::now();
int next = -1;
std::vector<int> gen;
const int total = (int) prompt.size() + n_gen - 1;
if (total > ctx) {
std::fprintf(stderr, "q36ref: %d positions do not fit --ctx %d\n", total, ctx);
return 2;
}
for (int i = 0; i < total; ++i) {
const int tok = i < (int) prompt.size() ? prompt[i] : next;
forward(m, s, tok, fl ? layers.data() : nullptr);
if (fl) std::fwrite(layers.data(), 4, layers.size(), fl);
if (i + 1 >= (int) prompt.size()) {
CK(cudaMemcpy(logits.data(), s.logits, (size_t) V * 4, cudaMemcpyDeviceToHost));
int a = 0, b = -1;
for (int j = 1; j < V; ++j)
if (logits[j] > logits[a]) a = j;
for (int j = 0; j < V; ++j)
if (j != a && (b < 0 || logits[j] > logits[b])) b = j;
next = gen.size() < forced.size() ? forced[gen.size()] : a;
gen.push_back(a);
if (ff) std::fwrite(logits.data(), 4, logits.size(), ff);
if (fg) {
const float rec[4] = {(float) a, logits[a], (float) b, logits[b]};
std::fwrite(rec, 4, 4, fg);
}
}
}
const double sec = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
if (fl) std::fclose(fl);
if (fg) std::fclose(fg);
if (ff) std::fclose(ff);
std::printf("gen:");
for (int t : gen) std::printf(" %d", t);
std::printf("\n");
std::fprintf(stderr, "q36ref: %zu prompt + %zu generated positions in %.2f s (%.2f tok/s, one token at a time)\n",
prompt.size(), gen.size(), sec, (double) total / sec);
return 0;
}