Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// Adapted from topk-moe.cu/common.cuh in llama.cpp
// 3cf03257f219afbe7334045ff7c6a06ac68c627d; finite F32, 512-expert/10-output path.
// MIT License
// Copyright (c) 2023-2026 The ggml authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "strata/kernels/native_router.hpp"
#include <cuda_runtime.h>
#include <atomic>
#include <cfloat>
#include <cstddef>
#include <cstdint>
#include <stdexcept>
namespace strata::kernels {
namespace {
std::atomic<bool> enabled{false};
__device__ __forceinline__ float warp_sum(float value) {
#pragma unroll
for (int mask = 16; mask; mask >>= 1) value += __shfl_xor_sync(0xffffffffu, value, mask, 32);
return value;
}
__device__ __forceinline__ float warp_max(float value) {
#pragma unroll
for (int mask = 16; mask; mask >>= 1) value = fmaxf(value, __shfl_xor_sync(0xffffffffu, value, mask, 32));
return value;
}
__launch_bounds__(256, 1)
__global__ void route(const float* __restrict__ logits, int32_t* __restrict__ ids,
float* __restrict__ weights) {
// Preserve the pinned 32x8 block geometry; only row zero is active here.
// blockIdx.x = the token (a multi-token launch; 0 for the single one)
logits += (size_t) blockIdx.x * 512; ids += (size_t) blockIdx.x * 10; weights += (size_t) blockIdx.x * 10;
if (threadIdx.y != 0) return;
const int lane = threadIdx.x;
float values[16];
#pragma unroll
for (int i = 0; i < 16; ++i) values[i] = logits[lane + i * 32];
__syncthreads();
float maximum = -INFINITY;
#pragma unroll
for (int i = 0; i < 16; ++i) maximum = max(maximum, values[i]);
maximum = warp_max(maximum);
float sum = 0.0f;
#pragma unroll
for (int i = 0; i < 16; ++i) {
values[i] = expf(values[i] - maximum);
sum += values[i];
}
const float reciprocal = 1.0f / warp_sum(sum);
#pragma unroll
for (int i = 0; i < 16; ++i) {
values[i] *= reciprocal;
if (__isnanf(values[i])) values[i] = -FLT_MAX;
}
float selected = 0.0f, selected_sum = 0.0f;
for (int rank = 0; rank < 10; ++rank) {
float best = values[0];
int expert = lane;
#pragma unroll
for (int i = 1; i < 16; ++i) {
if (values[i] > best) { best = values[i]; expert = lane + i * 32; }
}
#pragma unroll
for (int mask = 16; mask; mask >>= 1) {
const float other = __shfl_xor_sync(0xffffffffu, best, mask, 32);
const int other_id = __shfl_xor_sync(0xffffffffu, expert, mask, 32);
if (other > best || (other == best && other_id < expert)) { best = other; expert = other_id; }
}
if ((expert & 31) == lane) {
values[expert / 32] = -INFINITY;
ids[rank] = expert;
// Deliberately accumulate by WINNING EXPERT lane, not output rank.
// Multiple selected experts in one lane add in selection order.
selected_sum += best;
}
if (rank == lane) selected = best;
}
selected_sum = max(warp_sum(selected_sum), 6.103515625e-5f);
const float inverse_selected_sum = 1.0f / selected_sum;
if (lane < 10) weights[lane] = selected * inverse_selected_sum;
}
__launch_bounds__(256, 1)
__global__ void route_multi(const float* __restrict__ logits, int32_t* __restrict__ ids,
float* __restrict__ weights, int n_tok) {
const int tk = (int) blockIdx.x * 8 + (int) threadIdx.y;
if (tk >= n_tok) return;
logits += (size_t) tk * 512; ids += (size_t) tk * 10; weights += (size_t) tk * 10;
const int lane = threadIdx.x;
float values[16];
#pragma unroll
for (int i = 0; i < 16; ++i) values[i] = logits[lane + i * 32];
__syncwarp();
float maximum = -INFINITY;
#pragma unroll
for (int i = 0; i < 16; ++i) maximum = max(maximum, values[i]);
maximum = warp_max(maximum);
float sum = 0.0f;
#pragma unroll
for (int i = 0; i < 16; ++i) {
values[i] = expf(values[i] - maximum);
sum += values[i];
}
const float reciprocal = 1.0f / warp_sum(sum);
#pragma unroll
for (int i = 0; i < 16; ++i) {
values[i] *= reciprocal;
if (__isnanf(values[i])) values[i] = -FLT_MAX;
}
float selected = 0.0f, selected_sum = 0.0f;
for (int rank = 0; rank < 10; ++rank) {
float best = values[0];
int expert = lane;
#pragma unroll
for (int i = 1; i < 16; ++i) {
if (values[i] > best) { best = values[i]; expert = lane + i * 32; }
}
#pragma unroll
for (int mask = 16; mask; mask >>= 1) {
const float other = __shfl_xor_sync(0xffffffffu, best, mask, 32);
const int other_id = __shfl_xor_sync(0xffffffffu, expert, mask, 32);
if (other > best || (other == best && other_id < expert)) { best = other; expert = other_id; }
}
if ((expert & 31) == lane) {
values[expert / 32] = -INFINITY;
ids[rank] = expert;
selected_sum += best;
}
if (rank == lane) selected = best;
}
selected_sum = max(warp_sum(selected_sum), 6.103515625e-5f);
const float inverse_selected_sum = 1.0f / selected_sum;
if (lane < 10) weights[lane] = selected * inverse_selected_sum;
}
// The same contract for other (experts, top-k) shapes - Qwen3.6-35B-A3B's 256 / 8 - one warp per token, n_tok tokens
// (8 per block). The 512 / 10 path keeps its own kernels above (the Flash-Next bits), this template is only used
// for the other shapes.
template <int NE, int K>
__launch_bounds__(256, 1)
__global__ void route_topk(const float* __restrict__ logits, int32_t* __restrict__ ids, float* __restrict__ weights,
int n_tok) {
static_assert(NE % 32 == 0 && K <= 32, "experts in whole warps, at most 32 picks");
constexpr int V = NE / 32;
const int tk = (int) blockIdx.x * 8 + (int) threadIdx.y;
if (tk >= n_tok) return;
logits += (size_t) tk * NE; ids += (size_t) tk * K; weights += (size_t) tk * K;
const int lane = threadIdx.x;
float values[V];
#pragma unroll
for (int i = 0; i < V; ++i) values[i] = logits[lane + i * 32];
float maximum = -INFINITY;
#pragma unroll
for (int i = 0; i < V; ++i) maximum = max(maximum, values[i]);
maximum = warp_max(maximum);
float sum = 0.0f;
#pragma unroll
for (int i = 0; i < V; ++i) {
values[i] = expf(values[i] - maximum);
sum += values[i];
}
const float reciprocal = 1.0f / warp_sum(sum);
#pragma unroll
for (int i = 0; i < V; ++i) {
values[i] *= reciprocal;
if (__isnanf(values[i])) values[i] = -FLT_MAX;
}
float selected = 0.0f, selected_sum = 0.0f;
for (int rank = 0; rank < K; ++rank) {
float best = values[0];
int expert = lane;
#pragma unroll
for (int i = 1; i < V; ++i) {
if (values[i] > best) { best = values[i]; expert = lane + i * 32; }
}
#pragma unroll
for (int mask = 16; mask; mask >>= 1) {
const float other = __shfl_xor_sync(0xffffffffu, best, mask, 32);
const int other_id = __shfl_xor_sync(0xffffffffu, expert, mask, 32);
if (other > best || (other == best && other_id < expert)) { best = other; expert = other_id; }
}
if ((expert & 31) == lane) {
values[expert / 32] = -INFINITY;
ids[rank] = expert;
selected_sum += best;
}
if (rank == lane) selected = best;
}
selected_sum = max(warp_sum(selected_sum), 6.103515625e-5f);
const float inverse_selected_sum = 1.0f / selected_sum;
if (lane < K) weights[lane] = selected * inverse_selected_sum;
}
bool valid(const void* p, size_t bytes) {
const auto address = reinterpret_cast<uintptr_t>(p);
return p && address % 4 == 0 && bytes <= UINTPTR_MAX - address;
}
bool overlap(const void* a, size_t an, const void* b, size_t bn) {
const auto ap = reinterpret_cast<uintptr_t>(a), bp = reinterpret_cast<uintptr_t>(b);
return ap < bp + bn && bp < ap + an;
}
}
void native_router_set_enabled(bool value) { enabled.store(value, std::memory_order_relaxed); }
bool native_router_enabled() { return enabled.load(std::memory_order_relaxed); }
void native_router_top10(const float* logits, int32_t* ids, float* weights, void* stream) {
if (!stream || !valid(logits, 512 * 4) || !valid(ids, 10 * 4) || !valid(weights, 10 * 4)
|| overlap(logits, 512 * 4, ids, 10 * 4) || overlap(logits, 512 * 4, weights, 10 * 4)
|| overlap(ids, 10 * 4, weights, 10 * 4))
throw std::invalid_argument("native router requires a stream, aligned spans, and disjoint outputs");
route<<<1, dim3(32, 8), 0, static_cast<cudaStream_t>(stream)>>>(logits, ids, weights);
const auto error = cudaGetLastError();
if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
}
void native_router_top10_multi(const float* logits, int32_t* ids, float* weights, int n_tok, void* stream) {
if (!stream || n_tok < 1 || !valid(logits, (size_t) n_tok * 512 * 4) || !valid(ids, (size_t) n_tok * 10 * 4) ||
!valid(weights, (size_t) n_tok * 10 * 4))
throw std::invalid_argument("native router (multi) requires a stream and aligned [n,512]/[n,10] buffers");
route_multi<<<(unsigned) ((n_tok + 7) / 8), dim3(32, 8), 0, static_cast<cudaStream_t>(stream)>>>(logits, ids, weights, n_tok);
const auto error = cudaGetLastError();
if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
}
bool native_router_topk_supported(int n_expert, int k) { return (n_expert == 512 && k == 10) || (n_expert == 256 && k == 8); }
void native_router_topk(const float* logits, int32_t* ids, float* weights, int n_expert, int k, int n_tok, void* stream) {
if (n_expert == 512 && k == 10) { // Flash-Next: the pinned kernels above, unchanged
if (n_tok == 1) native_router_top10(logits, ids, weights, stream);
else native_router_top10_multi(logits, ids, weights, n_tok, stream);
return;
}
if (!stream || n_tok < 1 || !native_router_topk_supported(n_expert, k) ||
!valid(logits, (size_t) n_tok * n_expert * 4) || !valid(ids, (size_t) n_tok * k * 4) ||
!valid(weights, (size_t) n_tok * k * 4))
throw std::invalid_argument("native_router_topk: a stream, a supported (experts, k) and aligned buffers");
route_topk<256, 8><<<(unsigned) ((n_tok + 7) / 8), dim3(32, 8), 0, static_cast<cudaStream_t>(stream)>>>(
logits, ids, weights, n_tok);
const auto error = cudaGetLastError();
if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
}
// STRATA_ROUTE_RESIDENT (EXPERIMENTAL, opt-in, changes the output): residency-biased routing for 512 x 10.
// After the top-10: for the ranks lo..hi a non-resident expert is replaced by the best resident expert not already
// picked when its logit is within `margin` of the replaced one; the weights are then the softmax of the selected
// logits (the router's renormalisation). Tokens without a swap keep the router's exact bits.
// stats (device, 4 x uint64): [0] tail entries seen, [1] swaps, [2] non-resident entries before, [3] after.
//
// One warp per token. This used to be one THREAD per token scanning all 512 experts (4 ranks x 512 x 12 scalar
// ops on up to 6 threads): 45 us per layer, 2.2 ms of a 19.4 ms window, which is exactly what the option saved in
// PCIe experts (~2.4 ms) - the swap cost as much as the misses it removed. The ranks are still visited in order
// (a swap changes the set already picked), but the scan is coalesced (lane j reads f = j, j + 32, ...) and reduced
// within the warp. The bits are unchanged: a strictly greater logit wins, the smallest expert index breaks ties,
// and the softmax is the same expression.
__global__ void route_resident_k(const float* __restrict__ logits, int32_t* __restrict__ ids, float* __restrict__ weights,
const int32_t* __restrict__ res, int n_tok, float margin, int lo, int hi,
unsigned long long* __restrict__ stats) {
const int t = blockIdx.x;
if (t >= n_tok) return;
const int lane = (int) threadIdx.x;
const float* l = logits + (size_t) t * 512;
int32_t* id = ids + (size_t) t * 10;
int32_t my[10];
for (int r = 0; r < 10; ++r) my[r] = id[r];
int swaps = 0, tail = 0, before = 0;
if (lane == 0)
for (int r = 0; r < 10; ++r) before += res[my[r]] < 0;
for (int r = lo; r <= hi && r < 10; ++r) {
const int e = my[r];
if (res[e] >= 0) continue; // already resident: uniform across the warp
if (lane == 0) ++tail;
float bv = -INFINITY;
int bi = -1;
for (int f = lane; f < 512; f += 32) {
if (res[f] < 0) continue;
bool used = false;
for (int q = 0; q < 10; ++q) used |= my[q] == f;
if (used) continue;
if (l[f] > bv) { bv = l[f]; bi = f; }
}
#pragma unroll
for (int off = 16; off > 0; off >>= 1) {
const float ov = __shfl_down_sync(0xffffffffu, bv, off);
const int oi = __shfl_down_sync(0xffffffffu, bi, off);
if (ov > bv || (ov == bv && oi >= 0 && (bi < 0 || oi < bi))) { bv = ov; bi = oi; }
}
if (lane == 0 && bi >= 0 && l[e] - bv <= margin) { my[r] = bi; ++swaps; }
my[r] = __shfl_sync(0xffffffffu, my[r], 0); // all lanes keep the same picked set
}
int after = 0;
if (lane == 0)
for (int r = 0; r < 10; ++r) after += res[my[r]] < 0;
const int sw = __shfl_sync(0xffffffffu, swaps, 0);
if (sw) {
float m = -INFINITY;
for (int r = 0; r < 10; ++r) m = fmaxf(m, l[my[r]]);
float ex[10], sum = 0.0f;
for (int r = 0; r < 10; ++r) { ex[r] = expf(l[my[r]] - m); sum += ex[r]; }
for (int r = 0; r < 10; ++r) weights[(size_t) t * 10 + r] = ex[r] / sum;
}
if (lane == 0) {
for (int r = 0; r < 10; ++r) id[r] = my[r];
if (stats) {
atomicAdd(stats + 0, (unsigned long long) tail);
atomicAdd(stats + 1, (unsigned long long) sw);
atomicAdd(stats + 2, (unsigned long long) before);
atomicAdd(stats + 3, (unsigned long long) after);
}
}
}
void native_route_resident(const float* logits, int32_t* ids, float* weights, const int32_t* res_layer, int n_tok, float margin,
int rank_lo, int rank_hi, unsigned long long* stats, void* stream) {
if (!stream || n_tok < 1 || !res_layer) throw std::invalid_argument("native_route_resident: bad arguments");
route_resident_k<<<(unsigned) n_tok, 32, 0, static_cast<cudaStream_t>(stream)>>>(
logits, ids, weights, res_layer, n_tok, margin, rank_lo, rank_hi, stats);
const auto error = cudaGetLastError();
if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
}
}