FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
refactor(cpu-moe): split the executor into weight formats and gpu sync backends (#670)
Xiaoze Fan2010271266f1
diff --git a/docs/install_amd.md b/docs/install_amd.mdindex 81d1ad50..ed934ad6 100644--- a/docs/install_amd.md+++ b/docs/install_amd.md@@ -45,3 +45,11 @@ On ROCm, these paths use the built-in fallbacks; do not install the `[accel]`,`[fi]`, or `[sgl]` extras. Forcing `--attention-backend fi`, `fa`, or `trtllm`,or NVFP4 Marlin/b12x (e.g. `--quant-backend moe.nvfp4=marlin` or`--quant-backend moe.nvfp4=b12x`), fails with a ROCm-specific error.++## CPU and hybrid MoE++`--moe-strategy cpu`, `hybrid` and `--moe-cpu-layers` sync the GPU with the CPU+expert pool through HIP stream memory operations. ROCm 10 records them into CUDA+graphs itself; on ROCm 7.14, which does not, FreeToken adds the graph nodes+explicitly. When the startup probe of that path fails, CUDA graph capture stops+with an error; run those strategies with `--cuda-graph-max-bs 0`.diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 3988caf6..feb33fbb 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -813,9 +813,8 @@ class Engine:"""Build the persistent CPU MoE executor (decode-time expert compute).Must run before CUDA graph capture: the worker pool has to be live for the- eager warmup forward, and the pinned IO buffers / host-func task pointers- must be stable for the captured nodes. Buffers/tasks themselves are- allocated lazily on the first (eager) forward at each batch size.+ eager warmup forward, and the pinned IO block must be stable for the captured+ nodes."""from freetoken.moe.cpu_executor import CpuMoeExecutor@@ -828,7 +827,7 @@ class Engine:)# Decode batches never exceed max_running_req, but CUDA-graph padding can# round a batch up to the largest captured size; cover both.- max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, 1)+ max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)method = sample.quant_methodexecutor = CpuMoeExecutor(cache,@@ -1365,10 +1364,10 @@ def _cpu_moe_executor_viable(model_config) -> bool:"""Whether an automatic CPU-decode decision may target the CPU MoE executor.A default boot must degrade to GPU offload instead of crashing in CpuMoeExecutor after the whole load; explicit cpu/hybrid/--moe-cpu-layers picks still fail loudly."""- from freetoken.moe.cpu_executor import _WFMT_IDS, compiled_extension_supports+ from freetoken.moe.cpu_executor import compiled_extension_supportstry:- from freetoken.kernel import _cpu_moe # noqa: F401+ from freetoken.kernel import _cpu_moeexcept ImportError:return Falseact = getattr(model_config, "hidden_act", "silu")@@ -1379,7 +1378,7 @@ def _cpu_moe_executor_viable(model_config) -> bool:return Falseexpert_quant = getattr(model_config, "expert_quant", "none")fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16")- return fmt == "mxfp4" or fmt in _WFMT_IDS+ return fmt == "mxfp4" or fmt in _cpu_moe.weight_formats()def _pin_budget_bytes(reserved: int = 0) -> int | None:diff --git a/python/freetoken/kernel/csrc/cpu_moe/common.h b/python/freetoken/kernel/csrc/cpu_moe/common.hnew file mode 100644index 00000000..adf7e145--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/common.h@@ -0,0 +1,235 @@+// Numeric helpers and ISA detection shared by the CPU MoE formats.+#pragma once++#include <algorithm>+#include <cmath>+#include <cstdint>+#include <cstdlib>+#include <cstring>+#include <new>+#include <vector>++#if defined(__x86_64__) || defined(__i386__)+#include <immintrin.h>+#define CPU_MOE_X86 1+#else+#define CPU_MOE_X86 0+#endif++// Each SIMD kernel is compiled for its own ISA and picked at runtime (pick_isa), so the build sets no global -march.+#define CPU_MOE_TARGET(...) __attribute__((target(__VA_ARGS__)))++#if CPU_MOE_X86 && ((defined(__GNUC__) && __GNUC__ >= 10) || defined(__clang__))+#define CPU_MOE_HAS_AVX512BF16 1+#define CPU_MOE_HAS_AVX512VNNI 1+#endif++namespace freetoken::cpu_moe {++// Scratch rows are read with 64-byte SIMD loads, so their base is cache-line aligned whatever the heap state; std::vector alone guarantees 16 bytes.+template <class T>+struct CacheLineAllocator {+ using value_type = T;+ CacheLineAllocator() = default;+ template <class U>+ CacheLineAllocator(const CacheLineAllocator<U>&) {}+ T* allocate(size_t n) { return static_cast<T*>(::operator new(n * sizeof(T), std::align_val_t(64))); }+ void deallocate(T* p, size_t) { ::operator delete(p, std::align_val_t(64)); }+ template <class U>+ bool operator==(const CacheLineAllocator<U>&) const { return true; }+};++template <class T>+using AlignedVector = std::vector<T, CacheLineAllocator<T>>;++using bf16_t = uint16_t;++inline float bf16_to_f32(bf16_t v) {+ uint32_t u = static_cast<uint32_t>(v) << 16;+ float f;+ std::memcpy(&f, &u, sizeof(f));+ return f;+}++inline bf16_t f32_to_bf16(float f) {+ uint32_t u;+ std::memcpy(&u, &f, sizeof(u));+ // round-to-nearest-even+ const uint32_t lsb = (u >> 16) & 1u;+ u += 0x7fffu + lsb;+ return static_cast<bf16_t>(u >> 16);+}++// ACT_SWIGLUOAI is the clamped (up + 1) swiglu (gpt-oss "swigluoai" /+// MiniMax-M3): gate/up are combined jointly with the runtime alpha/limit+// scalars, so it is handled in the gated_rows epilogue (act_apply never sees it;+// the mxfp4 kernel additionally fuses its own copy of the same math).+// ACT_SWIGLU_CLAMP (GLM-5.3 "swiglu_limit") is the same clamped form WITHOUT+// the (up + 1) bias: clamp(gate, max=lim) * sigmoid(alpha*gate) * clamp(up, +-lim).+enum ActKind {+ ACT_SILU = 0,+ ACT_GELU = 1,+ ACT_GELU_TANH = 2,+ ACT_SWIGLUOAI = 3,+ ACT_SWIGLU_CLAMP = 4,+};++inline float act_apply(int act, float x) {+ if (act == ACT_SILU) return x / (1.0f + std::exp(-x));+ if (act == ACT_GELU)+ return 0.5f * x * (1.0f + std::erf(x * 0.70710678118654752440f));+ // gelu_tanh+ const float k0 = 0.79788456080286535588f; // sqrt(2/pi)+ const float inner = k0 * (x + 0.044715f * x * x * x);+ return 0.5f * x * (1.0f + std::tanh(inner));+}++// Software prefetch distance (bytes) ahead of the current weight row stream. The+// weight stream is the bandwidth bottleneck (read once, never reused); nudging the+// HW prefetcher with a few cache lines of lookahead raises sustained throughput.+constexpr int PF_AHEAD = 512;++const float kE2M1[16] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f,+ -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f};++inline float fp16_to_f32(uint16_t h) {+ const uint32_t sign = (uint32_t)(h & 0x8000u) << 16;+ uint32_t exp = (h >> 10) & 0x1Fu;+ uint32_t man = h & 0x3FFu;+ uint32_t f;+ if (exp == 0) {+ if (man == 0) {+ f = sign;+ } else {+ exp = 127 - 15 + 1;+ while ((man & 0x400u) == 0) {+ man <<= 1;+ --exp;+ }+ man &= 0x3FFu;+ f = sign | (exp << 23) | (man << 13);+ }+ } else if (exp == 0x1Fu) {+ f = sign | 0x7F800000u | (man << 13);+ } else {+ f = sign | ((exp + (127 - 15)) << 23) | (man << 13);+ }+ float out;+ std::memcpy(&out, &f, sizeof(out));+ return out;+}++// e4m3 (OCP "fn": finite, max-normal 448, exp bias 7). Decoded into a 256-entry LUT.+inline float e4m3_decode(uint8_t v) {+ const float sign = (v & 0x80u) ? -1.0f : 1.0f;+ const uint32_t exp = (v >> 3) & 0xFu;+ const uint32_t man = v & 0x7u;+ if (exp == 0) return sign * (man / 8.0f) * 0.015625f; // 2^(1-7) = 2^-6+ return sign * (1.0f + man / 8.0f) * std::ldexp(1.0f, (int)exp - 7);+}++#if CPU_MOE_X86+CPU_MOE_TARGET("avx2,fma")+inline float hsum256(__m256 v) {+ __m128 lo = _mm256_castps256_ps128(v);+ lo = _mm_add_ps(lo, _mm256_extractf128_ps(v, 1));+ lo = _mm_add_ps(lo, _mm_movehl_ps(lo, lo));+ lo = _mm_add_ss(lo, _mm_shuffle_ps(lo, lo, 0x55));+ return _mm_cvtss_f32(lo);+}++// AVX2 e2m1 nibble decode: codes (0..15) in 8 int32 lanes -> fp32. AVX2 vpermps is+// only 8-wide, so instead of a 16-entry LUT we use the e2m1 sign/magnitude symmetry:+// value = (code&8 ? - : +) * mag8[code&7], mag8 = e2m1[0..7]. The sign is bit 3 of+// the code shifted into the fp32 sign bit (bit 31). Bit-identical to the e2m1 LUT.+CPU_MOE_TARGET("avx2,fma")+inline __m256 e2m1_decode8(__m256i codes, __m256 mag8) {+ __m256 mag = _mm256_permutevar8x32_ps(mag8, _mm256_and_si256(codes, _mm256_set1_epi32(7)));+ __m256i sgn = _mm256_slli_epi32(_mm256_and_si256(codes, _mm256_set1_epi32(8)), 28);+ return _mm256_xor_ps(mag, _mm256_castsi256_ps(sgn));+}+#endif // CPU_MOE_X86++// SIMD tiers, ascending. Each format picks the highest tier <= the one chosen by+// pick_isa() that it implements (fp4 and fp8_block formats have no bf16-specific tier, so the+// avx512bf16 tier maps to their avx512 kernel).+enum IsaTier { ISA_SCALAR = 0, ISA_AVX2 = 1, ISA_AVX512 = 2, ISA_AVX512BF16 = 3 };++// Best tier the CPU+build supports, optionally capped DOWN by+// FREETOKEN_CPU_MOE_ISA={scalar,avx2,avx512,avx512bf16} (A/B testing on a machine+// that supports more). FREETOKEN_CPU_MOE_SCALAR=1 forces scalar (legacy alias).+inline IsaTier pick_isa() {+#if CPU_MOE_X86+ if (getenv("FREETOKEN_CPU_MOE_SCALAR")) return ISA_SCALAR;+ IsaTier best = ISA_SCALAR;+ if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma")) best = ISA_AVX2;+ if (best >= ISA_AVX2 && __builtin_cpu_supports("avx512f")) best = ISA_AVX512;+#ifdef CPU_MOE_HAS_AVX512BF16+ if (best >= ISA_AVX512 && __builtin_cpu_supports("avx512bf16")) best = ISA_AVX512BF16;+#endif+ if (const char* f = getenv("FREETOKEN_CPU_MOE_ISA")) {+ IsaTier want = best;+ if (!std::strcmp(f, "scalar")) want = ISA_SCALAR;+ else if (!std::strcmp(f, "avx2")) want = ISA_AVX2;+ else if (!std::strcmp(f, "avx512")) want = ISA_AVX512;+ else if (!std::strcmp(f, "avx512bf16")) want = ISA_AVX512BF16;+ if (want < best) best = want; // cap downward; never force above hw/build support+ }+ return best;+#else+ return ISA_SCALAR;+#endif+}++inline const char* tier_name(IsaTier t) {+#if CPU_MOE_X86+#ifdef CPU_MOE_HAS_AVX512BF16+ if (t >= ISA_AVX512BF16) return "avx512bf16";+#endif+ if (t >= ISA_AVX512) return "avx512f";+ if (t >= ISA_AVX2) return "avx2";+#endif+ (void)t;+ return "scalar";+}++// AVX-VNNI (VEX-256 VPDPBUSD) availability: Alder/Raptor Lake, Sapphire Rapids+, Zen5.+// Distinct from AVX-512 VNNI. Opt out with FREETOKEN_CPU_MOE_NO_VNNI=1 (A/B the W4A8 path).+inline bool cpu_has_avxvnni() {+#if CPU_MOE_X86+ const char* no = getenv("FREETOKEN_CPU_MOE_NO_VNNI");+ if (no && no[0] && no[0] != '0') return false; // ignore unset/empty/"0"+ return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("avxvnni");+#else+ return false;+#endif+}++// AVX-512 VNNI (512-bit VPDPBUSD): Cascade Lake+, Ice Lake, Sapphire/Emerald Rapids,+// Zen4+. 2x the 256-bit AVX-VNNI width. FREETOKEN_CPU_MOE_NO_AVX512VNNI=1 forces the+// 256-bit path (A/B the two W4A8 kernels on the same box); FREETOKEN_CPU_MOE_NO_VNNI=1+// still disables the whole W4A8 family (back to the faithful fp32 nvdot), so it is+// honored here too.+inline bool cpu_has_avx512vnni() {+#if CPU_MOE_X86 && defined(CPU_MOE_HAS_AVX512VNNI)+ const char* no = getenv("FREETOKEN_CPU_MOE_NO_AVX512VNNI");+ if (no && no[0] && no[0] != '0') return false;+ const char* no_vnni = getenv("FREETOKEN_CPU_MOE_NO_VNNI");+ if (no_vnni && no_vnni[0] && no_vnni[0] != '0') return false;+ return __builtin_cpu_supports("avx512vnni");+#else+ return false;+#endif+}++// Split an interleaved bf16 row into fp32 even/odd halves (even[m]=src[2m]).+// bf16->fp32 is exact, so this only reorders -- done once per token/route and+// reused across every output row of the GEMV.+inline void deinterleave_bf16_f32(const bf16_t* src, float* even, float* odd, int K) {+ for (int m = 0; m < K / 2; ++m) {+ even[m] = bf16_to_f32(src[2 * m]);+ odd[m] = bf16_to_f32(src[2 * m + 1]);+ }+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cppindex 6f85971e..c8b4d1b4 100644--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp@@ -1,32 +1,27 @@-// CPU-compute MoE executor for the "cpu" offload backend.+// CPU-compute MoE executor for the cpu and hybrid MoE strategies, and its Python binding.//// Decode ships activations to the CPU, computes the routed experts here (reading-// the pinned host expert banks at full RAM bandwidth), and ships the results-// back. To keep the whole decode path inside a single CUDA graph we expose-// submit/sync as host nodes via cudaLaunchHostFunc -- the callbacks only touch a-// CPU worker pool + pinned host buffers and never call any CUDA API.+// the pinned host expert banks at full RAM bandwidth), and ships the results back.+// One task (one MoE layer) is in flight at a time: submit() wakes the persistent worker+// pool, the pool runs pass 1 (gate_up) and pass 2 (down), and sync() waits for it.+// The GPU rings a doorbell to submit and waits on it to sync (GpuSync, gpu_sync.h), so the+// whole decode step stays in one CUDA graph; cudaLaunchHostFunc nodes are the fallback.//-// One task is in flight at a time (per MoE layer): submit() wakes the pool,-// sync() blocks the host-func thread until the pool drains. The heavy GEMV runs-// on the persistent worker threads, not the host-func thread.-//-// Weight formats: bf16, NVFP4, MXFP4, ds_fp4, Q4_0 and block-FP8 expert banks (see WFmt and-// the per-format bank schemas). Compute is FP32-accumulate; the intermediate is-// stored bf16 to match the GPU decode path. ISA is chosen once at construction-// (AVX-512-BF16 dpbf16 -> AVX-512F widening -> AVX2+FMA -> scalar).+// Weight formats live in kernels/, one file each behind the MoeFormat interface (format.h).+// Compute is FP32-accumulate; the intermediate is stored bf16 to match the GPU decode path.#include <algorithm>#include <atomic>#include <condition_variable>-#include <cmath>#include <cstdint>#include <chrono>-#include <cstdio>#include <cstring>+#include <map>#include <mutex>#include <stdexcept>#include <string>#include <thread>+#include <utility>#include <vector>#include <freetoken/hip_compat.h>@@ -40,1379 +35,99 @@#define CPU_MOE_HAS_AFFINITY 0#endif-#if defined(__x86_64__) || defined(__i386__)-#include <immintrin.h>-#define CPU_MOE_X86 1-#else-#define CPU_MOE_X86 0-#endif--namespace {--inline void cpu_relax() {-#if CPU_MOE_X86- _mm_pause();-#elif defined(__aarch64__)- asm volatile("yield" ::: "memory");-#endif-}--using bf16_t = uint16_t;--inline float bf16_to_f32(bf16_t v) {- uint32_t u = static_cast<uint32_t>(v) << 16;- float f;- std::memcpy(&f, &u, sizeof(f));- return f;-}--inline bf16_t f32_to_bf16(float f) {- uint32_t u;- std::memcpy(&u, &f, sizeof(u));- // round-to-nearest-even- const uint32_t lsb = (u >> 16) & 1u;- u += 0x7fffu + lsb;- return static_cast<bf16_t>(u >> 16);-}--// ACT_SWIGLUOAI is the clamped (up + 1) swiglu (gpt-oss "swigluoai" /-// MiniMax-M3): gate/up are combined jointly with the runtime alpha/limit-// scalars, so it is handled in the do_pass1 epilogue (act_apply never sees it;-// the mxfp4 kernel additionally fuses its own copy of the same math).-// ACT_SWIGLU_CLAMP (GLM-5.3 "swiglu_limit") is the same clamped form WITHOUT-// the (up + 1) bias: clamp(gate, max=lim) * sigmoid(alpha*gate) * clamp(up, +-lim).-enum ActKind {- ACT_SILU = 0,- ACT_GELU = 1,- ACT_GELU_TANH = 2,- ACT_SWIGLUOAI = 3,- ACT_SWIGLU_CLAMP = 4,-};--inline float act_apply(int act, float x) {- if (act == ACT_SILU) return x / (1.0f + std::exp(-x));- if (act == ACT_GELU)- return 0.5f * x * (1.0f + std::erf(x * 0.70710678118654752440f));- // gelu_tanh- const float k0 = 0.79788456080286535588f; // sqrt(2/pi)- const float inner = k0 * (x + 0.044715f * x * x * x);- return 0.5f * x * (1.0f + std::tanh(inner));-}--// ------------------------------- dot products --------------------------------// dot(weight[bf16], act[bf16], n) -> fp32. The selected impl is a function-// pointer chosen at runtime; the per-row call overhead is negligible vs n.--using dot_fn = float (*)(const bf16_t*, const bf16_t*, int);--float dot_scalar(const bf16_t* w, const bf16_t* x, int n) {- float acc = 0.0f;- for (int i = 0; i < n; ++i) acc += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);- return acc;-}--// Software prefetch distance (bytes) ahead of the current weight row stream. The-// weight stream is the bandwidth bottleneck (read once, never reused); nudging the-// HW prefetcher with a few cache lines of lookahead raises sustained throughput.-constexpr int PF_AHEAD = 512;--#if CPU_MOE_X86-__attribute__((target("avx512f")))-float dot_avx512f(const bf16_t* w, const bf16_t* x, int n) {- // 4 independent accumulators -> more in-flight loads (memory-level parallelism),- // which is what lifts a bandwidth-bound GEMV toward peak.- __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();- __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();- int i = 0;- for (; i + 64 <= n; i += 64) {- _mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);- for (int j = 0; j < 64; j += 16) {- __m256i wi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + i + j));- __m256i xi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x + i + j));- __m512 wf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(wi), 16));- __m512 xf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(xi), 16));- __m512& acc = (j == 0) ? a0 : (j == 16) ? a1 : (j == 32) ? a2 : a3;- acc = _mm512_fmadd_ps(wf, xf, acc);- }- }- for (; i + 16 <= n; i += 16) {- __m256i wi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + i));- __m256i xi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x + i));- __m512 wf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(wi), 16));- __m512 xf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(xi), 16));- a0 = _mm512_fmadd_ps(wf, xf, a0);- }- float s = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3)));- for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);- return s;-}--#if (defined(__GNUC__) && __GNUC__ >= 10) || defined(__clang__)-#define CPU_MOE_HAS_AVX512BF16 1-__attribute__((target("avx512bf16,avx512f")))-static inline __m512bh load_bh(const bf16_t* p) {- __m512i raw = _mm512_loadu_si512(reinterpret_cast<const void*>(p));- __m512bh out;- std::memcpy(&out, &raw, sizeof(out));- return out;-}--__attribute__((target("avx512bf16,avx512f")))-float dot_avx512bf16(const bf16_t* w, const bf16_t* x, int n) {- // 4 accumulators (128 bf16/iter) for memory-level parallelism + a prefetch nudge.- __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();- __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();- int i = 0;- for (; i + 128 <= n; i += 128) {- _mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);- a0 = _mm512_dpbf16_ps(a0, load_bh(w + i), load_bh(x + i));- a1 = _mm512_dpbf16_ps(a1, load_bh(w + i + 32), load_bh(x + i + 32));- a2 = _mm512_dpbf16_ps(a2, load_bh(w + i + 64), load_bh(x + i + 64));- a3 = _mm512_dpbf16_ps(a3, load_bh(w + i + 96), load_bh(x + i + 96));- }- for (; i + 32 <= n; i += 32) {- a0 = _mm512_dpbf16_ps(a0, load_bh(w + i), load_bh(x + i));- }- float s = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3)));- for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);- return s;-}-#endif // avx512bf16 available--// --------- AVX2 + FMA (256-bit fallback for CPUs without AVX-512) -----------// Covers Intel 12-14th gen / Arrow Lake (AVX-512 fused off) and AMD Zen<4. The-// bf16->fp32 widen is a zero-extend + <<16; with 4 independent accumulators the-// GEMV is memory-bandwidth bound, same as the AVX-512 path (just half the width).-__attribute__((target("avx2,fma")))-inline float hsum256(__m256 v) {- __m128 lo = _mm256_castps256_ps128(v);- lo = _mm_add_ps(lo, _mm256_extractf128_ps(v, 1));- lo = _mm_add_ps(lo, _mm_movehl_ps(lo, lo));- lo = _mm_add_ss(lo, _mm_shuffle_ps(lo, lo, 0x55));- return _mm_cvtss_f32(lo);-}--__attribute__((target("avx2,fma")))-float dot_avx2(const bf16_t* w, const bf16_t* x, int n) {- __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();- __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();- int i = 0;- for (; i + 32 <= n; i += 32) {- _mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);- for (int j = 0; j < 32; j += 8) {- __m128i wi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(w + i + j));- __m128i xi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(x + i + j));- __m256 wf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(wi), 16));- __m256 xf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(xi), 16));- __m256& acc = (j == 0) ? a0 : (j == 8) ? a1 : (j == 16) ? a2 : a3;- acc = _mm256_fmadd_ps(wf, xf, acc);- }- }- for (; i + 8 <= n; i += 8) {- __m128i wi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(w + i));- __m128i xi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(x + i));- __m256 wf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(wi), 16));- __m256 xf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(xi), 16));- a0 = _mm256_fmadd_ps(wf, xf, a0);- }- float s = hsum256(_mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3)));- for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);- return s;-}-#endif // CPU_MOE_X86+#include "format.h"+#include "gpu_sync.h"+#include "gpu_sync_io.h"-// --------------------------- NVFP4 (W4A16) dequant ----------------------------// Weights: e2m1 4-bit codes (2/byte, low nibble first), per-16 block scale in-// fp8-e4m3, per-output-row global scale in fp16. Dequant matches the GPU kernels-// (freetoken/kernel/triton/nvfp4_dequant.py): w = E2M1[code] * e4m3(scale) * global.-// Activations stay bf16 (W4A16); the GEMV dequantizes weights inside the K-loop.+namespace freetoken::cpu_moe {-const float kE2M1[16] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f,- -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f};+std::unique_ptr<MoeFormat> make_bf16_format(const MoeParams& p, const FormatArgs& a);+std::unique_ptr<MoeFormat> make_nvfp4_format(const MoeParams& p, const FormatArgs& a);+std::unique_ptr<MoeFormat> make_mxfp4_format(const MoeParams& p, const FormatArgs& a);+std::unique_ptr<MoeFormat> make_ds_fp4_format(const MoeParams& p, const FormatArgs& a);+std::unique_ptr<MoeFormat> make_q4_0_format(const MoeParams& p, const FormatArgs& a);+std::unique_ptr<MoeFormat> make_fp8_block_format(const MoeParams& p, const FormatArgs& a);-// e2m1 * 2 as exact int8 (all e2m1 values are multiples of 0.5). Used by the AVX-VNNI-// W4A8 path: nibble -> int8 weight via PSHUFB LUT, then VPDPBUSD against int8 activations;-// the *2 is undone by a 0.5 folded into the final scale. Mirrors ggml's kvalues_mxfp4.-alignas(16) const int8_t kE2M1x2[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};--inline float fp16_to_f32(uint16_t h) {- const uint32_t sign = (uint32_t)(h & 0x8000u) << 16;- uint32_t exp = (h >> 10) & 0x1Fu;- uint32_t man = h & 0x3FFu;- uint32_t f;- if (exp == 0) {- if (man == 0) {- f = sign;- } else {- exp = 127 - 15 + 1;- while ((man & 0x400u) == 0) {- man <<= 1;- --exp;- }- man &= 0x3FFu;- f = sign | (exp << 23) | (man << 13);- }- } else if (exp == 0x1Fu) {- f = sign | 0x7F800000u | (man << 13);- } else {- f = sign | ((exp + (127 - 15)) << 23) | (man << 13);- }- float out;- std::memcpy(&out, &f, sizeof(out));- return out;-}--// e4m3 (OCP "fn": finite, max-normal 448, exp bias 7). Decoded into a 256-entry LUT.-inline float e4m3_decode(uint8_t v) {- const float sign = (v & 0x80u) ? -1.0f : 1.0f;- const uint32_t exp = (v >> 3) & 0xFu;- const uint32_t man = v & 0x7u;- if (exp == 0) return sign * (man / 8.0f) * 0.015625f; // 2^(1-7) = 2^-6- return sign * (1.0f + man / 8.0f) * std::ldexp(1.0f, (int)exp - 7);-}--// ------------------------ block-FP8 (W8A16) dequant -------------------------// fp8-e4m3 weights row-major [rows, K] plus one bf16 scale per 128x128 block. The scale tensor is named weight_scale_inv but multiplies, as in kernel/triton/fp8_blockscale_moe.py.-// Whole 128-blocks read the activations as fp32 even/odd halves (xe[m] = x[2m], xo[m] = x[2m+1]); a ragged tail reads the bf16 row x.-// 0x7F/0xFF (NaN in torch float8_e4m3fn) decode to +-480 in every kernel, as e4m3_decode does.-using fp8_block_dot_fn = float (*)(const uint8_t*, const float*, const float*, const bf16_t*,- const bf16_t*, int, const float*);--constexpr int FP8_BLK = 128; // K elements covered by one weight scale--inline float fp8_block_tail(const uint8_t* w, const bf16_t* x, int k0, int K, const float* e4m3) {- float blk = 0.0f;- for (int k = k0; k < K; ++k) blk += e4m3[w[k]] * bf16_to_f32(x[k]);- return blk;-}--float dot_fp8_block_scalar(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,- const bf16_t* s, int K, const float* e4m3) {- float acc = 0.0f;- int b = 0, k0 = 0;- for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {- // 8 independent sums: a single running sum is bound by the add latency- float a[8] = {};- const uint8_t* wb = w + k0;- const float* pe = xe + k0 / 2;- const float* po = xo + k0 / 2;- for (int j = 0; j < FP8_BLK / 2; j += 4)- for (int l = 0; l < 4; ++l) {- a[2 * l] += e4m3[wb[2 * (j + l)]] * pe[j + l];- a[2 * l + 1] += e4m3[wb[2 * (j + l) + 1]] * po[j + l];- }- acc += (((a[0] + a[1]) + (a[2] + a[3])) + ((a[4] + a[5]) + (a[6] + a[7]))) *- bf16_to_f32(s[b]);- }- if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);- return acc;-}--#if CPU_MOE_X86-// A byte b read as fp16 bits (b & 0x80) << 8 | (b & 0x7F) << 7 is its e4m3 value times 2^-8, subnormals included, so the block scale carries the 256.-// In a 16-bit lane holding bytes (2m, 2m+1), srai 1 then & 0xBF80 builds that for byte 2m+1; a left shift by 8 first does byte 2m. Every AVX2 CPU has F16C.-__attribute__((target("avx2,fma,f16c")))-float dot_fp8_block_avx2(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,- const bf16_t* s, int K, const float* e4m3) {- const __m256i keep = _mm256_set1_epi16((short)0xBF80);- __m256 total = _mm256_setzero_ps();- int b = 0, k0 = 0;- for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {- _mm_prefetch(reinterpret_cast<const char*>(w + k0) + PF_AHEAD, _MM_HINT_T0);- __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();- __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();- for (int j = 0; j < FP8_BLK; j += 32) {- const __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j));- const __m256i odd = _mm256_and_si256(_mm256_srai_epi16(v, 1), keep);- const __m256i even = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v, 8), 1), keep);- const float* pe = xe + (k0 + j) / 2;- const float* po = xo + (k0 + j) / 2;- a0 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_castsi256_si128(even)), _mm256_loadu_ps(pe), a0);- a1 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_extracti128_si256(even, 1)), _mm256_loadu_ps(pe + 8), a1);- a2 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_castsi256_si128(odd)), _mm256_loadu_ps(po), a2);- a3 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_extracti128_si256(odd, 1)), _mm256_loadu_ps(po + 8), a3);- }- const __m256 blk = _mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3));- total = _mm256_fmadd_ps(blk, _mm256_set1_ps(bf16_to_f32(s[b]) * 256.0f), total);- }- float acc = hsum256(total);- if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);- return acc;-}--__attribute__((target("avx512f,avx2,fma,f16c")))-float dot_fp8_block_avx512f(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,- const bf16_t* s, int K, const float* e4m3) {- const __m256i keep = _mm256_set1_epi16((short)0xBF80);- __m512 total = _mm512_setzero_ps();- int b = 0, k0 = 0;- for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {- _mm_prefetch(reinterpret_cast<const char*>(w + k0) + PF_AHEAD, _MM_HINT_T0);- __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();- __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();- for (int j = 0; j < FP8_BLK; j += 64) {- const __m256i v0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j));- const __m256i v1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j + 32));- const __m256i o0 = _mm256_and_si256(_mm256_srai_epi16(v0, 1), keep);- const __m256i e0 = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v0, 8), 1), keep);- const __m256i o1 = _mm256_and_si256(_mm256_srai_epi16(v1, 1), keep);- const __m256i e1 = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v1, 8), 1), keep);- const float* pe = xe + (k0 + j) / 2;- const float* po = xo + (k0 + j) / 2;- a0 = _mm512_fmadd_ps(_mm512_cvtph_ps(e0), _mm512_loadu_ps(pe), a0);- a1 = _mm512_fmadd_ps(_mm512_cvtph_ps(o0), _mm512_loadu_ps(po), a1);- a2 = _mm512_fmadd_ps(_mm512_cvtph_ps(e1), _mm512_loadu_ps(pe + 16), a2);- a3 = _mm512_fmadd_ps(_mm512_cvtph_ps(o1), _mm512_loadu_ps(po + 16), a3);- }- const __m512 blk = _mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3));- total = _mm512_fmadd_ps(blk, _mm512_set1_ps(bf16_to_f32(s[b]) * 256.0f), total);- }- float acc = _mm512_reduce_add_ps(total);- if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);- return acc;-}-#endif // CPU_MOE_X86--// Activations pre-deinterleaved to fp32 (xe[m]=x[2m], xo[m]=x[2m+1]); see the-// ds_fp4 dot below for why (drops the hot loop to ~1.5 shuffle ops / 16 weights).-using nvdot_fn = float (*)(const uint8_t*, const uint8_t*, float, const float*, const float*,- int, const float*, const float*);--float dot_nvfp4_scalar(const uint8_t* packed, const uint8_t* scale, float global,- const float* xe, const float* xo, int K, const float* e2m1,- const float* e4m3) {- float acc = 0.0f;- const int nb = K / 16;- for (int b = 0; b < nb; ++b) {- const float bs = e4m3[scale[b]];- const uint8_t* pk = packed + (size_t)b * 8;- const float* xeb = xe + (size_t)b * 8; // 16 K -> 8 even + 8 odd- const float* xob = xo + (size_t)b * 8;- float bsum = 0.0f;- for (int j = 0; j < 8; ++j) {- const uint8_t byte = pk[j];- bsum += e2m1[byte & 0xF] * xeb[j];- bsum += e2m1[byte >> 4] * xob[j];- }- acc += bs * bsum;- }- return acc * global;-}--// ---- NVFP4 W4A8 (int8 activations) dot: nibble->int8 via LUT, per-16 act scale -----// asi8: int8 activations laid out per-16 block as [even(8), odd(8)]; asb[b] = per-block-// activation scale (absmax/127). Result folds the e2m1*2 -> *0.5 into the scale.-using nvi8dot_fn = float (*)(const uint8_t*, const uint8_t*, float, const int8_t*, int,- const float*, const float*);--[[maybe_unused]] float dot_nvfp4_i8_scalar(const uint8_t* packed, const uint8_t* scale,- float global, const int8_t* asi8, int K, const float* e4m3,- const float* asb) {- float acc = 0.0f;- const int nb = K / 16;- for (int b = 0; b < nb; ++b) {- const uint8_t* pk = packed + (size_t)b * 8;- const int8_t* ae = asi8 + (size_t)b * 16; // even(8)- const int8_t* ao = ae + 8; // odd(8)- int isum = 0;- for (int j = 0; j < 8; ++j) {- isum += (int)kE2M1x2[pk[j] & 0xF] * (int)ae[j];- isum += (int)kE2M1x2[pk[j] >> 4] * (int)ao[j];- }- acc += (e4m3[scale[b]] * asb[b]) * (float)isum;- }- return acc * (0.5f * global);-}--#if CPU_MOE_X86-// AVX2 e2m1 nibble decode: codes (0..15) in 8 int32 lanes -> fp32. AVX2 vpermps is-// only 8-wide, so instead of a 16-entry LUT we use the e2m1 sign/magnitude symmetry:-// value = (code&8 ? - : +) * mag8[code&7], mag8 = e2m1[0..7]. The sign is bit 3 of-// the code shifted into the fp32 sign bit (bit 31). Bit-identical to the e2m1 LUT.-__attribute__((target("avx2,fma")))-inline __m256 e2m1_decode8(__m256i codes, __m256 mag8) {- __m256 mag = _mm256_permutevar8x32_ps(mag8, _mm256_and_si256(codes, _mm256_set1_epi32(7)));- __m256i sgn = _mm256_slli_epi32(_mm256_and_si256(codes, _mm256_set1_epi32(8)), 28);- return _mm256_xor_ps(mag, _mm256_castsi256_ps(sgn));-}--// Two 16-K blocks (16 packed bytes) per iter: lo nibbles -> even-K, hi -> odd-K,-// gathered via two vpermps. The per-16 e4m3 scale differs across the two blocks, so-// it is applied per lane (low 8 lanes = block b, high 8 = block b+1).-__attribute__((target("avx512f")))-inline __m512 nvfp4_blk2(const uint8_t* pk, const float* xeb, const float* xob, __m512 lut,- __m512i loma, float s0, float s1) {- __m512i wi = _mm512_cvtepu8_epi32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(pk)));- __m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);- __m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);- __m512 prod = _mm512_fmadd_ps(vlo, _mm512_loadu_ps(xeb), _mm512_mul_ps(vhi, _mm512_loadu_ps(xob)));- // lanes 0-7 (block b) -> s0, lanes 8-15 (block b+1) -> s1 (pure AVX512F mask move)- __m512 scv = _mm512_mask_mov_ps(_mm512_set1_ps(s0), 0xFF00, _mm512_set1_ps(s1));- return _mm512_mul_ps(prod, scv);-}--__attribute__((target("avx512f")))-float dot_nvfp4_avx512(const uint8_t* packed, const uint8_t* scale, float global,- const float* xe, const float* xo, int K, const float* e2m1,- const float* e4m3) {- const __m512 lut = _mm512_loadu_ps(e2m1); // 16 e2m1 values- const __m512i loma = _mm512_set1_epi32(0xF);- __m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();- const int nb = K / 16; // 8 packed bytes + one e4m3 scale per block- int b = 0;- for (; b + 4 <= nb; b += 4) { // two blk2 calls -> 4 blocks, 2 accumulators- acc0 = _mm512_add_ps(acc0, nvfp4_blk2(packed + (size_t)b * 8, xe + (size_t)b * 8,- xo + (size_t)b * 8, lut, loma, e4m3[scale[b]], e4m3[scale[b + 1]]));- acc1 = _mm512_add_ps(acc1, nvfp4_blk2(packed + (size_t)(b + 2) * 8, xe + (size_t)(b + 2) * 8,- xo + (size_t)(b + 2) * 8, lut, loma, e4m3[scale[b + 2]], e4m3[scale[b + 3]]));- }- for (; b + 2 <= nb; b += 2)- acc0 = _mm512_add_ps(acc0, nvfp4_blk2(packed + (size_t)b * 8, xe + (size_t)b * 8,- xo + (size_t)b * 8, lut, loma, e4m3[scale[b]], e4m3[scale[b + 1]]));- float s = _mm512_reduce_add_ps(_mm512_add_ps(acc0, acc1));- for (; b < nb; ++b) { // odd final 16-K block- const uint8_t* pk = packed + (size_t)b * 8;- const float* xeb = xe + (size_t)b * 8;- const float* xob = xo + (size_t)b * 8;- float bsum = 0.0f;- for (int j = 0; j < 8; ++j) {- bsum += e2m1[pk[j] & 0xF] * xeb[j] + e2m1[pk[j] >> 4] * xob[j];- }- s += e4m3[scale[b]] * bsum;- }- return s * global;-}--// AVX2: one 16-K block (8 packed bytes) per call, 8 even + 8 odd lanes.-__attribute__((target("avx2,fma")))-inline __m256 nvfp4_blk_avx2(const uint8_t* pk, const float* xeb, const float* xob,- __m256 mag8, float sc) {- __m256i wi = _mm256_cvtepu8_epi32(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(pk)));- __m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);- __m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);- __m256 prod = _mm256_fmadd_ps(vlo, _mm256_loadu_ps(xeb), _mm256_mul_ps(vhi, _mm256_loadu_ps(xob)));- return _mm256_mul_ps(prod, _mm256_set1_ps(sc));-}--__attribute__((target("avx2,fma")))-float dot_nvfp4_avx2(const uint8_t* packed, const uint8_t* scale, float global,- const float* xe, const float* xo, int K, const float* e2m1,- const float* e4m3) {- const __m256 mag8 = _mm256_loadu_ps(e2m1); // e2m1[0..7] magnitudes- __m256 acc0 = _mm256_setzero_ps(), acc1 = _mm256_setzero_ps();- const int nb = K / 16;- int b = 0;- for (; b + 2 <= nb; b += 2) {- acc0 = _mm256_add_ps(acc0, nvfp4_blk_avx2(packed + (size_t)b * 8, xe + (size_t)b * 8,- xo + (size_t)b * 8, mag8, e4m3[scale[b]]));- acc1 = _mm256_add_ps(acc1, nvfp4_blk_avx2(packed + (size_t)(b + 1) * 8, xe + (size_t)(b + 1) * 8,- xo + (size_t)(b + 1) * 8, mag8, e4m3[scale[b + 1]]));- }- for (; b < nb; ++b)- acc0 = _mm256_add_ps(acc0, nvfp4_blk_avx2(packed + (size_t)b * 8, xe + (size_t)b * 8,- xo + (size_t)b * 8, mag8, e4m3[scale[b]]));- return hsum256(_mm256_add_ps(acc0, acc1)) * global;-}--// AVX-VNNI W4A8: decode 8 packed bytes (16 nibbles) of one 16-block to int8 [lo(8),hi(8)]-// via PSHUFB against the e2m1*2 LUT (replaces the 2 vpermps fp32 expands -- ~4x less-// port-5 traffic). lo=even-K weights, hi=odd-K, matching the [even(8),odd(8)] act layout.-__attribute__((target("avx2,avxvnni,fma")))-inline __m128i nvfp4_decode_block_i8(const uint8_t* pk, __m128i lut) {- __m128i b = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(pk)); // 8 bytes- __m128i lo = _mm_and_si128(b, _mm_set1_epi8(0x0F));- __m128i hi = _mm_and_si128(_mm_srli_epi16(b, 4), _mm_set1_epi8(0x0F));- return _mm_shuffle_epi8(lut, _mm_unpacklo_epi64(lo, hi)); // [lo(8),hi(8)] -> int8-}--// Two 16-blocks per VPDPBUSD (32 int8). Sign trick (ggml mul_add_epi8): |w|*(sign(w)*a)=w*a,-// so u8*s8 needs no offset/correction term. Per-block scale (e4m3 * act-scale) folded in fp32-// (lanes 0-3 -> block b, 4-7 -> block b+1). Bit-faithful weight; only the int8 activation quant-// (W4A8) differs from the bf16 reference.-__attribute__((target("avx2,avxvnni,fma")))-float dot_nvfp4_i8_vnni(const uint8_t* packed, const uint8_t* scale, float global,- const int8_t* asi8, int K, const float* e4m3, const float* asb) {- const __m128i lut = _mm_loadu_si128(reinterpret_cast<const __m128i*>(kE2M1x2));- __m256 accF = _mm256_setzero_ps();- const int nb = K / 16;- int b = 0;- for (; b + 2 <= nb; b += 2) {- __m128i wb = nvfp4_decode_block_i8(packed + (size_t)b * 8, lut);- __m128i wb1 = nvfp4_decode_block_i8(packed + (size_t)(b + 1) * 8, lut);- __m256i w = _mm256_set_m128i(wb1, wb); // [blk b | blk b+1]- __m256i a = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(asi8 + (size_t)b * 16));- __m256i aw = _mm256_sign_epi8(w, w); // |w| (u8 operand)- __m256i sa = _mm256_sign_epi8(a, w); // sign(w)*a (s8 operand)- __m256i di = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), aw, sa);- __m256 scv = _mm256_blend_ps(_mm256_set1_ps(e4m3[scale[b]] * asb[b]),- _mm256_set1_ps(e4m3[scale[b + 1]] * asb[b + 1]), 0xF0);- accF = _mm256_fmadd_ps(_mm256_cvtepi32_ps(di), scv, accF);- }- float s = hsum256(accF);- for (; b < nb; ++b) { // tail (odd block count)- const uint8_t* pk = packed + (size_t)b * 8;- const int8_t* ae = asi8 + (size_t)b * 16; const int8_t* ao = ae + 8;- int isum = 0;- for (int j = 0; j < 8; ++j)- isum += (int)kE2M1x2[pk[j] & 0xF] * (int)ae[j] + (int)kE2M1x2[pk[j] >> 4] * (int)ao[j];- s += (e4m3[scale[b]] * asb[b]) * (float)isum;- }- return s * (0.5f * global);-}--#if (defined(__GNUC__) && __GNUC__ >= 10) || defined(__clang__)-#define CPU_MOE_HAS_AVX512VNNI 1--// Software-prefetch distance for the W4A8 weight stream, in 16-K blocks (8 packed-// bytes each). Returns -1 when FREETOKEN_CPU_MOE_PF_BLOCKS is unset: the kernel then-// uses the built-in default min(512 blocks = 4 KB, 2 rows) -- 4 KB is the empirical-// optimum on large-row machines (Emerald Rapids sweep), while the 2-row cap keeps a small-row-// model's overshoot bounded (the executor works in 32-row tiles, so a fixed byte-// distance otherwise prefetches another worker's tile: duplicated DRAM traffic that-// regresses at the bandwidth ceiling). An EXPLICIT env value is honored verbatim-// (no clamp; 0 disables): the per-machine optimum can sit past the safe default (+20%-// at 4 KB on a 24-thread Ice Lake with 256B rows), so the escape hatch must reach it.-// Prefetch never faults, so overshooting a row/bank tail is safe.-static int nvfp4_pf_blocks() {- static const int v = [] {- const char* s = getenv("FREETOKEN_CPU_MOE_PF_BLOCKS");- return (s && s[0]) ? atoi(s) : -1;- }();- return v;-}--// AVX-512 VNNI W4A8: FOUR 16-K blocks per VPDPBUSD (64 int8) -- 2x the AVX-VNNI-// (256-bit) path. Decode 32 packed bytes -> 64 int8 with a single _mm512_shuffle_epi8-// (e2m1*2 LUT replicated to all 4 128-bit lanes). AVX-512 has no _mm512_sign_epi8, so-// the u8*s8 sign trick (|w| as u8, sign(w)*a as s8) uses abs_epi8 + a masked negate.-// Bit-faithful weight; only the int8 activation quant (W4A8) differs from bf16.-// One 4-block group (64 int8) -> scaled fp32 partial (16 lanes). Isolated as a helper so-// the caller can run several independent chains into separate accumulators (the decode ->-// dpbusd -> scale chain is long, so a single accumulator leaves the core latency-bound).-__attribute__((target("avx512f,avx512bw,avx512vnni,avx2")))-static inline __m512 nvfp4_i8_grp4(const uint8_t* packed, const uint8_t* scale,- const int8_t* asi8, const float* e4m3, const float* asb,- int b, __m512i lut, __m512i idx, __m512i mask0F,- __m512i idxsc) {- const __mmask64 hi_half = 0xFF00FF00FF00FF00ULL; // bytes 8-15 of each 128b lane -> hi nibbles- __m256i raw = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(packed + (size_t)b * 8));- __m512i src = _mm512_permutexvar_epi64(idx, _mm512_castsi256_si512(raw));- __m512i lo = _mm512_and_si512(src, mask0F);- __m512i hi = _mm512_and_si512(_mm512_srli_epi16(src, 4), mask0F);- __m512i comb = _mm512_mask_blend_epi8(hi_half, lo, hi);- __m512i w = _mm512_shuffle_epi8(lut, comb); // 64 int8 weights (e2m1*2)- // u8*s8 sign trick without _mm512_sign_epi8: aw=|w|, sa = (w<0 ? -a : a) (a moot at w==0)- __m512i a = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(asi8 + (size_t)b * 16));- __m512i aw = _mm512_abs_epi8(w);- __mmask64 neg = _mm512_movepi8_mask(w);- __m512i sa = _mm512_mask_sub_epi8(a, neg, _mm512_setzero_si512(), a);- __m512i di = _mm512_dpbusd_epi32(_mm512_setzero_si512(), aw, sa); // 16 int32 (groups of 4)- // int32[0..3]->blk b, [4..7]->b+1, [8..11]->b+2, [12..15]->b+3. The 4 block scales- // (e4m3 LUT x per-block act scale) are computed vectorized: a 4-byte load + epu8->epi32- // widen + one 4-lane LUT gather + one mul replaces 8 scalar loads + 4 scalar muls +- // a set_ps assembly, which otherwise dominates the per-group op count.- int sc_raw;- memcpy(&sc_raw, scale + b, 4);- __m128i sc4 = _mm_cvtepu8_epi32(_mm_cvtsi32_si128(sc_raw));- __m128 s4 = _mm_mul_ps(_mm_i32gather_ps(e4m3, sc4, 4), _mm_loadu_ps(asb + b));- __m512 scv = _mm512_permutexvar_ps(idxsc, _mm512_castps128_ps512(s4));- return _mm512_mul_ps(_mm512_cvtepi32_ps(di), scv);-}--__attribute__((target("avx512f,avx512bw,avx512vnni,avx2")))-float dot_nvfp4_i8_avx512vnni(const uint8_t* packed, const uint8_t* scale, float global,- const int8_t* asi8, int K, const float* e4m3, const float* asb) {- const __m512i lut = _mm512_broadcast_i32x4(- _mm_loadu_si128(reinterpret_cast<const __m128i*>(kE2M1x2)));- const __m512i idx = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0); // block i -> 128b lane i- const __m512i mask0F = _mm512_set1_epi8(0x0F);- const __m512i idxsc = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);- __m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();- __m512 acc2 = _mm512_setzero_ps(), acc3 = _mm512_setzero_ps();- const int nb = K / 16;- // Built-in default: min(4 KB, 2 rows) -- see nvfp4_pf_blocks(). An explicitly set- // env value is used verbatim so a per-machine sweep can reach operating points past- // the conservative default.- const int pfb = nvfp4_pf_blocks();- const int pf = (pfb < 0) ? std::min(512, 2 * nb) : pfb;- int b = 0;- // Four independent 4-block groups per iter -> four accumulator chains hide the ~10-op- // decode->dpbusd->scale latency (1-2 chains leave the loop latency-bound: the per-core- // rate sat at ~60% of the core's achievable DRAM stream rate).- for (; b + 16 <= nb; b += 16) {- // Prefetch the weight stream ahead: the interleaved decode lowers the L1-miss- // concurrency the HW prefetcher sustains on its own (134 -> 167 GB/s on Emerald Rapids).- if (pf > 0) {- _mm_prefetch(reinterpret_cast<const char*>(packed + ((size_t)b + (size_t)pf) * 8),- _MM_HINT_T0);- _mm_prefetch(reinterpret_cast<const char*>(packed + ((size_t)b + (size_t)pf) * 8 + 64),- _MM_HINT_T0);- }- acc0 = _mm512_add_ps(acc0, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b, lut, idx,- mask0F, idxsc));- acc1 = _mm512_add_ps(acc1, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b + 4, lut, idx,- mask0F, idxsc));- acc2 = _mm512_add_ps(acc2, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b + 8, lut, idx,- mask0F, idxsc));- acc3 = _mm512_add_ps(acc3, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b + 12, lut, idx,- mask0F, idxsc));- }- for (; b + 4 <= nb; b += 4)- acc0 = _mm512_add_ps(acc0, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b, lut, idx,- mask0F, idxsc));- float s = _mm512_reduce_add_ps(- _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)));- for (; b < nb; ++b) { // tail (<4 remaining 16-K blocks)- const uint8_t* pk = packed + (size_t)b * 8;- const int8_t* ae = asi8 + (size_t)b * 16; const int8_t* ao = ae + 8;- int isum = 0;- for (int j = 0; j < 8; ++j)- isum += (int)kE2M1x2[pk[j] & 0xF] * (int)ae[j] + (int)kE2M1x2[pk[j] >> 4] * (int)ao[j];- s += (e4m3[scale[b]] * asb[b]) * (float)isum;- }- return s * (0.5f * global);-}-#endif // avx512vnni available-#endif--// =====================================================================================-// CUDA stream memory operations (driver API, resolved via dlopen -- no link-time or-// toolchain dependence). The GPU side of the flag handshake: submit = WRITE_VALUE-// (done[slot]=0 then ready[slot]=1), sync = WAIT_VALUE(done[slot] >= 1). The wait is-// executed by the GPU front-end (no SM-resident kernel), so GPU "utilization" stays-// truthful during CPU compute windows -- a resident spin kernel pinned it at 99%,-// which laptop CPU/GPU dynamic power schedulers answered by clamping the CPU's max-// frequency (GEMV workers -1.5x: the reported edge regression). Availability is-// probed functionally at startup (memops_probe); anything unsupported (Windows WDDM,-// vGPU, old drivers) falls back to the cudaLaunchHostFunc path.-#if defined(_WIN32)-#include <windows.h>-static void* cumemop_dlopen() { return (void*)::LoadLibraryA("nvcuda.dll"); }-static void* cumemop_dlsym(void* h, const char* n) {- return (void*)::GetProcAddress((HMODULE)h, n);-}-#else-#include <dlfcn.h>-static void* cumemop_dlopen() {- void* h = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_LOCAL);- if (h == nullptr) h = dlopen("libcuda.so", RTLD_LAZY | RTLD_LOCAL);- return h;-}-static void* cumemop_dlsym(void* h, const char* n) { return dlsym(h, n); }-#endif--using cuMemOp64_fn = int (*)(void* stream, unsigned long long addr, unsigned long long value,- unsigned int flags);-static cuMemOp64_fn g_cu_write64 = nullptr;-static cuMemOp64_fn g_cu_wait64 = nullptr;-static constexpr unsigned int kCuWaitValueGeq = 0x0; // CU_STREAM_WAIT_VALUE_GEQ-static constexpr unsigned int kCuWriteDefault = 0x0; // CU_STREAM_WRITE_VALUE_DEFAULT--static bool cumemop_resolve() {- static bool resolved = [] {- void* h = cumemop_dlopen();- if (h == nullptr) return false;- // 11.7+ made the v2 entry points the default; older drivers export only the v1- // names with the same signature.- g_cu_write64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWriteValue64_v2"));- if (g_cu_write64 == nullptr)- g_cu_write64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWriteValue64"));- g_cu_wait64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWaitValue64_v2"));- if (g_cu_wait64 == nullptr)- g_cu_wait64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWaitValue64"));- return g_cu_write64 != nullptr && g_cu_wait64 != nullptr;- }();- return resolved;-}--// Functional probe on a scratch pinned int64: enqueue WRITE(7) + WAIT(>=7) + sync.-// Returns true only if the whole memop path works on THIS stream/device/driver.-static bool cumemops_probe(uintptr_t stream, uintptr_t scratch_addr) {- if (!cumemop_resolve()) return false;- auto* s = reinterpret_cast<void*>(stream);- if (g_cu_write64(s, (unsigned long long)scratch_addr, 7ULL, kCuWriteDefault) != 0) return false;- if (g_cu_wait64(s, (unsigned long long)scratch_addr, 7ULL, kCuWaitValueGeq) != 0) return false;- return cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream)) == cudaSuccess;-}--// GPU side of the flag handshake (see the block comment above): enqueued on the-// caller's (possibly capturing) stream; the WAIT immediate is the constant 1,-// replay-safe under CUDA graphs.-// The startup probe validates EAGER memops; a driver could still reject them at graph-// capture time. Those enqueue errors would otherwise be swallowed here and surface only-// as a later EndCapture failure -- log the first CUresult so triage is one step.-static void cumemop_check(int rc, const char* what) {- static std::atomic<bool> warned{false};- if (rc != 0 && !warned.exchange(true)) {- std::fprintf(stderr,- "[freetoken/cpu_moe] %s failed with CUresult=%d (first occurrence; "- "subsequent errors are not repeated). If this happened during CUDA "- "graph capture, the driver lacks capture support for stream memops -- "- "set FREETOKEN_CPU_MOE_FLAG_SYNC=0.\n",- what, rc);- }-}--static void cumemop_submit(uintptr_t stream, uintptr_t done_addr, uintptr_t ready_addr,- int64_t slot) {- auto* s = reinterpret_cast<void*>(stream);- // Order matters and is preserved by the front end: reset done BEFORE raising ready,- // so the coordinator's completion write for THIS step can never be wiped.- cumemop_check(g_cu_write64(s, (unsigned long long)(done_addr + (size_t)slot * 8), 0ULL,- kCuWriteDefault),- "cuStreamWriteValue64(done)");- cumemop_check(g_cu_write64(s, (unsigned long long)(ready_addr + (size_t)slot * 8), 1ULL,- kCuWriteDefault),- "cuStreamWriteValue64(ready)");-}--static void cumemop_sync(uintptr_t stream, uintptr_t done_addr, int64_t slot) {- cumemop_check(g_cu_wait64(reinterpret_cast<void*>(stream),- (unsigned long long)(done_addr + (size_t)slot * 8), 1ULL,- kCuWaitValueGeq),- "cuStreamWaitValue64(done)");-}--struct DotChoice {- dot_fn fn;- const char* name;-};--// SIMD tiers, ascending. Each format picks the highest tier <= the one chosen by-// pick_isa() that it implements (fp4 and fp8_block formats have no bf16-specific tier, so the-// avx512bf16 tier maps to their avx512 kernel).-enum IsaTier { ISA_SCALAR = 0, ISA_AVX2 = 1, ISA_AVX512 = 2, ISA_AVX512BF16 = 3 };--// Best tier the CPU+build supports, optionally capped DOWN by-// FREETOKEN_CPU_MOE_ISA={scalar,avx2,avx512,avx512bf16} (A/B testing on a machine-// that supports more). FREETOKEN_CPU_MOE_SCALAR=1 forces scalar (legacy alias).-inline IsaTier pick_isa() {-#if CPU_MOE_X86- if (getenv("FREETOKEN_CPU_MOE_SCALAR")) return ISA_SCALAR;- IsaTier best = ISA_SCALAR;- if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma")) best = ISA_AVX2;- if (best >= ISA_AVX2 && __builtin_cpu_supports("avx512f")) best = ISA_AVX512;-#ifdef CPU_MOE_HAS_AVX512BF16- if (best >= ISA_AVX512 && __builtin_cpu_supports("avx512bf16")) best = ISA_AVX512BF16;-#endif- if (const char* f = getenv("FREETOKEN_CPU_MOE_ISA")) {- IsaTier want = best;- if (!std::strcmp(f, "scalar")) want = ISA_SCALAR;- else if (!std::strcmp(f, "avx2")) want = ISA_AVX2;- else if (!std::strcmp(f, "avx512")) want = ISA_AVX512;- else if (!std::strcmp(f, "avx512bf16")) want = ISA_AVX512BF16;- if (want < best) best = want; // cap downward; never force above hw/build support- }- return best;-#else- return ISA_SCALAR;-#endif-}--DotChoice select_dot() {- const IsaTier t = pick_isa();-#if CPU_MOE_X86-#ifdef CPU_MOE_HAS_AVX512BF16- if (t >= ISA_AVX512BF16) return {dot_avx512bf16, "avx512bf16"};-#endif- if (t >= ISA_AVX512) return {dot_avx512f, "avx512f"};- if (t >= ISA_AVX2) return {dot_avx2, "avx2"};-#endif- (void)t;- return {dot_scalar, "scalar"};-}--nvdot_fn select_nvdot() {- const IsaTier t = pick_isa();-#if CPU_MOE_X86- if (t >= ISA_AVX512) return dot_nvfp4_avx512;- if (t >= ISA_AVX2) return dot_nvfp4_avx2;-#endif- (void)t;- return dot_nvfp4_scalar;-}--// AVX-VNNI (VEX-256 VPDPBUSD) availability: Alder/Raptor Lake, Sapphire Rapids+, Zen5.-// Distinct from AVX-512 VNNI. Opt out with FREETOKEN_CPU_MOE_NO_VNNI=1 (A/B the W4A8 path).-inline bool cpu_has_avxvnni() {-#if CPU_MOE_X86- const char* no = getenv("FREETOKEN_CPU_MOE_NO_VNNI");- if (no && no[0] && no[0] != '0') return false; // ignore unset/empty/"0"- return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("avxvnni");-#else- return false;-#endif-}+namespace {-// AVX-512 VNNI (512-bit VPDPBUSD): Cascade Lake+, Ice Lake, Sapphire/Emerald Rapids,-// Zen4+. 2x the 256-bit AVX-VNNI width. FREETOKEN_CPU_MOE_NO_AVX512VNNI=1 forces the-// 256-bit path (A/B the two W4A8 kernels on the same box); FREETOKEN_CPU_MOE_NO_VNNI=1-// still disables the whole W4A8 family (back to the faithful fp32 nvdot), so it is-// honored here too.-inline bool cpu_has_avx512vnni() {-#if CPU_MOE_X86 && defined(CPU_MOE_HAS_AVX512VNNI)- const char* no = getenv("FREETOKEN_CPU_MOE_NO_AVX512VNNI");- if (no && no[0] && no[0] != '0') return false;- const char* no_vnni = getenv("FREETOKEN_CPU_MOE_NO_VNNI");- if (no_vnni && no_vnni[0] && no_vnni[0] != '0') return false;- return __builtin_cpu_supports("avx512vnni");+inline void pin_this_thread(int cpu) {+#if CPU_MOE_HAS_AFFINITY+ cpu_set_t set;+ CPU_ZERO(&set);+ CPU_SET(cpu, &set);+ pthread_setaffinity_np(pthread_self(), sizeof(set), &set);#else- return false;-#endif-}--// Best W4A8 (int8-activation) nvfp4 dot, or nullptr if no SIMD VNNI (caller keeps the-// faithful fp32 nvdot path). The scalar i8 dot exists only as a correctness reference.-nvi8dot_fn select_nvi8dot() {-#if CPU_MOE_X86-#if defined(CPU_MOE_HAS_AVX512VNNI)- if (cpu_has_avx512vnni()) return dot_nvfp4_i8_avx512vnni;-#endif- if (cpu_has_avxvnni()) return dot_nvfp4_i8_vnni;-#endif- return nullptr;-}--// ----------------------- DeepSeek-V4 ds_fp4 (W4A8) ----------------------------// Row-major e2m1 (2/byte, low nibble first) + e8m0 per-32 block scale, no global-// (w = E2M1[code] * 2^(e8m0-127)); activations are FP8-e4m3 round-tripped (per-128-// block, ue8m0 scale) before each GEMM. Matches kernel/triton/dsv4 (fused_moe +-// fp8_linear): silu(clamp(gate,max=lim)) * clamp(up,-lim,lim), router weight on the-// down output.--// Activations are pre-deinterleaved to fp32 (xe[m]=x[2m], xo[m]=x[2m+1]) once per-// token/route and reused across all output rows. This drops the hot dot to one-// vpmovzxbd + two vpermps per 32 weights (1.5 shuffle ops / 16 vs 4 for a bf16,-// dup-permute, per-element gather), so the row-major fp4 GEMV stops being port-5-// bound and approaches the bf16 memory-bandwidth ceiling.-using dsdot_fn = float (*)(const uint8_t*, const uint8_t*, const float*, const float*, int,- const float*, const float*);--float dot_dsfp4_scalar(const uint8_t* packed, const uint8_t* scale, const float* xe,- const float* xo, int K, const float* e2m1, const float* e8m0) {- float acc = 0.0f;- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const float sc = e8m0[scale[b]];- const uint8_t* pk = packed + (size_t)b * 16; // 16 bytes = 32 codes- const float* xeb = xe + (size_t)b * 16;- const float* xob = xo + (size_t)b * 16;- float bsum = 0.0f;- for (int j = 0; j < 16; ++j) {- const uint8_t byte = pk[j];- bsum += e2m1[byte & 0xF] * xeb[j]; // low nibble -> even-K activation- bsum += e2m1[byte >> 4] * xob[j]; // high nibble -> odd-K activation- }- acc += sc * bsum;- }- return acc;-}--#if CPU_MOE_X86-// One 32-block: 16 bytes -> 16 low + 16 high nibble values via two vpermps, times-// the pre-split even/odd fp32 activations, folded by the per-32 e8m0 scale.-__attribute__((target("avx512f")))-inline __m512 dsfp4_blk(const uint8_t* pk, const float* xeb, const float* xob, __m512 lut,- __m512i loma, float sc) {- __m512i wi = _mm512_cvtepu8_epi32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(pk)));- __m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);- __m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);- __m512 prod = _mm512_fmadd_ps(vlo, _mm512_loadu_ps(xeb), _mm512_mul_ps(vhi, _mm512_loadu_ps(xob)));- return _mm512_mul_ps(prod, _mm512_set1_ps(sc));-}--__attribute__((target("avx512f")))-float dot_dsfp4_avx512(const uint8_t* packed, const uint8_t* scale, const float* xe,- const float* xo, int K, const float* e2m1, const float* e8m0) {- const __m512 lut = _mm512_loadu_ps(e2m1);- const __m512i loma = _mm512_set1_epi32(0xF);- __m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();- const int nb = K / 32; // 16 packed bytes + one e8m0 scale per block- int b = 0;- for (; b + 2 <= nb; b += 2) { // two independent accumulators hide FMA latency- acc0 = _mm512_add_ps(acc0, dsfp4_blk(packed + (size_t)b * 16, xe + (size_t)b * 16,- xo + (size_t)b * 16, lut, loma, e8m0[scale[b]]));- acc1 = _mm512_add_ps(acc1, dsfp4_blk(packed + (size_t)(b + 1) * 16, xe + (size_t)(b + 1) * 16,- xo + (size_t)(b + 1) * 16, lut, loma, e8m0[scale[b + 1]]));- }- for (; b < nb; ++b)- acc0 = _mm512_add_ps(acc0, dsfp4_blk(packed + (size_t)b * 16, xe + (size_t)b * 16,- xo + (size_t)b * 16, lut, loma, e8m0[scale[b]]));- return _mm512_reduce_add_ps(_mm512_add_ps(acc0, acc1));-}--// AVX2: a 32-K block is 16 bytes -> two 8-lane halves (8 even + 8 odd each).-__attribute__((target("avx2,fma")))-inline __m256 dsfp4_half_avx2(const uint8_t* pk, const float* xeb, const float* xob, __m256 mag8) {- __m256i wi = _mm256_cvtepu8_epi32(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(pk)));- __m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);- __m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);- return _mm256_fmadd_ps(vlo, _mm256_loadu_ps(xeb), _mm256_mul_ps(vhi, _mm256_loadu_ps(xob)));-}--__attribute__((target("avx2,fma")))-float dot_dsfp4_avx2(const uint8_t* packed, const uint8_t* scale, const float* xe,- const float* xo, int K, const float* e2m1, const float* e8m0) {- const __m256 mag8 = _mm256_loadu_ps(e2m1);- __m256 acc0 = _mm256_setzero_ps(), acc1 = _mm256_setzero_ps();- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const uint8_t* pk = packed + (size_t)b * 16;- const float* xeb = xe + (size_t)b * 16;- const float* xob = xo + (size_t)b * 16;- const __m256 sc = _mm256_set1_ps(e8m0[scale[b]]);- acc0 = _mm256_fmadd_ps(dsfp4_half_avx2(pk, xeb, xob, mag8), sc, acc0);- acc1 = _mm256_fmadd_ps(dsfp4_half_avx2(pk + 8, xeb + 8, xob + 8, mag8), sc, acc1);- }- return hsum256(_mm256_add_ps(acc0, acc1));-}-#endif--dsdot_fn select_dsdot() {- const IsaTier t = pick_isa();-#if CPU_MOE_X86- if (t >= ISA_AVX512) return dot_dsfp4_avx512;- if (t >= ISA_AVX2) return dot_dsfp4_avx2;-#endif- (void)t;- return dot_dsfp4_scalar;-}--// ------------------------- mxfp4 (gpt-oss) GEMV ------------------------------// Transposed split-K layout: blk[Kpairs, N2] (N innermost), scl[Kpairs/16, N2]-// e8m0 per 32-K. Computes out[c] = sum_kb (E2M1[lo]*x[2kb] + E2M1[hi]*x[2kb+1])-// * 2^(e8m0-127) for a contiguous column tile (blk/scl already offset to col 0 of-// the tile). Vectorized over N (16 columns / __m512), K stays the outer (cache--// sequential) loop. Used by both gate_up (K=H) and down (K=I).-using mxgemv_fn = void (*)(float*, const uint8_t*, const uint8_t*, const bf16_t*, int, int,- int, const float*, const float*);--void mxfp4_gemv_scalar(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,- int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {- for (int c = 0; c < ncol; ++c) out[c] = 0.0f;- for (int kb = 0; kb < Kpairs; ++kb) {- const uint8_t* w = blk + (size_t)kb * N2;- const uint8_t* s = scl + (size_t)(kb >> 4) * N2;- const float xl = bf16_to_f32(x[2 * kb]);- const float xh = bf16_to_f32(x[2 * kb + 1]);- for (int c = 0; c < ncol; ++c) {- const uint8_t byte = w[c];- out[c] += (e2m1[byte & 0xF] * xl + e2m1[byte >> 4] * xh) * e8m0[s[c]];- }- }-}--#if CPU_MOE_X86-__attribute__((target("avx512f")))-void mxfp4_gemv_avx512(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,- int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {- (void)e8m0; // e8m0[c]=2^(c-127) computed via bit construction (no gather)- const __m512 lut = _mm512_loadu_ps(e2m1);- const __m512i loma = _mm512_set1_epi32(0xF);- // K-outer / N-inner: each kb cache line is read once and all live column chunks- // (up to 4 -> 64 cols) accumulate from registers, so DRAM/L2 stream the tile once.- int c0 = 0;- for (; c0 + 16 <= ncol; c0 += 64) {- const int nchunk = std::min(4, (ncol - c0) / 16);- __m512 acc[4];- for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm512_setzero_ps();- for (int kblk = 0; kblk < Kpairs; kblk += 16) { // 16 K-pairs = 32 K = one scale row- __m512 sc[4];- for (int ci = 0; ci < nchunk; ++ci) {- __m128i sraw = _mm_loadu_si128(reinterpret_cast<const __m128i*>(- scl + (size_t)(kblk >> 4) * N2 + c0 + ci * 16));- sc[ci] = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu8_epi32(sraw), 23));- }- __m512 blk_acc[4];- for (int ci = 0; ci < nchunk; ++ci) blk_acc[ci] = _mm512_setzero_ps();- for (int kk = 0; kk < 16; ++kk) {- const int kb = kblk + kk;- const uint8_t* wbase = blk + (size_t)kb * N2 + c0;- // The transposed layout strides K by N2 bytes; prefetch ahead so the strided- // reads are not exposed to DRAM latency (the HW streamer misses big strides).- constexpr int PFD = 8;- if (kb + PFD < Kpairs)- _mm_prefetch(reinterpret_cast<const char*>(blk + (size_t)(kb + PFD) * N2 + c0),- _MM_HINT_T0);- const __m512 xl = _mm512_set1_ps(bf16_to_f32(x[2 * kb]));- const __m512 xh = _mm512_set1_ps(bf16_to_f32(x[2 * kb + 1]));- for (int ci = 0; ci < nchunk; ++ci) {- __m512i wi = _mm512_cvtepu8_epi32(- _mm_loadu_si128(reinterpret_cast<const __m128i*>(wbase + ci * 16)));- __m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);- __m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);- blk_acc[ci] = _mm512_fmadd_ps(vlo, xl, blk_acc[ci]);- blk_acc[ci] = _mm512_fmadd_ps(vhi, xh, blk_acc[ci]);- }- }- for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm512_fmadd_ps(blk_acc[ci], sc[ci], acc[ci]);- }- for (int ci = 0; ci < nchunk; ++ci) _mm512_storeu_ps(out + c0 + ci * 16, acc[ci]);- }- for (int c = c0; c < ncol; ++c) { // tail columns (< 16)- float o = 0.0f;- for (int kb = 0; kb < Kpairs; ++kb) {- const uint8_t byte = blk[(size_t)kb * N2 + c];- uint32_t bits = (uint32_t)scl[(size_t)(kb >> 4) * N2 + c] << 23;- float sc;- std::memcpy(&sc, &bits, 4);- o += (e2m1[byte & 0xF] * bf16_to_f32(x[2 * kb]) +- e2m1[byte >> 4] * bf16_to_f32(x[2 * kb + 1])) * sc;- }- out[c] = o;- }-}--__attribute__((target("avx2,fma")))-void mxfp4_gemv_avx2(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,- int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {- (void)e8m0; // e8m0[s]=2^(s-127) built via s<<23 (no gather)- const __m256 mag8 = _mm256_loadu_ps(e2m1);- int c0 = 0;- for (; c0 + 8 <= ncol; c0 += 32) { // up to 4 chunks of 8 = 32 cols- const int nchunk = std::min(4, (ncol - c0) / 8);- __m256 acc[4];- for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm256_setzero_ps();- for (int kblk = 0; kblk < Kpairs; kblk += 16) { // 16 K-pairs = one scale row- __m256 sc[4];- for (int ci = 0; ci < nchunk; ++ci) {- __m128i sraw = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(- scl + (size_t)(kblk >> 4) * N2 + c0 + ci * 8));- sc[ci] = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu8_epi32(sraw), 23));- }- __m256 blk_acc[4];- for (int ci = 0; ci < nchunk; ++ci) blk_acc[ci] = _mm256_setzero_ps();- for (int kk = 0; kk < 16; ++kk) {- const int kb = kblk + kk;- const uint8_t* wbase = blk + (size_t)kb * N2 + c0;- constexpr int PFD = 8;- if (kb + PFD < Kpairs)- _mm_prefetch(reinterpret_cast<const char*>(blk + (size_t)(kb + PFD) * N2 + c0),- _MM_HINT_T0);- const __m256 xl = _mm256_set1_ps(bf16_to_f32(x[2 * kb]));- const __m256 xh = _mm256_set1_ps(bf16_to_f32(x[2 * kb + 1]));- for (int ci = 0; ci < nchunk; ++ci) {- __m256i wi = _mm256_cvtepu8_epi32(- _mm_loadl_epi64(reinterpret_cast<const __m128i*>(wbase + ci * 8)));- __m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);- __m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);- blk_acc[ci] = _mm256_fmadd_ps(vlo, xl, blk_acc[ci]);- blk_acc[ci] = _mm256_fmadd_ps(vhi, xh, blk_acc[ci]);- }- }- for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm256_fmadd_ps(blk_acc[ci], sc[ci], acc[ci]);- }- for (int ci = 0; ci < nchunk; ++ci) _mm256_storeu_ps(out + c0 + ci * 8, acc[ci]);- }- for (int c = c0; c < ncol; ++c) { // tail columns (< 8); none when ncol%8==0- float o = 0.0f;- for (int kb = 0; kb < Kpairs; ++kb) {- const uint8_t byte = blk[(size_t)kb * N2 + c];- uint32_t bits = (uint32_t)scl[(size_t)(kb >> 4) * N2 + c] << 23;- float sc;- std::memcpy(&sc, &bits, 4);- o += (e2m1[byte & 0xF] * bf16_to_f32(x[2 * kb]) +- e2m1[byte >> 4] * bf16_to_f32(x[2 * kb + 1])) * sc;- }- out[c] = o;- }-}-#endif--fp8_block_dot_fn select_fp8_block_dot() {- const IsaTier t = pick_isa();-#if CPU_MOE_X86- // the avx512bf16 tier keeps the fp32 kernel: on Sapphire Rapids it is faster than widening to bf16 for dpbf16- if (t >= ISA_AVX512) return dot_fp8_block_avx512f;- if (t >= ISA_AVX2) return dot_fp8_block_avx2;+ (void)cpu;#endif- (void)t;- return dot_fp8_block_scalar;}-mxgemv_fn select_mxgemv() {- const IsaTier t = pick_isa();+inline void cpu_relax() {#if CPU_MOE_X86- if (t >= ISA_AVX512) return mxfp4_gemv_avx512;- if (t >= ISA_AVX2) return mxfp4_gemv_avx2;+ _mm_pause();+#elif defined(__aarch64__)+ asm volatile("yield" ::: "memory");#endif- (void)t;- return mxfp4_gemv_scalar;-}--// Round a clamped |x|<=448 to nearest float8-e4m3 (RNE), back to fp32. Matches-// torch.float8_e4m3fn / triton .to(float8e4nv).-inline float e4m3_round(float x) {- const float sign = x < 0.0f ? -1.0f : 1.0f;- const float a = std::fabs(x);- if (a == 0.0f) return 0.0f;- if (a >= 448.0f) return sign * 448.0f;- int e;- std::frexp(a, &e); // a in [2^(e-1), 2^e)- float step = std::ldexp(1.0f, e - 4);- const float min_step = std::ldexp(1.0f, -9); // e4m3 subnormal step (2^-9)- if (step < min_step) step = min_step;- float r = std::nearbyint(a / step) * step;- if (r > 448.0f) r = 448.0f;- return sign * r;-}--// IEEE ceil(log2(v)) for v>0 (matches dsv4 _log2_ceil / fast_round_scale).-inline int ceil_log2_pos(float v) {- uint32_t bits;- std::memcpy(&bits, &v, sizeof(bits));- const int exp = (int)((bits >> 23) & 0xFF);- const int man = (int)(bits & 0x7FFFFF);- return exp - 127 + (man != 0 ? 1 : 0);-}--// Split an interleaved bf16 row into fp32 even/odd halves (even[m]=src[2m]).-// bf16->fp32 is exact, so this only reorders -- done once per token/route and-// reused across every output row of the GEMV.-inline void deinterleave_bf16_f32(const bf16_t* src, float* even, float* odd, int K) {- for (int m = 0; m < K / 2; ++m) {- even[m] = bf16_to_f32(src[2 * m]);- odd[m] = bf16_to_f32(src[2 * m + 1]);- }-}--// DeepSeek-V4 / V4.1 activation FP8 round-trip (bf16 in/out): per `block` (the checkpoint's-// fp8 block: 128 on V4, 32 on V4.1), s = 2^ceil(log2(max(|x|,1e-4)/448));-// y = round_e4m3(clamp(x/s,+-448)) * s.-void fp8_roundtrip_bf16(const bf16_t* src, bf16_t* dst, int K, int block) {- for (int b0 = 0; b0 < K; b0 += block) {- const int b1 = std::min(K, b0 + block);- float amax = 1e-4f;- for (int i = b0; i < b1; ++i) amax = std::max(amax, std::fabs(bf16_to_f32(src[i])));- const float s = std::ldexp(1.0f, ceil_log2_pos(amax * (1.0f / 448.0f)));- const float inv_s = 1.0f / s;- for (int i = b0; i < b1; ++i) {- float q = bf16_to_f32(src[i]) * inv_s;- q = std::min(448.0f, std::max(-448.0f, q));- dst[i] = f32_to_bf16(e4m3_round(q) * s);- }- }}-// --------------------------------- executor -----------------------------------struct CpuMoeExecutor;+constexpr int kMaxTopK = 64;+// One MoE layer on the first num_tokens rows of the pinned IO.+// It is also the doorbell word the GPU writes, a constant baked into each captured graph; num_tokens >= 1 keeps it nonzero, since 0 means no doorbell.struct MoeTask {- CpuMoeExecutor* exec;- int layer_id;- int num_tokens;- const bf16_t* x; // [num_tokens, H]- const int32_t* ids; // [num_tokens, top_k] (raw expert ids; <0 = skip)- const float* w; // [num_tokens, top_k]- bf16_t* y; // [num_tokens, H]+ uint64_t layer : 32;+ uint64_t num_tokens : 32;};+static_assert(sizeof(MoeTask) == sizeof(uint64_t), "MoeTask must pack into one 64-bit word");-// Output-row tiling. Small enough to give every worker independent work even at-// batch size 1; large enough to amortize the atomic work-grab.-//-// Bandwidth notes (Sapphire Rapids 8480+, 13 cores): the two passes already read-// every expert weight byte exactly once per token (each output row block is owned-// by one worker), and x stays hot in L1 across a (token,expert)'s rows -- so the-// kernel is single-read bandwidth-optimal at bs=1 (~205 GB/s vs ~55 GB/s PCIe).-// One worker per *physical* core, pinned, is the sweet spot; SMT oversubscription-// thrashes the spin-barrier. Deferred (not worth it here / for this workload):-// - AMX-bf16: a GEMM tile engine; decode is M=1 GEMV so tiles sit idle. It would-// only pay off in a grouped/batched (dedup) path.-// - expert dedup for bs>1: read each distinct expert once and GEMM its tokens.-// Helps locality+bytes when bs is large; decode batches here are tiny (<=4).-// - NUMA: a single node is assumed. Multi-socket machines would split each-// expert's K dimension per node (banks are already per-row contiguous).-constexpr int IBLK = 32;-constexpr int HBLK = 32;--// -------------------------------- Q4_0 (W4A8) ---------------------------------// Native GGUF Q4_0 experts (gemma4 GGUF): per-32 block = fp16 scale d + 16 packed-// bytes; byte j holds element j in its low nibble and j+16 in its high nibble, so a-// block's storage order is [lo0..lo15, hi0..hi15] and w = (nibble - 8) * d. Matches-// the reference dequant (models/gguf/dequant.py) and the packed banks the GPU offload-// path streams.-//-// llama.cpp ggml_vec_dot_q4_0_q8_0: W4A8. The activation is pre-quantized to Q8_0-// (per-32-block int8 ``aq`` + fp32 scale ``asb``); each block unpacks its 16 bytes to-// 32 int8 weights in [-8,7] (bytes_from_nibbles_32: low nibbles -> elems 0..15, high-// -> 16..31) and runs an integer block dot -- VPDPBUSD (AVX-VNNI) or VPMADDUBSW+VPMADDWD-// (AVX2) with the ggml sign trick |w|*(sign(w)*a)=w*a, or a scalar int loop -- then-// scales the block sum by wd*xd in fp32. No fp weight dequant / shuffle chain. The GPU-// offload path (ggml_moe_a8_vec / MMVQ) is also W4A8, so cpu and hybrid stay close.-using q4dot_fn = float (*)(const uint8_t*, const int8_t*, const float*, int);--float q4_0_dot_i8_scalar(const uint8_t* w, const int8_t* aq, const float* asb, int K) {- float acc = 0.0f;- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const uint8_t* blk = w + (size_t)b * 18;- uint16_t dh;- std::memcpy(&dh, blk, sizeof(dh));- const uint8_t* q = blk + 2; // 16 nibble bytes- const int8_t* a = aq + (size_t)b * 32;- int isum = 0;- for (int j = 0; j < 16; ++j) {- isum += ((int)(q[j] & 0x0F) - 8) * (int)a[j]; // elem j- isum += ((int)(q[j] >> 4) - 8) * (int)a[16 + j]; // elem 16+j- }- acc += fp16_to_f32(dh) * asb[b] * (float)isum;- }- return acc;-}--#if CPU_MOE_X86-// fp16 block scale -> fp32 via HW F16C (single value in lane 0).-__attribute__((target("f16c")))-static inline float q4_scale(uint16_t h) {- return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128((int)h)));+inline uint64_t to_word(MoeTask t) {+ uint64_t w;+ std::memcpy(&w, &t, sizeof(w));+ return w;}-// Unpack one Q4_0 block's 16 bytes -> 32 int8 weights in [-8,7] (elems 0..15 = low-// nibbles, 16..31 = high nibbles). ``eight`` = _mm256_set1_epi8(8).-__attribute__((target("avx2")))-static inline __m256i q4_unpack32(const uint8_t* blk, __m128i mask, __m256i eight) {- const __m128i qb = _mm_loadu_si128(reinterpret_cast<const __m128i*>(blk + 2));- const __m128i lo = _mm_and_si128(qb, mask);- const __m128i hi = _mm_and_si128(_mm_srli_epi16(qb, 4), mask);- return _mm256_sub_epi8(_mm256_set_m128i(hi, lo), eight);+inline MoeTask from_word(uint64_t w) {+ MoeTask t;+ std::memcpy(&t, &w, sizeof(t));+ return t;}-// AVX2 W4A8 (llama.cpp non-VNNI mul_sum_i8_pairs): integer block dot via VPMADDUBSW +-// VPMADDWD (sign trick), scaled by wd*xd. |aw*sa| pair sums <= 8*127*2 < 32767 -> no-// int16 saturation. This is the fast path on AVX2 CPUs without AVX-VNNI (and the-// avx512-tier fallback, since the block dot is 256-bit either way).-__attribute__((target("avx2,fma,f16c")))-float q4_0_dot_i8_avx2(const uint8_t* w, const int8_t* aq, const float* asb, int K) {- const __m128i mask = _mm_set1_epi8(0x0F);- const __m256i eight = _mm256_set1_epi8(8);- const __m256i ones16 = _mm256_set1_epi16(1);- __m256 accF = _mm256_setzero_ps();- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const uint8_t* blk = w + (size_t)b * 18;- _mm_prefetch(reinterpret_cast<const char*>(blk) + 512, _MM_HINT_T0);- uint16_t dh;- std::memcpy(&dh, blk, sizeof(dh));- __m256i wq = q4_unpack32(blk, mask, eight);- __m256i a = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(aq + (size_t)b * 32));- __m256i aw = _mm256_sign_epi8(wq, wq); // |wq| (unsigned operand)- __m256i sa = _mm256_sign_epi8(a, wq); // sign(wq) * a (signed operand)- __m256i d32 = _mm256_madd_epi16(_mm256_maddubs_epi16(aw, sa), ones16); // 8 int32- accF = _mm256_fmadd_ps(_mm256_cvtepi32_ps(d32), _mm256_set1_ps(q4_scale(dh) * asb[b]), accF);- }- return hsum256(accF);-}--// AVX-VNNI W4A8: one VPDPBUSD per block (the fast path on modern CPUs).-__attribute__((target("avx2,avxvnni,fma,f16c")))-float q4_0_dot_i8_vnni(const uint8_t* w, const int8_t* aq, const float* asb, int K) {- const __m128i mask = _mm_set1_epi8(0x0F);- const __m256i eight = _mm256_set1_epi8(8);- __m256 accF = _mm256_setzero_ps();- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const uint8_t* blk = w + (size_t)b * 18;- _mm_prefetch(reinterpret_cast<const char*>(blk) + 512, _MM_HINT_T0);- uint16_t dh;- std::memcpy(&dh, blk, sizeof(dh));- __m256i wq = q4_unpack32(blk, mask, eight);- __m256i a = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(aq + (size_t)b * 32));- __m256i aw = _mm256_sign_epi8(wq, wq); // |wq| (unsigned operand)- __m256i sa = _mm256_sign_epi8(a, wq); // sign(wq) * a (signed operand)- __m256i di = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), aw, sa);- // All 32 elems of the block share wd*xd; distribute over di's 8 partial sums and- // reduce at the end (equivalent to scale * block_total).- accF = _mm256_fmadd_ps(_mm256_cvtepi32_ps(di), _mm256_set1_ps(q4_scale(dh) * asb[b]), accF);- }- return hsum256(accF);-}-#endif // CPU_MOE_X86--// All tiers are W4A8 (int8 activations pre-quantized to Q8_0). AVX-VNNI is orthogonal to-// the ISA tier (gated by cpu_has_avxvnni() / FREETOKEN_CPU_MOE_NO_VNNI), so it wins when-// present; otherwise the 256-bit VPMADDUBSW kernel covers both the avx2 and avx512 tiers.-q4dot_fn select_q4dot() {- const IsaTier t = pick_isa();-#if CPU_MOE_X86- if (cpu_has_avxvnni()) return q4_0_dot_i8_vnni;- if (t >= ISA_AVX2) return q4_0_dot_i8_avx2;-#endif- (void)t;- return q4_0_dot_i8_scalar;-}--enum WFmt { WF_BF16 = 0, WF_NVFP4 = 1, WF_MXFP4 = 2, WF_DSFP4 = 3, WF_Q4_0 = 4,- WF_FP8_BLOCK = 5 };+// Every weight format this module serves, keyed by the name cpu_executor.py passes; weight_formats() exports the names.+const std::pair<const char*, std::unique_ptr<MoeFormat> (*)(const MoeParams&, const FormatArgs&)> kFormats[] = {+ {"bf16", make_bf16_format}, {"nvfp4", make_nvfp4_format}, {"mxfp4_triton", make_mxfp4_format},+ {"ds_fp4", make_ds_fp4_format}, {"q4_0", make_q4_0_format}, {"fp8_block", make_fp8_block_format},+};-// Each ctor pointer arg is the address of a CPU int64 array of length-// num_layers (one base address per layer, built by cpu_executor.py's-// _make_table), not a single flat bank. tbl_at resolves-// tbl[layer_id] once per task/pass; a null table (bank unused by this fmt, ptr-// arg 0) resolves to nullptr without dereferencing.-inline const void* tbl_at(const uint64_t* tbl, int layer_id) {- return tbl ? reinterpret_cast<const void*>(tbl[layer_id]) : nullptr;+std::unique_ptr<MoeFormat> make_format(const std::string& name, const MoeParams& p, const FormatArgs& a) {+ for (const auto& [n, make] : kFormats)+ if (name == n) return make(p, a);+ throw std::invalid_argument("unknown CPU MoE weight format " + name);}struct CpuMoeExecutor {+ // A host-function callback gets one pointer, so it carries the executor with the task.+ struct HostTask {+ CpuMoeExecutor* exec;+ MoeTask task;+ };++ const int num_layers, num_experts;+ const bool apply_on_input;+ MoeParams params; // the format keeps a reference; declared before fmt+ std::unique_ptr<MoeFormat> fmt;+ bool preps_inter = false;+ bool rounds_routes = false;+ int n_iblk = 0, n_hblk = 0; // kRowBlock blocks per intermediate row and per output row++ // Pinned IO shared by every layer and batch size (one task in flight): [max_tokens, H] x and y, [max_tokens, top_k] ids and w.+ const bf16_t* io_x;+ const int32_t* io_ids;+ const float* io_w;+ bf16_t* io_y;+ AlignedVector<bf16_t> g_scratch; // [max_tokens * top_k * I] intermediate++ // Worker pool.int num_threads;- int num_layers, num_experts, top_k;- int H, I;- int act, apply_on_input;- int fmt; // WFmt- bool needs_di = false; // pre-deinterleave activations to fp32 (nvfp4/ds_fp4/fp8_block)- // Per-layer pointer tables (one base address per layer, see tbl_at). gate_up_tbl- // doubles as the bf16 gate_up table and the nvfp4/mxfp4/q4_0/ds_fp4 packed-gate_up- // table (down_tbl likewise for down); which reinterpretation applies is picked by- // fmt at each resolve site (see gemm1_dot/gemm2_dot/do_pass1_mxfp4/do_pass1_dsfp4).- const uint64_t* gate_up_tbl; // bf16: [E,2I,H] rows; else: packed e2m1/mxfp4-blocks- const uint64_t* down_tbl; // bf16: [E,H,I] rows; else: packed e2m1/mxfp4-blocks- const uint64_t* gu_scale_tbl; // nvfp4/mxfp4/ds_fp4: [E,2I,*] block scales- const uint64_t* gu_global_tbl; // nvfp4: [E,2I] fp16 row globals- const uint64_t* dn_scale_tbl; // nvfp4/mxfp4/ds_fp4: [E,H,*] block scales- const uint64_t* dn_global_tbl; // nvfp4: [E,H] fp16 row globals- const uint64_t* gu_bias_tbl; // mxfp4: [E,2I] bf16 biases- const uint64_t* dn_bias_tbl; // mxfp4: [E,H] bf16 biases- float swiglu_alpha;- float swiglu_limit; // +inf == no clamp- dot_fn dot;- nvdot_fn nvdot;- nvi8dot_fn nvi8dot = nullptr; // AVX-VNNI W4A8 nvfp4 dot (nullptr -> use fp32 nvdot)- bool use_vnni = false; // nvfp4 + AVX-VNNI: decode via int8 VPDPBUSD (W4A8)- bool use_q4a8 = false; // q4_0: always W4A8 (llama.cpp Q4_0 x Q8_0); int8 pre-quant- dsdot_fn dsdot;- mxgemv_fn mxgemv;- q4dot_fn q4dot;- fp8_block_dot_fn fp8_block_dot;- // fp8_block: bf16 scale per 128x128 weight block. These are the scale tensor's- // trailing strides, including any padding used by the host bank layout.- int fp8_block_gu_scale_row = 0, fp8_block_dn_scale_row = 0;- // ds_fp4: the caller already FP8-round-tripped the input activations on the GPU- // (same reference grid), so submit() must not repeat it on the host-callback- // thread. That scalar per-element pass is single-threaded ON THE DECODE CRITICAL- // PATH (~0.3ms/layer at H=4096, every worker and the GPU waiting on it); moving- // it to a captured GPU elementwise kernel removes it while keeping the official- // W4A8 numerics bit-exact. Set via set_input_prequant (see cpu_executor.py).- bool input_prequant = false;- // ds_fp4 activation round-trip block (the checkpoint's fp8 block); set via set_act_block.- int act_block = 128;- // Q4_0 packed-row byte strides (H/32*18 for gate_up over K=H, I/32*18 for down over K=I).- int q4_gu_row_bytes = 0, q4_dn_row_bytes = 0;- float e2m1_lut[16];- float e4m3_lut[256];- float e8m0_lut[256]; // mxfp4 block scale: 2^(s-127), s clamped to [0,254]- const char* isa;-- std::vector<bf16_t> g_scratch; // [max_tokens * top_k * I] intermediate- std::vector<bf16_t> xq_scratch; // [max_tokens * H] ds_fp4 fp8-roundtripped input- // nvfp4/ds_fp4/fp8_block activations pre-deinterleaved to fp32 (even/odd K) for the row-major dot.- std::vector<float> xe_scratch, xo_scratch; // [max_tokens * H/2] (input)- std::vector<float> ge_scratch, go_scratch; // [max_tokens*top_k*I/2] (intermediate)- // AVX-VNNI W4A8: per-16-block int8 activations [even(8),odd(8)] + per-block scale.- std::vector<int8_t> xi8_scratch, gi8_scratch; // [max_tokens*H], [max_tokens*top_k*I]- std::vector<float> xas_scratch, gas_scratch; // [max_tokens*H/16], [..*top_k*I/16]- std::string isa_str;-+ std::vector<int> core_ids; // worker tid -> logical CPU to pin to (may be empty)std::vector<std::thread> workers;std::mutex task_mtx;std::condition_variable task_cv;@@ -1421,7 +136,7 @@ struct CpuMoeExecutor {std::atomic<bool> stop{false}; // spinning workers read it without the lockuint64_t cur_gen = 0;- MoeTask* cur_task = nullptr;+ MoeTask cur_task{};// Separate cache lines: idle workers poll `submitted` while busy workers hit p1_next/p2_next.alignas(64) std::atomic<uint64_t> submitted{0};alignas(64) std::atomic<uint64_t> completed{0};@@ -1431,264 +146,56 @@ struct CpuMoeExecutor {std::atomic<bool> spin_wait{false};bool coord_pinned = false;+ // Work split of the current task.alignas(64) std::atomic<int64_t> p1_next{0};std::atomic<int64_t> p2_next{0};- std::atomic<int64_t> prt_next{0}; // intermediate-row prep phase (prep_g_row)+ std::atomic<int64_t> prt_next{0}; // intermediate-row prep phase (prep_inter)int64_t p1_total = 0, p2_total = 0, prt_total = 0;- int n_iblk = 0, n_hblk = 0;std::atomic<int> done_count{0};std::atomic<int> bar_count{0};std::atomic<int> bar_sense{0};- std::vector<MoeTask*> owned_tasks; // persistent task descriptors (graph-stable)- std::vector<int> core_ids; // worker tid -> logical CPU to pin to (may be empty)-// ---- Flag-based GPU<->CPU handshake (replaces the per-layer cudaLaunchHostFunc pair) ----- // A tiny GPU kernel bumps ready_flags[slot] at submit; this coordinator thread busy-polls- // it, runs the slot's task on the worker pool, and sets done_flags[slot], which a GPU- // spin-wait kernel polls at sync. This removes the ~2x30-50us host-func dispatch round- // trips per MoE layer per decode step that otherwise idle the GPU (~6 ms/step on a- // 75-layer model). One slot per (layer, decode batch size) pair -- the Python side- // allocates slots as tasks are created. Flags live in mapped-pinned host memory (UVA:- // the same pointers are used by the GPU kernels and by this thread).+ // The GPU rings the doorbell with a task word at submit; this coordinator thread busy-polls it, runs the task on the worker pool, and completes it, which the GPU waits on at sync.+ // This removes the ~2x30-50us host-func dispatch round trips per MoE layer per decode step that otherwise idle the GPU (~6 ms/step on a 75-layer model).+ std::unique_ptr<GpuSync> gpu_sync;+ std::unique_ptr<KernelIo> kernel_io; // replaces the copies around the doorbell once enable_kernel_io() runsstd::thread coord_thread;std::atomic<bool> coord_stop{false};- volatile int64_t* ready_flags = nullptr; // GPU increments, this thread polls- volatile int64_t* done_flags = nullptr; // this thread sets, GPU spin-waits- int coord_num_slots = 0;- std::vector<MoeTask*> flag_task; // slot -> task (registered lazily)- std::vector<int64_t> flag_served; // slot -> completed dispatch count (tests/debug)- std::mutex flag_task_mtx;-- // Portable ordering for the flag handshake: "ready observed => the DMA'd inputs that- // preceded the bump are visible" and "y stores are visible before done". Plain- // volatile loads lean on x86 TSO; acquire/release makes it hold on aarch64 too- // (GH200/Jetson) at zero x86 cost. (MSVC branch is x86-only today: compiler barrier- // + TSO.)- static int64_t flag_load_acquire(const volatile int64_t* p) {-#if defined(_MSC_VER)- const int64_t v = *p;- _ReadWriteBarrier();- return v;-#else- return __atomic_load_n(const_cast<const int64_t*>(p), __ATOMIC_ACQUIRE);-#endif- }-- static void flag_store_release(volatile int64_t* p, int64_t v) {-#if defined(_MSC_VER)- _ReadWriteBarrier();- *p = v;-#else- __atomic_store_n(const_cast<int64_t*>(p), v, __ATOMIC_RELEASE);-#endif- }+ std::atomic<int64_t> flag_served{0}; // completed doorbells (tests, watchdog)+ std::map<uint64_t, HostTask> host_tasks; // host-function userData, keyed by doorbell word (graph-stable)CpuMoeExecutor(int num_threads_, int num_layers_, int num_experts_, int top_k_,- int hidden_size, int inter_size, int max_tokens, int activation_id,- int apply_router_weight_on_input, int weight_format,- uintptr_t gate_up_ptr, uintptr_t down_ptr, uintptr_t gate_up_scale_ptr,- uintptr_t gate_up_global_ptr, uintptr_t down_scale_ptr,- uintptr_t down_global_ptr, uintptr_t gate_up_bias_ptr,- uintptr_t down_bias_ptr, double swiglu_alpha_, double swiglu_limit_,- std::vector<int> core_ids_, int fp8_block_gu_scale_stride_,- int fp8_block_dn_scale_stride_)- : num_threads(num_threads_ > 0 ? num_threads_ : 1),- num_layers(num_layers_),+ int hidden_size, int inter_size, int max_tokens_, int activation_id,+ int apply_router_weight_on_input, const std::string& weight_format,+ std::map<std::string, uint64_t> banks, std::map<std::string, int64_t> options,+ double swiglu_alpha_, double swiglu_limit_,+ std::vector<int> core_ids_, uintptr_t x_ptr, uintptr_t ids_ptr, uintptr_t w_ptr,+ uintptr_t y_ptr)+ : num_layers(num_layers_),num_experts(num_experts_),- top_k(top_k_),- H(hidden_size),- I(inter_size),- act(activation_id),- apply_on_input(apply_router_weight_on_input),- fmt(weight_format),- gate_up_tbl(reinterpret_cast<const uint64_t*>(gate_up_ptr)),- down_tbl(reinterpret_cast<const uint64_t*>(down_ptr)),- gu_scale_tbl(reinterpret_cast<const uint64_t*>(gate_up_scale_ptr)),- gu_global_tbl(reinterpret_cast<const uint64_t*>(gate_up_global_ptr)),- dn_scale_tbl(reinterpret_cast<const uint64_t*>(down_scale_ptr)),- dn_global_tbl(reinterpret_cast<const uint64_t*>(down_global_ptr)),- gu_bias_tbl(reinterpret_cast<const uint64_t*>(gate_up_bias_ptr)),- dn_bias_tbl(reinterpret_cast<const uint64_t*>(down_bias_ptr)),- swiglu_alpha(static_cast<float>(swiglu_alpha_)),- swiglu_limit(static_cast<float>(swiglu_limit_)),- fp8_block_gu_scale_row(fp8_block_gu_scale_stride_),- fp8_block_dn_scale_row(fp8_block_dn_scale_stride_),+ apply_on_input(apply_router_weight_on_input != 0),+ params{top_k_, max_tokens_, hidden_size, inter_size, activation_id, static_cast<float>(swiglu_alpha_),+ static_cast<float>(swiglu_limit_)},+ io_x(reinterpret_cast<const bf16_t*>(x_ptr)),+ io_ids(reinterpret_cast<const int32_t*>(ids_ptr)),+ io_w(reinterpret_cast<const float*>(w_ptr)),+ io_y(reinterpret_cast<bf16_t*>(y_ptr)),+ num_threads(num_threads_ > 0 ? num_threads_ : 1),core_ids(std::move(core_ids_)) {- DotChoice c = select_dot();- dot = c.fn;- nvdot = select_nvdot();- dsdot = select_dsdot();- mxgemv = select_mxgemv();- q4dot = select_q4dot();- fp8_block_dot = select_fp8_block_dot();- if (weight_format == WF_Q4_0) {- if (H % 32 != 0 || I % 32 != 0)- throw std::runtime_error("Q4_0 CPU MoE requires H and I to be multiples of 32");- q4_gu_row_bytes = (H / 32) * 18; // K = H (gate_up rows)- q4_dn_row_bytes = (I / 32) * 18; // K = I (down rows)- }- isa = c.name;- // nvfp4 (AVX-VNNI only): W4A8 int8 decode when the CPU supports it. q4_0 is always- // W4A8 (activations pre-quantized to Q8_0); select_q4dot picks VPDPBUSD / VPMADDUBSW- // / scalar for the tier, so the tag reflects which of those q4dot resolved to.- nvi8dot = select_nvi8dot();- use_vnni = (weight_format == WF_NVFP4) && (nvi8dot != nullptr);- use_q4a8 = (weight_format == WF_Q4_0);- const char* q4tag = use_q4a8 ? (cpu_has_avxvnni() ? "+vnni(q4_0-w4a8)" : "+q4_0-w4a8") : "";- const char* vnni_tag =- cpu_has_avx512vnni() ? "+avx512vnni(nvfp4-w4a8)" : "+vnni(nvfp4-w4a8)";- isa_str = std::string(c.name) + (use_vnni ? vnni_tag : "") + q4tag;- isa = isa_str.c_str();- for (int i = 0; i < 16; ++i) e2m1_lut[i] = kE2M1[i];- for (int i = 0; i < 256; ++i) e4m3_lut[i] = e4m3_decode((uint8_t)i);- // e8m0 (mxfp4 block scale) = 2^(s-127); the GPU GEMV clamps s to [0,254].- for (int i = 0; i < 256; ++i) e8m0_lut[i] = std::ldexp(1.0f, std::min(i, 254) - 127);- g_scratch.assign(static_cast<size_t>(max_tokens) * top_k * I, 0);- // Row-major fp4 (nvfp4/ds_fp4) and fp8_block pre-deinterleave activations to fp32 even/odd.- needs_di = (fmt == WF_NVFP4 || fmt == WF_DSFP4 || fmt == WF_FP8_BLOCK);- if (needs_di) {- if (fmt == WF_DSFP4) xq_scratch.assign(static_cast<size_t>(max_tokens) * H, 0);- xe_scratch.assign(static_cast<size_t>(max_tokens) * (H / 2), 0);- xo_scratch.assign(static_cast<size_t>(max_tokens) * (H / 2), 0);- ge_scratch.assign(static_cast<size_t>(max_tokens) * top_k * (I / 2), 0);- go_scratch.assign(static_cast<size_t>(max_tokens) * top_k * (I / 2), 0);- if (use_vnni) {- xi8_scratch.assign(static_cast<size_t>(max_tokens) * H, 0);- xas_scratch.assign(static_cast<size_t>(max_tokens) * (H / 16), 0);- gi8_scratch.assign(static_cast<size_t>(max_tokens) * top_k * I, 0);- gas_scratch.assign(static_cast<size_t>(max_tokens) * top_k * (I / 16), 0);- }- }- // q4_0 W4A8: per-32-block Q8_0 activations (int8 + fp32 scale) for input + intermediate.- if (use_q4a8) {- xi8_scratch.assign(static_cast<size_t>(max_tokens) * H, 0);- xas_scratch.assign(static_cast<size_t>(max_tokens) * (H / 32), 0);- gi8_scratch.assign(static_cast<size_t>(max_tokens) * top_k * I, 0);- gas_scratch.assign(static_cast<size_t>(max_tokens) * top_k * (I / 32), 0);- }+ if (params.top_k > kMaxTopK) throw std::invalid_argument("CPU MoE supports top_k <= " + std::to_string(kMaxTopK));+ fmt = make_format(weight_format, params, FormatArgs{weight_format, std::move(banks), std::move(options)});+ preps_inter = fmt->preps_inter();+ rounds_routes = fmt->rounds_routes();+ n_iblk = (params.I + kRowBlock - 1) / kRowBlock;+ n_hblk = (params.H + kRowBlock - 1) / kRowBlock;+ g_scratch.assign(static_cast<size_t>(params.max_tokens) * params.top_k * params.I, 0);for (int t = 0; t < num_threads; ++t)workers.emplace_back([this, t] { worker_loop(t); });}- // Quantize a bf16 activation row to Q8_0 (llama.cpp): per-32-block symmetric int8 in- // natural order + one fp32 scale (amax/127) per block. Done once per token/route,- // amortized over every output row that the q4_0 W4A8 GEMV reads. K % 32 == 0.- void quant_q8_0(const bf16_t* x, int K, int8_t* aq, float* asb) {- const int nb = K / 32;- for (int b = 0; b < nb; ++b) {- const bf16_t* xb = x + (size_t)b * 32;- float xf[32], amax = 0.0f;- for (int j = 0; j < 32; ++j) {- xf[j] = bf16_to_f32(xb[j]);- amax = std::max(amax, std::fabs(xf[j]));- }- const float d = amax > 0.0f ? amax / 127.0f : 1.0f;- asb[b] = d;- const float inv = amax > 0.0f ? 1.0f / d : 0.0f;- int8_t* o = aq + (size_t)b * 32;- for (int j = 0; j < 32; ++j)- o[j] = (int8_t)std::max(-127, std::min(127, (int)std::lround(xf[j] * inv)));- }- }-- // Quantize the pre-deinterleaved fp32 even/odd activations to per-16-block int8 in the- // [even(8),odd(8)] layout the VNNI dot expects. Done once per token/route (amortized over- // every output row), so a scalar pass is fine relative to the GEMV.- void quant_i8_pg16(const float* xe, const float* xo, int K, int8_t* asi8, float* asb) {- const int nb = K / 16;- for (int b = 0; b < nb; ++b) {- const float* xeb = xe + (size_t)b * 8;- const float* xob = xo + (size_t)b * 8;- float amax = 0.0f;- for (int j = 0; j < 8; ++j)- amax = std::max(amax, std::max(std::fabs(xeb[j]), std::fabs(xob[j])));- const float s = amax > 0.0f ? amax / 127.0f : 1.0f;- asb[b] = s;- const float inv = 1.0f / s;- int8_t* ae = asi8 + (size_t)b * 16;- for (int j = 0; j < 8; ++j) {- int qe = (int)std::lround(xeb[j] * inv), qo = (int)std::lround(xob[j] * inv);- ae[j] = (int8_t)std::max(-127, std::min(127, qe));- ae[8 + j] = (int8_t)std::max(-127, std::min(127, qo));- }- }- }-- // gate_up output row `row` (in [0, 2I)) dotted with activation over K = H. ``e`` is- // the layer-local expert row (0..num_experts); the layer bases (already resolved- // once per task/pass by the caller via tbl_at) pick the layer's own tensors.- // bf16 uses the interleaved bf16 row; nvfp4 uses the pre-split fp32 even/odd halves- // (or, with AVX-VNNI, the per-16-block int8 activations).- // fp8_block uses the halves for whole 128-blocks and the bf16 row for a ragged tail.- inline float gemm1_dot(const bf16_t* gate_up_l, const uint8_t* gu_packed_l,- const uint8_t* gu_scale_l, const uint16_t* gu_global_l, int e, int row,- const bf16_t* x, const float* xe, const float* xo, const int8_t* xi8,- const float* xas) {- if (fmt == WF_BF16) {- const bf16_t* w = gate_up_l + ((size_t)e * (2 * I) + row) * H;- return dot(w, x, H);- }- if (fmt == WF_Q4_0) {- const uint8_t* w =- gu_packed_l + ((size_t)e * (2 * I) + row) * (size_t)q4_gu_row_bytes;- return q4dot(w, xi8, xas, H); // W4A8: int8 activations (Q8_0), scale in xas- }- if (fmt == WF_FP8_BLOCK) {- const uint8_t* w = gu_packed_l + ((size_t)e * (2 * I) + row) * (size_t)H;- const bf16_t* sc = reinterpret_cast<const bf16_t*>(gu_scale_l) +- ((size_t)e * ((2 * I + FP8_BLK - 1) / FP8_BLK) + row / FP8_BLK) *- (size_t)fp8_block_gu_scale_row;- return fp8_block_dot(w, xe, xo, x, sc, H, e4m3_lut);- }- const size_t r = (size_t)e * (2 * I) + row;- if (use_vnni)- return nvi8dot(gu_packed_l + r * (size_t)(H / 2), gu_scale_l + r * (size_t)(H / 16),- fp16_to_f32(gu_global_l[r]), xi8, H, e4m3_lut, xas);- return nvdot(gu_packed_l + r * (size_t)(H / 2), gu_scale_l + r * (size_t)(H / 16),- fp16_to_f32(gu_global_l[r]), xe, xo, H, e2m1_lut, e4m3_lut);- }-- // down output row `row` (in [0, H)) dotted with the intermediate over K = I. Same- // layer-local-base convention as gemm1_dot.- inline float gemm2_dot(const bf16_t* down_l, const uint8_t* dn_packed_l,- const uint8_t* dn_scale_l, const uint16_t* dn_global_l, int e, int row,- const bf16_t* g, const float* ge, const float* go, const int8_t* gi8,- const float* gas) {- if (fmt == WF_BF16) {- const bf16_t* w = down_l + ((size_t)e * H + row) * I;- return dot(w, g, I);- }- if (fmt == WF_Q4_0) {- const uint8_t* w = dn_packed_l + ((size_t)e * H + row) * (size_t)q4_dn_row_bytes;- return q4dot(w, gi8, gas, I); // W4A8: int8 activations (Q8_0), scale in gas- }- if (fmt == WF_FP8_BLOCK) {- const uint8_t* w = dn_packed_l + ((size_t)e * H + row) * (size_t)I;- const bf16_t* sc = reinterpret_cast<const bf16_t*>(dn_scale_l) +- ((size_t)e * ((H + FP8_BLK - 1) / FP8_BLK) + row / FP8_BLK) *- (size_t)fp8_block_dn_scale_row;- return fp8_block_dot(w, ge, go, g, sc, I, e4m3_lut);- }- const size_t r = (size_t)e * H + row;- if (use_vnni)- return nvi8dot(dn_packed_l + r * (size_t)(I / 2), dn_scale_l + r * (size_t)(I / 16),- fp16_to_f32(dn_global_l[r]), gi8, I, e4m3_lut, gas);- return nvdot(dn_packed_l + r * (size_t)(I / 2), dn_scale_l + r * (size_t)(I / 16),- fp16_to_f32(dn_global_l[r]), ge, go, I, e2m1_lut, e4m3_lut);- }-void pin_self(int tid) {-#if CPU_MOE_HAS_AFFINITY- if (core_ids.empty()) return;- const int cpu = core_ids[tid % static_cast<int>(core_ids.size())];- cpu_set_t set;- CPU_ZERO(&set);- CPU_SET(cpu, &set);- pthread_setaffinity_np(pthread_self(), sizeof(set), &set);-#else- (void)tid;-#endif+ if (!core_ids.empty()) pin_this_thread(core_ids[tid % static_cast<int>(core_ids.size())]);}~CpuMoeExecutor() {@@ -1701,30 +208,23 @@ struct CpuMoeExecutor {task_cv.notify_all();for (auto& th : workers)if (th.joinable()) th.join();- for (MoeTask* t : owned_tasks) delete t;}- uintptr_t create_task(int layer_id, int num_tokens, uintptr_t x_ptr,- uintptr_t ids_ptr, uintptr_t w_ptr, uintptr_t y_ptr) {- MoeTask* t = new MoeTask{this,- layer_id,- num_tokens,- reinterpret_cast<const bf16_t*>(x_ptr),- reinterpret_cast<const int32_t*>(ids_ptr),- reinterpret_cast<const float*>(w_ptr),- reinterpret_cast<bf16_t*>(y_ptr)};- owned_tasks.push_back(t);- return reinterpret_cast<uintptr_t>(t);+ MoeTask task(int layer_id, int num_tokens) const {+ if (layer_id < 0 || layer_id >= num_layers || num_tokens < 1 || num_tokens > params.max_tokens)+ throw std::out_of_range("CPU MoE task (layer " + std::to_string(layer_id) + ", " ++ std::to_string(num_tokens) + " tokens) is outside the executor's layers or IO");+ return MoeTask{static_cast<uint64_t>(layer_id), static_cast<uint64_t>(num_tokens)};}- const char* isa_name() const { return isa; }+ std::string isa_name() const { return fmt->isa_name(); }- const char* fp8_block_isa_name() const {-#if CPU_MOE_X86- if (fp8_block_dot == dot_fp8_block_avx512f) return "avx512f";- if (fp8_block_dot == dot_fp8_block_avx2) return "avx2";-#endif- return "scalar";+ // Whether any route of the task names an expert; do_pass1 and do_pass2 skip the others.+ bool has_routes(const MoeTask& t) const {+ const int64_t n = static_cast<int64_t>(t.num_tokens) * params.top_k;+ for (int64_t i = 0; i < n; ++i)+ if (io_ids[i] >= 0 && io_ids[i] < num_experts) return true;+ return false;}void barrier(int& local_sense) {@@ -1733,275 +233,58 @@ struct CpuMoeExecutor {bar_count.store(0);bar_sense.store(local_sense);} else {- while (bar_sense.load() != local_sense) {-#if CPU_MOE_X86- _mm_pause();-#endif- }+ while (bar_sense.load() != local_sense) cpu_relax();}}- void do_pass1(const MoeTask* t, int64_t p) {- if (fmt == WF_MXFP4) {- do_pass1_mxfp4(t, p);- return;- }- if (fmt == WF_DSFP4) {- do_pass1_dsfp4(t, p);- return;- }+ // Pass 1 item p = one kRowBlock block of one (token, route)'s intermediate row.+ void do_pass1(const MoeTask& t, int64_t p) {+ const int top_k = params.top_k, H = params.H, I = params.I;const int64_t ib = p % n_iblk;const int64_t tk = p / n_iblk;const int k = static_cast<int>(tk % top_k);const int tok = static_cast<int>(tk / top_k);- const int e = t->ids[static_cast<size_t>(tok) * top_k + k];+ const int e = io_ids[static_cast<size_t>(tok) * top_k + k];if (e < 0 || e >= num_experts) return;- const float w_in = apply_on_input ? t->w[static_cast<size_t>(tok) * top_k + k] : 1.0f;- // Resolve this task's layer base once; row indexing below is layer-local (e).- const bf16_t* gate_up_l = reinterpret_cast<const bf16_t*>(tbl_at(gate_up_tbl, t->layer_id));- const uint8_t* gu_packed_l = reinterpret_cast<const uint8_t*>(tbl_at(gate_up_tbl, t->layer_id));- const uint8_t* gu_scale_l = reinterpret_cast<const uint8_t*>(tbl_at(gu_scale_tbl, t->layer_id));- const uint16_t* gu_global_l =- reinterpret_cast<const uint16_t*>(tbl_at(gu_global_tbl, t->layer_id));- const bf16_t* x_row = t->x + (size_t)tok * H;- const float* xe = needs_di ? xe_scratch.data() + (size_t)tok * (H / 2) : nullptr;- const float* xo = needs_di ? xo_scratch.data() + (size_t)tok * (H / 2) : nullptr;- const int8_t* xi8 =- (use_vnni || use_q4a8) ? xi8_scratch.data() + (size_t)tok * H : nullptr;- const float* xas = use_vnni ? xas_scratch.data() + (size_t)tok * (H / 16)- : use_q4a8 ? xas_scratch.data() + (size_t)tok * (H / 32)- : nullptr;- bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I;- const int i0 = static_cast<int>(ib) * IBLK;- const int i1 = std::min(I, i0 + IBLK);- const bool clamped = act == ACT_SWIGLUOAI || act == ACT_SWIGLU_CLAMP;- const float up_bias = act == ACT_SWIGLUOAI ? 1.0f : 0.0f;- const float lim = swiglu_limit, alpha = swiglu_alpha;- for (int i = i0; i < i1; ++i) {- // gate = row i, up = row I+i- float gate =- gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, i, x_row, xe, xo, xi8, xas) * w_in;- float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row,- xe, xo, xi8, xas) * w_in;- if (clamped) {- // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + up_bias)- // -- swigluoai carries the +1 up bias (gpt-oss/MiniMax); swiglu_clamp- // (GLM-5.3) does not. lim == +inf: no clamp.- if (gate > lim) gate = lim;- if (up > lim) up = lim;- else if (up < -lim) up = -lim;- const float glu = gate / (1.0f + std::exp(-gate * alpha));- g_row[i] = f32_to_bf16(glu * (up + up_bias));- } else {- g_row[i] = f32_to_bf16(act_apply(act, gate) * up);- }- }+ const float w_in = apply_on_input ? io_w[static_cast<size_t>(tok) * top_k + k] : 1.0f;+ const int i0 = static_cast<int>(ib) * kRowBlock;+ const int i1 = std::min(I, i0 + kRowBlock);+ fmt->gate_up(static_cast<int>(t.layer), e, tok, i0, i1, io_x + (size_t)tok * H, w_in,+ g_scratch.data() + ((size_t)tok * top_k + k) * I);}- void do_pass2(const MoeTask* t, int64_t p) {- if (fmt == WF_MXFP4) {- do_pass2_mxfp4(t, p);- return;- }- if (fmt == WF_DSFP4) {- do_pass2_dsfp4(t, p);- return;- }- const int64_t hb = p % n_hblk;- const int tok = static_cast<int>(p / n_hblk);- const int h0 = static_cast<int>(hb) * HBLK;- const int h1 = std::min(H, h0 + HBLK);- // Resolve this task's layer base once; row indexing below is layer-local (e).- const bf16_t* down_l = reinterpret_cast<const bf16_t*>(tbl_at(down_tbl, t->layer_id));- const uint8_t* dn_packed_l = reinterpret_cast<const uint8_t*>(tbl_at(down_tbl, t->layer_id));- const uint8_t* dn_scale_l = reinterpret_cast<const uint8_t*>(tbl_at(dn_scale_tbl, t->layer_id));- const uint16_t* dn_global_l =- reinterpret_cast<const uint16_t*>(tbl_at(dn_global_tbl, t->layer_id));- bf16_t* y_row = t->y + (size_t)tok * H;- for (int h = h0; h < h1; ++h) {- float acc = 0.0f;- for (int k = 0; k < top_k; ++k) {- const int e = t->ids[static_cast<size_t>(tok) * top_k + k];- if (e < 0 || e >= num_experts) continue;- const float w_out = apply_on_input ? 1.0f : t->w[static_cast<size_t>(tok) * top_k + k];- const size_t gr = (size_t)tok * top_k + k;- const bf16_t* g_row = g_scratch.data() + gr * I;- const float* ge = needs_di ? ge_scratch.data() + gr * (I / 2) : nullptr;- const float* go = needs_di ? go_scratch.data() + gr * (I / 2) : nullptr;- const int8_t* gi8 = (use_vnni || use_q4a8) ? gi8_scratch.data() + gr * I : nullptr;- const float* gas = use_vnni ? gas_scratch.data() + gr * (I / 16)- : use_q4a8 ? gas_scratch.data() + gr * (I / 32)- : nullptr;- acc += gemm2_dot(down_l, dn_packed_l, dn_scale_l, dn_global_l, e, h, g_row, ge, go, gi8,- gas) * w_out;- }- y_row[h] = f32_to_bf16(acc);- }- }-- // ----------------------------- mxfp4 (gpt-oss) ------------------------------ // Transposed split-K layout (N innermost), so the GEMV streams K and accumulates- // a contiguous block of N output columns -> cache-line-efficient, no repack, no- // extra host memory. Pass 1 fuses gate_up + clamped-swiglu(+bias); pass 2 fuses- // down(+bias) * router-weight, summed over the token's routes.- //- // Dequant: w = E2M1[code] * 2^(e8m0_scale - 127); two codes per byte (low nibble- // first), one e8m0 scale per 32 contiguous K. Matches kernel/triton/mxfp4_moe.py.-- void do_pass1_mxfp4(const MoeTask* t, int64_t p) {- const int64_t ib = p % n_iblk;- const int64_t tk = p / n_iblk;- const int k = static_cast<int>(tk % top_k);- const int tok = static_cast<int>(tk / top_k);- const int e = t->ids[static_cast<size_t>(tok) * top_k + k];- if (e < 0 || e >= num_experts) return;- // Resolve this task's layer base once; row indexing below is layer-local (e).- const uint8_t* gu_packed_l = reinterpret_cast<const uint8_t*>(tbl_at(gate_up_tbl, t->layer_id));- const uint8_t* gu_scale_l = reinterpret_cast<const uint8_t*>(tbl_at(gu_scale_tbl, t->layer_id));- const bf16_t* gu_bias_l = reinterpret_cast<const bf16_t*>(tbl_at(gu_bias_tbl, t->layer_id));- const int N2 = 2 * I; // gate_up output width (gate/up interleaved)- const int Hh = H / 2; // packed-K rows- const int i0 = static_cast<int>(ib) * IBLK;- const int i1 = std::min(I, i0 + IBLK);- const int nunit = i1 - i0; // intermediate units owned by this tile- const int col0 = 2 * i0; // first gate_up column- const int ncol = 2 * nunit; // gate_up columns owned by this tile- const bf16_t* x_row = t->x + (size_t)tok * H;- const uint8_t* blk_e = gu_packed_l + (size_t)e * Hh * N2;- const uint8_t* scl_e = gu_scale_l + (size_t)e * (size_t)(H / 32) * N2;- float gu[2 * IBLK];- mxgemv(gu, blk_e + col0, scl_e + col0, x_row, Hh, N2, ncol, e2m1_lut, e8m0_lut);- const bf16_t* bias_e = gu_bias_l + (size_t)e * N2 + col0;- bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I;- const float lim = swiglu_limit, alpha = swiglu_alpha;- for (int j = 0; j < nunit; ++j) {- float gate = gu[2 * j] + bf16_to_f32(bias_e[2 * j]);- float up = gu[2 * j + 1] + bf16_to_f32(bias_e[2 * j + 1]);- if (gate > lim) gate = lim;- if (up > lim) up = lim;- else if (up < -lim) up = -lim;- const float glu = gate / (1.0f + std::exp(-gate * alpha)); // gate * sigmoid(alpha*gate)- g_row[i0 + j] = f32_to_bf16(glu * (up + 1.0f));- }- }-- void do_pass2_mxfp4(const MoeTask* t, int64_t p) {+ // Pass 2 item p = one kRowBlock block of one token's output row, summed over its routes.+ void do_pass2(const MoeTask& t, int64_t p) {+ const int top_k = params.top_k, H = params.H, I = params.I;const int64_t hb = p % n_hblk;const int tok = static_cast<int>(p / n_hblk);- const int Ih = I / 2; // packed-K rows- const int h0 = static_cast<int>(hb) * HBLK;- const int h1 = std::min(H, h0 + HBLK);- const int nh = h1 - h0;- // Resolve this task's layer base once; row indexing below is layer-local (e).- const uint8_t* dn_packed_l = reinterpret_cast<const uint8_t*>(tbl_at(down_tbl, t->layer_id));- const uint8_t* dn_scale_l = reinterpret_cast<const uint8_t*>(tbl_at(dn_scale_tbl, t->layer_id));- const bf16_t* dn_bias_l = reinterpret_cast<const bf16_t*>(tbl_at(dn_bias_tbl, t->layer_id));- float acc[HBLK];- for (int c = 0; c < nh; ++c) acc[c] = 0.0f;+ const int h0 = static_cast<int>(hb) * kRowBlock;+ const int h1 = std::min(H, h0 + kRowBlock);+ DownRoute routes[kMaxTopK];+ float w_out[kMaxTopK];+ int n = 0;for (int k = 0; k < top_k; ++k) {- const int e = t->ids[static_cast<size_t>(tok) * top_k + k];+ const int e = io_ids[static_cast<size_t>(tok) * top_k + k];if (e < 0 || e >= num_experts) continue;- const float wt = t->w[static_cast<size_t>(tok) * top_k + k];- const bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I;- const uint8_t* blk_e = dn_packed_l + (size_t)e * Ih * H;- const uint8_t* scl_e = dn_scale_l + (size_t)e * (size_t)(I / 32) * H;- float part[HBLK];- mxgemv(part, blk_e + h0, scl_e + h0, g_row, Ih, H, nh, e2m1_lut, e8m0_lut);- const bf16_t* bias_e = dn_bias_l + (size_t)e * H + h0;- for (int c = 0; c < nh; ++c) acc[c] += (part[c] + bf16_to_f32(bias_e[c])) * wt;- }- bf16_t* y_row = t->y + (size_t)tok * H;- for (int c = 0; c < nh; ++c) y_row[h0 + c] = f32_to_bf16(acc[c]);- }-- // ----------------------------- ds_fp4 (DSV4) -------------------------------- // Row-major e2m1 + e8m0/32 (no global); silu-swiglu with clamp; FP8-roundtripped- // activations (x once in submit -> xq_scratch; the intermediate g in a dedicated- // round-trip phase between the two passes). Router weight applies on the down output.-- void do_pass1_dsfp4(const MoeTask* t, int64_t p) {- const int64_t ib = p % n_iblk;- const int64_t tk = p / n_iblk;- const int k = static_cast<int>(tk % top_k);- const int tok = static_cast<int>(tk / top_k);- const int e = t->ids[static_cast<size_t>(tok) * top_k + k];- if (e < 0 || e >= num_experts) return;- // Resolve this task's layer base once; row indexing below is layer-local (e).- const uint8_t* gu_packed_l = reinterpret_cast<const uint8_t*>(tbl_at(gate_up_tbl, t->layer_id));- const uint8_t* gu_scale_l = reinterpret_cast<const uint8_t*>(tbl_at(gu_scale_tbl, t->layer_id));- const int N2 = 2 * I, Hh = H / 2, Hs = H / 32;- // fp8-roundtripped input, pre-deinterleaved to fp32 even/odd halves.- const float* xe = xe_scratch.data() + (size_t)tok * (H / 2);- const float* xo = xo_scratch.data() + (size_t)tok * (H / 2);- const uint8_t* gp = gu_packed_l + (size_t)e * N2 * Hh;- const uint8_t* gs = gu_scale_l + (size_t)e * N2 * Hs;- bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I;- const int i0 = static_cast<int>(ib) * IBLK;- const int i1 = std::min(I, i0 + IBLK);- const float lim = swiglu_limit;- for (int i = i0; i < i1; ++i) {- // gate_up is stored bf16 by the reference GEMV before swiglu; round to match.- float gate = bf16_to_f32(f32_to_bf16(- dsdot(gp + (size_t)i * Hh, gs + (size_t)i * Hs, xe, xo, H, e2m1_lut, e8m0_lut)));- float up = bf16_to_f32(f32_to_bf16(dsdot(- gp + (size_t)(I + i) * Hh, gs + (size_t)(I + i) * Hs, xe, xo, H, e2m1_lut, e8m0_lut)));- if (lim > 0.0f) {- if (gate > lim) gate = lim;- if (up > lim) up = lim;- else if (up < -lim) up = -lim;- }- const float glu = gate / (1.0f + std::exp(-gate)); // silu(gate)- g_row[i] = f32_to_bf16(glu * up);- }- }-- // Prepare one intermediate row (token,route) for the down GEMV, reused across every down output row.- // ds_fp4 first FP8 round-trips it (DSV4 act_quant); nvfp4/ds_fp4/fp8_block deinterleave it to fp32 even/odd; q4_0 Q8_0-quantizes it.- void prep_g_row(int64_t r) {- bf16_t* g = g_scratch.data() + (size_t)r * I;- if (use_q4a8) { // q4_0 W4A8: Q8_0-quantize the intermediate row for the down GEMV.- quant_q8_0(g, I, gi8_scratch.data() + (size_t)r * I,- gas_scratch.data() + (size_t)r * (I / 32));- return;- }- if (fmt == WF_DSFP4) fp8_roundtrip_bf16(g, g, I, act_block);- float* ge = ge_scratch.data() + (size_t)r * (I / 2);- float* go = go_scratch.data() + (size_t)r * (I / 2);- deinterleave_bf16_f32(g, ge, go, I);- if (use_vnni)- quant_i8_pg16(ge, go, I, gi8_scratch.data() + (size_t)r * I,- gas_scratch.data() + (size_t)r * (I / 16));- }-- void do_pass2_dsfp4(const MoeTask* t, int64_t p) {- const int64_t hb = p % n_hblk;- const int tok = static_cast<int>(p / n_hblk);- const int Ih = I / 2, Is = I / 32;- const int h0 = static_cast<int>(hb) * HBLK;- const int h1 = std::min(H, h0 + HBLK);- // Resolve this task's layer base once; row indexing below is layer-local (e).- const uint8_t* dn_packed_l = reinterpret_cast<const uint8_t*>(tbl_at(down_tbl, t->layer_id));- const uint8_t* dn_scale_l = reinterpret_cast<const uint8_t*>(tbl_at(dn_scale_tbl, t->layer_id));- bf16_t* y_row = t->y + (size_t)tok * H;+ const size_t r = (size_t)tok * top_k + k;+ routes[n] = {e, static_cast<int64_t>(r), g_scratch.data() + r * I};+ w_out[n] = apply_on_input ? 1.0f : io_w[r];+ ++n;+ }+ float part[kMaxTopK * kRowBlock];+ if (n > 0) fmt->down(static_cast<int>(t.layer), h0, h1, routes, n, part);+ bf16_t* y_row = io_y + (size_t)tok * H;for (int h = h0; h < h1; ++h) {float acc = 0.0f;- for (int k = 0; k < top_k; ++k) {- const int e = t->ids[static_cast<size_t>(tok) * top_k + k];- if (e < 0 || e >= num_experts) continue;- const float wt = t->w[static_cast<size_t>(tok) * top_k + k];- const float* ge = ge_scratch.data() + ((size_t)tok * top_k + k) * (I / 2);- const float* go = go_scratch.data() + ((size_t)tok * top_k + k) * (I / 2);- const uint8_t* dp = dn_packed_l + (size_t)e * (size_t)H * Ih + (size_t)h * Ih;- const uint8_t* ds = dn_scale_l + (size_t)e * (size_t)H * Is + (size_t)h * Is;- // The reference rounds each route's weighted down output to bf16 before the- // fp32 sum over routes (down [T, top_k, H] bf16 -> .sum(dim=1)).- acc += bf16_to_f32(f32_to_bf16(dsdot(dp, ds, ge, go, I, e2m1_lut, e8m0_lut) * wt));+ for (int j = 0; j < n; ++j) {+ const float v = part[j * kRowBlock + h - h0] * w_out[j];+ acc += rounds_routes ? bf16_to_f32(f32_to_bf16(v)) : v;}y_row[h] = f32_to_bf16(acc);}}- void run_task_body(const MoeTask* t) {+ void run_task_body(const MoeTask& t) {int local_sense = 0;for (;;) {int64_t p = p1_next.fetch_add(1, std::memory_order_relaxed);@@ -2009,13 +292,13 @@ struct CpuMoeExecutor {do_pass1(t, p);}barrier(local_sense);- // Prepare the intermediate rows (per token,route) for the down GEMV, see prep_g_row.+ // Prepare the intermediate rows (per token,route) for the down GEMV, see prep_inter.// Needs all of pass1 done (a full row spans every iblk).- if (needs_di || use_q4a8) {+ if (preps_inter) {for (;;) {int64_t r = prt_next.fetch_add(1, std::memory_order_relaxed);if (r >= prt_total) break;- prep_g_row(r);+ fmt->prep_inter(r, g_scratch.data() + (size_t)r * params.I);}barrier(local_sense);}@@ -2051,8 +334,7 @@ struct CpuMoeExecutor {}// submit() writes cur_task before the release store to `submitted`.my_gen = gen;- MoeTask* t = cur_task;- run_task_body(t);+ run_task_body(cur_task);last_active = wclk::now();if (done_count.fetch_add(1) + 1 == num_threads) {completed.store(my_gen, std::memory_order_release);@@ -2064,69 +346,20 @@ struct CpuMoeExecutor {}}- void submit(MoeTask* t) {- n_iblk = (I + IBLK - 1) / IBLK;- n_hblk = (H + HBLK - 1) / HBLK;- // Grow the per-token intermediate scratch if a larger batch shows up than the- // construction-time hint (CUDA-graph capture warms the largest bs first, so- // this happens at most once, before any capture, while the pool is idle).- const size_t need = static_cast<size_t>(t->num_tokens) * top_k * I;- if (need > g_scratch.size()) g_scratch.resize(need);- p1_total = static_cast<int64_t>(t->num_tokens) * top_k * n_iblk;- p2_total = static_cast<int64_t>(t->num_tokens) * n_hblk;- prt_total = (needs_di || use_q4a8) ? static_cast<int64_t>(t->num_tokens) * top_k : 0;+ void submit(MoeTask t) {+ const int num_tokens = static_cast<int>(t.num_tokens);+ p1_total = static_cast<int64_t>(num_tokens) * params.top_k * n_iblk;+ p2_total = static_cast<int64_t>(num_tokens) * n_hblk;+ prt_total = preps_inter ? static_cast<int64_t>(num_tokens) * params.top_k : 0;p1_next.store(0, std::memory_order_relaxed);p2_next.store(0, std::memory_order_relaxed);prt_next.store(0, std::memory_order_relaxed);done_count.store(0, std::memory_order_relaxed);bar_count.store(0, std::memory_order_relaxed);bar_sense.store(0, std::memory_order_relaxed);- // Deinterleave the per-token input once, up front (ds_fp4 FP8-round-trips it first).+ // Prepare the per-token input once, up front.// Single-threaded: tiny for decode, and done before the workers are woken below.- if (needs_di) {- const size_t xn = static_cast<size_t>(t->num_tokens) * H;- if (xn / 2 > xe_scratch.size()) {- xe_scratch.resize(xn / 2);- xo_scratch.resize(xn / 2);- ge_scratch.resize(static_cast<size_t>(t->num_tokens) * top_k * (I / 2));- go_scratch.resize(static_cast<size_t>(t->num_tokens) * top_k * (I / 2));- if (use_vnni) {- xi8_scratch.resize(xn);- xas_scratch.resize(xn / 16);- gi8_scratch.resize(static_cast<size_t>(t->num_tokens) * top_k * I);- gas_scratch.resize(static_cast<size_t>(t->num_tokens) * top_k * (I / 16));- }- }- const bool ds = (fmt == WF_DSFP4) && !input_prequant;- if (ds && xn > xq_scratch.size()) xq_scratch.resize(xn);- for (int tok = 0; tok < t->num_tokens; ++tok) {- const bf16_t* src = t->x + (size_t)tok * H;- if (ds) { // DSV4 FP8-round-trips the input before the gate_up GEMV- bf16_t* xq = xq_scratch.data() + (size_t)tok * H;- fp8_roundtrip_bf16(src, xq, H, act_block);- src = xq;- }- float* xe = xe_scratch.data() + (size_t)tok * (H / 2);- float* xo = xo_scratch.data() + (size_t)tok * (H / 2);- deinterleave_bf16_f32(src, xe, xo, H);- if (use_vnni)- quant_i8_pg16(xe, xo, H, xi8_scratch.data() + (size_t)tok * H,- xas_scratch.data() + (size_t)tok * (H / 16));- }- }- // q4_0 W4A8: Q8_0-quantize the per-token input once (single-threaded, tiny for decode).- if (use_q4a8) {- const size_t xn = static_cast<size_t>(t->num_tokens) * H;- if (xn > xi8_scratch.size()) {- xi8_scratch.resize(xn);- xas_scratch.resize(xn / 32);- gi8_scratch.resize(static_cast<size_t>(t->num_tokens) * top_k * I);- gas_scratch.resize(static_cast<size_t>(t->num_tokens) * top_k * (I / 32));- }- for (int tok = 0; tok < t->num_tokens; ++tok)- quant_q8_0(t->x + (size_t)tok * H, H, xi8_scratch.data() + (size_t)tok * H,- xas_scratch.data() + (size_t)tok * (H / 32));- }+ for (int tok = 0; tok < num_tokens; ++tok) fmt->prep_input(tok, io_x + (size_t)tok * params.H);{std::lock_guard<std::mutex> lk(task_mtx);cur_task = t;@@ -2153,62 +386,56 @@ struct CpuMoeExecutor {sync_cv.wait(lk, [&] { return completed.load(std::memory_order_acquire) >= target; });}- void submit_with_cuda_stream(uintptr_t stream, uintptr_t task) {- cudaLaunchHostFunc(reinterpret_cast<cudaStream_t>(stream), &CpuMoeExecutor::submit_cb,- reinterpret_cast<void*>(task));+ void submit_with_cuda_stream(uintptr_t stream, int layer_id, int num_tokens) {+ const MoeTask t = task(layer_id, num_tokens);+ HostTask& h = host_tasks.try_emplace(to_word(t), HostTask{this, t}).first->second;+ cudaLaunchHostFunc(reinterpret_cast<cudaStream_t>(stream), &CpuMoeExecutor::submit_cb, &h);}- void sync_with_cuda_stream(uintptr_t stream, uintptr_t task) {- cudaLaunchHostFunc(reinterpret_cast<cudaStream_t>(stream), &CpuMoeExecutor::sync_cb,- reinterpret_cast<void*>(task));+ void sync_with_cuda_stream(uintptr_t stream) {+ cudaLaunchHostFunc(reinterpret_cast<cudaStream_t>(stream), &CpuMoeExecutor::sync_cb, this);}- // Register a (layer, batch-size) slot's task so the coordinator can dispatch it on a- // flag bump.- void register_flag_task(int slot, uintptr_t task) {- std::lock_guard<std::mutex> lk(flag_task_mtx);- if (static_cast<int>(flag_task.size()) <= slot) flag_task.resize(slot + 1, nullptr);- flag_task[slot] = reinterpret_cast<MoeTask*>(task);- }-- int64_t flag_served_count(int slot) const {- return (slot >= 0 && slot < static_cast<int>(flag_served.size())) ? flag_served[slot] : 0;- }-- // Start the busy-poll coordinator over the mapped-pinned flag arrays. ``pin_core`` >= 0+ // Start the busy-poll coordinator over the doorbell. ``pin_core`` >= 0// pins the coordinator to that logical CPU (the worker auto-sizing reserves it), so its// polling never migrates onto / contends with a GEMV worker's core.- void start_flag_coordinator(uintptr_t ready_ptr, uintptr_t done_ptr, int num_slots,- int pin_core) {- ready_flags = reinterpret_cast<volatile int64_t*>(ready_ptr);- done_flags = reinterpret_cast<volatile int64_t*>(done_ptr);- coord_num_slots = num_slots;- {- std::lock_guard<std::mutex> lk(flag_task_mtx);- if (static_cast<int>(flag_task.size()) < num_slots) flag_task.resize(num_slots, nullptr);- }- flag_served.assign(num_slots, 0);+ void start_flag_coordinator(int pin_core) {+ gpu_sync = std::make_unique<GpuSync>();coord_stop.store(false);coord_pinned = CPU_MOE_HAS_AFFINITY && pin_core >= 0;coord_thread = std::thread([this, pin_core] {-#if CPU_MOE_HAS_AFFINITY- if (pin_core >= 0) {- cpu_set_t set;- CPU_ZERO(&set);- CPU_SET(pin_core, &set);- pthread_setaffinity_np(pthread_self(), sizeof(set), &set);- }-#endif+ if (pin_core >= 0) pin_this_thread(pin_core);coordinator_loop();});}+ void flag_signal(uintptr_t stream, int layer_id, int num_tokens) {+ gpu_sync->gpu_submit(stream, to_word(task(layer_id, num_tokens)));+ }+ void flag_wait(uintptr_t stream) { gpu_sync->gpu_wait(stream); }+ void enable_kernel_io() {+ kernel_io = std::make_unique<KernelIo>(gpu_sync->ready_word(), gpu_sync->done_word(), io_x, io_ids, io_w, io_y, params.H,+ params.top_k);+ }+ void flag_publish(uintptr_t stream, uintptr_t x, uintptr_t ids, uintptr_t w, int layer_id, int num_tokens) {+ kernel_io->publish(stream, reinterpret_cast<const void*>(x), reinterpret_cast<const int32_t*>(ids),+ reinterpret_cast<const float*>(w), num_tokens, to_word(task(layer_id, num_tokens)));+ }+ void flag_combine(uintptr_t stream, uintptr_t out, int num_tokens, bool accumulate) {+ kernel_io->combine(stream, reinterpret_cast<void*>(out), num_tokens, accumulate);+ }+ bool flag_pending() const { return gpu_sync && gpu_sync->pending(); }+ // Complete a doorbell the coordinator never answered so the stream can make progress.+ void poison_flag() {+ if (gpu_sync) gpu_sync->cpu_done();+ }+void coordinator_loop() {// Idle backoff (edge-device friendliness): spin hot only while decode traffic is- // flowing, with a TIME-based hot window (an iteration count varies 2-6x with slot- // count and pause cost, so some configs dozed every token). 50 ms since the last- // flag comfortably covers intra- and inter-token gaps -- including host-func-only- // stretches (prefill bursts, --moe-cpu-layers subsets) -- so steady decode never+ // flowing, with a TIME-based hot window (an iteration count varies 2-6x with pause+ // cost, so some configs dozed every token). 50 ms since the last+ // flag comfortably covers intra- and inter-token gaps -- including prefill bursts and+ // --moe-cpu-layers gaps -- so steady decode never// sleeps and keeps the ~us-level wakeup. Past it, i.e. the engine is actually idle,// escalate to timed sleeps capped at 2 ms: a dozing coordinator costs <0.1% of a// core instead of 100%, and the only price is a <=2 ms discovery delay on the FIRST@@ -2222,30 +449,19 @@ struct CpuMoeExecutor {int64_t sleep_us = 100;bool dozing = false;while (!coord_stop.load(std::memory_order_relaxed)) {- bool any = false;- for (int L = 0; L < coord_num_slots; ++L) {- // Binary handshake (memop-compatible: the GPU-side WAIT compares against an- // immediate baked at graph capture, so the protocol resets per step instead of- // counting). Acquire: everything the GPU made visible before setting ready --- // the D2H input copies -- is visible to the worker pool after this read.- if (flag_load_acquire(&ready_flags[L]) != 0) {- flag_store_release(&ready_flags[L], 0); // consume this step's doorbell- MoeTask* t;- {- std::lock_guard<std::mutex> lk(flag_task_mtx);- t = (L < static_cast<int>(flag_task.size())) ? flag_task[L] : nullptr;- }- if (t != nullptr) {- submit(t);- sync(coord_pinned && spin_wait.load(std::memory_order_relaxed));- }- // Release: the workers' y stores are visible before the GPU sees done.- flag_store_release(&done_flags[L], 1);- if (L < static_cast<int>(flag_served.size())) ++flag_served[L];- any = true;+ const uint64_t word = gpu_sync->cpu_poll();+ if (word != 0) {+ const MoeTask t = from_word(word);+ if (has_routes(t)) {+ submit(t);+ sync(coord_pinned && spin_wait.load(std::memory_order_relaxed));+ } else {+ // What the pool would write for a task without routes, minus its fork and join.+ std::memset(io_y, 0, sizeof(bf16_t) * t.num_tokens * params.H);}- }- if (any) {+ // Release: the workers' y stores are visible before the GPU sees done.+ gpu_sync->cpu_done();+ flag_served.fetch_add(1, std::memory_order_relaxed);last_active = coord_clock::now();empty_polls = 0;sleep_us = 100;@@ -2254,15 +470,13 @@ struct CpuMoeExecutor {}// Hot phase: pause-spin, consulting the clock only every 1024 empty polls (the// amortization must gate the CLOCK, not the sleep -- gating the sleep decision- // let 1023/1024 idle iterations hot-scan all slots, ~20% of a core at 992 slots).+ // let 1023/1024 idle iterations hot-poll).// Doze phase: sleep on EVERY iteration until traffic returns; one wake-up scan// (~0.5 us) per 2 ms sleep is ~0.03% duty.if (!dozing) {if ((++empty_polls & 1023u) != 0 ||coord_clock::now() - last_active < kHotWindow) {-#if CPU_MOE_X86- _mm_pause();-#endif+ cpu_relax();continue;}dozing = true;@@ -2270,92 +484,79 @@ struct CpuMoeExecutor {std::this_thread::sleep_for(std::chrono::microseconds(sleep_us));sleep_us = std::min<int64_t>(sleep_us * 2, kSleepCapUs);}- // Teardown: release any in-flight (or future) spin-wait immediately so a replay- // caught mid-shutdown exits its sync kernel now instead of owning the watchdog- // stall. Runs before the destructor's join() returns, while the flag arrays are- // still alive on the Python side.- for (int L = 0; L < coord_num_slots; ++L) {- flag_store_release(&done_flags[L], INT64_MAX);- }+ // Teardown: complete any in-flight wait so a replay caught mid-shutdown does not stall+ // until the watchdog fires. Runs before the destructor's join() returns, while the doorbell is still alive.+ gpu_sync->cpu_done();}// Eager (non-graph) path: run one task to completion on the pool.- void run_task(uintptr_t task) {- MoeTask* t = reinterpret_cast<MoeTask*>(task);- submit(t);+ void run(int layer_id, int num_tokens) {+ submit(task(layer_id, num_tokens));sync();}static void CUDART_CB submit_cb(void* ud) {- MoeTask* t = reinterpret_cast<MoeTask*>(ud);- t->exec->submit(t);- }- static void CUDART_CB sync_cb(void* ud) {- MoeTask* t = reinterpret_cast<MoeTask*>(ud);- t->exec->sync();+ HostTask* h = reinterpret_cast<HostTask*>(ud);+ h->exec->submit(h->task);}+ static void CUDART_CB sync_cb(void* ud) { reinterpret_cast<CpuMoeExecutor*>(ud)->sync(); }};} // namespace+} // namespace freetoken::cpu_moePYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {namespace py = pybind11;+ using freetoken::cpu_moe::CpuMoeExecutor;+ using freetoken::cpu_moe::GpuSync;py::class_<CpuMoeExecutor>(m, "CpuMoeExecutor")- .def(py::init<int, int, int, int, int, int, int, int, int, int, uintptr_t, uintptr_t,- uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t,- double, double, std::vector<int>, int, int>(),+ .def(py::init<int, int, int, int, int, int, int, int, int, std::string,+ std::map<std::string, uint64_t>, std::map<std::string, int64_t>,+ double, double, std::vector<int>, uintptr_t, uintptr_t, uintptr_t, uintptr_t>(),py::arg("num_threads"), py::arg("num_layers"), py::arg("num_experts"),py::arg("top_k"), py::arg("hidden_size"), py::arg("inter_size"),py::arg("max_tokens"), py::arg("activation_id"),py::arg("apply_router_weight_on_input"), py::arg("weight_format"),- py::arg("gate_up_ptr"), py::arg("down_ptr"), py::arg("gate_up_scale_ptr"),- py::arg("gate_up_global_ptr"), py::arg("down_scale_ptr"),- py::arg("down_global_ptr"), py::arg("gate_up_bias_ptr"),- py::arg("down_bias_ptr"), py::arg("swiglu_alpha"), py::arg("swiglu_limit"),- py::arg("core_ids"), py::arg("fp8_block_gu_scale_stride") = 0,- py::arg("fp8_block_dn_scale_stride") = 0)- .def("create_task", &CpuMoeExecutor::create_task, py::arg("layer_id"),- py::arg("num_tokens"), py::arg("x_ptr"), py::arg("ids_ptr"), py::arg("w_ptr"),+ py::arg("banks"), py::arg("options"), py::arg("swiglu_alpha"), py::arg("swiglu_limit"),+ py::arg("core_ids"), py::arg("x_ptr"), py::arg("ids_ptr"), py::arg("w_ptr"),py::arg("y_ptr")).def("submit_with_cuda_stream", &CpuMoeExecutor::submit_with_cuda_stream,- py::arg("stream"), py::arg("task"), py::call_guard<py::gil_scoped_release>())+ py::arg("stream"), py::arg("layer_id"), py::arg("num_tokens"),+ py::call_guard<py::gil_scoped_release>()).def("sync_with_cuda_stream", &CpuMoeExecutor::sync_with_cuda_stream,- py::arg("stream"), py::arg("task"), py::call_guard<py::gil_scoped_release>())- .def("run_task", &CpuMoeExecutor::run_task, py::arg("task"),+ py::arg("stream"), py::call_guard<py::gil_scoped_release>())+ .def("run", &CpuMoeExecutor::run, py::arg("layer_id"), py::arg("num_tokens"),py::call_guard<py::gil_scoped_release>())- .def("register_flag_task", &CpuMoeExecutor::register_flag_task,- py::arg("slot"), py::arg("task"))- .def("flag_served_count", &CpuMoeExecutor::flag_served_count, py::arg("slot"))- .def("start_flag_coordinator", &CpuMoeExecutor::start_flag_coordinator,- py::arg("ready_ptr"), py::arg("done_ptr"), py::arg("num_slots"),- py::arg("pin_core"))- .def("set_input_prequant",- [](CpuMoeExecutor& e, bool v) { e.input_prequant = v; },- py::arg("value"))- .def("set_act_block",- [](CpuMoeExecutor& e, int v) { e.act_block = v; },- py::arg("value"))+ .def("flag_served", [](const CpuMoeExecutor& e) { return e.flag_served.load(); })+ .def("start_flag_coordinator", &CpuMoeExecutor::start_flag_coordinator, py::arg("pin_core"))+ .def("flag_signal", &CpuMoeExecutor::flag_signal, py::arg("stream"), py::arg("layer_id"),+ py::arg("num_tokens"))+ .def("flag_wait", &CpuMoeExecutor::flag_wait, py::arg("stream"))+ .def("flag_pending", &CpuMoeExecutor::flag_pending)+ .def("enable_kernel_io", &CpuMoeExecutor::enable_kernel_io)+ .def("flag_publish", &CpuMoeExecutor::flag_publish, py::arg("stream"), py::arg("x"), py::arg("ids"), py::arg("w"),+ py::arg("layer_id"), py::arg("num_tokens"))+ .def("flag_combine", &CpuMoeExecutor::flag_combine, py::arg("stream"), py::arg("out"), py::arg("num_tokens"),+ py::arg("accumulate"))+ .def("poison_flag", &CpuMoeExecutor::poison_flag).def("isa_name", &CpuMoeExecutor::isa_name)- .def("fp8_block_isa_name", &CpuMoeExecutor::fp8_block_isa_name).def("set_spin_wait",[](CpuMoeExecutor& e, bool v) { e.spin_wait.store(v, std::memory_order_relaxed); },py::arg("enabled")).def("get_spin_wait", [](const CpuMoeExecutor& e) { return e.spin_wait.load(); });- m.def("memops_probe", &cumemops_probe, py::arg("stream"), py::arg("scratch_addr"));- m.def("memop_submit", &cumemop_submit, py::arg("stream"), py::arg("done_addr"),- py::arg("ready_addr"), py::arg("slot"));- m.def("memop_sync", &cumemop_sync, py::arg("stream"), py::arg("done_addr"),- py::arg("slot"));+ m.def("memops_probe", &GpuSync::probe, py::arg("stream"));+ m.def("kernel_io_probe", &freetoken::cpu_moe::KernelIo::probe, py::arg("stream"));+ m.def("host_func_capture_safe", &GpuSync::host_func_capture_safe);// ABI capability marker: the highest ActKind this build implements in the// GENERIC epilogue. CpuMoeExecutor.__init__ probes it before requesting an act// id the epilogue must handle -- a prebuilt .so from before ACT_SWIGLUOAI// accepts id 3 without error and silently computes the wrong activation// (act_apply falls through to gelu_tanh); the probe turns a stale extension// into a loud rebuild instruction instead of wrong model outputs.- m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLU_CLAMP); });- // Weight-format ABI probe. A stale prebuilt .so accepts a newer WFmt id without- // complaint and then falls through to the wrong dequant branch -- silently wrong- // numbers, not a crash. The Python executor refuses to build against an extension- // that predates the format it was asked for.- m.def("max_weight_format_id", []() { return static_cast<int>(WF_FP8_BLOCK); });+ m.def("max_generic_act_id", []() { return static_cast<int>(freetoken::cpu_moe::ACT_SWIGLU_CLAMP); });+ m.def("weight_formats", []() {+ py::list names;+ for (const auto& f : freetoken::cpu_moe::kFormats) names.append(f.first);+ return names;+ });}diff --git a/python/freetoken/kernel/csrc/cpu_moe/format.h b/python/freetoken/kernel/csrc/cpu_moe/format.hnew file mode 100644index 00000000..286ae110--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/format.h@@ -0,0 +1,131 @@+// The interface between the CPU MoE executor and one expert weight format.+#pragma once++#include <map>+#include <memory>+#include <stdexcept>+#include <string>++#include "common.h"++namespace freetoken::cpu_moe {++// Output-row tiling. Small enough to give every worker independent work even at+// batch size 1; large enough to amortize the atomic work-grab.+//+// Bandwidth notes (Sapphire Rapids 8480+, 13 cores): the two passes already read+// every expert weight byte exactly once per token (each output row block is owned+// by one worker), and x stays hot in L1 across a (token,expert)'s rows -- so the+// kernel is single-read bandwidth-optimal at bs=1 (~205 GB/s vs ~55 GB/s PCIe).+// One worker per *physical* core, pinned, is the sweet spot; SMT oversubscription+// thrashes the spin-barrier. Deferred (not worth it here / for this workload):+// - AMX-bf16: a GEMM tile engine; decode is M=1 GEMV so tiles sit idle. It would+// only pay off in a grouped/batched (dedup) path.+// - expert dedup for bs>1: read each distinct expert once and GEMM its tokens.+// Helps locality+bytes when bs is large; decode batches here are tiny (<=4).+// - NUMA: a single node is assumed. Multi-socket machines would split each+// expert's K dimension per node (banks are already per-row contiguous).+constexpr int kRowBlock = 32;++struct MoeParams {+ int top_k = 0, max_tokens = 0;+ int H = 0, I = 0;+ int act = ACT_SILU;+ float swiglu_alpha = 1.0f;+ float swiglu_limit = INFINITY; // +inf == no clamp+};++// What cpu_executor.py passes for one format, by name: banks map canonical roles (gate_up, gate_up_scale, ...) to per-layer tables, options hold small integers.+// A format reads what it needs once, in its constructor.+struct FormatArgs {+ std::string format;+ std::map<std::string, uint64_t> banks;+ std::map<std::string, int64_t> options;++ const uint64_t* bank(const std::string& role) const {+ const auto it = banks.find(role);+ if (it == banks.end() || it->second == 0)+ throw std::invalid_argument("CPU MoE " + format + " experts need the " + role + " bank");+ return reinterpret_cast<const uint64_t*>(it->second);+ }+ int64_t option(const std::string& name) const {+ const auto it = options.find(name);+ if (it == options.end()) throw std::invalid_argument("CPU MoE " + format + " experts need the " + name + " option");+ return it->second;+ }+ int64_t option(const std::string& name, int64_t fallback) const {+ const auto it = options.find(name);+ return it == options.end() ? fallback : it->second;+ }+};++// A bank table is the address of a CPU int64 array of length+// num_layers (one base address per layer, built by cpu_executor.py's+// _make_table), not a single flat bank. tbl_at resolves+// tbl[layer_id] once per task/pass.+inline const void* tbl_at(const uint64_t* tbl, int layer_id) {+ return reinterpret_cast<const void*>(tbl[layer_id]);+}++// One route of a pass 2 item: expert e on intermediate row r = tok * top_k + k, whose bf16 values are g.+struct DownRoute {+ int e;+ int64_t r;+ const bf16_t* g;+};++// g[i] for i in [i0, i1) from the gate row i and the up row I + i; dot(row) is the row's GEMV.+// ACT_SWIGLUOAI and ACT_SWIGLU_CLAMP clamp both inputs; the other kinds go through act_apply.+template <class RowDot>+inline void gated_rows(const MoeParams& p, int i0, int i1, float w_in, bf16_t* g, RowDot dot) {+ const bool clamped = p.act == ACT_SWIGLUOAI || p.act == ACT_SWIGLU_CLAMP;+ const float up_bias = p.act == ACT_SWIGLUOAI ? 1.0f : 0.0f;+ const float lim = p.swiglu_limit, alpha = p.swiglu_alpha;+ for (int i = i0; i < i1; ++i) {+ // gate = row i, up = row I+i+ float gate = dot(i) * w_in;+ float up = dot(p.I + i) * w_in;+ if (clamped) {+ // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + up_bias)+ // -- swigluoai carries the +1 up bias (gpt-oss/MiniMax); swiglu_clamp+ // (GLM-5.3) does not. lim == +inf: no clamp.+ if (gate > lim) gate = lim;+ if (up > lim) up = lim;+ else if (up < -lim) up = -lim;+ const float glu = gate / (1.0f + std::exp(-gate * alpha));+ g[i] = f32_to_bf16(glu * (up + up_bias));+ } else {+ g[i] = f32_to_bf16(act_apply(p.act, gate) * up);+ }+ }+}++// One expert weight format: its kernels, its activation layout and the scratch for it.+// The executor owns the threads and the work split.+// Scratch is sized once from max_tokens; gate_up, down and prep_inter run on the worker threads, prep_input before they wake.+class MoeFormat {+ public:+ explicit MoeFormat(const MoeParams& p) : p(p) {}+ virtual ~MoeFormat() = default;++ virtual std::string isa_name() const = 0;+ // Once per input row x[tok] before pass 1.+ virtual void prep_input(int /*tok*/, const bf16_t* /*x*/) {}+ // Whether pass 2 needs prep_inter on every intermediate row.+ virtual bool preps_inter() const { return false; }+ virtual void prep_inter(int64_t /*r*/, bf16_t* /*g*/) {}+ // Pass 1: g[i] for i in [i0, i1) (at most kRowBlock rows), expert e of `layer` on token tok.+ // w_in scales the gate and up dots: the router weight when it applies on the input, else 1.+ virtual void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* x, float w_in,+ bf16_t* g) const = 0;+ // Pass 2: out[j * kRowBlock + h - h0] for h in [h0, h1) (at most kRowBlock rows) and each of the n routes, before the router weight.+ // Visiting the routes' rows h together keeps n weight streams in flight, which the down GEMV needs to reach DRAM bandwidth.+ virtual void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const = 0;+ // Round each router-weighted down output to bf16 before the sum over routes.+ virtual bool rounds_routes() const { return false; }++ protected:+ const MoeParams& p;+};++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/gpu_sync.h b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync.hnew file mode 100644index 00000000..829a8f34--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync.h@@ -0,0 +1,78 @@+// GPU <-> CPU handshake for the CPU MoE coordinator, one implementation per GPU runtime (gpu_sync_cuda.cpp, gpu_sync_hip.cpp).+//+// A backend keeps two contracts, in eager mode and in graph replay:+// 1. submit, GPU -> CPU: after gpu_submit(stream, task), cpu_poll() returns task once, and every earlier write of the stream is visible to the CPU.+// 2. wait, CPU -> GPU: gpu_wait(stream) holds later work of the stream until cpu_done(), and that work sees every CPU write made before cpu_done().+// task is a nonzero word, opaque here; the executor encodes the layer and token count in it, as a constant baked into each captured graph.+// One task is in flight at a time: decode runs on one stream and every layer waits before the next one signals.+// KernelIo (gpu_sync_io.h) may replace the copies around submit and wait: its kernel copies the inputs into the IO rows and does gpu_submit's writes itself.+// It skips a step without a CPU route: done = 1 and no ready, so cpu_poll() never sees the step and gpu_wait() passes at once.+#pragma once++#include <cstdint>+#include <memory>++#if defined(_MSC_VER)+#include <intrin.h>+#endif++namespace freetoken::cpu_moe {++// Portable ordering for the flag handshake: "ready observed => the DMA'd inputs that+// preceded the bump are visible" and "y stores are visible before done". Plain+// volatile loads lean on x86 TSO; acquire/release makes it hold on aarch64 too+// (GH200/Jetson) at zero x86 cost. (MSVC branch is x86-only today: compiler barrier+// + TSO.)+inline int64_t flag_load_acquire(const volatile int64_t* p) {+#if defined(_MSC_VER)+ const int64_t v = *p;+ _ReadWriteBarrier();+ return v;+#else+ return __atomic_load_n(const_cast<const int64_t*>(p), __ATOMIC_ACQUIRE);+#endif+}++inline void flag_store_release(volatile int64_t* p, int64_t v) {+#if defined(_MSC_VER)+ _ReadWriteBarrier();+ *p = v;+#else+ __atomic_store_n(const_cast<int64_t*>(p), v, __ATOMIC_RELEASE);+#endif+}++class GpuSync {+ public:+ // Allocates the handshake memory; throws if this runtime cannot.+ GpuSync();+ ~GpuSync();+ GpuSync(const GpuSync&) = delete;+ GpuSync& operator=(const GpuSync&) = delete;++ // Whether both contracts hold on this device and stream, including graph replay where the runtime needs a real check.+ static bool probe(uintptr_t stream);+ // Whether the cudaLaunchHostFunc fallback is replayed correctly from a captured graph.+ static bool host_func_capture_safe();++ // 1. submit. gpu_submit() is enqueued on `stream`, which may be capturing; cpu_poll() returns 0 when nothing is submitted.+ void gpu_submit(uintptr_t stream, uint64_t task);+ uint64_t cpu_poll();++ // 2. wait. cpu_done() also unblocks the stream on teardown and from the watchdog.+ void gpu_wait(uintptr_t stream);+ void cpu_done();++ // Submitted but not polled yet (watchdog only).+ bool pending() const;++ // Device addresses of the two words, for KernelIo.+ uintptr_t ready_word() const;+ uintptr_t done_word() const;++ private:+ struct Impl;+ std::unique_ptr<Impl> impl_;+};++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_cuda.cpp b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_cuda.cppnew file mode 100644index 00000000..a2a089bf--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_cuda.cpp@@ -0,0 +1,166 @@+#include <atomic>+#include <cstdint>+#include <cstdio>+#include <cstring>+#include <stdexcept>+#include <string>++#if defined(_WIN32)+#include <windows.h>+#else+#include <dlfcn.h>+#endif++#include <cuda_runtime_api.h>++#include "gpu_sync.h"++namespace freetoken::cpu_moe {+namespace {++// =====================================================================================+// CUDA stream memory operations (driver API, resolved via dlopen -- no link-time or+// toolchain dependence). The GPU side of the flag handshake: submit = WRITE_VALUE+// (done=0 then ready=task word), sync = WAIT_VALUE(done >= 1). The wait is+// executed by the GPU front-end (no SM-resident kernel), so GPU "utilization" stays+// truthful during CPU compute windows -- a resident spin kernel pinned it at 99%,+// which laptop CPU/GPU dynamic power schedulers answered by clamping the CPU's max+// frequency (GEMV workers -1.5x: the reported edge regression). Availability is+// probed functionally at startup (memops_probe); anything unsupported (some driver models,+// vGPU, old drivers) falls back to the cudaLaunchHostFunc path.+#if defined(_WIN32)+static void* cumemop_dlopen() { return (void*)::LoadLibraryA("nvcuda.dll"); }+static void* cumemop_dlsym(void* h, const char* n) {+ return (void*)::GetProcAddress((HMODULE)h, n);+}+#else+static void* cumemop_dlopen() {+ void* h = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_LOCAL);+ if (h == nullptr) h = dlopen("libcuda.so", RTLD_LAZY | RTLD_LOCAL);+ return h;+}+static void* cumemop_dlsym(void* h, const char* n) { return dlsym(h, n); }+#endif++using cuMemOp64_fn = int (*)(void* stream, unsigned long long addr, unsigned long long value,+ unsigned int flags);+static cuMemOp64_fn g_cu_write64 = nullptr;+static cuMemOp64_fn g_cu_wait64 = nullptr;+static constexpr unsigned int kCuWaitValueGeq = 0x0; // CU_STREAM_WAIT_VALUE_GEQ+static constexpr unsigned int kCuWriteDefault = 0x0; // CU_STREAM_WRITE_VALUE_DEFAULT++static bool cumemop_resolve() {+ static bool resolved = [] {+ void* h = cumemop_dlopen();+ if (h == nullptr) return false;+ // 11.7+ made the v2 entry points the default; older drivers export only the v1+ // names with the same signature.+ g_cu_write64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWriteValue64_v2"));+ if (g_cu_write64 == nullptr)+ g_cu_write64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWriteValue64"));+ g_cu_wait64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWaitValue64_v2"));+ if (g_cu_wait64 == nullptr)+ g_cu_wait64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWaitValue64"));+ return g_cu_write64 != nullptr && g_cu_wait64 != nullptr;+ }();+ return resolved;+}++// GPU side of the flag handshake (see the block comment above): enqueued on the+// caller's (possibly capturing) stream; the WAIT immediate is the constant 1,+// replay-safe under CUDA graphs.+// The startup probe validates EAGER memops; a driver could still reject them at graph+// capture time. Those enqueue errors would otherwise be swallowed here and surface only+// as a later EndCapture failure -- log the first CUresult so triage is one step.+static void cumemop_check(int rc, const char* what) {+ static std::atomic<bool> warned{false};+ if (rc != 0 && !warned.exchange(true)) {+ std::fprintf(stderr,+ "[freetoken/cpu_moe] %s failed with CUresult=%d (first occurrence; "+ "subsequent errors are not repeated). If this happened during CUDA "+ "graph capture, the driver lacks capture support for stream memops -- "+ "set FREETOKEN_CPU_MOE_FLAG_SYNC=0.\n",+ what, rc);+ }+}++} // namespace++// ready and done live on separate cache lines of one mapped-pinned block; with UVA the GPU memops and the coordinator use the same addresses.+struct GpuSync::Impl {+ void* block = nullptr;+ void* device_block = nullptr;+ volatile int64_t* ready = nullptr;+ volatile int64_t* done = nullptr;+};++GpuSync::GpuSync() : impl_(std::make_unique<Impl>()) {+ if (!cumemop_resolve()) throw std::runtime_error("CPU MoE flag sync: CUDA stream memops are unavailable");+ const cudaError_t rc = cudaHostAlloc(&impl_->block, 128, cudaHostAllocPortable | cudaHostAllocMapped);+ if (rc != cudaSuccess)+ throw std::runtime_error(std::string("CPU MoE flag allocation failed: ") + cudaGetErrorString(rc));+ std::memset(impl_->block, 0, 128);+ impl_->ready = static_cast<volatile int64_t*>(impl_->block);+ impl_->done = impl_->ready + 8;+ const cudaError_t dev_rc = cudaHostGetDevicePointer(&impl_->device_block, impl_->block, 0);+ if (dev_rc != cudaSuccess) {+ cudaFreeHost(impl_->block);+ throw std::runtime_error(std::string("CPU MoE flag device address: ") + cudaGetErrorString(dev_rc));+ }+}++GpuSync::~GpuSync() { cudaFreeHost(impl_->block); }++// Functional probe on a scratch pinned int64: enqueue WRITE(7) + WAIT(>=7) + sync.+// Returns true only if the whole memop path works on THIS stream/device/driver.+bool GpuSync::probe(uintptr_t stream) {+ if (!cumemop_resolve()) return false;+ void* scratch = nullptr;+ if (cudaHostAlloc(&scratch, sizeof(int64_t), cudaHostAllocPortable | cudaHostAllocMapped) != cudaSuccess)+ return false;+ *static_cast<int64_t*>(scratch) = 0;+ auto* s = reinterpret_cast<void*>(stream);+ const auto addr = reinterpret_cast<unsigned long long>(scratch);+ const bool ok = g_cu_write64(s, addr, 7ULL, kCuWriteDefault) == 0 && g_cu_wait64(s, addr, 7ULL, kCuWaitValueGeq) == 0 &&+ cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream)) == cudaSuccess;+ // a failed enqueue can leave the WRITE pending on the scratch word+ cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream));+ cudaFreeHost(scratch);+ return ok;+}++bool GpuSync::host_func_capture_safe() { return true; }++void GpuSync::gpu_submit(uintptr_t stream, uint64_t task) {+ auto* s = reinterpret_cast<void*>(stream);+ // Order matters and is preserved by the front end: reset done BEFORE raising ready,+ // so the coordinator's completion write for THIS step can never be wiped.+ cumemop_check(g_cu_write64(s, reinterpret_cast<unsigned long long>(impl_->done), 0ULL, kCuWriteDefault),+ "cuStreamWriteValue64(done)");+ cumemop_check(g_cu_write64(s, reinterpret_cast<unsigned long long>(impl_->ready), task, kCuWriteDefault),+ "cuStreamWriteValue64(ready)");+}++void GpuSync::gpu_wait(uintptr_t stream) {+ cumemop_check(g_cu_wait64(reinterpret_cast<void*>(stream), reinterpret_cast<unsigned long long>(impl_->done), 1ULL,+ kCuWaitValueGeq),+ "cuStreamWaitValue64(done)");+}++uint64_t GpuSync::cpu_poll() {+ // Acquire: everything the GPU made visible before writing ready -- the D2H input copies -- is visible to the worker pool after this read.+ const int64_t task = flag_load_acquire(impl_->ready);+ if (task == 0) return 0;+ flag_store_release(impl_->ready, 0); // consume this step's doorbell+ return static_cast<uint64_t>(task);+}++void GpuSync::cpu_done() { flag_store_release(impl_->done, 1); }++bool GpuSync::pending() const { return flag_load_acquire(impl_->ready) != 0 && flag_load_acquire(impl_->done) == 0; }++uintptr_t GpuSync::ready_word() const { return reinterpret_cast<uintptr_t>(impl_->device_block); }++uintptr_t GpuSync::done_word() const { return reinterpret_cast<uintptr_t>(impl_->device_block) + 8 * sizeof(int64_t); }++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_hip.cpp b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_hip.cppnew file mode 100644index 00000000..31ff9be5--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_hip.cpp@@ -0,0 +1,275 @@+// HIP stream memory operations for the CPU MoE flag handshake.+//+// HIP WAIT addresses must come from hipMallocSignalMemory, one 64-bit word per allocation.+//+// ROCm 10 records hipStreamWrite/WaitValue64 into the graph under stream capture, one node per call.+// ROCm 7.14 runs them while the stream is being captured but records nothing, so there we add explicit batch-memop nodes and splice them into the stream's dependency set.+// Which of the two applies is measured once at runtime (records_native_memops), not read from the version number.+// The probe then does a real capture + instantiate + replay through the chosen path.+#include <array>+#include <atomic>+#include <chrono>+#include <cstdint>+#include <map>+#include <stdexcept>+#include <string>+#include <thread>++#include <freetoken/hip_compat.h>++#include "gpu_sync.h"++namespace freetoken::cpu_moe {++namespace {++void hip_check(hipError_t rc, const char* what) {+ if (rc != hipSuccess)+ throw std::runtime_error(std::string("CPU MoE HIP flag sync: ") + what + " failed: " + hipGetErrorString(rc));+}++// Capability failures must not poison eager fallback or hide a caller's earlier error.+struct ProbeErrorScope {+ ProbeErrorScope() { hip_check(hipPeekAtLastError(), "pending error before probe"); }+ ~ProbeErrorScope() { (void)hipGetLastError(); }+};++hipError_t alloc_signal(uint64_t** out) {+ void* storage = nullptr;+ const hipError_t rc = hipExtMallocWithFlags(&storage, sizeof(uint64_t), hipMallocSignalMemory);+ if (rc == hipSuccess) {+ *out = static_cast<uint64_t*>(storage);+ __atomic_store_n(*out, 0ULL, __ATOMIC_RELEASE);+ }+ return rc;+}++void free_signal(uint64_t* signal) {+ if (signal != nullptr) (void)hipFree(signal);+}++volatile int64_t* as_flag(uint64_t* signal) { return reinterpret_cast<volatile int64_t*>(signal); }++bool capturing(hipStream_t stream) {+ hipStreamCaptureStatus status = hipStreamCaptureStatusNone;+ hip_check(hipStreamIsCapturing(stream, &status), "capture status query");+ return status == hipStreamCaptureStatusActive;+}++// Whether this runtime records a native hipStreamWriteValue64 issued under capture as a graph node.+bool records_native_memops() {+ static const bool native = [] {+ ProbeErrorScope errors;+ uint64_t* word = nullptr;+ hipStream_t stream = nullptr;+ hipGraph_t graph = nullptr;+ hipError_t rc = alloc_signal(&word);+ if (rc == hipSuccess) rc = hipStreamCreateWithFlags(&stream, hipStreamNonBlocking);+ if (rc == hipSuccess) rc = hipStreamBeginCapture(stream, hipStreamCaptureModeRelaxed);+ const bool capture_started = rc == hipSuccess;+ if (rc == hipSuccess) rc = hipStreamWriteValue64(stream, word, 1, hipStreamWriteValueDefault);+ if (capture_started) {+ const hipError_t end_rc = hipStreamEndCapture(stream, &graph);+ if (rc == hipSuccess) rc = end_rc;+ }+ size_t nodes = 0;+ if (rc == hipSuccess) rc = hipGraphGetNodes(graph, nullptr, &nodes);+ if (graph != nullptr) (void)hipGraphDestroy(graph);+ if (stream != nullptr) {+ (void)hipStreamSynchronize(stream);+ (void)hipStreamDestroy(stream);+ }+ free_signal(word);+ return rc == hipSuccess && nodes == 1;+ }();+ return native;+}++// Appends one batch-memop node to the capture of `stream`; the node keeps a pointer to `ops`.+hipError_t add_capture_memop_node(hipStream_t stream, hipStreamBatchMemOpParams* ops, unsigned int count) {+ hipStreamCaptureStatus status = hipStreamCaptureStatusNone;+ hipGraph_t graph = nullptr;+ const hipGraphNode_t* dependencies = nullptr;+ size_t dependency_count = 0;+ hipError_t rc = hipStreamGetCaptureInfo_v2(stream, &status, nullptr, &graph, &dependencies, &dependency_count);+ if (rc != hipSuccess) return rc;+ if (status != hipStreamCaptureStatusActive || graph == nullptr) return hipErrorIllegalState;++ hipCtx_t context = nullptr;+#if defined(__GNUC__)+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"+#endif+ rc = hipCtxGetCurrent(&context);+#if defined(__GNUC__)+#pragma GCC diagnostic pop+#endif+ if (rc != hipSuccess) return rc;++ hipBatchMemOpNodeParams params{};+ params.ctx = context;+ params.count = count;+ params.paramArray = ops;+ hipGraphNode_t node = nullptr;+ rc = hipGraphAddBatchMemOpNode(&node, graph, dependencies, dependency_count, ¶ms);+ if (rc != hipSuccess) return rc;+ rc = hipStreamUpdateCaptureDependencies(stream, &node, 1, hipStreamSetCaptureDependencies);+ // EndCapture rejects a node that was added but never joined to the stream.+ if (rc != hipSuccess && node != nullptr) (void)hipGraphDestroyNode(node);+ return rc;+}++hipStreamBatchMemOpParams write_op(uint64_t* address, uint64_t value) {+ hipStreamBatchMemOpParams op{};+ op.writeValue.operation = hipStreamMemOpWriteValue64;+ op.writeValue.address = reinterpret_cast<hipDeviceptr_t>(address);+ op.writeValue.value64 = value;+ return op;+}++hipStreamBatchMemOpParams wait_op(uint64_t* address) {+ hipStreamBatchMemOpParams op{};+ op.waitValue.operation = hipStreamMemOpWaitValue64;+ op.waitValue.address = reinterpret_cast<hipDeviceptr_t>(address);+ op.waitValue.value64 = 1;+ op.waitValue.flags = hipStreamWaitValueGte;+ return op;+}++bool needs_explicit_nodes(hipStream_t stream) { return capturing(stream) && !records_native_memops(); }++// node_op is null for a native call, else the parameter slot of an explicit node, which the node keeps a pointer to.+// The ops of one batch node run in parallel, so each write gets its own node and consecutive writes stay in order.+hipError_t write_value(hipStream_t stream, uint64_t* address, uint64_t value, hipStreamBatchMemOpParams* node_op) {+ if (node_op == nullptr) return hipStreamWriteValue64(stream, address, value, hipStreamWriteValueDefault);+ *node_op = write_op(address, value);+ return add_capture_memop_node(stream, node_op, 1);+}++hipError_t wait_done(hipStream_t stream, uint64_t* done, hipStreamBatchMemOpParams* node_op) {+ if (node_op == nullptr) return hipStreamWaitValue64(stream, done, 1, hipStreamWaitValueGte, UINT64_MAX);+ *node_op = wait_op(done);+ return add_capture_memop_node(stream, node_op, 1);+}++} // namespace++struct GpuSync::Impl {+ uint64_t* ready = nullptr; // signal memory+ uint64_t* done = nullptr;+ // Explicit nodes keep these addresses, so one parameter pair per distinct task, never moved (std::map nodes are stable).+ std::map<uint64_t, std::array<hipStreamBatchMemOpParams, 2>> submit_ops; // done = 0, ready = task+ hipStreamBatchMemOpParams wait_op{};+};++GpuSync::GpuSync() : impl_(std::make_unique<Impl>()) {+ hipError_t rc = alloc_signal(&impl_->ready);+ if (rc == hipSuccess) rc = alloc_signal(&impl_->done);+ if (rc != hipSuccess) {+ free_signal(impl_->ready);+ free_signal(impl_->done);+ hip_check(rc, "signal allocation");+ }+ // Measure now: the measurement captures and allocates, which a graph capture in progress would reject.+ (void)records_native_memops();+}++GpuSync::~GpuSync() {+ free_signal(impl_->ready);+ free_signal(impl_->done);+}++bool GpuSync::probe(uintptr_t /*stream*/) {+ ProbeErrorScope errors;+ int device = 0;+ int can_wait = 0;+ if (hipGetDevice(&device) != hipSuccess ||+ hipDeviceGetAttribute(&can_wait, hipDeviceAttributeCanUseStreamWaitValue, device) != hipSuccess || can_wait == 0)+ return false;+ const bool explicit_nodes = !records_native_memops();++ uint64_t* ready = nullptr;+ uint64_t* done = nullptr;+ hipStream_t stream = nullptr;+ hipGraph_t graph = nullptr;+ hipGraphExec_t graph_exec = nullptr;+ std::array<hipStreamBatchMemOpParams, 2> node_ops{};+ hipError_t rc = alloc_signal(&ready);+ if (rc == hipSuccess) rc = alloc_signal(&done);+ if (rc == hipSuccess) rc = hipStreamCreateWithFlags(&stream, hipStreamNonBlocking);+ // A runtime that runs the WAIT during capture instead of recording it must not block here.+ if (rc == hipSuccess) __atomic_store_n(done, 1ULL, __ATOMIC_RELEASE);+ if (rc == hipSuccess) rc = hipStreamBeginCapture(stream, hipStreamCaptureModeRelaxed);+ const bool capture_started = rc == hipSuccess;+ if (rc == hipSuccess) rc = write_value(stream, ready, 1, explicit_nodes ? &node_ops[0] : nullptr);+ if (rc == hipSuccess) rc = wait_done(stream, done, explicit_nodes ? &node_ops[1] : nullptr);+ // End the capture even on failure so its stream and graph are not left active during cleanup.+ if (capture_started) {+ const hipError_t end_rc = hipStreamEndCapture(stream, &graph);+ if (rc == hipSuccess) rc = end_rc;+ }+ if (rc == hipSuccess) rc = hipStreamSynchronize(stream);+ size_t nodes = 0;+ if (rc == hipSuccess) rc = hipGraphGetNodes(graph, nullptr, &nodes);+ if (rc == hipSuccess && nodes != 2) rc = hipErrorNotSupported;+ if (rc == hipSuccess) rc = hipGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0);++ bool served = false;+ if (rc == hipSuccess) {+ __atomic_store_n(ready, 0ULL, __ATOMIC_RELEASE);+ __atomic_store_n(done, 0ULL, __ATOMIC_RELEASE);+ std::thread responder([&] {+ const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);+ while (__atomic_load_n(ready, __ATOMIC_ACQUIRE) == 0 && std::chrono::steady_clock::now() < deadline)+ std::this_thread::yield();+ served = __atomic_load_n(ready, __ATOMIC_ACQUIRE) == 1;+ // The probe graph never resets done, so this also releases a launch that arrives after the deadline.+ __atomic_store_n(done, 1ULL, __ATOMIC_RELEASE);+ });+ rc = hipGraphLaunch(graph_exec, stream);+ if (rc == hipSuccess) rc = hipStreamSynchronize(stream);+ responder.join();+ }+ const bool ok = rc == hipSuccess && served;++ if (graph_exec != nullptr) (void)hipGraphExecDestroy(graph_exec);+ if (graph != nullptr) (void)hipGraphDestroy(graph);+ if (stream != nullptr) (void)hipStreamDestroy(stream);+ free_signal(ready);+ free_signal(done);+ return ok;+}++void GpuSync::gpu_submit(uintptr_t stream, uint64_t task) {+ auto s = reinterpret_cast<hipStream_t>(stream);+ hipStreamBatchMemOpParams* ops = needs_explicit_nodes(s) ? impl_->submit_ops[task].data() : nullptr;+ // Reset done before raising ready, so the coordinator's completion for this step can never be wiped.+ hip_check(write_value(s, impl_->done, 0, ops), "write done");+ hip_check(write_value(s, impl_->ready, task, ops ? ops + 1 : nullptr), "write ready");+}++void GpuSync::gpu_wait(uintptr_t stream) {+ auto s = reinterpret_cast<hipStream_t>(stream);+ hip_check(wait_done(s, impl_->done, needs_explicit_nodes(s) ? &impl_->wait_op : nullptr), "wait done");+}++bool GpuSync::host_func_capture_safe() { return false; }++uint64_t GpuSync::cpu_poll() {+ const int64_t task = flag_load_acquire(as_flag(impl_->ready));+ if (task == 0) return 0;+ flag_store_release(as_flag(impl_->ready), 0);+ return static_cast<uint64_t>(task);+}++void GpuSync::cpu_done() { flag_store_release(as_flag(impl_->done), 1); }++bool GpuSync::pending() const {+ return flag_load_acquire(as_flag(impl_->ready)) != 0 && flag_load_acquire(as_flag(impl_->done)) == 0;+}++uintptr_t GpuSync::ready_word() const { return reinterpret_cast<uintptr_t>(impl_->ready); }++uintptr_t GpuSync::done_word() const { return reinterpret_cast<uintptr_t>(impl_->done); }++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_io.cu b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_io.cunew file mode 100644index 00000000..b41961f0--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_io.cu@@ -0,0 +1,199 @@+#include <algorithm>+#include <cstring>+#include <stdexcept>++#include <freetoken/hip_compat.h>++#include "gpu_sync_io.h"++namespace freetoken::cpu_moe {+namespace {++__device__ inline float bf16_bits_to_float(uint16_t b) { return __uint_as_float(static_cast<uint32_t>(b) << 16); }++// Round to nearest even, the same rounding as a torch bf16 add.+__device__ inline uint16_t float_to_bf16_bits(float f) {+ const uint32_t u = __float_as_uint(f);+ if ((u & 0x7fffffffu) > 0x7f800000u) return 0x7fc0;+ return static_cast<uint16_t>((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);+}++// One block. Copies x/ids/w into the IO rows, then done = 0 and ready = task.+// A step without a CPU route writes done = 1 and rings nothing: the coordinator stays asleep and the GPU wait passes at once.+__global__ void publish_kernel(const uint16_t* __restrict__ x, const int32_t* __restrict__ ids, const float* __restrict__ w,+ int64_t n_x, int64_t n_ids, uint16_t* io_x, int32_t* io_ids, float* io_w,+ volatile int64_t* ready, volatile int64_t* done, int64_t task, int32_t* has_cpu) {+ __shared__ int any;+ if (threadIdx.x == 0) any = 0;+ __syncthreads();+ for (int64_t i = threadIdx.x; i < n_ids; i += blockDim.x)+ if (ids[i] >= 0) any = 1;+ __syncthreads();+ if (!any) {+ if (threadIdx.x == 0) {+ *has_cpu = 0;+ *done = 1;+ __threadfence_system();+ }+ return;+ }+ const bool vec = (reinterpret_cast<uintptr_t>(x) | reinterpret_cast<uintptr_t>(io_x)) % 16 == 0 && n_x % 8 == 0;+ if (vec) {+ for (int64_t i = threadIdx.x; i < n_x / 8; i += blockDim.x)+ reinterpret_cast<uint4*>(io_x)[i] = reinterpret_cast<const uint4*>(x)[i];+ } else {+ for (int64_t i = threadIdx.x; i < n_x; i += blockDim.x) io_x[i] = x[i];+ }+ for (int64_t i = threadIdx.x; i < n_ids; i += blockDim.x) {+ io_ids[i] = ids[i];+ io_w[i] = w[i];+ }+ __threadfence_system();+ __syncthreads();+ if (threadIdx.x == 0) {+ *has_cpu = 1;+ *done = 0;+ __threadfence_system();+ *ready = task;+ __threadfence_system();+ }+}++// out = CPU rows, or out += CPU rows with accumulate; a step publish skipped contributes zeros.+// io_y is volatile: the same host rows hold a new step on every call.+__global__ void combine_kernel(uint16_t* out, const volatile uint16_t* io_y, const int32_t* has_cpu, int64_t n, bool accumulate) {+ const bool on = *has_cpu != 0;+ if (accumulate && !on) return;+ for (int64_t i = blockIdx.x * static_cast<int64_t>(blockDim.x) + threadIdx.x; i < n; i += static_cast<int64_t>(gridDim.x) * blockDim.x) {+ float v = accumulate ? bf16_bits_to_float(out[i]) : 0.0f;+ if (on) v += bf16_bits_to_float(io_y[i]);+ out[i] = float_to_bf16_bits(v);+ }+}++cudaError_t launch_publish(cudaStream_t stream, const uint16_t* x, const int32_t* ids, const float* w, int64_t n_x, int64_t n_ids,+ uint16_t* io_x, int32_t* io_ids, float* io_w, volatile int64_t* ready, volatile int64_t* done, int64_t task,+ int32_t* has_cpu) {+ publish_kernel<<<1, 256, 0, stream>>>(x, ids, w, n_x, n_ids, io_x, io_ids, io_w, ready, done, task, has_cpu);+ return cudaGetLastError();+}++cudaError_t launch_combine(cudaStream_t stream, uint16_t* out, const uint16_t* io_y, const int32_t* has_cpu, int64_t n, bool accumulate) {+ if (n == 0) return cudaSuccess;+ const auto blocks = static_cast<unsigned>(std::min<int64_t>((n + 255) / 256, 1024));+ combine_kernel<<<blocks, 256, 0, stream>>>(out, io_y, has_cpu, n, accumulate);+ return cudaGetLastError();+}++void check(cudaError_t rc, const char* what) {+ if (rc != cudaSuccess) throw std::runtime_error(std::string("CPU MoE kernel IO: ") + what + ": " + cudaGetErrorString(rc));+}++template <typename T>+T* device_address(const void* host) {+ void* dev = nullptr;+ check(cudaHostGetDevicePointer(&dev, const_cast<void*>(host), 0), "device address of the IO rows");+ return static_cast<T*>(dev);+}++} // namespace++KernelIo::KernelIo(uintptr_t ready, uintptr_t done, const void* x, const int32_t* ids, const float* w, const void* y, int hidden,+ int top_k)+ : ready_(reinterpret_cast<volatile int64_t*>(ready)),+ done_(reinterpret_cast<volatile int64_t*>(done)),+ x_(device_address<uint16_t>(x)),+ ids_(device_address<int32_t>(ids)),+ w_(device_address<float>(w)),+ y_(device_address<const uint16_t>(y)),+ hidden_(hidden),+ top_k_(top_k) {+ check(cudaMalloc(reinterpret_cast<void**>(&has_cpu_), sizeof(int32_t)), "has_cpu allocation");+ const cudaError_t rc = cudaMemset(has_cpu_, 0, sizeof(int32_t));+ if (rc != cudaSuccess) {+ (void)cudaFree(has_cpu_);+ check(rc, "has_cpu reset");+ }+}++KernelIo::~KernelIo() { (void)cudaFree(has_cpu_); }++void KernelIo::publish(uintptr_t stream, const void* x, const int32_t* ids, const float* w, int num_tokens, uint64_t task) {+ check(launch_publish(reinterpret_cast<cudaStream_t>(stream), static_cast<const uint16_t*>(x), ids, w,+ static_cast<int64_t>(num_tokens) * hidden_, static_cast<int64_t>(num_tokens) * top_k_, x_, ids_, w_, ready_, done_,+ static_cast<int64_t>(task), has_cpu_),+ "publish launch");+}++void KernelIo::combine(uintptr_t stream, void* out, int num_tokens, bool accumulate) {+ check(launch_combine(reinterpret_cast<cudaStream_t>(stream), static_cast<uint16_t*>(out), y_, has_cpu_,+ static_cast<int64_t>(num_tokens) * hidden_, accumulate),+ "combine launch");+}++// A ringing step, its combine and a skipped step on scratch mapped memory, checking every word they write.+std::string KernelIo::probe(uintptr_t stream_handle) {+ struct alignas(16) Area {+ int64_t ready, done;+ int32_t has_cpu, pad[3];+ uint16_t x[8], io_x[8], io_y[8], out[8];+ int32_t ids[2], io_ids[2];+ float w[2], io_w[2];+ };+ const auto stream = reinterpret_cast<cudaStream_t>(stream_handle);+ void* host = nullptr;+ void* dev = nullptr;+ if (cudaHostAlloc(&host, sizeof(Area), cudaHostAllocPortable | cudaHostAllocMapped) != cudaSuccess)+ return "pinned scratch allocation failed";+ std::string reason;+ auto ok = [&](cudaError_t rc, const char* what) {+ if (rc != cudaSuccess && reason.empty()) reason = std::string(what) + ": " + cudaGetErrorString(rc);+ return rc == cudaSuccess;+ };+ auto fail = [&](const char* why) {+ if (reason.empty()) reason = why;+ };+ if (ok(cudaHostGetDevicePointer(&dev, host, 0), "device address of the scratch")) {+ auto* h = static_cast<Area*>(host);+ auto* d = static_cast<Area*>(dev);+ std::memset(h, 0, sizeof(Area));+ for (int i = 0; i < 8; ++i) {+ h->x[i] = static_cast<uint16_t>(0x3f80 + i);+ h->io_y[i] = 0x3f80; // 1.0+ h->out[i] = 0x4000; // 2.0+ }+ h->ids[0] = 3;+ h->ids[1] = -1;+ h->w[0] = 0.25f;+ h->w[1] = 0.75f;+ auto publish = [&] {+ return launch_publish(stream, d->x, d->ids, d->w, 8, 2, d->io_x, d->io_ids, d->io_w, &d->ready, &d->done, 5, &d->has_cpu);+ };++ if (ok(publish(), "publish launch") && ok(cudaStreamSynchronize(stream), "publish")) {+ if (h->ready != 5 || h->done != 0 || h->has_cpu != 1) fail("publish wrote wrong handshake words");+ if (std::memcmp(h->io_x, h->x, sizeof(h->x)) || std::memcmp(h->io_ids, h->ids, sizeof(h->ids)) ||+ std::memcmp(h->io_w, h->w, sizeof(h->w)))+ fail("publish wrote wrong IO rows");+ }+ if (reason.empty() && ok(launch_combine(stream, d->out, d->io_y, &d->has_cpu, 8, true), "combine launch") &&+ ok(cudaStreamSynchronize(stream), "combine")) {+ for (int i = 0; i < 8; ++i)+ if (h->out[i] != 0x4040) fail("combine returned a wrong sum"); // 3.0+ }+ if (reason.empty()) {+ h->ready = 0;+ h->ids[0] = -1;+ if (ok(publish(), "skip launch") && ok(launch_combine(stream, d->out, d->io_y, &d->has_cpu, 8, false), "skip combine launch") &&+ ok(cudaStreamSynchronize(stream), "skipped step")) {+ bool zeros = true;+ for (int i = 0; i < 8; ++i) zeros = zeros && h->out[i] == 0;+ if (h->ready != 0 || h->done != 1 || h->has_cpu != 0 || !zeros) fail("a skipped step wrote wrong words");+ }+ }+ }+ (void)cudaFreeHost(host);+ return reason;+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_io.h b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_io.hnew file mode 100644index 00000000..6eacb393--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/gpu_sync_io.h@@ -0,0 +1,39 @@+// Kernel IO for the GPU <-> CPU handshake (gpu_sync.h): kernels on the executor's mapped pinned IO rows in place of the copies around submit and wait.+// One source for both runtimes: nvcc builds gpu_sync_io.cu on CUDA, hipcc on ROCm through freetoken/hip_compat.h.+#pragma once++#include <cstdint>+#include <string>++namespace freetoken::cpu_moe {++class KernelIo {+ public:+ // ready, done: device addresses of GpuSync's words. x, ids, w, y: host addresses of the pinned IO rows. Throws if setup fails.+ KernelIo(uintptr_t ready, uintptr_t done, const void* x, const int32_t* ids, const float* w, const void* y, int hidden, int top_k);+ ~KernelIo();+ KernelIo(const KernelIo&) = delete;+ KernelIo& operator=(const KernelIo&) = delete;++ // Copies the step's device inputs into the IO rows and does gpu_submit's writes, or skips the step when no route goes to the CPU.+ // x [n, hidden] bf16, ids [n, top_k] int32 (< 0 = not a CPU route), w [n, top_k] fp32.+ void publish(uintptr_t stream, const void* x, const int32_t* ids, const float* w, int num_tokens, uint64_t task);+ // Follows gpu_wait(): writes the CPU rows to out [n, hidden] bf16, or adds them to it; a skipped step gives zeros.+ void combine(uintptr_t stream, void* out, int num_tokens, bool accumulate);++ // Whether both kernels run and write the right words on this device: "" or the reason.+ static std::string probe(uintptr_t stream);++ private:+ volatile int64_t* ready_;+ volatile int64_t* done_;+ uint16_t* x_;+ int32_t* ids_;+ float* w_;+ const uint16_t* y_;+ int hidden_;+ int top_k_;+ int32_t* has_cpu_ = nullptr; // device word: whether the last published step has CPU routes+};++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/bf16.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/bf16.cppnew file mode 100644index 00000000..273c3946--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/bf16.cpp@@ -0,0 +1,159 @@+#include "../format.h"++namespace freetoken::cpu_moe {+namespace {++// ------------------------------- dot products -------------------------------+// dot(weight[bf16], act[bf16], n) -> fp32. The selected impl is a function+// pointer chosen at runtime; the per-row call overhead is negligible vs n.++using dot_fn = float (*)(const bf16_t*, const bf16_t*, int);++float dot_scalar(const bf16_t* w, const bf16_t* x, int n) {+ float acc = 0.0f;+ for (int i = 0; i < n; ++i) acc += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);+ return acc;+}++#if CPU_MOE_X86+CPU_MOE_TARGET("avx512f")+float dot_avx512f(const bf16_t* w, const bf16_t* x, int n) {+ // 4 independent accumulators -> more in-flight loads (memory-level parallelism),+ // which is what lifts a bandwidth-bound GEMV toward peak.+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();+ int i = 0;+ for (; i + 64 <= n; i += 64) {+ _mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);+ for (int j = 0; j < 64; j += 16) {+ __m256i wi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + i + j));+ __m256i xi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x + i + j));+ __m512 wf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(wi), 16));+ __m512 xf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(xi), 16));+ __m512& acc = (j == 0) ? a0 : (j == 16) ? a1 : (j == 32) ? a2 : a3;+ acc = _mm512_fmadd_ps(wf, xf, acc);+ }+ }+ for (; i + 16 <= n; i += 16) {+ __m256i wi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + i));+ __m256i xi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x + i));+ __m512 wf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(wi), 16));+ __m512 xf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(xi), 16));+ a0 = _mm512_fmadd_ps(wf, xf, a0);+ }+ float s = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3)));+ for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);+ return s;+}++#ifdef CPU_MOE_HAS_AVX512BF16+CPU_MOE_TARGET("avx512bf16,avx512f")+static inline __m512bh load_bh(const bf16_t* p) {+ __m512i raw = _mm512_loadu_si512(reinterpret_cast<const void*>(p));+ __m512bh out;+ std::memcpy(&out, &raw, sizeof(out));+ return out;+}++CPU_MOE_TARGET("avx512bf16,avx512f")+float dot_avx512bf16(const bf16_t* w, const bf16_t* x, int n) {+ // 4 accumulators (128 bf16/iter) for memory-level parallelism + a prefetch nudge.+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();+ int i = 0;+ for (; i + 128 <= n; i += 128) {+ _mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);+ a0 = _mm512_dpbf16_ps(a0, load_bh(w + i), load_bh(x + i));+ a1 = _mm512_dpbf16_ps(a1, load_bh(w + i + 32), load_bh(x + i + 32));+ a2 = _mm512_dpbf16_ps(a2, load_bh(w + i + 64), load_bh(x + i + 64));+ a3 = _mm512_dpbf16_ps(a3, load_bh(w + i + 96), load_bh(x + i + 96));+ }+ for (; i + 32 <= n; i += 32) {+ a0 = _mm512_dpbf16_ps(a0, load_bh(w + i), load_bh(x + i));+ }+ float s = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3)));+ for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);+ return s;+}+#endif // avx512bf16 available++// --------- AVX2 + FMA (256-bit fallback for CPUs without AVX-512) ----------+// Covers Intel 12-14th gen / Arrow Lake (AVX-512 fused off) and AMD Zen<4. The+// bf16->fp32 widen is a zero-extend + <<16; with 4 independent accumulators the+// GEMV is memory-bandwidth bound, same as the AVX-512 path (just half the width).+CPU_MOE_TARGET("avx2,fma")+float dot_avx2(const bf16_t* w, const bf16_t* x, int n) {+ __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();+ __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();+ int i = 0;+ for (; i + 32 <= n; i += 32) {+ _mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);+ for (int j = 0; j < 32; j += 8) {+ __m128i wi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(w + i + j));+ __m128i xi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(x + i + j));+ __m256 wf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(wi), 16));+ __m256 xf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(xi), 16));+ __m256& acc = (j == 0) ? a0 : (j == 8) ? a1 : (j == 16) ? a2 : a3;+ acc = _mm256_fmadd_ps(wf, xf, acc);+ }+ }+ for (; i + 8 <= n; i += 8) {+ __m128i wi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(w + i));+ __m128i xi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(x + i));+ __m256 wf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(wi), 16));+ __m256 xf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(xi), 16));+ a0 = _mm256_fmadd_ps(wf, xf, a0);+ }+ float s = hsum256(_mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3)));+ for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);+ return s;+}+#endif // CPU_MOE_X86++dot_fn select_dot() {+ const IsaTier t = pick_isa();+#if CPU_MOE_X86+#ifdef CPU_MOE_HAS_AVX512BF16+ if (t >= ISA_AVX512BF16) return dot_avx512bf16;+#endif+ if (t >= ISA_AVX512) return dot_avx512f;+ if (t >= ISA_AVX2) return dot_avx2;+#endif+ (void)t;+ return dot_scalar;+}++class Bf16Format final : public MoeFormat {+ public:+ Bf16Format(const MoeParams& p, const FormatArgs& a)+ : MoeFormat(p), dot_(select_dot()), gate_up_(a.bank("gate_up")), down_(a.bank("down")) {}++ std::string isa_name() const override { return tier_name(pick_isa()); }++ void gate_up(int layer, int e, int /*tok*/, int i0, int i1, const bf16_t* x, float w_in,+ bf16_t* g) const override {+ const bf16_t* w = static_cast<const bf16_t*>(tbl_at(gate_up_, layer)) + (size_t)e * (2 * p.I) * p.H;+ gated_rows(p, i0, i1, w_in, g, [&](int row) { return dot_(w + (size_t)row * p.H, x, p.H); });+ }++ void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {+ const bf16_t* w = static_cast<const bf16_t*>(tbl_at(down_, layer));+ for (int h = h0; h < h1; ++h)+ for (int j = 0; j < n; ++j)+ out[j * kRowBlock + h - h0] = dot_(w + ((size_t)routes[j].e * p.H + h) * p.I, routes[j].g, p.I);+ }++ private:+ dot_fn dot_;+ // bf16 rows: gate_up [E, 2I, H], down [E, H, I].+ const uint64_t* gate_up_;+ const uint64_t* down_;+};++} // namespace++std::unique_ptr<MoeFormat> make_bf16_format(const MoeParams& p, const FormatArgs& a) {+ return std::make_unique<Bf16Format>(p, a);+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/ds_fp4.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/ds_fp4.cppnew file mode 100644index 00000000..a7afe980--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/ds_fp4.cpp@@ -0,0 +1,269 @@+#include <vector>++#include "../format.h"++namespace freetoken::cpu_moe {+namespace {++// ----------------------- DeepSeek-V4 ds_fp4 (W4A8) ---------------------------+// Row-major e2m1 (2/byte, low nibble first) + e8m0 per-32 block scale, no global+// (w = E2M1[code] * 2^(e8m0-127)); activations are FP8-e4m3 round-tripped (per-128+// block, ue8m0 scale) before each GEMM. Matches kernel/triton/dsv4 (fused_moe ++// fp8_linear): silu(clamp(gate,max=lim)) * clamp(up,-lim,lim), router weight on the+// down output.++// Activations are pre-deinterleaved to fp32 (xe[m]=x[2m], xo[m]=x[2m+1]) once per+// token/route and reused across all output rows. This drops the hot dot to one+// vpmovzxbd + two vpermps per 32 weights (1.5 shuffle ops / 16 vs 4 for a bf16,+// dup-permute, per-element gather), so the row-major fp4 GEMV stops being port-5+// bound and approaches the bf16 memory-bandwidth ceiling.+using dsdot_fn = float (*)(const uint8_t*, const uint8_t*, const float*, const float*, int,+ const float*, const float*);++float dot_dsfp4_scalar(const uint8_t* packed, const uint8_t* scale, const float* xe,+ const float* xo, int K, const float* e2m1, const float* e8m0) {+ float acc = 0.0f;+ const int nb = K / 32;+ for (int b = 0; b < nb; ++b) {+ const float sc = e8m0[scale[b]];+ const uint8_t* pk = packed + (size_t)b * 16; // 16 bytes = 32 codes+ const float* xeb = xe + (size_t)b * 16;+ const float* xob = xo + (size_t)b * 16;+ float bsum = 0.0f;+ for (int j = 0; j < 16; ++j) {+ const uint8_t byte = pk[j];+ bsum += e2m1[byte & 0xF] * xeb[j]; // low nibble -> even-K activation+ bsum += e2m1[byte >> 4] * xob[j]; // high nibble -> odd-K activation+ }+ acc += sc * bsum;+ }+ return acc;+}++#if CPU_MOE_X86+// One 32-block: 16 bytes -> 16 low + 16 high nibble values via two vpermps, times+// the pre-split even/odd fp32 activations, folded by the per-32 e8m0 scale.+CPU_MOE_TARGET("avx512f")+inline __m512 dsfp4_blk(const uint8_t* pk, const float* xeb, const float* xob, __m512 lut,+ __m512i loma, float sc) {+ __m512i wi = _mm512_cvtepu8_epi32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(pk)));+ __m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);+ __m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);+ __m512 prod = _mm512_fmadd_ps(vlo, _mm512_loadu_ps(xeb), _mm512_mul_ps(vhi, _mm512_loadu_ps(xob)));+ return _mm512_mul_ps(prod, _mm512_set1_ps(sc));+}++CPU_MOE_TARGET("avx512f")+float dot_dsfp4_avx512(const uint8_t* packed, const uint8_t* scale, const float* xe,+ const float* xo, int K, const float* e2m1, const float* e8m0) {+ const __m512 lut = _mm512_loadu_ps(e2m1);+ const __m512i loma = _mm512_set1_epi32(0xF);+ __m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();+ const int nb = K / 32; // 16 packed bytes + one e8m0 scale per block+ int b = 0;+ for (; b + 2 <= nb; b += 2) { // two independent accumulators hide FMA latency+ acc0 = _mm512_add_ps(acc0, dsfp4_blk(packed + (size_t)b * 16, xe + (size_t)b * 16,+ xo + (size_t)b * 16, lut, loma, e8m0[scale[b]]));+ acc1 = _mm512_add_ps(acc1, dsfp4_blk(packed + (size_t)(b + 1) * 16, xe + (size_t)(b + 1) * 16,+ xo + (size_t)(b + 1) * 16, lut, loma, e8m0[scale[b + 1]]));+ }+ for (; b < nb; ++b)+ acc0 = _mm512_add_ps(acc0, dsfp4_blk(packed + (size_t)b * 16, xe + (size_t)b * 16,+ xo + (size_t)b * 16, lut, loma, e8m0[scale[b]]));+ return _mm512_reduce_add_ps(_mm512_add_ps(acc0, acc1));+}++// AVX2: a 32-K block is 16 bytes -> two 8-lane halves (8 even + 8 odd each).+CPU_MOE_TARGET("avx2,fma")+inline __m256 dsfp4_half_avx2(const uint8_t* pk, const float* xeb, const float* xob, __m256 mag8) {+ __m256i wi = _mm256_cvtepu8_epi32(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(pk)));+ __m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);+ __m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);+ return _mm256_fmadd_ps(vlo, _mm256_loadu_ps(xeb), _mm256_mul_ps(vhi, _mm256_loadu_ps(xob)));+}++CPU_MOE_TARGET("avx2,fma")+float dot_dsfp4_avx2(const uint8_t* packed, const uint8_t* scale, const float* xe,+ const float* xo, int K, const float* e2m1, const float* e8m0) {+ const __m256 mag8 = _mm256_loadu_ps(e2m1);+ __m256 acc0 = _mm256_setzero_ps(), acc1 = _mm256_setzero_ps();+ const int nb = K / 32;+ for (int b = 0; b < nb; ++b) {+ const uint8_t* pk = packed + (size_t)b * 16;+ const float* xeb = xe + (size_t)b * 16;+ const float* xob = xo + (size_t)b * 16;+ const __m256 sc = _mm256_set1_ps(e8m0[scale[b]]);+ acc0 = _mm256_fmadd_ps(dsfp4_half_avx2(pk, xeb, xob, mag8), sc, acc0);+ acc1 = _mm256_fmadd_ps(dsfp4_half_avx2(pk + 8, xeb + 8, xob + 8, mag8), sc, acc1);+ }+ return hsum256(_mm256_add_ps(acc0, acc1));+}+#endif++dsdot_fn select_dsdot() {+ const IsaTier t = pick_isa();+#if CPU_MOE_X86+ if (t >= ISA_AVX512) return dot_dsfp4_avx512;+ if (t >= ISA_AVX2) return dot_dsfp4_avx2;+#endif+ (void)t;+ return dot_dsfp4_scalar;+}++// Round a clamped |x|<=448 to nearest float8-e4m3 (RNE), back to fp32. Matches+// torch.float8_e4m3fn / triton .to(float8e4nv).+inline float e4m3_round(float x) {+ const float sign = x < 0.0f ? -1.0f : 1.0f;+ const float a = std::fabs(x);+ if (a == 0.0f) return 0.0f;+ if (a >= 448.0f) return sign * 448.0f;+ int e;+ std::frexp(a, &e); // a in [2^(e-1), 2^e)+ float step = std::ldexp(1.0f, e - 4);+ const float min_step = std::ldexp(1.0f, -9); // e4m3 subnormal step (2^-9)+ if (step < min_step) step = min_step;+ float r = std::nearbyint(a / step) * step;+ if (r > 448.0f) r = 448.0f;+ return sign * r;+}++// IEEE ceil(log2(v)) for v>0 (matches dsv4 _log2_ceil / fast_round_scale).+inline int ceil_log2_pos(float v) {+ uint32_t bits;+ std::memcpy(&bits, &v, sizeof(bits));+ const int exp = (int)((bits >> 23) & 0xFF);+ const int man = (int)(bits & 0x7FFFFF);+ return exp - 127 + (man != 0 ? 1 : 0);+}++// DeepSeek-V4 / V4.1 activation FP8 round-trip (bf16 in/out): per `block` (the checkpoint's+// fp8 block: 128 on V4, 32 on V4.1), s = 2^ceil(log2(max(|x|,1e-4)/448));+// y = round_e4m3(clamp(x/s,+-448)) * s.+void fp8_roundtrip_bf16(const bf16_t* src, bf16_t* dst, int K, int block) {+ for (int b0 = 0; b0 < K; b0 += block) {+ const int b1 = std::min(K, b0 + block);+ float amax = 1e-4f;+ for (int i = b0; i < b1; ++i) amax = std::max(amax, std::fabs(bf16_to_f32(src[i])));+ const float s = std::ldexp(1.0f, ceil_log2_pos(amax * (1.0f / 448.0f)));+ const float inv_s = 1.0f / s;+ for (int i = b0; i < b1; ++i) {+ float q = bf16_to_f32(src[i]) * inv_s;+ q = std::min(448.0f, std::max(-448.0f, q));+ dst[i] = f32_to_bf16(e4m3_round(q) * s);+ }+ }+}++class DsFp4Format final : public MoeFormat {+ public:+ DsFp4Format(const MoeParams& p, const FormatArgs& a) : MoeFormat(p), dot_(select_dsdot()) {+ gate_up_ = a.bank("gate_up");+ gu_scale_ = a.bank("gate_up_scale");+ down_ = a.bank("down");+ dn_scale_ = a.bank("down_scale");+ act_block_ = static_cast<int>(a.option("act_block"));+ input_prequant_ = a.option("input_prequant", 0) != 0;+ const size_t tokens = p.max_tokens, routes = (size_t)p.max_tokens * p.top_k;+ xq_.resize(tokens * p.H);+ xe_.resize(tokens * (p.H / 2));+ xo_.resize(tokens * (p.H / 2));+ ge_.resize(routes * (p.I / 2));+ go_.resize(routes * (p.I / 2));+ for (int i = 0; i < 16; ++i) e2m1_lut_[i] = kE2M1[i];+ for (int i = 0; i < 256; ++i) e8m0_lut_[i] = std::ldexp(1.0f, std::min(i, 254) - 127);+ }++ std::string isa_name() const override { return tier_name(pick_isa()); }+++ // DSV4 FP8-round-trips the input before the gate_up GEMV, unless the GPU already did.+ void prep_input(int tok, const bf16_t* x) override {+ const bf16_t* src = x;+ if (!input_prequant_) {+ bf16_t* xq = xq_.data() + (size_t)tok * p.H;+ fp8_roundtrip_bf16(x, xq, p.H, act_block_);+ src = xq;+ }+ deinterleave_bf16_f32(src, xe_.data() + (size_t)tok * (p.H / 2), xo_.data() + (size_t)tok * (p.H / 2), p.H);+ }++ bool preps_inter() const override { return true; }++ void prep_inter(int64_t r, bf16_t* g) override {+ fp8_roundtrip_bf16(g, g, p.I, act_block_);+ deinterleave_bf16_f32(g, ge_.data() + (size_t)r * (p.I / 2), go_.data() + (size_t)r * (p.I / 2), p.I);+ }++ void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* /*x*/, float /*w_in*/,+ bf16_t* g) const override {+ const int H = p.H, I = p.I;+ const uint8_t* gu_packed_l = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));+ const uint8_t* gu_scale_l = static_cast<const uint8_t*>(tbl_at(gu_scale_, layer));+ const int N2 = 2 * I, Hh = H / 2, Hs = H / 32;+ const float* xe = xe_.data() + (size_t)tok * (H / 2);+ const float* xo = xo_.data() + (size_t)tok * (H / 2);+ const uint8_t* gp = gu_packed_l + (size_t)e * N2 * Hh;+ const uint8_t* gs = gu_scale_l + (size_t)e * N2 * Hs;+ const float lim = p.swiglu_limit;+ for (int i = i0; i < i1; ++i) {+ // gate_up is stored bf16 by the reference GEMV before swiglu; round to match.+ float gate = bf16_to_f32(f32_to_bf16(+ dot_(gp + (size_t)i * Hh, gs + (size_t)i * Hs, xe, xo, H, e2m1_lut_, e8m0_lut_)));+ float up = bf16_to_f32(f32_to_bf16(dot_(+ gp + (size_t)(I + i) * Hh, gs + (size_t)(I + i) * Hs, xe, xo, H, e2m1_lut_, e8m0_lut_)));+ if (lim > 0.0f) {+ if (gate > lim) gate = lim;+ if (up > lim) up = lim;+ else if (up < -lim) up = -lim;+ }+ const float glu = gate / (1.0f + std::exp(-gate)); // silu(gate)+ g[i] = f32_to_bf16(glu * up);+ }+ }++ void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {+ const int H = p.H, I = p.I, Ih = I / 2, Is = I / 32;+ const uint8_t* dn_packed_l = static_cast<const uint8_t*>(tbl_at(down_, layer));+ const uint8_t* dn_scale_l = static_cast<const uint8_t*>(tbl_at(dn_scale_, layer));+ for (int h = h0; h < h1; ++h) {+ for (int j = 0; j < n; ++j) {+ const size_t e = (size_t)routes[j].e, r = (size_t)routes[j].r;+ const uint8_t* dp = dn_packed_l + e * (size_t)H * Ih + (size_t)h * Ih;+ const uint8_t* ds = dn_scale_l + e * (size_t)H * Is + (size_t)h * Is;+ out[j * kRowBlock + h - h0] = dot_(dp, ds, ge_.data() + r * Ih, go_.data() + r * Ih, I, e2m1_lut_, e8m0_lut_);+ }+ }+ }++ // The reference rounds each route's weighted down output to bf16 before the fp32 sum over routes.+ bool rounds_routes() const override { return true; }++ private:+ dsdot_fn dot_;+ // e2m1 rows [E,2I,H/2] with e8m0 scales per 32 along K [E,2I,H/32]; down likewise with K = I.+ const uint64_t* gate_up_ = nullptr;+ const uint64_t* gu_scale_ = nullptr;+ const uint64_t* down_ = nullptr;+ const uint64_t* dn_scale_ = nullptr;+ // Activation FP8 round-trip block, the checkpoint's fp8 block.+ int act_block_ = 128;+ // The caller already FP8-round-tripped the input activations on the GPU+ // (same reference grid), so prep_input must not repeat it on the host-callback+ // thread. That scalar per-element pass is single-threaded ON THE DECODE CRITICAL+ // PATH (~0.3ms/layer at H=4096, every worker and the GPU waiting on it); moving+ // it to a captured GPU elementwise kernel removes it while keeping the official+ // W4A8 numerics bit-exact (see cpu_executor.py).+ bool input_prequant_ = false;+ float e2m1_lut_[16];+ float e8m0_lut_[256];+ AlignedVector<bf16_t> xq_;+ AlignedVector<float> xe_, xo_, ge_, go_;+};++} // namespace++std::unique_ptr<MoeFormat> make_ds_fp4_format(const MoeParams& p, const FormatArgs& a) {+ return std::make_unique<DsFp4Format>(p, a);+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/fp8_block.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/fp8_block.cppnew file mode 100644index 00000000..d27fb186--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/fp8_block.cpp@@ -0,0 +1,203 @@+#include <string>+#include <vector>++#include "../format.h"++namespace freetoken::cpu_moe {+namespace {++// ------------------------ block-FP8 (W8A16) dequant ------------------------+// fp8-e4m3 weights row-major [rows, K] plus one bf16 scale per 128x128 block. The scale tensor is named weight_scale_inv but multiplies, as in kernel/triton/fp8_blockscale_moe.py.+// Whole 128-blocks read the activations as fp32 even/odd halves (xe[m] = x[2m], xo[m] = x[2m+1]); a ragged tail reads the bf16 row x.+// 0x7F/0xFF (NaN in torch float8_e4m3fn) decode to +-480 in every kernel, as e4m3_decode does.+using fp8_block_dot_fn = float (*)(const uint8_t*, const float*, const float*, const bf16_t*,+ const bf16_t*, int, const float*);++constexpr int FP8_BLK = 128; // K elements covered by one weight scale++inline float fp8_block_tail(const uint8_t* w, const bf16_t* x, int k0, int K, const float* e4m3) {+ float blk = 0.0f;+ for (int k = k0; k < K; ++k) blk += e4m3[w[k]] * bf16_to_f32(x[k]);+ return blk;+}++float dot_fp8_block_scalar(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,+ const bf16_t* s, int K, const float* e4m3) {+ float acc = 0.0f;+ int b = 0, k0 = 0;+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {+ // 8 independent sums: a single running sum is bound by the add latency+ float a[8] = {};+ const uint8_t* wb = w + k0;+ const float* pe = xe + k0 / 2;+ const float* po = xo + k0 / 2;+ for (int j = 0; j < FP8_BLK / 2; j += 4)+ for (int l = 0; l < 4; ++l) {+ a[2 * l] += e4m3[wb[2 * (j + l)]] * pe[j + l];+ a[2 * l + 1] += e4m3[wb[2 * (j + l) + 1]] * po[j + l];+ }+ acc += (((a[0] + a[1]) + (a[2] + a[3])) + ((a[4] + a[5]) + (a[6] + a[7]))) *+ bf16_to_f32(s[b]);+ }+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);+ return acc;+}++#if CPU_MOE_X86+// A byte b read as fp16 bits (b & 0x80) << 8 | (b & 0x7F) << 7 is its e4m3 value times 2^-8, subnormals included, so the block scale carries the 256.+// In a 16-bit lane holding bytes (2m, 2m+1), srai 1 then & 0xBF80 builds that for byte 2m+1; a left shift by 8 first does byte 2m. Every AVX2 CPU has F16C.+CPU_MOE_TARGET("avx2,fma,f16c")+float dot_fp8_block_avx2(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,+ const bf16_t* s, int K, const float* e4m3) {+ const __m256i keep = _mm256_set1_epi16((short)0xBF80);+ __m256 total = _mm256_setzero_ps();+ int b = 0, k0 = 0;+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {+ _mm_prefetch(reinterpret_cast<const char*>(w + k0) + PF_AHEAD, _MM_HINT_T0);+ __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();+ __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();+ for (int j = 0; j < FP8_BLK; j += 32) {+ const __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j));+ const __m256i odd = _mm256_and_si256(_mm256_srai_epi16(v, 1), keep);+ const __m256i even = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v, 8), 1), keep);+ const float* pe = xe + (k0 + j) / 2;+ const float* po = xo + (k0 + j) / 2;+ a0 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_castsi256_si128(even)), _mm256_loadu_ps(pe), a0);+ a1 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_extracti128_si256(even, 1)), _mm256_loadu_ps(pe + 8), a1);+ a2 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_castsi256_si128(odd)), _mm256_loadu_ps(po), a2);+ a3 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_extracti128_si256(odd, 1)), _mm256_loadu_ps(po + 8), a3);+ }+ const __m256 blk = _mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3));+ total = _mm256_fmadd_ps(blk, _mm256_set1_ps(bf16_to_f32(s[b]) * 256.0f), total);+ }+ float acc = hsum256(total);+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);+ return acc;+}++CPU_MOE_TARGET("avx512f,avx2,fma,f16c")+float dot_fp8_block_avx512f(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,+ const bf16_t* s, int K, const float* e4m3) {+ const __m256i keep = _mm256_set1_epi16((short)0xBF80);+ __m512 total = _mm512_setzero_ps();+ int b = 0, k0 = 0;+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {+ _mm_prefetch(reinterpret_cast<const char*>(w + k0) + PF_AHEAD, _MM_HINT_T0);+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();+ for (int j = 0; j < FP8_BLK; j += 64) {+ const __m256i v0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j));+ const __m256i v1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j + 32));+ const __m256i o0 = _mm256_and_si256(_mm256_srai_epi16(v0, 1), keep);+ const __m256i e0 = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v0, 8), 1), keep);+ const __m256i o1 = _mm256_and_si256(_mm256_srai_epi16(v1, 1), keep);+ const __m256i e1 = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v1, 8), 1), keep);+ const float* pe = xe + (k0 + j) / 2;+ const float* po = xo + (k0 + j) / 2;+ a0 = _mm512_fmadd_ps(_mm512_cvtph_ps(e0), _mm512_loadu_ps(pe), a0);+ a1 = _mm512_fmadd_ps(_mm512_cvtph_ps(o0), _mm512_loadu_ps(po), a1);+ a2 = _mm512_fmadd_ps(_mm512_cvtph_ps(e1), _mm512_loadu_ps(pe + 16), a2);+ a3 = _mm512_fmadd_ps(_mm512_cvtph_ps(o1), _mm512_loadu_ps(po + 16), a3);+ }+ const __m512 blk = _mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3));+ total = _mm512_fmadd_ps(blk, _mm512_set1_ps(bf16_to_f32(s[b]) * 256.0f), total);+ }+ float acc = _mm512_reduce_add_ps(total);+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);+ return acc;+}+#endif // CPU_MOE_X86++fp8_block_dot_fn select_fp8_block_dot() {+ const IsaTier t = pick_isa();+#if CPU_MOE_X86+ // the avx512bf16 tier keeps the fp32 kernel: on Sapphire Rapids it is faster than widening to bf16 for dpbf16+ if (t >= ISA_AVX512) return dot_fp8_block_avx512f;+ if (t >= ISA_AVX2) return dot_fp8_block_avx2;+#endif+ (void)t;+ return dot_fp8_block_scalar;+}++class Fp8BlockFormat final : public MoeFormat {+ public:+ Fp8BlockFormat(const MoeParams& p, const FormatArgs& a) : MoeFormat(p), dot_(select_fp8_block_dot()) {+ gate_up_ = a.bank("gate_up");+ gu_scale_ = a.bank("gate_up_scale");+ down_ = a.bank("down");+ dn_scale_ = a.bank("down_scale");+ gu_scale_stride_ = a.option("gu_scale_stride");+ dn_scale_stride_ = a.option("dn_scale_stride");+ const size_t tokens = p.max_tokens, routes = (size_t)p.max_tokens * p.top_k;+ xe_.resize(tokens * (p.H / 2));+ xo_.resize(tokens * (p.H / 2));+ ge_.resize(routes * (p.I / 2));+ go_.resize(routes * (p.I / 2));+ for (int i = 0; i < 256; ++i) e4m3_lut_[i] = e4m3_decode((uint8_t)i);+ }++ std::string isa_name() const override { return tier_name(pick_isa()); }+++ void prep_input(int tok, const bf16_t* x) override {+ deinterleave_bf16_f32(x, xe_.data() + (size_t)tok * (p.H / 2), xo_.data() + (size_t)tok * (p.H / 2), p.H);+ }++ bool preps_inter() const override { return true; }++ void prep_inter(int64_t r, bf16_t* g) override {+ deinterleave_bf16_f32(g, ge_.data() + (size_t)r * (p.I / 2), go_.data() + (size_t)r * (p.I / 2), p.I);+ }++ // The halves cover whole 128-blocks; the bf16 row covers a ragged tail.+ void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* x, float w_in,+ bf16_t* g) const override {+ const int H = p.H, I = p.I;+ const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));+ const bf16_t* scale = static_cast<const bf16_t*>(tbl_at(gu_scale_, layer));+ const float* xe = xe_.data() + (size_t)tok * (H / 2);+ const float* xo = xo_.data() + (size_t)tok * (H / 2);+ gated_rows(p, i0, i1, w_in, g, [&](int row) {+ const uint8_t* w = packed + ((size_t)e * (2 * I) + row) * (size_t)H;+ const bf16_t* sc = scale + ((size_t)e * ((2 * I + FP8_BLK - 1) / FP8_BLK) + row / FP8_BLK) *+ (size_t)gu_scale_stride_;+ return dot_(w, xe, xo, x, sc, H, e4m3_lut_);+ });+ }++ void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {+ const int H = p.H, I = p.I;+ const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(down_, layer));+ const bf16_t* scale = static_cast<const bf16_t*>(tbl_at(dn_scale_, layer));+ for (int h = h0; h < h1; ++h) {+ for (int j = 0; j < n; ++j) {+ const size_t e = (size_t)routes[j].e, r = (size_t)routes[j].r;+ const uint8_t* w = packed + (e * H + h) * (size_t)I;+ const bf16_t* sc = scale + (e * ((H + FP8_BLK - 1) / FP8_BLK) + h / FP8_BLK) *+ (size_t)dn_scale_stride_;+ out[j * kRowBlock + h - h0] =+ dot_(w, ge_.data() + r * (I / 2), go_.data() + r * (I / 2), routes[j].g, sc, I, e4m3_lut_);+ }+ }+ }++ private:+ fp8_block_dot_fn dot_;+ // e4m3 rows [E,2I,H] and [E,H,I] with one bf16 scale per 128x128 block.+ const uint64_t* gate_up_ = nullptr;+ const uint64_t* gu_scale_ = nullptr;+ const uint64_t* down_ = nullptr;+ const uint64_t* dn_scale_ = nullptr;+ // Row strides of the scale tensors, including any padding of the host bank layout.+ int64_t gu_scale_stride_ = 0, dn_scale_stride_ = 0;+ float e4m3_lut_[256];+ AlignedVector<float> xe_, xo_, ge_, go_;+};++} // namespace++std::unique_ptr<MoeFormat> make_fp8_block_format(const MoeParams& p, const FormatArgs& a) {+ return std::make_unique<Fp8BlockFormat>(p, a);+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/mxfp4.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/mxfp4.cppnew file mode 100644index 00000000..484d6dc0--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/mxfp4.cpp@@ -0,0 +1,247 @@+#include "../format.h"++namespace freetoken::cpu_moe {+namespace {++// ------------------------- mxfp4 (gpt-oss) GEMV -----------------------------+// Transposed split-K layout: blk[Kpairs, N2] (N innermost), scl[Kpairs/16, N2]+// e8m0 per 32-K. Computes out[c] = sum_kb (E2M1[lo]*x[2kb] + E2M1[hi]*x[2kb+1])+// * 2^(e8m0-127) for a contiguous column tile (blk/scl already offset to col 0 of+// the tile). Vectorized over N (16 columns / __m512), K stays the outer (cache-+// sequential) loop. Used by both gate_up (K=H) and down (K=I).+using mxgemv_fn = void (*)(float*, const uint8_t*, const uint8_t*, const bf16_t*, int, int,+ int, const float*, const float*);++void mxfp4_gemv_scalar(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,+ int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {+ for (int c = 0; c < ncol; ++c) out[c] = 0.0f;+ for (int kb = 0; kb < Kpairs; ++kb) {+ const uint8_t* w = blk + (size_t)kb * N2;+ const uint8_t* s = scl + (size_t)(kb >> 4) * N2;+ const float xl = bf16_to_f32(x[2 * kb]);+ const float xh = bf16_to_f32(x[2 * kb + 1]);+ for (int c = 0; c < ncol; ++c) {+ const uint8_t byte = w[c];+ out[c] += (e2m1[byte & 0xF] * xl + e2m1[byte >> 4] * xh) * e8m0[s[c]];+ }+ }+}++#if CPU_MOE_X86+CPU_MOE_TARGET("avx512f")+void mxfp4_gemv_avx512(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,+ int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {+ (void)e8m0; // e8m0[c]=2^(c-127) computed via bit construction (no gather)+ const __m512 lut = _mm512_loadu_ps(e2m1);+ const __m512i loma = _mm512_set1_epi32(0xF);+ // K-outer / N-inner: each kb cache line is read once and all live column chunks+ // (up to 4 -> 64 cols) accumulate from registers, so DRAM/L2 stream the tile once.+ int c0 = 0;+ for (; c0 + 16 <= ncol; c0 += 64) {+ const int nchunk = std::min(4, (ncol - c0) / 16);+ __m512 acc[4];+ for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm512_setzero_ps();+ for (int kblk = 0; kblk < Kpairs; kblk += 16) { // 16 K-pairs = 32 K = one scale row+ __m512 sc[4];+ for (int ci = 0; ci < nchunk; ++ci) {+ __m128i sraw = _mm_loadu_si128(reinterpret_cast<const __m128i*>(+ scl + (size_t)(kblk >> 4) * N2 + c0 + ci * 16));+ sc[ci] = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu8_epi32(sraw), 23));+ }+ __m512 blk_acc[4];+ for (int ci = 0; ci < nchunk; ++ci) blk_acc[ci] = _mm512_setzero_ps();+ for (int kk = 0; kk < 16; ++kk) {+ const int kb = kblk + kk;+ const uint8_t* wbase = blk + (size_t)kb * N2 + c0;+ // The transposed layout strides K by N2 bytes; prefetch ahead so the strided+ // reads are not exposed to DRAM latency (the HW streamer misses big strides).+ constexpr int PFD = 8;+ if (kb + PFD < Kpairs)+ _mm_prefetch(reinterpret_cast<const char*>(blk + (size_t)(kb + PFD) * N2 + c0),+ _MM_HINT_T0);+ const __m512 xl = _mm512_set1_ps(bf16_to_f32(x[2 * kb]));+ const __m512 xh = _mm512_set1_ps(bf16_to_f32(x[2 * kb + 1]));+ for (int ci = 0; ci < nchunk; ++ci) {+ __m512i wi = _mm512_cvtepu8_epi32(+ _mm_loadu_si128(reinterpret_cast<const __m128i*>(wbase + ci * 16)));+ __m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);+ __m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);+ blk_acc[ci] = _mm512_fmadd_ps(vlo, xl, blk_acc[ci]);+ blk_acc[ci] = _mm512_fmadd_ps(vhi, xh, blk_acc[ci]);+ }+ }+ for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm512_fmadd_ps(blk_acc[ci], sc[ci], acc[ci]);+ }+ for (int ci = 0; ci < nchunk; ++ci) _mm512_storeu_ps(out + c0 + ci * 16, acc[ci]);+ }+ for (int c = c0; c < ncol; ++c) { // tail columns (< 16)+ float o = 0.0f;+ for (int kb = 0; kb < Kpairs; ++kb) {+ const uint8_t byte = blk[(size_t)kb * N2 + c];+ uint32_t bits = (uint32_t)scl[(size_t)(kb >> 4) * N2 + c] << 23;+ float sc;+ std::memcpy(&sc, &bits, 4);+ o += (e2m1[byte & 0xF] * bf16_to_f32(x[2 * kb]) ++ e2m1[byte >> 4] * bf16_to_f32(x[2 * kb + 1])) * sc;+ }+ out[c] = o;+ }+}++CPU_MOE_TARGET("avx2,fma")+void mxfp4_gemv_avx2(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,+ int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {+ (void)e8m0; // e8m0[s]=2^(s-127) built via s<<23 (no gather)+ const __m256 mag8 = _mm256_loadu_ps(e2m1);+ int c0 = 0;+ for (; c0 + 8 <= ncol; c0 += 32) { // up to 4 chunks of 8 = 32 cols+ const int nchunk = std::min(4, (ncol - c0) / 8);+ __m256 acc[4];+ for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm256_setzero_ps();+ for (int kblk = 0; kblk < Kpairs; kblk += 16) { // 16 K-pairs = one scale row+ __m256 sc[4];+ for (int ci = 0; ci < nchunk; ++ci) {+ __m128i sraw = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(+ scl + (size_t)(kblk >> 4) * N2 + c0 + ci * 8));+ sc[ci] = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu8_epi32(sraw), 23));+ }+ __m256 blk_acc[4];+ for (int ci = 0; ci < nchunk; ++ci) blk_acc[ci] = _mm256_setzero_ps();+ for (int kk = 0; kk < 16; ++kk) {+ const int kb = kblk + kk;+ const uint8_t* wbase = blk + (size_t)kb * N2 + c0;+ constexpr int PFD = 8;+ if (kb + PFD < Kpairs)+ _mm_prefetch(reinterpret_cast<const char*>(blk + (size_t)(kb + PFD) * N2 + c0),+ _MM_HINT_T0);+ const __m256 xl = _mm256_set1_ps(bf16_to_f32(x[2 * kb]));+ const __m256 xh = _mm256_set1_ps(bf16_to_f32(x[2 * kb + 1]));+ for (int ci = 0; ci < nchunk; ++ci) {+ __m256i wi = _mm256_cvtepu8_epi32(+ _mm_loadl_epi64(reinterpret_cast<const __m128i*>(wbase + ci * 8)));+ __m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);+ __m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);+ blk_acc[ci] = _mm256_fmadd_ps(vlo, xl, blk_acc[ci]);+ blk_acc[ci] = _mm256_fmadd_ps(vhi, xh, blk_acc[ci]);+ }+ }+ for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm256_fmadd_ps(blk_acc[ci], sc[ci], acc[ci]);+ }+ for (int ci = 0; ci < nchunk; ++ci) _mm256_storeu_ps(out + c0 + ci * 8, acc[ci]);+ }+ for (int c = c0; c < ncol; ++c) { // tail columns (< 8); none when ncol%8==0+ float o = 0.0f;+ for (int kb = 0; kb < Kpairs; ++kb) {+ const uint8_t byte = blk[(size_t)kb * N2 + c];+ uint32_t bits = (uint32_t)scl[(size_t)(kb >> 4) * N2 + c] << 23;+ float sc;+ std::memcpy(&sc, &bits, 4);+ o += (e2m1[byte & 0xF] * bf16_to_f32(x[2 * kb]) ++ e2m1[byte >> 4] * bf16_to_f32(x[2 * kb + 1])) * sc;+ }+ out[c] = o;+ }+}+#endif++mxgemv_fn select_mxgemv() {+ const IsaTier t = pick_isa();+#if CPU_MOE_X86+ if (t >= ISA_AVX512) return mxfp4_gemv_avx512;+ if (t >= ISA_AVX2) return mxfp4_gemv_avx2;+#endif+ (void)t;+ return mxfp4_gemv_scalar;+}++// ----------------------------- mxfp4 (gpt-oss) -----------------------------+// Transposed split-K layout (N innermost), so the GEMV streams K and accumulates+// a contiguous block of N output columns -> cache-line-efficient, no repack, no+// extra host memory. Pass 1 fuses gate_up + clamped-swiglu(+bias); pass 2 adds the+// down bias, and the executor applies the router weight and sums the routes.+//+// Dequant: w = E2M1[code] * 2^(e8m0_scale - 127); two codes per byte (low nibble+// first), one e8m0 scale per 32 contiguous K. Matches kernel/triton/mxfp4_moe.py.+class Mxfp4Format final : public MoeFormat {+ public:+ Mxfp4Format(const MoeParams& p, const FormatArgs& a) : MoeFormat(p), gemv_(select_mxgemv()) {+ gate_up_ = a.bank("gate_up");+ gu_scale_ = a.bank("gate_up_scale");+ gu_bias_ = a.bank("gate_up_bias");+ down_ = a.bank("down");+ dn_scale_ = a.bank("down_scale");+ dn_bias_ = a.bank("down_bias");+ for (int i = 0; i < 16; ++i) e2m1_lut_[i] = kE2M1[i];+ // e8m0 (mxfp4 block scale) = 2^(s-127); the GPU GEMV clamps s to [0,254].+ for (int i = 0; i < 256; ++i) e8m0_lut_[i] = std::ldexp(1.0f, std::min(i, 254) - 127);+ }++ std::string isa_name() const override { return tier_name(pick_isa()); }++ void gate_up(int layer, int e, int /*tok*/, int i0, int i1, const bf16_t* x, float /*w_in*/,+ bf16_t* g) const override {+ const int H = p.H, I = p.I;+ const uint8_t* gu_packed_l = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));+ const uint8_t* gu_scale_l = static_cast<const uint8_t*>(tbl_at(gu_scale_, layer));+ const bf16_t* gu_bias_l = static_cast<const bf16_t*>(tbl_at(gu_bias_, layer));+ const int N2 = 2 * I; // gate_up output width (gate/up interleaved)+ const int Hh = H / 2; // packed-K rows+ const int nunit = i1 - i0; // intermediate units owned by this tile+ const int col0 = 2 * i0; // first gate_up column+ const int ncol = 2 * nunit; // gate_up columns owned by this tile+ const uint8_t* blk_e = gu_packed_l + (size_t)e * Hh * N2;+ const uint8_t* scl_e = gu_scale_l + (size_t)e * (size_t)(H / 32) * N2;+ float gu[2 * kRowBlock];+ gemv_(gu, blk_e + col0, scl_e + col0, x, Hh, N2, ncol, e2m1_lut_, e8m0_lut_);+ const bf16_t* bias_e = gu_bias_l + (size_t)e * N2 + col0;+ const float lim = p.swiglu_limit, alpha = p.swiglu_alpha;+ for (int j = 0; j < nunit; ++j) {+ float gate = gu[2 * j] + bf16_to_f32(bias_e[2 * j]);+ float up = gu[2 * j + 1] + bf16_to_f32(bias_e[2 * j + 1]);+ if (gate > lim) gate = lim;+ if (up > lim) up = lim;+ else if (up < -lim) up = -lim;+ const float glu = gate / (1.0f + std::exp(-gate * alpha)); // gate * sigmoid(alpha*gate)+ g[i0 + j] = f32_to_bf16(glu * (up + 1.0f));+ }+ }++ // The transposed layout already streams a whole column block per call, so the routes go one after another.+ void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {+ const int H = p.H, I = p.I;+ const int Ih = I / 2; // packed-K rows+ const int nh = h1 - h0;+ const uint8_t* dn_packed_l = static_cast<const uint8_t*>(tbl_at(down_, layer));+ const uint8_t* dn_scale_l = static_cast<const uint8_t*>(tbl_at(dn_scale_, layer));+ const bf16_t* dn_bias_l = static_cast<const bf16_t*>(tbl_at(dn_bias_, layer));+ for (int j = 0; j < n; ++j) {+ const size_t e = (size_t)routes[j].e;+ const uint8_t* blk_e = dn_packed_l + e * Ih * H;+ const uint8_t* scl_e = dn_scale_l + e * (size_t)(I / 32) * H;+ float* part = out + j * kRowBlock;+ gemv_(part, blk_e + h0, scl_e + h0, routes[j].g, Ih, H, nh, e2m1_lut_, e8m0_lut_);+ const bf16_t* bias_e = dn_bias_l + e * H + h0;+ for (int c = 0; c < nh; ++c) part[c] += bf16_to_f32(bias_e[c]);+ }+ }++ private:+ mxgemv_fn gemv_;+ // gpt-oss split-K layout, N innermost: blocks [E,H/2,2I], e8m0 scales [E,H/32,2I], bf16 bias [E,2I]; down likewise with K = I, N = H.+ const uint64_t* gate_up_ = nullptr;+ const uint64_t* gu_scale_ = nullptr;+ const uint64_t* gu_bias_ = nullptr;+ const uint64_t* down_ = nullptr;+ const uint64_t* dn_scale_ = nullptr;+ const uint64_t* dn_bias_ = nullptr;+ float e2m1_lut_[16];+ float e8m0_lut_[256];+};++} // namespace++std::unique_ptr<MoeFormat> make_mxfp4_format(const MoeParams& p, const FormatArgs& a) {+ return std::make_unique<Mxfp4Format>(p, a);+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cppnew file mode 100644index 00000000..429955fa--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp@@ -0,0 +1,466 @@+#include <vector>++#include "../format.h"++namespace freetoken::cpu_moe {+namespace {++// --------------------------- NVFP4 (W4A16) dequant ---------------------------+// Weights: e2m1 4-bit codes (2/byte, low nibble first), per-16 block scale in+// fp8-e4m3, per-output-row global scale in fp16. Dequant matches the GPU kernels+// (freetoken/kernel/triton/nvfp4_dequant.py): w = E2M1[code] * e4m3(scale) * global.+// Activations stay bf16 (W4A16); the GEMV dequantizes weights inside the K-loop.+++// e2m1 * 2 as exact int8 (all e2m1 values are multiples of 0.5). Used by the AVX-VNNI+// W4A8 path: nibble -> int8 weight via PSHUFB LUT, then VPDPBUSD against int8 activations;+// the *2 is undone by a 0.5 folded into the final scale. Mirrors ggml's kvalues_mxfp4.+alignas(16) const int8_t kE2M1x2[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};++// Activations pre-deinterleaved to fp32 (xe[m]=x[2m], xo[m]=x[2m+1]); see the+// ds_fp4 dot in ds_fp4.cpp for why (drops the hot loop to ~1.5 shuffle ops / 16 weights).+using nvdot_fn = float (*)(const uint8_t*, const uint8_t*, float, const float*, const float*,+ int, const float*, const float*);++float dot_nvfp4_scalar(const uint8_t* packed, const uint8_t* scale, float global,+ const float* xe, const float* xo, int K, const float* e2m1,+ const float* e4m3) {+ float acc = 0.0f;+ const int nb = K / 16;+ for (int b = 0; b < nb; ++b) {+ const float bs = e4m3[scale[b]];+ const uint8_t* pk = packed + (size_t)b * 8;+ const float* xeb = xe + (size_t)b * 8; // 16 K -> 8 even + 8 odd+ const float* xob = xo + (size_t)b * 8;+ float bsum = 0.0f;+ for (int j = 0; j < 8; ++j) {+ const uint8_t byte = pk[j];+ bsum += e2m1[byte & 0xF] * xeb[j];+ bsum += e2m1[byte >> 4] * xob[j];+ }+ acc += bs * bsum;+ }+ return acc * global;+}++// ---- NVFP4 W4A8 (int8 activations) dot: nibble->int8 via LUT, per-16 act scale ----+// asi8: int8 activations laid out per-16 block as [even(8), odd(8)]; asb[b] = per-block+// activation scale (absmax/127). Result folds the e2m1*2 -> *0.5 into the scale.+using nvi8dot_fn = float (*)(const uint8_t*, const uint8_t*, float, const int8_t*, int,+ const float*, const float*);++[[maybe_unused]] float dot_nvfp4_i8_scalar(const uint8_t* packed, const uint8_t* scale,+ float global, const int8_t* asi8, int K, const float* e4m3,+ const float* asb) {+ float acc = 0.0f;+ const int nb = K / 16;+ for (int b = 0; b < nb; ++b) {+ const uint8_t* pk = packed + (size_t)b * 8;+ const int8_t* ae = asi8 + (size_t)b * 16; // even(8)+ const int8_t* ao = ae + 8; // odd(8)+ int isum = 0;+ for (int j = 0; j < 8; ++j) {+ isum += (int)kE2M1x2[pk[j] & 0xF] * (int)ae[j];+ isum += (int)kE2M1x2[pk[j] >> 4] * (int)ao[j];+ }+ acc += (e4m3[scale[b]] * asb[b]) * (float)isum;+ }+ return acc * (0.5f * global);+}++#if CPU_MOE_X86+// Two 16-K blocks (16 packed bytes) per iter: lo nibbles -> even-K, hi -> odd-K,+// gathered via two vpermps. The per-16 e4m3 scale differs across the two blocks, so+// it is applied per lane (low 8 lanes = block b, high 8 = block b+1).+CPU_MOE_TARGET("avx512f")+inline __m512 nvfp4_blk2(const uint8_t* pk, const float* xeb, const float* xob, __m512 lut,+ __m512i loma, float s0, float s1) {+ __m512i wi = _mm512_cvtepu8_epi32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(pk)));+ __m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);+ __m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);+ __m512 prod = _mm512_fmadd_ps(vlo, _mm512_loadu_ps(xeb), _mm512_mul_ps(vhi, _mm512_loadu_ps(xob)));+ // lanes 0-7 (block b) -> s0, lanes 8-15 (block b+1) -> s1 (pure AVX512F mask move)+ __m512 scv = _mm512_mask_mov_ps(_mm512_set1_ps(s0), 0xFF00, _mm512_set1_ps(s1));+ return _mm512_mul_ps(prod, scv);+}++CPU_MOE_TARGET("avx512f")+float dot_nvfp4_avx512(const uint8_t* packed, const uint8_t* scale, float global,+ const float* xe, const float* xo, int K, const float* e2m1,+ const float* e4m3) {+ const __m512 lut = _mm512_loadu_ps(e2m1); // 16 e2m1 values+ const __m512i loma = _mm512_set1_epi32(0xF);+ __m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();+ const int nb = K / 16; // 8 packed bytes + one e4m3 scale per block+ int b = 0;+ for (; b + 4 <= nb; b += 4) { // two blk2 calls -> 4 blocks, 2 accumulators+ acc0 = _mm512_add_ps(acc0, nvfp4_blk2(packed + (size_t)b * 8, xe + (size_t)b * 8,+ xo + (size_t)b * 8, lut, loma, e4m3[scale[b]], e4m3[scale[b + 1]]));+ acc1 = _mm512_add_ps(acc1, nvfp4_blk2(packed + (size_t)(b + 2) * 8, xe + (size_t)(b + 2) * 8,+ xo + (size_t)(b + 2) * 8, lut, loma, e4m3[scale[b + 2]], e4m3[scale[b + 3]]));+ }+ for (; b + 2 <= nb; b += 2)+ acc0 = _mm512_add_ps(acc0, nvfp4_blk2(packed + (size_t)b * 8, xe + (size_t)b * 8,+ xo + (size_t)b * 8, lut, loma, e4m3[scale[b]], e4m3[scale[b + 1]]));+ float s = _mm512_reduce_add_ps(_mm512_add_ps(acc0, acc1));+ for (; b < nb; ++b) { // odd final 16-K block+ const uint8_t* pk = packed + (size_t)b * 8;+ const float* xeb = xe + (size_t)b * 8;+ const float* xob = xo + (size_t)b * 8;+ float bsum = 0.0f;+ for (int j = 0; j < 8; ++j) {+ bsum += e2m1[pk[j] & 0xF] * xeb[j] + e2m1[pk[j] >> 4] * xob[j];+ }+ s += e4m3[scale[b]] * bsum;+ }+ return s * global;+}++// AVX2: one 16-K block (8 packed bytes) per call, 8 even + 8 odd lanes.+CPU_MOE_TARGET("avx2,fma")+inline __m256 nvfp4_blk_avx2(const uint8_t* pk, const float* xeb, const float* xob,+ __m256 mag8, float sc) {+ __m256i wi = _mm256_cvtepu8_epi32(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(pk)));+ __m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);+ __m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);+ __m256 prod = _mm256_fmadd_ps(vlo, _mm256_loadu_ps(xeb), _mm256_mul_ps(vhi, _mm256_loadu_ps(xob)));+ return _mm256_mul_ps(prod, _mm256_set1_ps(sc));+}++CPU_MOE_TARGET("avx2,fma")+float dot_nvfp4_avx2(const uint8_t* packed, const uint8_t* scale, float global,+ const float* xe, const float* xo, int K, const float* e2m1,+ const float* e4m3) {+ const __m256 mag8 = _mm256_loadu_ps(e2m1); // e2m1[0..7] magnitudes+ __m256 acc0 = _mm256_setzero_ps(), acc1 = _mm256_setzero_ps();+ const int nb = K / 16;+ int b = 0;+ for (; b + 2 <= nb; b += 2) {+ acc0 = _mm256_add_ps(acc0, nvfp4_blk_avx2(packed + (size_t)b * 8, xe + (size_t)b * 8,+ xo + (size_t)b * 8, mag8, e4m3[scale[b]]));+ acc1 = _mm256_add_ps(acc1, nvfp4_blk_avx2(packed + (size_t)(b + 1) * 8, xe + (size_t)(b + 1) * 8,+ xo + (size_t)(b + 1) * 8, mag8, e4m3[scale[b + 1]]));+ }+ for (; b < nb; ++b)+ acc0 = _mm256_add_ps(acc0, nvfp4_blk_avx2(packed + (size_t)b * 8, xe + (size_t)b * 8,+ xo + (size_t)b * 8, mag8, e4m3[scale[b]]));+ return hsum256(_mm256_add_ps(acc0, acc1)) * global;+}++// AVX-VNNI W4A8: decode 8 packed bytes (16 nibbles) of one 16-block to int8 [lo(8),hi(8)]+// via PSHUFB against the e2m1*2 LUT (replaces the 2 vpermps fp32 expands -- ~4x less+// port-5 traffic). lo=even-K weights, hi=odd-K, matching the [even(8),odd(8)] act layout.+CPU_MOE_TARGET("avx2,avxvnni,fma")+inline __m128i nvfp4_decode_block_i8(const uint8_t* pk, __m128i lut) {+ __m128i b = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(pk)); // 8 bytes+ __m128i lo = _mm_and_si128(b, _mm_set1_epi8(0x0F));+ __m128i hi = _mm_and_si128(_mm_srli_epi16(b, 4), _mm_set1_epi8(0x0F));+ return _mm_shuffle_epi8(lut, _mm_unpacklo_epi64(lo, hi)); // [lo(8),hi(8)] -> int8+}++// Two 16-blocks per VPDPBUSD (32 int8). Sign trick (ggml mul_add_epi8): |w|*(sign(w)*a)=w*a,+// so u8*s8 needs no offset/correction term. Per-block scale (e4m3 * act-scale) folded in fp32+// (lanes 0-3 -> block b, 4-7 -> block b+1). Bit-faithful weight; only the int8 activation quant+// (W4A8) differs from the bf16 reference.+CPU_MOE_TARGET("avx2,avxvnni,fma")+float dot_nvfp4_i8_vnni(const uint8_t* packed, const uint8_t* scale, float global,+ const int8_t* asi8, int K, const float* e4m3, const float* asb) {+ const __m128i lut = _mm_loadu_si128(reinterpret_cast<const __m128i*>(kE2M1x2));+ __m256 accF = _mm256_setzero_ps();+ const int nb = K / 16;+ int b = 0;+ for (; b + 2 <= nb; b += 2) {+ __m128i wb = nvfp4_decode_block_i8(packed + (size_t)b * 8, lut);+ __m128i wb1 = nvfp4_decode_block_i8(packed + (size_t)(b + 1) * 8, lut);+ __m256i w = _mm256_set_m128i(wb1, wb); // [blk b | blk b+1]+ __m256i a = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(asi8 + (size_t)b * 16));+ __m256i aw = _mm256_sign_epi8(w, w); // |w| (u8 operand)+ __m256i sa = _mm256_sign_epi8(a, w); // sign(w)*a (s8 operand)+ __m256i di = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), aw, sa);+ __m256 scv = _mm256_blend_ps(_mm256_set1_ps(e4m3[scale[b]] * asb[b]),+ _mm256_set1_ps(e4m3[scale[b + 1]] * asb[b + 1]), 0xF0);+ accF = _mm256_fmadd_ps(_mm256_cvtepi32_ps(di), scv, accF);+ }+ float s = hsum256(accF);+ for (; b < nb; ++b) { // tail (odd block count)+ const uint8_t* pk = packed + (size_t)b * 8;+ const int8_t* ae = asi8 + (size_t)b * 16; const int8_t* ao = ae + 8;+ int isum = 0;+ for (int j = 0; j < 8; ++j)+ isum += (int)kE2M1x2[pk[j] & 0xF] * (int)ae[j] + (int)kE2M1x2[pk[j] >> 4] * (int)ao[j];+ s += (e4m3[scale[b]] * asb[b]) * (float)isum;+ }+ return s * (0.5f * global);+}++#ifdef CPU_MOE_HAS_AVX512VNNI++// Software-prefetch distance for the W4A8 weight stream, in 16-K blocks (8 packed+// bytes each). Returns -1 when FREETOKEN_CPU_MOE_PF_BLOCKS is unset: the kernel then+// uses the built-in default min(512 blocks = 4 KB, 2 rows) -- 4 KB is the empirical+// optimum on large-row machines (Emerald Rapids sweep), while the 2-row cap keeps a small-row+// model's overshoot bounded (the executor works in 32-row tiles, so a fixed byte+// distance otherwise prefetches another worker's tile: duplicated DRAM traffic that+// regresses at the bandwidth ceiling). An EXPLICIT env value is honored verbatim+// (no clamp; 0 disables): the per-machine optimum can sit past the safe default (+20%+// at 4 KB on a 24-thread Ice Lake with 256B rows), so the escape hatch must reach it.+// Prefetch never faults, so overshooting a row/bank tail is safe.+static int nvfp4_pf_blocks() {+ static const int v = [] {+ const char* s = getenv("FREETOKEN_CPU_MOE_PF_BLOCKS");+ return (s && s[0]) ? atoi(s) : -1;+ }();+ return v;+}++// AVX-512 VNNI W4A8: FOUR 16-K blocks per VPDPBUSD (64 int8) -- 2x the AVX-VNNI+// (256-bit) path. Decode 32 packed bytes -> 64 int8 with a single _mm512_shuffle_epi8+// (e2m1*2 LUT replicated to all 4 128-bit lanes). AVX-512 has no _mm512_sign_epi8, so+// the u8*s8 sign trick (|w| as u8, sign(w)*a as s8) uses abs_epi8 + a masked negate.+// Bit-faithful weight; only the int8 activation quant (W4A8) differs from bf16.+// One 4-block group (64 int8) -> scaled fp32 partial (16 lanes). Isolated as a helper so+// the caller can run several independent chains into separate accumulators (the decode ->+// dpbusd -> scale chain is long, so a single accumulator leaves the core latency-bound).+CPU_MOE_TARGET("avx512f,avx512bw,avx512vnni,avx2")+static inline __m512 nvfp4_i8_grp4(const uint8_t* packed, const uint8_t* scale,+ const int8_t* asi8, const float* e4m3, const float* asb,+ int b, __m512i lut, __m512i idx, __m512i mask0F,+ __m512i idxsc) {+ const __mmask64 hi_half = 0xFF00FF00FF00FF00ULL; // bytes 8-15 of each 128b lane -> hi nibbles+ __m256i raw = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(packed + (size_t)b * 8));+ __m512i src = _mm512_permutexvar_epi64(idx, _mm512_castsi256_si512(raw));+ __m512i lo = _mm512_and_si512(src, mask0F);+ __m512i hi = _mm512_and_si512(_mm512_srli_epi16(src, 4), mask0F);+ __m512i comb = _mm512_mask_blend_epi8(hi_half, lo, hi);+ __m512i w = _mm512_shuffle_epi8(lut, comb); // 64 int8 weights (e2m1*2)+ // u8*s8 sign trick without _mm512_sign_epi8: aw=|w|, sa = (w<0 ? -a : a) (a moot at w==0)+ __m512i a = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(asi8 + (size_t)b * 16));+ __m512i aw = _mm512_abs_epi8(w);+ __mmask64 neg = _mm512_movepi8_mask(w);+ __m512i sa = _mm512_mask_sub_epi8(a, neg, _mm512_setzero_si512(), a);+ __m512i di = _mm512_dpbusd_epi32(_mm512_setzero_si512(), aw, sa); // 16 int32 (groups of 4)+ // int32[0..3]->blk b, [4..7]->b+1, [8..11]->b+2, [12..15]->b+3. The 4 block scales+ // (e4m3 LUT x per-block act scale) are computed vectorized: a 4-byte load + epu8->epi32+ // widen + one 4-lane LUT gather + one mul replaces 8 scalar loads + 4 scalar muls ++ // a set_ps assembly, which otherwise dominates the per-group op count.+ int sc_raw;+ memcpy(&sc_raw, scale + b, 4);+ __m128i sc4 = _mm_cvtepu8_epi32(_mm_cvtsi32_si128(sc_raw));+ __m128 s4 = _mm_mul_ps(_mm_i32gather_ps(e4m3, sc4, 4), _mm_loadu_ps(asb + b));+ __m512 scv = _mm512_permutexvar_ps(idxsc, _mm512_castps128_ps512(s4));+ return _mm512_mul_ps(_mm512_cvtepi32_ps(di), scv);+}++CPU_MOE_TARGET("avx512f,avx512bw,avx512vnni,avx2")+float dot_nvfp4_i8_avx512vnni(const uint8_t* packed, const uint8_t* scale, float global,+ const int8_t* asi8, int K, const float* e4m3, const float* asb) {+ const __m512i lut = _mm512_broadcast_i32x4(+ _mm_loadu_si128(reinterpret_cast<const __m128i*>(kE2M1x2)));+ const __m512i idx = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0); // block i -> 128b lane i+ const __m512i mask0F = _mm512_set1_epi8(0x0F);+ const __m512i idxsc = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ __m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();+ __m512 acc2 = _mm512_setzero_ps(), acc3 = _mm512_setzero_ps();+ const int nb = K / 16;+ // Built-in default: min(4 KB, 2 rows) -- see nvfp4_pf_blocks(). An explicitly set+ // env value is used verbatim so a per-machine sweep can reach operating points past+ // the conservative default.+ const int pfb = nvfp4_pf_blocks();+ const int pf = (pfb < 0) ? std::min(512, 2 * nb) : pfb;+ int b = 0;+ // Four independent 4-block groups per iter -> four accumulator chains hide the ~10-op+ // decode->dpbusd->scale latency (1-2 chains leave the loop latency-bound: the per-core+ // rate sat at ~60% of the core's achievable DRAM stream rate).+ for (; b + 16 <= nb; b += 16) {+ // Prefetch the weight stream ahead: the interleaved decode lowers the L1-miss+ // concurrency the HW prefetcher sustains on its own (134 -> 167 GB/s on Emerald Rapids).+ if (pf > 0) {+ _mm_prefetch(reinterpret_cast<const char*>(packed + ((size_t)b + (size_t)pf) * 8),+ _MM_HINT_T0);+ _mm_prefetch(reinterpret_cast<const char*>(packed + ((size_t)b + (size_t)pf) * 8 + 64),+ _MM_HINT_T0);+ }+ acc0 = _mm512_add_ps(acc0, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b, lut, idx,+ mask0F, idxsc));+ acc1 = _mm512_add_ps(acc1, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b + 4, lut, idx,+ mask0F, idxsc));+ acc2 = _mm512_add_ps(acc2, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b + 8, lut, idx,+ mask0F, idxsc));+ acc3 = _mm512_add_ps(acc3, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b + 12, lut, idx,+ mask0F, idxsc));+ }+ for (; b + 4 <= nb; b += 4)+ acc0 = _mm512_add_ps(acc0, nvfp4_i8_grp4(packed, scale, asi8, e4m3, asb, b, lut, idx,+ mask0F, idxsc));+ float s = _mm512_reduce_add_ps(+ _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)));+ for (; b < nb; ++b) { // tail (<4 remaining 16-K blocks)+ const uint8_t* pk = packed + (size_t)b * 8;+ const int8_t* ae = asi8 + (size_t)b * 16; const int8_t* ao = ae + 8;+ int isum = 0;+ for (int j = 0; j < 8; ++j)+ isum += (int)kE2M1x2[pk[j] & 0xF] * (int)ae[j] + (int)kE2M1x2[pk[j] >> 4] * (int)ao[j];+ s += (e4m3[scale[b]] * asb[b]) * (float)isum;+ }+ return s * (0.5f * global);+}+#endif // avx512vnni available+#endif++nvdot_fn select_nvdot() {+ const IsaTier t = pick_isa();+#if CPU_MOE_X86+ if (t >= ISA_AVX512) return dot_nvfp4_avx512;+ if (t >= ISA_AVX2) return dot_nvfp4_avx2;+#endif+ (void)t;+ return dot_nvfp4_scalar;+}++// Best W4A8 (int8-activation) nvfp4 dot, or nullptr if no SIMD VNNI (caller keeps the+// faithful fp32 nvdot path). The scalar i8 dot exists only as a correctness reference.+nvi8dot_fn select_nvi8dot() {+#if CPU_MOE_X86+#if defined(CPU_MOE_HAS_AVX512VNNI)+ if (cpu_has_avx512vnni()) return dot_nvfp4_i8_avx512vnni;+#endif+ if (cpu_has_avxvnni()) return dot_nvfp4_i8_vnni;+#endif+ return nullptr;+}++// Quantize the pre-deinterleaved fp32 even/odd activations to per-16-block int8 in the+// [even(8),odd(8)] layout the VNNI dot expects. Done once per token/route (amortized over+// every output row), so a scalar pass is fine relative to the GEMV.+void quant_i8_pg16(const float* xe, const float* xo, int K, int8_t* asi8, float* asb) {+ const int nb = K / 16;+ for (int b = 0; b < nb; ++b) {+ const float* xeb = xe + (size_t)b * 8;+ const float* xob = xo + (size_t)b * 8;+ float amax = 0.0f;+ for (int j = 0; j < 8; ++j)+ amax = std::max(amax, std::max(std::fabs(xeb[j]), std::fabs(xob[j])));+ const float s = amax > 0.0f ? amax / 127.0f : 1.0f;+ asb[b] = s;+ const float inv = 1.0f / s;+ int8_t* ae = asi8 + (size_t)b * 16;+ for (int j = 0; j < 8; ++j) {+ int qe = (int)std::lround(xeb[j] * inv), qo = (int)std::lround(xob[j] * inv);+ ae[j] = (int8_t)std::max(-127, std::min(127, qe));+ ae[8 + j] = (int8_t)std::max(-127, std::min(127, qo));+ }+ }+}++class Nvfp4Format final : public MoeFormat {+ public:+ Nvfp4Format(const MoeParams& p, const FormatArgs& a)+ : MoeFormat(p), nvdot_(select_nvdot()), nvi8dot_(select_nvi8dot()) {+ gate_up_ = a.bank("gate_up");+ gu_scale_ = a.bank("gate_up_scale");+ gu_global_ = a.bank("gate_up_global");+ down_ = a.bank("down");+ dn_scale_ = a.bank("down_scale");+ dn_global_ = a.bank("down_global");+ const size_t tokens = p.max_tokens, routes = (size_t)p.max_tokens * p.top_k;+ xe_.resize(tokens * (p.H / 2));+ xo_.resize(tokens * (p.H / 2));+ ge_.resize(routes * (p.I / 2));+ go_.resize(routes * (p.I / 2));+ if (nvi8dot_ != nullptr) {+ xi8_.resize(tokens * p.H);+ xas_.resize(tokens * (p.H / 16));+ gi8_.resize(routes * p.I);+ gas_.resize(routes * (p.I / 16));+ }+ isa_ = tier_name(pick_isa());+ if (nvi8dot_ != nullptr) isa_ += cpu_has_avx512vnni() ? "+avx512vnni(nvfp4-w4a8)" : "+vnni(nvfp4-w4a8)";+ for (int i = 0; i < 16; ++i) e2m1_lut_[i] = kE2M1[i];+ for (int i = 0; i < 256; ++i) e4m3_lut_[i] = e4m3_decode((uint8_t)i);+ }++ std::string isa_name() const override { return isa_; }+++ void prep_input(int tok, const bf16_t* x) override {+ float* xe = xe_.data() + (size_t)tok * (p.H / 2);+ float* xo = xo_.data() + (size_t)tok * (p.H / 2);+ deinterleave_bf16_f32(x, xe, xo, p.H);+ if (nvi8dot_ != nullptr)+ quant_i8_pg16(xe, xo, p.H, xi8_.data() + (size_t)tok * p.H, xas_.data() + (size_t)tok * (p.H / 16));+ }++ bool preps_inter() const override { return true; }++ void prep_inter(int64_t r, bf16_t* g) override {+ float* ge = ge_.data() + (size_t)r * (p.I / 2);+ float* go = go_.data() + (size_t)r * (p.I / 2);+ deinterleave_bf16_f32(g, ge, go, p.I);+ if (nvi8dot_ != nullptr)+ quant_i8_pg16(ge, go, p.I, gi8_.data() + (size_t)r * p.I, gas_.data() + (size_t)r * (p.I / 16));+ }++ void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* /*x*/, float w_in,+ bf16_t* g) const override {+ const int H = p.H;+ const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));+ const uint8_t* scale = static_cast<const uint8_t*>(tbl_at(gu_scale_, layer));+ const uint16_t* global = static_cast<const uint16_t*>(tbl_at(gu_global_, layer));+ const float* xe = xe_.data() + (size_t)tok * (H / 2);+ const float* xo = xo_.data() + (size_t)tok * (H / 2);+ const int8_t* xi8 = nvi8dot_ != nullptr ? xi8_.data() + (size_t)tok * H : nullptr;+ const float* xas = nvi8dot_ != nullptr ? xas_.data() + (size_t)tok * (H / 16) : nullptr;+ gated_rows(p, i0, i1, w_in, g, [&](int row) {+ const size_t r = (size_t)e * (2 * p.I) + row;+ if (nvi8dot_ != nullptr)+ return nvi8dot_(packed + r * (size_t)(H / 2), scale + r * (size_t)(H / 16),+ fp16_to_f32(global[r]), xi8, H, e4m3_lut_, xas);+ return nvdot_(packed + r * (size_t)(H / 2), scale + r * (size_t)(H / 16),+ fp16_to_f32(global[r]), xe, xo, H, e2m1_lut_, e4m3_lut_);+ });+ }++ void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {+ const int I = p.I;+ const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(down_, layer));+ const uint8_t* scale = static_cast<const uint8_t*>(tbl_at(dn_scale_, layer));+ const uint16_t* global = static_cast<const uint16_t*>(tbl_at(dn_global_, layer));+ for (int h = h0; h < h1; ++h) {+ for (int j = 0; j < n; ++j) {+ const size_t r = (size_t)routes[j].r;+ const size_t row = (size_t)routes[j].e * p.H + h;+ out[j * kRowBlock + h - h0] = nvi8dot_ != nullptr+ ? nvi8dot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), fp16_to_f32(global[row]),+ gi8_.data() + r * I, I, e4m3_lut_, gas_.data() + r * (I / 16))+ : nvdot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), fp16_to_f32(global[row]),+ ge_.data() + r * (I / 2), go_.data() + r * (I / 2), I, e2m1_lut_, e4m3_lut_);+ }+ }+ }++ private:+ nvdot_fn nvdot_;+ nvi8dot_fn nvi8dot_; // AVX-VNNI W4A8 nvfp4 dot (nullptr -> use fp32 nvdot)+ // ModelOpt rows: packed e2m1 [E,2I,H/2], e4m3 scales per 16 along K [E,2I,H/16], fp16 row globals [E,2I]; down likewise with K = I.+ const uint64_t* gate_up_ = nullptr;+ const uint64_t* gu_scale_ = nullptr;+ const uint64_t* gu_global_ = nullptr;+ const uint64_t* down_ = nullptr;+ const uint64_t* dn_scale_ = nullptr;+ const uint64_t* dn_global_ = nullptr;+ std::string isa_;+ float e2m1_lut_[16];+ float e4m3_lut_[256];+ // Activations pre-deinterleaved to fp32 even/odd for the row-major dot; with VNNI also per-16-block int8 + scale.+ AlignedVector<float> xe_, xo_, ge_, go_;+ AlignedVector<int8_t> xi8_, gi8_;+ AlignedVector<float> xas_, gas_;+};++} // namespace++std::unique_ptr<MoeFormat> make_nvfp4_format(const MoeParams& p, const FormatArgs& a) {+ return std::make_unique<Nvfp4Format>(p, a);+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/q4_0.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/q4_0.cppnew file mode 100644index 00000000..600a194e--- /dev/null+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/q4_0.cpp@@ -0,0 +1,216 @@+#include <stdexcept>+#include <vector>++#include "../format.h"++namespace freetoken::cpu_moe {+namespace {++// -------------------------------- Q4_0 (W4A8) --------------------------------+// Native GGUF Q4_0 experts (gemma4 GGUF): per-32 block = fp16 scale d + 16 packed+// bytes; byte j holds element j in its low nibble and j+16 in its high nibble, so a+// block's storage order is [lo0..lo15, hi0..hi15] and w = (nibble - 8) * d. Matches+// the reference dequant (models/gguf/dequant.py) and the packed banks the GPU offload+// path streams.+//+// llama.cpp ggml_vec_dot_q4_0_q8_0: W4A8. The activation is pre-quantized to Q8_0+// (per-32-block int8 ``aq`` + fp32 scale ``asb``); each block unpacks its 16 bytes to+// 32 int8 weights in [-8,7] (bytes_from_nibbles_32: low nibbles -> elems 0..15, high+// -> 16..31) and runs an integer block dot -- VPDPBUSD (AVX-VNNI) or VPMADDUBSW+VPMADDWD+// (AVX2) with the ggml sign trick |w|*(sign(w)*a)=w*a, or a scalar int loop -- then+// scales the block sum by wd*xd in fp32. No fp weight dequant / shuffle chain. The GPU+// offload path (ggml_moe_a8_vec / MMVQ) is also W4A8, so cpu and hybrid stay close.+using q4dot_fn = float (*)(const uint8_t*, const int8_t*, const float*, int);++float q4_0_dot_i8_scalar(const uint8_t* w, const int8_t* aq, const float* asb, int K) {+ float acc = 0.0f;+ const int nb = K / 32;+ for (int b = 0; b < nb; ++b) {+ const uint8_t* blk = w + (size_t)b * 18;+ uint16_t dh;+ std::memcpy(&dh, blk, sizeof(dh));+ const uint8_t* q = blk + 2; // 16 nibble bytes+ const int8_t* a = aq + (size_t)b * 32;+ int isum = 0;+ for (int j = 0; j < 16; ++j) {+ isum += ((int)(q[j] & 0x0F) - 8) * (int)a[j]; // elem j+ isum += ((int)(q[j] >> 4) - 8) * (int)a[16 + j]; // elem 16+j+ }+ acc += fp16_to_f32(dh) * asb[b] * (float)isum;+ }+ return acc;+}++#if CPU_MOE_X86+// fp16 block scale -> fp32 via HW F16C (single value in lane 0).+CPU_MOE_TARGET("f16c")+static inline float q4_scale(uint16_t h) {+ return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128((int)h)));+}++// Unpack one Q4_0 block's 16 bytes -> 32 int8 weights in [-8,7] (elems 0..15 = low+// nibbles, 16..31 = high nibbles). ``eight`` = _mm256_set1_epi8(8).+CPU_MOE_TARGET("avx2")+static inline __m256i q4_unpack32(const uint8_t* blk, __m128i mask, __m256i eight) {+ const __m128i qb = _mm_loadu_si128(reinterpret_cast<const __m128i*>(blk + 2));+ const __m128i lo = _mm_and_si128(qb, mask);+ const __m128i hi = _mm_and_si128(_mm_srli_epi16(qb, 4), mask);+ return _mm256_sub_epi8(_mm256_set_m128i(hi, lo), eight);+}++// AVX2 W4A8 (llama.cpp non-VNNI mul_sum_i8_pairs): integer block dot via VPMADDUBSW ++// VPMADDWD (sign trick), scaled by wd*xd. |aw*sa| pair sums <= 8*127*2 < 32767 -> no+// int16 saturation. This is the fast path on AVX2 CPUs without AVX-VNNI (and the+// avx512-tier fallback, since the block dot is 256-bit either way).+CPU_MOE_TARGET("avx2,fma,f16c")+float q4_0_dot_i8_avx2(const uint8_t* w, const int8_t* aq, const float* asb, int K) {+ const __m128i mask = _mm_set1_epi8(0x0F);+ const __m256i eight = _mm256_set1_epi8(8);+ const __m256i ones16 = _mm256_set1_epi16(1);+ __m256 accF = _mm256_setzero_ps();+ const int nb = K / 32;+ for (int b = 0; b < nb; ++b) {+ const uint8_t* blk = w + (size_t)b * 18;+ _mm_prefetch(reinterpret_cast<const char*>(blk) + 512, _MM_HINT_T0);+ uint16_t dh;+ std::memcpy(&dh, blk, sizeof(dh));+ __m256i wq = q4_unpack32(blk, mask, eight);+ __m256i a = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(aq + (size_t)b * 32));+ __m256i aw = _mm256_sign_epi8(wq, wq); // |wq| (unsigned operand)+ __m256i sa = _mm256_sign_epi8(a, wq); // sign(wq) * a (signed operand)+ __m256i d32 = _mm256_madd_epi16(_mm256_maddubs_epi16(aw, sa), ones16); // 8 int32+ accF = _mm256_fmadd_ps(_mm256_cvtepi32_ps(d32), _mm256_set1_ps(q4_scale(dh) * asb[b]), accF);+ }+ return hsum256(accF);+}++// AVX-VNNI W4A8: one VPDPBUSD per block (the fast path on modern CPUs).+CPU_MOE_TARGET("avx2,avxvnni,fma,f16c")+float q4_0_dot_i8_vnni(const uint8_t* w, const int8_t* aq, const float* asb, int K) {+ const __m128i mask = _mm_set1_epi8(0x0F);+ const __m256i eight = _mm256_set1_epi8(8);+ __m256 accF = _mm256_setzero_ps();+ const int nb = K / 32;+ for (int b = 0; b < nb; ++b) {+ const uint8_t* blk = w + (size_t)b * 18;+ _mm_prefetch(reinterpret_cast<const char*>(blk) + 512, _MM_HINT_T0);+ uint16_t dh;+ std::memcpy(&dh, blk, sizeof(dh));+ __m256i wq = q4_unpack32(blk, mask, eight);+ __m256i a = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(aq + (size_t)b * 32));+ __m256i aw = _mm256_sign_epi8(wq, wq); // |wq| (unsigned operand)+ __m256i sa = _mm256_sign_epi8(a, wq); // sign(wq) * a (signed operand)+ __m256i di = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), aw, sa);+ // All 32 elems of the block share wd*xd; distribute over di's 8 partial sums and+ // reduce at the end (equivalent to scale * block_total).+ accF = _mm256_fmadd_ps(_mm256_cvtepi32_ps(di), _mm256_set1_ps(q4_scale(dh) * asb[b]), accF);+ }+ return hsum256(accF);+}+#endif // CPU_MOE_X86++// All tiers are W4A8 (int8 activations pre-quantized to Q8_0). AVX-VNNI is orthogonal to+// the ISA tier (gated by cpu_has_avxvnni() / FREETOKEN_CPU_MOE_NO_VNNI), so it wins when+// present; otherwise the 256-bit VPMADDUBSW kernel covers both the avx2 and avx512 tiers.+q4dot_fn select_q4dot() {+ const IsaTier t = pick_isa();+#if CPU_MOE_X86+ if (cpu_has_avxvnni()) return q4_0_dot_i8_vnni;+ if (t >= ISA_AVX2) return q4_0_dot_i8_avx2;+#endif+ (void)t;+ return q4_0_dot_i8_scalar;+}++// Quantize a bf16 activation row to Q8_0 (llama.cpp): per-32-block symmetric int8 in+// natural order + one fp32 scale (amax/127) per block. Done once per token/route,+// amortized over every output row that the q4_0 W4A8 GEMV reads. K % 32 == 0.+void quant_q8_0(const bf16_t* x, int K, int8_t* aq, float* asb) {+ const int nb = K / 32;+ for (int b = 0; b < nb; ++b) {+ const bf16_t* xb = x + (size_t)b * 32;+ float xf[32], amax = 0.0f;+ for (int j = 0; j < 32; ++j) {+ xf[j] = bf16_to_f32(xb[j]);+ amax = std::max(amax, std::fabs(xf[j]));+ }+ const float d = amax > 0.0f ? amax / 127.0f : 1.0f;+ asb[b] = d;+ const float inv = amax > 0.0f ? 1.0f / d : 0.0f;+ int8_t* o = aq + (size_t)b * 32;+ for (int j = 0; j < 32; ++j)+ o[j] = (int8_t)std::max(-127, std::min(127, (int)std::lround(xf[j] * inv)));+ }+}++class Q4_0Format final : public MoeFormat {+ public:+ Q4_0Format(const MoeParams& p, const FormatArgs& a) : MoeFormat(p), dot_(select_q4dot()) {+ gate_up_ = a.bank("gate_up");+ down_ = a.bank("down");+ const size_t tokens = p.max_tokens, routes = (size_t)p.max_tokens * p.top_k;+ xi8_.resize(tokens * p.H);+ xas_.resize(tokens * (p.H / 32));+ gi8_.resize(routes * p.I);+ gas_.resize(routes * (p.I / 32));+ if (p.H % 32 != 0 || p.I % 32 != 0)+ throw std::runtime_error("Q4_0 CPU MoE requires H and I to be multiples of 32");+ gu_row_bytes_ = (p.H / 32) * 18; // K = H (gate_up rows)+ dn_row_bytes_ = (p.I / 32) * 18; // K = I (down rows)+ // select_q4dot picks VPDPBUSD / VPMADDUBSW / scalar for the tier, so the tag reflects which of those it resolved to.+ isa_ = std::string(tier_name(pick_isa())) + (cpu_has_avxvnni() ? "+vnni(q4_0-w4a8)" : "+q4_0-w4a8");+ }++ std::string isa_name() const override { return isa_; }+++ void prep_input(int tok, const bf16_t* x) override {+ quant_q8_0(x, p.H, xi8_.data() + (size_t)tok * p.H, xas_.data() + (size_t)tok * (p.H / 32));+ }++ bool preps_inter() const override { return true; }++ void prep_inter(int64_t r, bf16_t* g) override {+ quant_q8_0(g, p.I, gi8_.data() + (size_t)r * p.I, gas_.data() + (size_t)r * (p.I / 32));+ }++ void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* /*x*/, float w_in,+ bf16_t* g) const override {+ const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));+ const int8_t* xi8 = xi8_.data() + (size_t)tok * p.H;+ const float* xas = xas_.data() + (size_t)tok * (p.H / 32);+ gated_rows(p, i0, i1, w_in, g, [&](int row) {+ return dot_(packed + ((size_t)e * (2 * p.I) + row) * (size_t)gu_row_bytes_, xi8, xas, p.H);+ });+ }++ void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {+ const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(down_, layer));+ for (int h = h0; h < h1; ++h) {+ for (int j = 0; j < n; ++j) {+ const size_t r = (size_t)routes[j].r;+ out[j * kRowBlock + h - h0] = dot_(packed + ((size_t)routes[j].e * p.H + h) * (size_t)dn_row_bytes_,+ gi8_.data() + r * p.I, gas_.data() + r * (p.I / 32), p.I);+ }+ }+ }++ private:+ q4dot_fn dot_;+ // GGUF Q4_0 rows of 32-element blocks (fp16 scale + 16 nibble bytes): gate_up [E,2I,H/32*18], down [E,H,I/32*18].+ const uint64_t* gate_up_ = nullptr;+ const uint64_t* down_ = nullptr;+ int gu_row_bytes_ = 0, dn_row_bytes_ = 0;+ std::string isa_;+ // Q8_0 activations: per-32-block int8 + fp32 scale, for the input and the intermediate.+ AlignedVector<int8_t> xi8_, gi8_;+ AlignedVector<float> xas_, gas_;+};++} // namespace++std::unique_ptr<MoeFormat> make_q4_0_format(const MoeParams& p, const FormatArgs& a) {+ return std::make_unique<Q4_0Format>(p, a);+}++} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/include/freetoken/hip_compat.h b/python/freetoken/kernel/csrc/include/freetoken/hip_compat.hindex 1898b411..8abcd1a0 100644--- a/python/freetoken/kernel/csrc/include/freetoken/hip_compat.h+++ b/python/freetoken/kernel/csrc/include/freetoken/hip_compat.h@@ -44,6 +44,18 @@#define cudaGetLastError hipGetLastError#endif+#ifndef cudaMalloc+#define cudaMalloc hipMalloc+#endif++#ifndef cudaFree+#define cudaFree hipFree+#endif++#ifndef cudaMemset+#define cudaMemset hipMemset+#endif+#ifndef cudaMallocHost#define cudaMallocHost hipHostMalloc#endifdiff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.pyindex 4918a032..82f09a9b 100644--- a/python/freetoken/layers/moe.py+++ b/python/freetoken/layers/moe.py@@ -346,8 +346,9 @@ class OffloadMoELayer(MoELayer):alphas=cache.alphas_for_slots(self.layer_id),is_prefill=False,)- cpu_routed = cpu_routed_early if not _HYBRID_OVERLAP else executor.decode_sync(pending)- return gpu_routed + cpu_routed+ if not _HYBRID_OVERLAP:+ return gpu_routed + cpu_routed_early+ return executor.decode_sync(pending, add_to=gpu_routed)def _prefill_routed(self,diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.pyindex e70cad31..9430ab2a 100644--- a/python/freetoken/moe/benchbw.py+++ b/python/freetoken/moe/benchbw.py@@ -490,17 +490,16 @@ def _cpu_moe_step_fns(ex, wl: Workload, E: int):disjoint experts (working set >> LLC) -- the DRAM-bound regime hybrid decode actuallyruns in; ``run()`` executes one step (releases the GIL inside the extension)."""bs, top_k = 1, wl.top_k- io = ex._io_for(bs)+ io = ex._ioio["x"].copy_(torch.randn(bs, wl.hidden, dtype=torch.bfloat16) * 0.1)io["w"].copy_(torch.rand(bs, top_k, dtype=torch.float32))- task = ex._task_for(0, bs)base = torch.arange(top_k, dtype=torch.int32)def set_ids(i: int) -> None:io["ids"].copy_((((i * top_k) + base) % E).view(bs, top_k))def run() -> None:- ex._ext.run_task(task)+ ex._ext.run(0, bs)return set_ids, run@@ -566,7 +565,7 @@ def measure_overlap_bw(fmt: str, wl: Workload, device: torch.device,backend its bandwidth-matched fetch split: fetched : cpu-computed = pcie_ov : cpu_ov.Both sides hammer their own pinned banks flat out for ``seconds`` from a shared- barrier: a worker thread loops bs=1 CPU decode steps (``run_task`` releases the GIL)+ barrier: a worker thread loops bs=1 CPU decode steps (``run`` releases the GIL)while the main thread loops full-layer ``copy_missing`` gathers (synchronized per copyso the DMA is really in flight, not just enqueued). Each side reports bytes / its ownelapsed; the two windows differ by at most one CPU step + one gather.@@ -663,7 +662,7 @@ def _bench_format(fmt: str, wl: Workload, device: torch.device, threshold: floatentry["isa_sweep"] = c["isa_sweep"]# The tier label comes from the bf16 dot chooser, so it can over-report tiers# the actual per-format kernel doesn't distinguish. Disclose the real story.- if fmt == "nvfp4": # WF_NVFP4 is the only use_vnni format+ if fmt == "nvfp4": # nvfp4 is the only use_vnni format_note(entry, "isa tier labels are nominal: this W4A8 kernel rides AVX-VNNI ""regardless of the forced tier")elif fmt in ("mxfp4_triton", "ds_fp4", "fp8_block"):diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex f519687e..bb9ee563 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -1,18 +1,15 @@"""Python wrapper around the ``_cpu_moe`` C++ executor (--moe-strategy cpu).-Owns the persistent CPU worker pool, the per-batch-size pinned IO buffers, and-the per-(layer, batch-size) host-func task descriptors. ``decode`` issues the-whole CUDA-graph-capturable sequence on the current stream:+Owns the persistent CPU worker pool and one pinned IO block of ``max_tokens`` rows that+every layer and batch size shares (one task is in flight at a time). ``decode`` issues+the whole CUDA-graph-capturable sequence on the current stream:- D2H (hidden, topk_ids, topk_weights -> pinned)- -> submit host node (cudaLaunchHostFunc: enqueue MoE task to the pool)- -> sync host node (cudaLaunchHostFunc: spin until the pool drains)+ D2H (hidden, topk_ids, topk_weights -> pinned IO rows [:bs])+ -> submit (doorbell naming (layer, bs), or a cudaLaunchHostFunc host node)+ -> sync (wait for the CPU pool to finish)-> H2D (pinned expert output -> GPU)-Buffers and tasks are allocated lazily per batch size. GraphRunner runs an eager-``model.forward()`` at each batch size immediately before capturing it, so the-first (eager) call materializes the stable pinned buffers + task pointers that-the subsequent capture embeds in its host/memcpy nodes.+With the doorbell on CUDA, a publish kernel does the D2H and the doorbell, and skips the step when no route goes to the CPU; a combine kernel does the H2D (csrc/cpu_moe/gpu_sync_io.h)."""from __future__ import annotations@@ -35,28 +32,27 @@ logger = init_logger(__name__)# on a 75-layer model). Instead the GPU raises a mapped-pinned "ready" flag at submit; a# persistent CPU coordinator (in _cpu_moe) polls it, runs the layer, and sets a "done"# flag the GPU waits on at sync -- no host-func round-trip. Both GPU-side operations are-# STREAM MEMORY OPERATIONS (cuStreamWriteValue64 / cuStreamWaitValue64, resolved from the-# driver at runtime): they execute on the GPU front end with no SM-resident kernel, so+# STREAM MEMORY OPERATIONS (cuStreamWriteValue64 / cuStreamWaitValue64 on CUDA, see+# gpu_sync.h): they execute on the GPU front end with no SM-resident kernel, so# GPU utilization stays truthful during the CPU compute window. (The first cut used a# spin-wait kernel; that pinned reported utilization at 99% and laptop CPU/GPU dynamic# power schedulers responded by clamping the CPU frequency -- a net decode regression on-# power-coupled edge devices.) Each (layer, decode batch size) pair gets its own flag-# slot, so every captured decode graph rides the handshake. Where memops are unavailable-# (Windows WDDM, vGPU, old drivers -- functionally probed at startup) or the slot-# capacity is exceeded, decode keeps the host-func path (functional, slower). A Python-# watchdog thread turns a wedged coordinator into a loud RuntimeError (via err[] +-# raise_if_unhealthy) instead of an indefinite stream stall.+# power-coupled edge devices.) The doorbell word names the (layer, decode batch size)+# task, so one ready/done pair serves every layer and every captured graph (gpu_sync.h).+# Where memops are unavailable (some driver models, vGPU, old drivers -- functionally probed at+# startup), decode keeps the host-func path (functional, slower). ROCm 7.14 cannot replay+# that path from a graph, so capture fails loudly there instead. A Python watchdog thread+# turns a wedged coordinator into a loud RuntimeError (via raise_if_unhealthy) instead of+# an indefinite stream stall.# Caveat: the coordinator busy-polls one core while decode traffic flows (idle backoff# otherwise); FREETOKEN_CPU_MOE_FLAG_SYNC=0 opts out entirely._FLAG_SYNC = os.getenv("FREETOKEN_CPU_MOE_FLAG_SYNC", "1") != "0"-# Flag slots per MoE layer: covers this many distinct decode batch sizes (captured graph-# sizes plus any eager padded sizes); more than that is unheard of, and the overflow-# just keeps the host-func path for the extra combos.-_FLAG_SLOTS_PER_LAYER = 16# Pool threads spin up to 50 ms before they park. FREETOKEN_CPU_MOE_SPIN=0 opts out._SPIN_WAIT = os.getenv("FREETOKEN_CPU_MOE_SPIN", "1") != "0"+# ds_fp4 runs its input FP8 round-trip on the GPU when it can; tests turn this off to compare against the CPU round-trip.+_DSFP4_GPU_PREQUANT = True-# Activation ids must match ActKind in csrc/cpu_moe/cpu_moe_ext.cpp. Id 3 is the+# Activation ids must match ActKind in csrc/cpu_moe/common.h. Id 3 is the# clamped (up + 1) swiglu: "swigluoai" runs it in the generic GEMV epilogue,# "gpt_oss_swiglu" is the same math fused inside the mxfp4 kernel._ACT_IDS = {@@ -70,10 +66,6 @@ _ACT_IDS = {"swiglu_clamp": 4,}-# Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp.-_WFMT_IDS = {"bf16": 0, "nvfp4": 1, "mxfp4_triton": 2, "ds_fp4": 3, "q4_0": 4,- "fp8_block": 5}-def compiled_extension_supports(activation: str) -> bool:"""Whether the compiled ``_cpu_moe`` extension can serve ``activation``@@ -157,7 +149,7 @@ def resolve_threads_and_affinity(requested: int) -> tuple[int, list[int]]:class CpuMoeExecutor:"""Decode-time CPU expert compute over an ``OffloadMoeCache``'s host banks- (bf16, nvfp4, mxfp4_triton, ds_fp4, q4_0 or fp8_block — see ``_WFMT_IDS`` / ``_resolve_banks``)."""+ (the formats ``_cpu_moe.weight_formats()`` lists — see ``_resolve_banks``)."""def __init__(self,@@ -178,26 +170,20 @@ class CpuMoeExecutor:from freetoken.moe.legacy_format import canonical_rolefmt = fmt or cache.quant_format- if fmt not in _WFMT_IDS:+ if fmt not in _cpu_moe.weight_formats():raise NotImplementedError(f"--moe-strategy cpu/hybrid computes experts on the CPU and supports "- f"{sorted(_WFMT_IDS)} formats, but this checkpoint's experts are "+ f"{sorted(_cpu_moe.weight_formats())} formats, but this checkpoint's experts are "f"{fmt!r}; use --moe-strategy offload (GPU-side dequant) instead.")if activation not in _ACT_IDS:raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}")+ if apply_router_weight_on_input and fmt in ("mxfp4_triton", "ds_fp4"):+ raise NotImplementedError(f"CPU MoE backend: {fmt} experts apply the router weight on the output only")# ABI probe: a stale prebuilt _cpu_moe.so accepts newer act ids without# error and silently computes the wrong activation in the generic# epilogue -- fail loudly with the rebuild instruction instead. (mxfp4# handles its act inside the kernel and predates the marker.)- # A prebuilt .so that predates this format would accept the id and then take- # the wrong dequant branch -- silently wrong output rather than a crash.- if _WFMT_IDS[fmt] > getattr(_cpu_moe, "max_weight_format_id", lambda: 4)():- raise RuntimeError(- f"the compiled _cpu_moe extension predates the {fmt!r} expert format; "- "rebuild it with `python setup.py build_ext --inplace` (or reinstall "- "the wheel) before serving this checkpoint on the cpu/hybrid backend."- )if _ACT_IDS[activation] >= 3 and fmt != "mxfp4_triton":supported = getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()if _ACT_IDS[activation] > supported:@@ -218,28 +204,42 @@ class CpuMoeExecutor:# The per-layer tensors and their pointer tables must outlive the executor# (C++ holds raw addresses into both).self._banks: list[torch.Tensor] = []- ptrs, (self.H, self.I) = self._resolve_banks(+ banks, options, (self.H, self.I) = self._resolve_banks({canonical_role(name): per_layer for name, per_layer in cache.bank_sources.items()}, fmt)+ # ds_fp4: the reference FP8 activation round-trip is a scalar per-element chain+ # that the C++ side runs single-threaded on the CUDA host-callback thread --+ # straight on the decode critical path (~0.3ms/layer at H=4096, every worker and+ # the GPU waiting on it). When a GPU is present we run the numerically identical+ # round-trip as a captured GPU kernel BEFORE the D2H (see decode_submit) and tell+ # the C++ side to skip its own. Measured on DeepSeek-V4-Flash bs=1 decode:+ # 12.85 -> 15.65 tok/s, output bit-identical (tests/moe/test_dsfp4_prequant.py).+ self._gpu_prequant = fmt == "ds_fp4" and device.type == "cuda" and _DSFP4_GPU_PREQUANT+ # the W4A8 activation round-trip block follows the checkpoint's fp8 block (128 on V4, 32 on V4.1)+ self._act_block = act_block+ if fmt == "ds_fp4":+ assert act_block is not None, "ds_fp4 experts need act_block, the checkpoint's fp8 activation quant block"+ options.update(act_block=int(act_block), input_prequant=int(self._gpu_prequant))+ if self._gpu_prequant:+ logger.info_rank0(+ "ds_fp4: input FP8 round-trip moved to the GPU "+ "(bit-identical grid; the CPU-side scalar round-trip is skipped)"+ )+# Decide the flag handshake up front (env + device + a functional stream-memop# probe): its coordinator needs a core of its own, which the auto thread sizing# below reserves (a coordinator time-slicing against the GEMV workers measurably# destabilizes throughput on fully-subscribed boxes).self._flag_sync = _FLAG_SYNC and device.type == "cuda"- self._cpu_moe = _cpu_moe # module ref for the decode-path memop calls- if self._flag_sync:- probe_scratch = alloc_pinned_tensor(1, dtype=torch.int64)- probe_scratch.zero_()- if not _cpu_moe.memops_probe(- torch.cuda.current_stream().cuda_stream, probe_scratch.data_ptr()- ):- logger.info_rank0(- "cpu-moe flag handshake unavailable: CUDA stream memory operations "- "are not supported here (Windows WDDM / vGPU / old driver); using "- "the cudaLaunchHostFunc sync"- )- self._flag_sync = False+ if self._flag_sync and not _cpu_moe.memops_probe(torch.cuda.current_stream().cuda_stream):+ logger.info_rank0(+ "cpu-moe flag handshake unavailable: stream memory operations failed the "+ "startup probe; using the cudaLaunchHostFunc sync"+ )+ self._flag_sync = False+ # ROCm 7.14 returns wrong results when it replays host-function nodes, so there only the doorbell is graph-safe.+ self._graph_capture_safe = self._flag_sync or _cpu_moe.host_func_capture_safe()nthreads, core_ids = resolve_threads_and_affinity(num_threads)coord_core = -1@@ -250,6 +250,17 @@ class CpuMoeExecutor:nthreads -= 1core_ids = core_ids[:-1]self._coord_core = coord_core+ alloc_io = (+ (lambda *shape, dtype: torch.empty(*shape, dtype=dtype))+ if device.type == "cpu"+ else alloc_pinned_tensor+ )+ self._io = {+ "x": alloc_io(self.max_tokens, self.H, dtype=torch.bfloat16),+ "ids": alloc_io(self.max_tokens, self.top_k, dtype=torch.int32),+ "w": alloc_io(self.max_tokens, self.top_k, dtype=torch.float32),+ "y": alloc_io(self.max_tokens, self.H, dtype=torch.bfloat16),+ }self._ext = _cpu_moe.CpuMoeExecutor(num_threads=nthreads,num_layers=self.num_layers,@@ -260,11 +271,16 @@ class CpuMoeExecutor:max_tokens=self.max_tokens,activation_id=_ACT_IDS[activation],apply_router_weight_on_input=1 if apply_router_weight_on_input else 0,- weight_format=_WFMT_IDS[fmt],+ weight_format=fmt,+ banks=banks,+ options=options,swiglu_alpha=float(swiglu_alpha),swiglu_limit=float(swiglu_limit) if swiglu_limit is not None else float("inf"),core_ids=core_ids,- **ptrs,+ x_ptr=self._io["x"].data_ptr(),+ ids_ptr=self._io["ids"].data_ptr(),+ w_ptr=self._io["w"].data_ptr(),+ y_ptr=self._io["y"].data_ptr(),)self.num_threads = nthreadsself.core_ids = core_ids@@ -281,34 +297,24 @@ class CpuMoeExecutor:)torch.set_num_threads(clamp)- self._io: dict[int, dict[str, torch.Tensor]] = {}- self._tasks: dict[tuple[int, int], int] = {}-- # Flag-based handshake: mapped-pinned ready/done/err int64 arrays (one slot per- # (MoE layer, decode batch size) pair, allocated as tasks are created) + a- # persistent CPU coordinator that polls ready[], runs the slot's task on the- # pool, and sets done[]. Binary per-step protocol (GPU memops: done=0, ready=1;- # coordinator: consume ready, run, done=1; GPU waits done>=1 -- the WAIT- # immediate is constant, so CUDA-graph replays are safe). err[] is raised by the- # WATCHDOG thread when a ready flag stays unanswered (dead coordinator): it- # poisons done to unblock the stream and raise_if_unhealthy() turns the step- # into a loud error instead of silent stale output. Buffers are kept alive on- # self so the coordinator's pinned pointers stay valid for the executor's- # lifetime (flag_sync itself was decided above, before thread sizing).- self._ready = self._done = self._err = None- self._flag_slots: dict[tuple[int, int], int] = {} # (layer_id, bs) -> slot- self._flag_capacity = self.num_layers * _FLAG_SLOTS_PER_LAYER+ # Flag-based handshake: the extension owns the ready/done words (gpu_sync.h) and a+ # persistent CPU coordinator that polls the doorbell, runs the task it names on the+ # pool, and sets done for the GPU. The WATCHDOG thread marks the executor unhealthy+ # when a doorbell stays unanswered (dead coordinator): it poisons done to unblock the+ # stream and raise_if_unhealthy() turns the step into a loud error instead of silent+ # stale output (flag_sync itself was decided above, before thread sizing).+ self._unhealthy = False+ self._kernel_io = Falseif self._flag_sync:- self._ready = alloc_pinned_tensor(self._flag_capacity, dtype=torch.int64)- self._done = alloc_pinned_tensor(self._flag_capacity, dtype=torch.int64)- self._err = alloc_pinned_tensor(self._flag_capacity, dtype=torch.int64)- self._ready.zero_()- self._done.zero_()- self._err.zero_()- self._ext.start_flag_coordinator(- self._ready.data_ptr(), self._done.data_ptr(), self._flag_capacity,- self._coord_core,- )+ self._ext.start_flag_coordinator(self._coord_core)+ # ROCm keeps the copies until the kernel IO has been checked on AMD GPUs.+ if torch.version.hip is None:+ reason = _cpu_moe.kernel_io_probe(torch.cuda.current_stream().cuda_stream)+ if reason:+ logger.warning_rank0(f"cpu-moe kernel IO unavailable ({reason}); using D2H/H2D copies around the doorbell")+ else:+ self._ext.enable_kernel_io()+ self._kernel_io = Trueself._watchdog_stop = False# The thread target holds a WEAKREF and re-derefs it each tick: a bound# method would strong-reference the executor forever (the loop never ends@@ -325,32 +331,14 @@ class CpuMoeExecutor:)self._watchdog.start()- # ds_fp4: the reference FP8 activation round-trip is a scalar per-element chain- # that the C++ side runs single-threaded on the CUDA host-callback thread --- # straight on the decode critical path (~0.3ms/layer at H=4096, every worker and- # the GPU waiting on it). When a GPU is present we run the numerically identical- # round-trip as a captured GPU kernel BEFORE the D2H (see decode_submit) and tell- # the C++ side to skip its own. Measured on DeepSeek-V4-Flash bs=1 decode:- # 12.85 -> 15.65 tok/s, output bit-identical (tests/moe/test_dsfp4_prequant.py).- self._gpu_prequant = fmt == "ds_fp4" and device.type == "cuda"- # the W4A8 activation round-trip block follows the checkpoint's fp8 block (128 on V4, 32 on V4.1)- self._act_block = act_block- if fmt == "ds_fp4":- assert act_block is not None, "ds_fp4 experts need act_block, the checkpoint's fp8 activation quant block"- self._ext.set_act_block(act_block)- if self._gpu_prequant:- self._ext.set_input_prequant(True)- logger.info_rank0(- "ds_fp4: input FP8 round-trip moved to the GPU "- "(bit-identical grid; the CPU-side scalar round-trip is skipped)"- )logger.info_rank0(f"CPU MoE executor ready: threads={nthreads} (pinned to cores "f"{core_ids[0]}..{core_ids[-1]}) isa={self.isa} fmt={fmt} "f"H={self.H} I={self.I} experts={self.num_experts} layers={self.num_layers} "f"top_k={self.top_k} act={activation} max_tokens={self.max_tokens} "- f"spin_wait={self.spin_wait}"+ f"spin_wait={self.spin_wait} "+ f"handshake={'kernel' if self._kernel_io else 'memop' if self._flag_sync else 'host-func'}")def _make_table(self, layers: list[torch.Tensor]) -> torch.Tensor:@@ -368,7 +356,11 @@ class CpuMoeExecutor:self._banks.extend(layers)return table- def _resolve_fp8_block_banks(self, banks: dict) -> tuple[dict, tuple[int, int]]:+ def _tables(self, **roles: list[torch.Tensor]) -> dict[str, int]:+ """Per-layer table addresses keyed by canonical bank role."""+ return {role: self._make_table(layers).data_ptr() for role, layers in roles.items()}++ def _resolve_fp8_block_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:"""Block-FP8 schema: row-major e4m3 + one bf16 scale per 128x128 block; the scale banks pad their last dim, so C++ gets the real row stride."""gu, gus = banks["gate_up"], banks["gate_up_scale"]dn, dns = banks["down"], banks["down_scale"]@@ -380,28 +372,19 @@ class CpuMoeExecutor:assert tuple(dn[0].shape[1:]) == (H, I), (dn[0].shape, H, I)assert gus[0].shape[1] == (2 * I + 127) // 128 and gus[0].shape[2] >= (H + 127) // 128, (gus[0].shape, H, I)assert dns[0].shape[1] == (H + 127) // 128 and dns[0].shape[2] >= (I + 127) // 128, (dns[0].shape, H, I)- ptrs = dict(- gate_up_ptr=self._make_table(gu).data_ptr(),- down_ptr=self._make_table(dn).data_ptr(),- gate_up_scale_ptr=self._make_table(gus).data_ptr(),- gate_up_global_ptr=0,- down_scale_ptr=self._make_table(dns).data_ptr(),- down_global_ptr=0,- gate_up_bias_ptr=0,- down_bias_ptr=0,- fp8_block_gu_scale_stride=int(gus[0].stride(1)),- fp8_block_dn_scale_stride=int(dns[0].stride(1)),- )- return ptrs, (H, I)+ banks = self._tables(gate_up=gu, down=dn, gate_up_scale=gus, down_scale=dns)+ options = dict(gu_scale_stride=int(gus[0].stride(1)), dn_scale_stride=int(dns[0].stride(1)))+ return banks, options, (H, I)- def _resolve_banks(self, banks: dict, fmt: str) -> tuple[dict, tuple[int, int]]:- """Return (pointer kwargs for the C++ ctor, (H, I)) for the given format.+ def _resolve_banks(self, banks: dict, fmt: str) -> tuple[dict, dict, tuple[int, int]]:+ """Return (banks, options, (H, I)) for the C++ ctor: banks maps each canonical role+ the format reads to its per-layer table's address, options holds its integer settings.``banks[name]`` is a list of ``num_layers`` ``[num_experts, ...]`` tensors(the per-layer host bank contract); shapes are read from the first layer so- per-partition (TP) sizes are exact. Unused pointers are 0. Every pointer kwarg- is actually a per-layer table's address (see ``_make_table``), not a single- bank's -- the C++ ctor resolves ``tbl[layer_id]`` per task.+ per-partition (TP) sizes are exact. Every bank address is a per-layer table+ (see ``_make_table``), not a single bank's -- the C++ side resolves+ ``tbl[layer_id]`` per task."""if fmt == "bf16":gate_up = banks["gate_up"]@@ -414,17 +397,8 @@ class CpuMoeExecutor:I = int(gate_up[0].shape[1] // 2)assert gate_up[0].shape[1] == 2 * Iassert tuple(down[0].shape[1:]) == (H, I), (down[0].shape, H, I)- ptrs = dict(- gate_up_ptr=self._make_table(gate_up).data_ptr(),- down_ptr=self._make_table(down).data_ptr(),- gate_up_scale_ptr=0,- gate_up_global_ptr=0,- down_scale_ptr=0,- down_global_ptr=0,- gate_up_bias_ptr=0,- down_bias_ptr=0,- )- return ptrs, (H, I)+ banks = self._tables(gate_up=gate_up, down=down)+ return banks, {}, (H, I)if fmt == "q4_0":return self._resolve_q4_0_banks(banks)@@ -452,19 +426,10 @@ class CpuMoeExecutor:assert tuple(gus[0].shape[1:]) == (2 * I, H // 16), (gus[0].shape, I, H)assert tuple(dns[0].shape[1:]) == (H, I // 16), (dns[0].shape, H, I)assert tuple(gug[0].shape[1:]) == (2 * I,) and tuple(dng[0].shape[1:]) == (H,)- ptrs = dict(- gate_up_ptr=self._make_table(gup).data_ptr(),- down_ptr=self._make_table(dnp).data_ptr(),- gate_up_scale_ptr=self._make_table(gus).data_ptr(),- gate_up_global_ptr=self._make_table(gug).data_ptr(),- down_scale_ptr=self._make_table(dns).data_ptr(),- down_global_ptr=self._make_table(dng).data_ptr(),- gate_up_bias_ptr=0,- down_bias_ptr=0,- )- return ptrs, (H, I)+ banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, gate_up_global=gug, down_scale=dns, down_global=dng)+ return banks, {}, (H, I)- def _resolve_q4_0_banks(self, banks: dict) -> tuple[dict, tuple[int, int]]:+ def _resolve_q4_0_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:"""Native GGUF Q4_0 schema (gemma4 GGUF): per-32 blocks (fp16 scale + 16 nibblebytes), row-major over K -- the *same* packed banks the GPU offload path streams.gate_up is [S, 2I, H//32*18], down is [S, H, I//32*18]; the C++ W4A16 GEMV reads a@@ -479,19 +444,10 @@ class CpuMoeExecutor:assert H % 32 == 0 and I % 32 == 0, (H, I)assert int(gate_up[0].shape[2]) == (H // 32) * 18, (gate_up[0].shape, H)assert int(down[0].shape[2]) == (I // 32) * 18, (down[0].shape, I)- ptrs = dict(- gate_up_ptr=self._make_table(gate_up).data_ptr(),- down_ptr=self._make_table(down).data_ptr(),- gate_up_scale_ptr=0,- gate_up_global_ptr=0,- down_scale_ptr=0,- down_global_ptr=0,- gate_up_bias_ptr=0,- down_bias_ptr=0,- )- return ptrs, (H, I)+ banks = self._tables(gate_up=gate_up, down=down)+ return banks, {}, (H, I)- def _resolve_mxfp4_banks(self, banks: dict) -> tuple[dict, tuple[int, int]]:+ def _resolve_mxfp4_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:"""gpt-oss mxfp4 ``mxfp4_triton`` schema: transposed split-K blocks/scales(N innermost) + per-output-row biases. The C++ kernel streams K andaccumulates a contiguous N-block, so the GPU-tiled layout is read in place@@ -510,19 +466,10 @@ class CpuMoeExecutor:assert tuple(gus[0].shape[1:]) == (H // 32, 2 * I), (gus[0].shape, H, I)assert tuple(dns[0].shape[1:]) == (I // 32, H), (dns[0].shape, H, I)assert tuple(gob[0].shape[1:]) == (2 * I,) and tuple(dob[0].shape[1:]) == (H,)- ptrs = dict(- gate_up_ptr=self._make_table(gub).data_ptr(),- down_ptr=self._make_table(dnb).data_ptr(),- gate_up_scale_ptr=self._make_table(gus).data_ptr(),- gate_up_global_ptr=0,- down_scale_ptr=self._make_table(dns).data_ptr(),- down_global_ptr=0,- gate_up_bias_ptr=self._make_table(gob).data_ptr(),- down_bias_ptr=self._make_table(dob).data_ptr(),- )- return ptrs, (H, I)+ banks = self._tables(gate_up=gub, down=dnb, gate_up_scale=gus, down_scale=dns, gate_up_bias=gob, down_bias=dob)+ return banks, {}, (H, I)- def _resolve_dsfp4_banks(self, banks: dict) -> tuple[dict, tuple[int, int]]:+ def _resolve_dsfp4_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:"""DeepSeek-V4 ``ds_fp4`` schema: row-major e2m1 (2/byte) + e8m0 per-32 blockscales, no global, no bias. Layout matches nvfp4 (K contiguous per output row),so the C++ GEMV reads it in place. The kernel additionally FP8-round-trips the@@ -538,57 +485,8 @@ class CpuMoeExecutor:assert tuple(dnp[0].shape[1:]) == (H, I // 2), (dnp[0].shape, H, I)assert tuple(gus[0].shape[1:]) == (2 * I, H // 32), (gus[0].shape, I, H)assert tuple(dns[0].shape[1:]) == (H, I // 32), (dns[0].shape, H, I)- ptrs = dict(- gate_up_ptr=self._make_table(gup).data_ptr(),- down_ptr=self._make_table(dnp).data_ptr(),- gate_up_scale_ptr=self._make_table(gus).data_ptr(),- gate_up_global_ptr=0,- down_scale_ptr=self._make_table(dns).data_ptr(),- down_global_ptr=0,- gate_up_bias_ptr=0,- down_bias_ptr=0,- )- return ptrs, (H, I)-- def _io_for(self, bs: int) -> dict[str, torch.Tensor]:- io = self._io.get(bs)- if io is None:- alloc_io = (- (lambda *shape, dtype: torch.empty(*shape, dtype=dtype))- if self.device.type == "cpu"- else alloc_pinned_tensor- )- io = {- "x": alloc_io(bs, self.H, dtype=torch.bfloat16),- "ids": alloc_io(bs, self.top_k, dtype=torch.int32),- "w": alloc_io(bs, self.top_k, dtype=torch.float32),- "y": alloc_io(bs, self.H, dtype=torch.bfloat16),- }- self._io[bs] = io- return io-- def _task_for(self, layer_id: int, bs: int) -> int:- key = (layer_id, bs)- task = self._tasks.get(key)- if task is None:- io = self._io_for(bs)- task = self._ext.create_task(- layer_id,- bs,- io["x"].data_ptr(),- io["ids"].data_ptr(),- io["w"].data_ptr(),- io["y"].data_ptr(),- )- self._tasks[key] = task- # Allocate this (layer, bs) combo a flag slot and register its task with the- # coordinator. Combos past the slot capacity keep the host-func path.- if self._flag_sync and key not in self._flag_slots:- slot = len(self._flag_slots)- if slot < self._flag_capacity:- self._flag_slots[key] = slot- self._ext.register_flag_task(slot, task)- return task+ banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, down_scale=dns)+ return banks, {}, (H, I)def decode(self,@@ -612,7 +510,7 @@ class CpuMoeExecutor:topk_weights: torch.Tensor,topk_ids: torch.Tensor,) -> tuple:- """Issue the D2H copies + the CPU-pool submit host node, then return without+ """Issue the D2H copies + the CPU-pool submit host node (or the publish kernel), then return withoutwaiting. Lets a caller (the hybrid backend) enqueue GPU work between this and:meth:`decode_sync` so the CPU compute overlaps the GPU GEMM / PCIe fetch.@@ -621,7 +519,14 @@ class CpuMoeExecutor::meth:`decode_sync`. The output tensor is allocated here so it stays live (anddistinct from the interleaved GPU work) across the overlap window."""bs = hidden_states.shape[0]- io = self._io_for(bs)+ if bs > self.max_tokens:+ raise ValueError(f"CPU MoE decode batch {bs} exceeds the executor's max_tokens {self.max_tokens}")+ if not self._graph_capture_safe and torch.cuda.is_current_stream_capturing():+ raise RuntimeError(+ "CPU MoE decode cannot be captured into a CUDA graph here: the stream-memop "+ "doorbell is off or failed its startup probe, and the host-function sync is not "+ "replay-safe on this runtime; run with --cuda-graph-max-bs 0"+ )if self._gpu_prequant:# DSV4: apply the reference FP8 round-trip on the GPU (the same kernel the@@ -631,91 +536,86 @@ class CpuMoeExecutor:hidden_states = act_quant_fp8_roundtrip(hidden_states, block=self._act_block)+ stream = torch.cuda.current_stream().cuda_stream+ if self._kernel_io and hidden_states.dtype == torch.bfloat16:+ x = hidden_states.contiguous()+ ids = topk_ids.to(torch.int32).contiguous()+ w = topk_weights.to(torch.float32).contiguous()+ assert x.shape == (bs, self.H) and ids.shape == w.shape == (bs, self.top_k), (x.shape, ids.shape, w.shape)+ self._ext.flag_publish(stream, x.data_ptr(), ids.data_ptr(), w.data_ptr(), layer_id, bs)+ return (bs, torch.empty_like(x), True)+# D2H: ship this step's activations + routing to pinned host memory.- io["x"].copy_(hidden_states, non_blocking=True)- io["ids"].copy_(topk_ids.to(torch.int32), non_blocking=True)- io["w"].copy_(topk_weights.to(torch.float32), non_blocking=True)+ io = self._io+ io["x"][:bs].copy_(hidden_states, non_blocking=True)+ io["ids"][:bs].copy_(topk_ids.to(torch.int32), non_blocking=True)+ io["w"][:bs].copy_(topk_weights.to(torch.float32), non_blocking=True)- task = self._task_for(layer_id, bs)out = torch.empty_like(hidden_states)- slot = self._flag_slots.get((layer_id, bs)) if self._flag_sync else None- if slot is not None:- # Front-end memops: done[slot]=0 then ready[slot]=1 (the coordinator's- # doorbell). No kernel launched; no host-func round trip.- self._cpu_moe.memop_submit(- torch.cuda.current_stream().cuda_stream,- self._done.data_ptr(), self._ready.data_ptr(), slot,- )+ if self._flag_sync:+ # Front-end memops: reset done, then ring the doorbell with (layer, bs).+ # No kernel launched; no host-func round trip.+ self._ext.flag_signal(stream, layer_id, bs)else:- stream = torch.cuda.current_stream().cuda_stream- self._ext.submit_with_cuda_stream(stream, task)- return (bs, task, out, slot)+ self._ext.submit_with_cuda_stream(stream, layer_id, bs)+ return (bs, out, False)- def decode_sync(self, pending: tuple) -> torch.Tensor:+ def decode_sync(self, pending: tuple, add_to: torch.Tensor | None = None) -> torch.Tensor:"""Issue the CPU-pool sync + the H2D result copy for a prior :meth:`decode_submit`,and return the GPU output tensor. With flag-sync the wait is a front-end stream- memop on done[slot] (set by the CPU coordinator); otherwise a cudaLaunchHostFunc."""- bs, task, out, slot = pending- if slot is not None:- # Front-end WAIT(done[slot] >= 1): blocks this stream's later nodes without+ memop on done (set by the CPU coordinator); otherwise a cudaLaunchHostFunc.+ With ``add_to``, return ``add_to`` + the CPU output; the kernel path adds in place into ``add_to``."""+ bs, out, kernel_io = pending+ stream = torch.cuda.current_stream().cuda_stream+ if self._flag_sync:+ # Front-end WAIT(done >= 1): blocks this stream's later nodes without# occupying an SM, so GPU utilization stays truthful during the CPU window.- self._cpu_moe.memop_sync(- torch.cuda.current_stream().cuda_stream, self._done.data_ptr(), slot,- )+ self._ext.flag_wait(stream)else:- stream = torch.cuda.current_stream().cuda_stream- self._ext.sync_with_cuda_stream(stream, task)- io = self._io[bs]- out.copy_(io["y"], non_blocking=True)- return out-- def _watchdog_tick(self, suspects: dict) -> None:+ self._ext.sync_with_cuda_stream(stream)+ if kernel_io:+ if add_to is not None and add_to.dtype == torch.bfloat16 and add_to.is_contiguous() and add_to.shape == out.shape:+ self._ext.flag_combine(stream, add_to.data_ptr(), bs, True)+ return add_to+ self._ext.flag_combine(stream, out.data_ptr(), bs, False)+ return out if add_to is None else add_to + out+ out.copy_(self._io["y"][:bs], non_blocking=True)+ return out if add_to is None else add_to + out++ def _watchdog_tick(self, suspect: dict) -> None:"""One watchdog sampling round (called every 2 s by ``_watchdog_main``).- A slot is only declared dead when THREE things hold across >=10 s: its doorbell- is still pending (ready==1 && done==0), it was already pending when first- suspected, and the coordinator has served NOTHING on it since (flag_served_count- unchanged). The served-count criterion kills the false-positive window: two- point samples can land on the same slot's (independent, us-scale) pending- windows under heavy external load, but a coordinator that made progress in- between is alive by definition. ``suspects`` maps slot -> (first_seen,- served_at_first_sight) and persists across ticks."""- stuck = (self._ready == 1) & (self._done == 0)- if not bool(stuck.any()):- suspects.clear()+ The doorbell is only declared dead when THREE things hold across >=10 s: it is+ still pending (ready set, done==0), it was already pending when first suspected, and+ the coordinator has served NOTHING since (flag_served unchanged). The served-count+ criterion kills the false-positive window: two point samples can land on+ (independent, us-scale) pending windows under heavy external load, but a+ coordinator that made progress in between is alive by definition. ``suspect``+ holds (first_seen, served_at_first_sight) and persists across ticks."""+ if not self._ext.flag_pending():+ suspect.clear()returnnow = time.monotonic()- pending = set(stuck.nonzero().flatten().tolist())- for slot in list(suspects):- if slot not in pending:- del suspects[slot]- dead = []- for slot in pending:- served = self._ext.flag_served_count(slot)- first_seen, served_then = suspects.get(slot, (None, None))- if first_seen is None or served != served_then:- suspects[slot] = (now, served) # new suspect, or alive-but-loaded: rearm- continue- if now - first_seen >= 10.0:- dead.append(slot)- if not dead:+ served = self._ext.flag_served()+ if not suspect or suspect["served"] != served:+ suspect.update(first_seen=now, served=served) # new suspect, or alive-but-loaded: rearm+ return+ if now - suspect["first_seen"] < 10.0:returnlogger.error(- f"cpu-moe flag watchdog: slots {dead} unanswered for >10s with no coordinator "- "progress (wedged/dead); poisoning done[] and failing the next step"+ "cpu-moe flag watchdog: doorbell unanswered for >10s with no coordinator "+ "progress (wedged/dead); poisoning done and failing the next step")- for i in dead:- self._err[i] = 1- for i in dead:- self._done[i] = 1 # after err: unblock the stream into a checked failure- suspects.pop(i, None)+ self._unhealthy = True+ self._ext.poison_flag() # after the mark: unblock the stream into a checked failure+ suspect.clear()def raise_if_unhealthy(self) -> None:"""Raise if the flag watchdog fired (a doorbell stayed unanswered because thecoordinator never responded). Called by the engine once per forward -- a single- pinned read -- so a dead coordinator surfaces as a loud error on the next step+ flag read -- so a dead coordinator surfaces as a loud error on the next stepinstead of silently shipping stale expert outputs."""- if self._err is not None and bool((self._err != 0).any()):+ if self._unhealthy:raise RuntimeError("CPU MoE flag-handshake watchdog fired: a decode step's doorbell was ""never answered by the coordinator thread (its outputs cannot be "@@ -728,13 +628,13 @@ class CpuMoeExecutor:def _watchdog_main(executor_ref) -> None:"""Watchdog daemon body: weakref-deref per tick so the thread never keeps a deadexecutor alive (see the start site in ``CpuMoeExecutor.__init__``)."""- suspects: dict = {}+ suspect: dict = {}while True:time.sleep(2.0)executor = executor_ref()- if executor is None or executor._watchdog_stop or executor._ready is None:+ if executor is None or executor._watchdog_stop:returntry:- executor._watchdog_tick(suspects)+ executor._watchdog_tick(suspect)finally:del executor # drop the strong ref before the next sleepdiff --git a/setup.py b/setup.pyindex b8e4693c..b9683731 100644--- a/setup.py+++ b/setup.py@@ -77,15 +77,22 @@ IS_ROCM = _is_rocm()if IS_ROCM:runtime_include_dirs, runtime_library_dirs, runtime_lib = _rocm_paths()runtime_link_args = [f"-Wl,-rpath,{runtime_library_dirs[0]}"]- # These extensions contain host code only. BuildExtension supplies the ROCm+ # Host sources. BuildExtension supplies the ROCm# platform defines to the C++ compiler; offload architecture flags belong on# HIP device sources and would be rejected by the host compiler here.extra_compile = ["-O3", "-std=c++20"]+ # Device sources (.cu, built by hipcc); BuildExtension adds the offload architectures.+ device_compile = ["-O3", "-std=c++20"]else:runtime_include_dirs, runtime_library_dirs = _cuda_runtime_paths()runtime_lib = "cudart"runtime_link_args = []extra_compile = ["-O3", "-std=c++20"]+ # Device sources (.cu, built by nvcc): SASS for each major arch, which also runs on the newer minors of that major,+ # plus PTX for the oldest arch nvcc 13 supports, which the driver builds for any other GPU at load.+ # Naming archs also stops torch from picking them, which fails on a build machine without a GPU.+ device_compile = ["-O3", "-std=c++20"] + [f"-gencode=arch=compute_{a},code=sm_{a}" for a in (75, 80, 90, 100, 120)]+ device_compile.append("-gencode=arch=compute_75,code=compute_75")_check_toolchain()@@ -112,11 +119,17 @@ setup(name="freetoken.kernel._cpu_moe",sources=["python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp",+ f"python/freetoken/kernel/csrc/cpu_moe/gpu_sync_{'hip' if IS_ROCM else 'cuda'}.cpp",+ "python/freetoken/kernel/csrc/cpu_moe/gpu_sync_io.cu",+ *sorted(+ str(p.relative_to(ROOT))+ for p in (ROOT / "python/freetoken/kernel/csrc/cpu_moe/kernels").glob("*.cpp")+ ),],include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs],library_dirs=runtime_library_dirs,libraries=[runtime_lib],- extra_compile_args=extra_compile + ["-pthread"],+ extra_compile_args={"cxx": extra_compile + ["-pthread"], "nvcc": device_compile},extra_link_args=runtime_link_args,),# disk-backed row store (PLE / Engram tables); Linux-only until the TableFile/BatchReader seams grow Windows bodiesdiff --git a/tests/moe/hip_probe_faults.cpp b/tests/moe/hip_probe_faults.cppnew file mode 100644index 00000000..4426bf0d--- /dev/null+++ b/tests/moe/hip_probe_faults.cpp@@ -0,0 +1,75 @@+#include <hip/hip_runtime.h>+#include <dlfcn.h>++#include <atomic>+#include <cstdlib>+#include <cstring>++namespace {+std::atomic<int> begins{0}, ends{0}, injected{0}, removed_nodes{0}, last_end_rc{0};++bool mode(const char* value) {+ const char* selected = std::getenv("FREETOKEN_TEST_HIP_PROBE_FAULT");+ return selected != nullptr && std::strcmp(selected, value) == 0;+}++template <typename Function>+Function real(const char* name) {+ return reinterpret_cast<Function>(dlsym(RTLD_NEXT, name));+}+} // namespace++extern "C" hipError_t hipStreamBeginCapture(hipStream_t stream, hipStreamCaptureMode capture_mode) {+ const hipError_t rc = real<decltype(&hipStreamBeginCapture)>("hipStreamBeginCapture")(stream, capture_mode);+ if (rc == hipSuccess) ++begins;+ return rc;+}++extern "C" hipError_t hipStreamWriteValue64(hipStream_t stream, void* address, uint64_t value, unsigned flags) {+ // Force the compatibility path on runtimes that normally record native writes.+ if (mode("dependency") && begins == 1) {+ ++injected;+ return hipSuccess;+ }+ return real<decltype(&hipStreamWriteValue64)>("hipStreamWriteValue64")(stream, address, value, flags);+}++extern "C" hipError_t hipStreamUpdateCaptureDependencies(+ hipStream_t stream, hipGraphNode_t* dependencies, size_t count, unsigned flags) {+ if (mode("dependency")) {+ ++injected;+ return hipErrorNotSupported;+ }+ return real<decltype(&hipStreamUpdateCaptureDependencies)>("hipStreamUpdateCaptureDependencies")(+ stream, dependencies, count, flags);+}++extern "C" hipError_t hipGraphDestroyNode(hipGraphNode_t node) {+ const hipError_t rc = real<decltype(&hipGraphDestroyNode)>("hipGraphDestroyNode")(node);+ if (rc == hipSuccess) ++removed_nodes;+ return rc;+}++extern "C" hipError_t hipStreamEndCapture(hipStream_t stream, hipGraph_t* graph) {+ ++ends;+ hipError_t rc = real<decltype(&hipStreamEndCapture)>("hipStreamEndCapture")(stream, graph);+ if (mode("end-capture") && ends == 2 && rc == hipSuccess) {+ // A real invalid API call also sets HIP's thread-local last-error state.+ rc = real<decltype(&hipGraphGetNodes)>("hipGraphGetNodes")(*graph, nullptr, nullptr);+ ++injected;+ }+ last_end_rc = static_cast<int>(rc);+ return rc;+}++extern "C" int probe_test_injected() { return injected; }+extern "C" int probe_test_begins() { return begins; }+extern "C" int probe_test_ends() { return ends; }+extern "C" int probe_test_removed_nodes() { return removed_nodes; }+extern "C" int probe_test_last_end_rc() { return last_end_rc; }+extern "C" int probe_test_peek_error() {+ return static_cast<int>(real<decltype(&hipPeekAtLastError)>("hipPeekAtLastError")());+}+extern "C" int probe_test_make_error() {+ return static_cast<int>(real<decltype(&hipGraphGetNodes)>("hipGraphGetNodes")(nullptr, nullptr, nullptr));+}diff --git a/tests/moe/test_cpu_moe.py b/tests/moe/test_cpu_moe.pyindex 547c94c6..38e79b27 100644--- a/tests/moe/test_cpu_moe.py+++ b/tests/moe/test_cpu_moe.py@@ -440,6 +440,111 @@ def _reference(cache, layer, hidden, ids, w, act="silu", apply_in=False):return out[email protected](scope="module")+def hip_probe_fault_library(tmp_path_factory):+ import importlib.util+ import os+ from pathlib import Path+ import shutil+ import subprocess++ if not torch.version.hip:+ pytest.skip("requires the HIP sync backend")+ compiler = shutil.which("c++")+ if compiler is None:+ pytest.skip("requires a C++ compiler for HIP fault injection")+ candidates = [Path(os.getenv("ROCM_HOME", "/opt/rocm")), Path("/opt/rocm")]+ spec = importlib.util.find_spec("_rocm_sdk_core")+ if spec and spec.submodule_search_locations:+ candidates.append(Path(next(iter(spec.submodule_search_locations))))+ include = next((root / "include" for root in candidates if (root / "include/hip/hip_runtime.h").exists()), None)+ if include is None:+ pytest.skip("requires HIP SDK headers for fault injection")+ library = tmp_path_factory.mktemp("hip_probe") / "faults.so"+ command = [+ compiler, "-std=c++20", "-shared", "-fPIC", "-D__HIP_PLATFORM_AMD__=1",+ f"-I{include}", str(Path(__file__).with_name("hip_probe_faults.cpp")),+ "-ldl", "-o", str(library),+ ]+ result = subprocess.run(command, capture_output=True, text=True, timeout=60)+ assert result.returncode == 0, result.stdout + result.stderr+ return library++[email protected]("fault", ["dependency", "end-capture", "pending"])+def test_cpu_moe_hip_probe_error_recovery(hip_probe_fault_library, fault):+ """Rejected probes leave eager usable and preserve errors predating the probe."""+ import os+ from pathlib import Path+ import subprocess+ import sys+ import textwrap++ script = textwrap.dedent("""+ import ctypes+ import os+ import runpy+ import sys++ import pytest+ import torch+ from freetoken.kernel import _cpu_moe+ from freetoken.moe.cpu_executor import CpuMoeExecutor++ shim = ctypes.CDLL(os.environ["LD_PRELOAD"])+ stream_ptr = torch.cuda.current_stream().cuda_stream+ fault = os.environ["FREETOKEN_TEST_HIP_PROBE_FAULT"]+ if fault == "pending":+ assert shim.probe_test_make_error() != 0+ pending = shim.probe_test_peek_error()+ with pytest.raises(RuntimeError, match="pending error before probe"):+ _cpu_moe.memops_probe(stream_ptr)+ assert shim.probe_test_peek_error() == pending+ assert shim.probe_test_begins() == 0+ sys.exit(0)++ helpers = runpy.run_path(sys.argv[1])+ torch.manual_seed(17)+ cache = helpers["_make_cache"](2, 8, 256, 128, scale=0.03)+ ex = CpuMoeExecutor(+ cache, top_k=4, activation="silu", apply_router_weight_on_input=False,+ num_threads=4, max_tokens=3, device=torch.device("cuda"),+ )+ assert not ex._flag_sync and not ex._graph_capture_safe+ assert shim.probe_test_injected() > 0+ assert shim.probe_test_begins() == shim.probe_test_ends()+ assert shim.probe_test_peek_error() == 0+ if fault == "dependency":+ assert shim.probe_test_removed_nodes() == 1+ assert shim.probe_test_last_end_rc() == 0+ else:+ assert shim.probe_test_last_end_rc() != 0++ for index in range(2):+ torch.manual_seed(23 + index)+ hidden = (torch.randn(3, 256) * 0.3).bfloat16().cuda()+ ids = torch.randint(0, 8, (3, 4), dtype=torch.int32).cuda()+ weights = torch.rand(3, 4).cuda()+ actual = ex.decode(1, hidden, weights, ids).float().cpu()+ expected = helpers["_reference"](cache, 1, hidden, ids.cpu(), weights.cpu())+ relative = (actual - expected).abs().max() / (expected.abs().max() + 1e-6)+ assert relative < 2e-2, relative++ stream = torch.cuda.Stream()+ with pytest.raises(RuntimeError, match="--cuda-graph-max-bs 0"):+ with torch.cuda.graph(torch.cuda.CUDAGraph(), stream=stream):+ ex.decode(1, hidden, weights, ids)+ assert torch.isfinite(ex.decode(1, hidden, weights, ids).cpu()).all()+ """)+ env = dict(os.environ, LD_PRELOAD=str(hip_probe_fault_library),+ FREETOKEN_TEST_HIP_PROBE_FAULT=fault, FREETOKEN_CPU_MOE_FLAG_SYNC="1")+ result = subprocess.run(+ [sys.executable, "-c", script, str(Path(__file__).resolve())],+ env=env, capture_output=True, text=True, timeout=30,+ )+ assert result.returncode == 0, result.stdout + result.stderr++def test_cpu_moe_decode_cuda_graph_replay():from freetoken.moe.cpu_executor import CpuMoeExecutor@@ -467,7 +572,7 @@ def test_cpu_moe_decode_cuda_graph_replay():ids = torch.randint(0, E, (bs, top_k), device=dev, dtype=torch.int32)w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)- # Eager forward: also materializes the per-bs pinned buffers + per-(layer,bs) task.+ # Eager forward.out_eager = ex.decode(layer, hidden, w, ids)torch.cuda.synchronize()ref_eager = _reference(cache, layer, hidden, ids.cpu(), w.cpu())@@ -500,11 +605,9 @@ def test_cpu_moe_decode_cuda_graph_replay():# must actually have exercised it -- guard against a silent fallback to the# host-func path making this test vacuous for the capture-embedded memops.if ex._flag_sync:- assert (layer, bs) in ex._flag_slots, "flag slot expected for the decode task"- slot = ex._flag_slots[(layer, bs)]- assert ex._ext.flag_served_count(slot) >= 4, "1 eager + 3 replay dispatches expected"- assert int(ex._done[slot]) == 1 and int(ex._ready[slot]) == 0, "handshake at rest"- assert int(ex._err.sum()) == 0, "watchdog must not fire in normal operation"+ assert ex._ext.flag_served() >= 4, "1 eager + 3 replay dispatches expected"+ assert not ex._ext.flag_pending(), "handshake at rest"+ assert not ex._unhealthy, "watchdog must not fire in normal operation"ex.raise_if_unhealthy()print("cpu moe cuda graph replay OK")diff --git a/tests/moe/test_cpu_moe_fp8_block.py b/tests/moe/test_cpu_moe_fp8_block.pyindex 06f954d8..baecb86f 100644--- a/tests/moe/test_cpu_moe_fp8_block.py+++ b/tests/moe/test_cpu_moe_fp8_block.py@@ -74,11 +74,11 @@ def test_cpu_fp8_block_accepts_qwen38_padded_scale_bank():device=torch.device("cpu"),)hidden = torch.randn(1, H, dtype=torch.bfloat16)- io = ex._io_for(1)+ io = ex._ioio["x"].copy_(hidden)io["ids"].zero_()io["w"].fill_(1.0)- ex._ext.run_task(ex._task_for(0, 1))+ ex._ext.run(0, 1)cpu_out = io["y"][0].float().to(torch.float64)banks = cache.bank_sourcesgate_up = _reference_fp8_block_gemv(banks["gate_up"][0], banks["gate_up_scale"][0], hidden[0])@@ -140,11 +140,8 @@ def _reference_fp8_block_gemv(weights, scales, x):@pytest.mark.parametrize("codes", ["finite", "subnormal"])- "isa, expected_isa",- [("scalar", "scalar"), ("avx2", "avx2"), ("avx512", "avx512f"), ("avx512bf16", "avx512f")],-)-def test_cpu_fp8_block_matches_float64_reference(monkeypatch, isa, expected_isa, codes):[email protected]("isa", ["scalar", "avx2", "avx512"])+def test_cpu_fp8_block_matches_float64_reference(monkeypatch, isa, codes):from freetoken.moe.cpu_executor import CpuMoeExecutorH, I = 259, 257 # ragged K for both GEMVs; 257 down rows exercise many reductions@@ -159,17 +156,16 @@ def test_cpu_fp8_block_matches_float64_reference(monkeypatch, isa, expected_isa,max_tokens=1,device=torch.device("cpu"),)- tiers = {"scalar": 0, "avx2": 1, "avx512": 2, "avx512bf16": 3}+ tiers = {"scalar": 0, "avx2": 1, "avx512": 2}if tiers[isa] > ("scalar", "avx2", "avx512f", "avx512bf16").index(ex.isa):pytest.skip(f"requested {isa}, host/build selected {ex.isa}")- assert ex._ext.fp8_block_isa_name() == expected_isahidden = torch.randn(1, H, dtype=torch.bfloat16)- io = ex._io_for(1)+ io = ex._ioio["x"].copy_(hidden)io["ids"].zero_()io["w"].fill_(1.0)- ex._ext.run_task(ex._task_for(0, 1))+ ex._ext.run(0, 1)cpu_out = io["y"][0].float().to(torch.float64)banks = cache.bank_sources@@ -233,9 +229,8 @@ def test_cpu_fp8_block_matches_gpu_decode_kernel(bs, isa, monkeypatch):max_tokens=bs,device=dev,)- got = ex._ext.fp8_block_isa_name()- if got != _ISA_TIERS[isa]:- pytest.skip(f"asked for {isa}, kernel selected {got}: not supported here")+ if ex.isa != _ISA_TIERS[isa]:+ pytest.skip(f"asked for {isa}, host/build selected {ex.isa}: not supported here")hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16)ids = torch.stack([torch.randperm(E, device=dev)[:top_k] for _ in range(bs)]).to(torch.int32)@@ -254,7 +249,7 @@ def test_cpu_fp8_block_matches_gpu_decode_kernel(bs, isa, monkeypatch):).float()rel = (cpu_out - gpu_out).abs().max() / (gpu_out.abs().max() + 1e-6)- assert rel < 2e-2, f"bs={bs} isa={got} rel err {rel.item()}"+ assert rel < 2e-2, f"bs={bs} isa={ex.isa} rel err {rel.item()}"def test_fp16_bits_carry_every_e4m3_byte():diff --git a/tests/moe/test_cpu_moe_q4_0.py b/tests/moe/test_cpu_moe_q4_0.pyindex b5ec2f09..7babb5d4 100644--- a/tests/moe/test_cpu_moe_q4_0.py+++ b/tests/moe/test_cpu_moe_q4_0.py@@ -1,7 +1,7 @@"""CPU MoE executor -- native GGUF Q4_0 experts (``--moe-backend cpu``/``hybrid`` forthe gemma4 GGUF checkpoint).-The CPU W4A16 GEMV (``q4_0_dot`` in csrc/cpu_moe/cpu_moe_ext.cpp) reads the *same*+The CPU W4A16 GEMV (``q4_0_dot`` in csrc/cpu_moe/kernels/q4_0.cpp) reads the *same*packed Q4_0 banks the GPU offload path streams and dequantizes weights inside theK-loop. We check it against the reference dequant (models/gguf/dequant.py) + theproduction bf16 GPU decode kernel on byte-identical banks: both are W4A16, so thediff --git a/tests/moe/test_dsfp4_prequant.py b/tests/moe/test_dsfp4_prequant.pyindex 11519502..8187402d 100644--- a/tests/moe/test_dsfp4_prequant.py+++ b/tests/moe/test_dsfp4_prequant.py@@ -17,8 +17,9 @@ H, I, E, TOPK = 1024, 512, 16, 4@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")-def test_dsfp4_gpu_prequant_bit_parity():+def test_dsfp4_gpu_prequant_bit_parity(monkeypatch):from freetoken.kernel.pinned import alloc_pinned_tensor+ from freetoken.moe import cpu_executorfrom freetoken.moe.cpu_executor import CpuMoeExecutortorch.manual_seed(0)@@ -44,14 +45,13 @@ def test_dsfp4_gpu_prequant_bit_parity():wts = torch.rand(2, TOPK, device=dev)def run(cpu_roundtrip: bool) -> torch.Tensor:+ # control: the original CPU-side scalar round-trip+ monkeypatch.setattr(cpu_executor, "_DSFP4_GPU_PREQUANT", not cpu_roundtrip)ex = CpuMoeExecutor(cache, top_k=TOPK, activation="silu", apply_router_weight_on_input=False,num_threads=4, max_tokens=2, device=dev, swiglu_limit=10.0, act_block=128,)- assert ex._gpu_prequant, "ds_fp4 on CUDA should enable GPU prequant"- if cpu_roundtrip: # control: the original CPU-side scalar round-trip- ex._gpu_prequant = False- ex._ext.set_input_prequant(False)+ assert ex._gpu_prequant != cpu_roundtrip, "ds_fp4 on CUDA should enable GPU prequant"out = ex.decode(0, x, wts, ids.clone()).clone()torch.cuda.synchronize()del ex