FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(cpu-moe): serve block-fp8 experts on the cpu and hybrid backends (#662)
Xiaoze Fan95ac86f21567
diff --git a/docs/cli.md b/docs/cli.mdindex a15a34b7..2166e9e8 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -87,7 +87,7 @@ See [models.md](models.md#moe-strategies) for what each strategy does.| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family strategies) || `--kv-reserve-tokens` | 8192 | KV token floor reserved before `--moe-cache-auto` fills experts || `--moe-cpu-threads` | physical cores | CPU worker threads for the cpu/hybrid executor |-| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, nvfp4, mxfp4), so fp8 experts cannot use it |+| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, fp8_block, nvfp4, mxfp4) || `--moe-hybrid-max-fetch` | auto | With `hybrid`: max experts fetched over PCIe per layer per step; rest computed on CPU || `--moe-prefill-hit-d2d` | off | Prefill: copy cache-hit experts device-side, stream only misses (CUDA >= 13) || `--disable-moe-prefill-overlap` | overlap on | Disable the two-buffer prefill copy overlap |diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cppindex 86765013..6f85971e 100644--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp@@ -10,7 +10,7 @@// sync() blocks the host-func thread until the pool drains. The heavy GEMV runs// on the persistent worker threads, not the host-func thread.//-// Weight formats: bf16, NVFP4, MXFP4, ds_fp4 and Q4_0 expert banks (see WFmt and+// Weight formats: bf16, NVFP4, MXFP4, ds_fp4, Q4_0 and block-FP8 expert banks (see WFmt and// the per-format bank schemas). Compute is FP32-accumulate; the intermediate is// stored bf16 to match the GPU decode path. ISA is chosen once at construction// (AVX-512-BF16 dpbf16 -> AVX-512F widening -> AVX2+FMA -> scalar).@@ -271,6 +271,108 @@ inline float e4m3_decode(uint8_t v) {return sign * (1.0f + man / 8.0f) * std::ldexp(1.0f, (int)exp - 7);}+// ------------------------ block-FP8 (W8A16) dequant ------------------------+// fp8-e4m3 weights row-major [rows, K] plus one bf16 scale per 128x128 block. The scale tensor is named weight_scale_inv but multiplies, as in kernel/triton/fp8_blockscale_moe.py.+// Whole 128-blocks read the activations as fp32 even/odd halves (xe[m] = x[2m], xo[m] = x[2m+1]); a ragged tail reads the bf16 row x.+// 0x7F/0xFF (NaN in torch float8_e4m3fn) decode to +-480 in every kernel, as e4m3_decode does.+using fp8_block_dot_fn = float (*)(const uint8_t*, const float*, const float*, const bf16_t*,+ const bf16_t*, int, const float*);++constexpr int FP8_BLK = 128; // K elements covered by one weight scale++inline float fp8_block_tail(const uint8_t* w, const bf16_t* x, int k0, int K, const float* e4m3) {+ float blk = 0.0f;+ for (int k = k0; k < K; ++k) blk += e4m3[w[k]] * bf16_to_f32(x[k]);+ return blk;+}++float dot_fp8_block_scalar(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,+ const bf16_t* s, int K, const float* e4m3) {+ float acc = 0.0f;+ int b = 0, k0 = 0;+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {+ // 8 independent sums: a single running sum is bound by the add latency+ float a[8] = {};+ const uint8_t* wb = w + k0;+ const float* pe = xe + k0 / 2;+ const float* po = xo + k0 / 2;+ for (int j = 0; j < FP8_BLK / 2; j += 4)+ for (int l = 0; l < 4; ++l) {+ a[2 * l] += e4m3[wb[2 * (j + l)]] * pe[j + l];+ a[2 * l + 1] += e4m3[wb[2 * (j + l) + 1]] * po[j + l];+ }+ acc += (((a[0] + a[1]) + (a[2] + a[3])) + ((a[4] + a[5]) + (a[6] + a[7]))) *+ bf16_to_f32(s[b]);+ }+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);+ return acc;+}++#if CPU_MOE_X86+// A byte b read as fp16 bits (b & 0x80) << 8 | (b & 0x7F) << 7 is its e4m3 value times 2^-8, subnormals included, so the block scale carries the 256.+// In a 16-bit lane holding bytes (2m, 2m+1), srai 1 then & 0xBF80 builds that for byte 2m+1; a left shift by 8 first does byte 2m. Every AVX2 CPU has F16C.+__attribute__((target("avx2,fma,f16c")))+float dot_fp8_block_avx2(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,+ const bf16_t* s, int K, const float* e4m3) {+ const __m256i keep = _mm256_set1_epi16((short)0xBF80);+ __m256 total = _mm256_setzero_ps();+ int b = 0, k0 = 0;+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {+ _mm_prefetch(reinterpret_cast<const char*>(w + k0) + PF_AHEAD, _MM_HINT_T0);+ __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();+ __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();+ for (int j = 0; j < FP8_BLK; j += 32) {+ const __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j));+ const __m256i odd = _mm256_and_si256(_mm256_srai_epi16(v, 1), keep);+ const __m256i even = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v, 8), 1), keep);+ const float* pe = xe + (k0 + j) / 2;+ const float* po = xo + (k0 + j) / 2;+ a0 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_castsi256_si128(even)), _mm256_loadu_ps(pe), a0);+ a1 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_extracti128_si256(even, 1)), _mm256_loadu_ps(pe + 8), a1);+ a2 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_castsi256_si128(odd)), _mm256_loadu_ps(po), a2);+ a3 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_extracti128_si256(odd, 1)), _mm256_loadu_ps(po + 8), a3);+ }+ const __m256 blk = _mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3));+ total = _mm256_fmadd_ps(blk, _mm256_set1_ps(bf16_to_f32(s[b]) * 256.0f), total);+ }+ float acc = hsum256(total);+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);+ return acc;+}++__attribute__((target("avx512f,avx2,fma,f16c")))+float dot_fp8_block_avx512f(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,+ const bf16_t* s, int K, const float* e4m3) {+ const __m256i keep = _mm256_set1_epi16((short)0xBF80);+ __m512 total = _mm512_setzero_ps();+ int b = 0, k0 = 0;+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {+ _mm_prefetch(reinterpret_cast<const char*>(w + k0) + PF_AHEAD, _MM_HINT_T0);+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();+ for (int j = 0; j < FP8_BLK; j += 64) {+ const __m256i v0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j));+ const __m256i v1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j + 32));+ const __m256i o0 = _mm256_and_si256(_mm256_srai_epi16(v0, 1), keep);+ const __m256i e0 = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v0, 8), 1), keep);+ const __m256i o1 = _mm256_and_si256(_mm256_srai_epi16(v1, 1), keep);+ const __m256i e1 = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v1, 8), 1), keep);+ const float* pe = xe + (k0 + j) / 2;+ const float* po = xo + (k0 + j) / 2;+ a0 = _mm512_fmadd_ps(_mm512_cvtph_ps(e0), _mm512_loadu_ps(pe), a0);+ a1 = _mm512_fmadd_ps(_mm512_cvtph_ps(o0), _mm512_loadu_ps(po), a1);+ a2 = _mm512_fmadd_ps(_mm512_cvtph_ps(e1), _mm512_loadu_ps(pe + 16), a2);+ a3 = _mm512_fmadd_ps(_mm512_cvtph_ps(o1), _mm512_loadu_ps(po + 16), a3);+ }+ const __m512 blk = _mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3));+ total = _mm512_fmadd_ps(blk, _mm512_set1_ps(bf16_to_f32(s[b]) * 256.0f), total);+ }+ float acc = _mm512_reduce_add_ps(total);+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);+ return acc;+}+#endif // CPU_MOE_X86+// Activations pre-deinterleaved to fp32 (xe[m]=x[2m], xo[m]=x[2m+1]); see the// ds_fp4 dot below for why (drops the hot loop to ~1.5 shuffle ops / 16 weights).using nvdot_fn = float (*)(const uint8_t*, const uint8_t*, float, const float*, const float*,@@ -677,7 +779,7 @@ struct DotChoice {};// SIMD tiers, ascending. Each format picks the highest tier <= the one chosen by-// pick_isa() that it implements (fp4 formats have no bf16-specific tier, so the+// pick_isa() that it implements (fp4 and fp8_block formats have no bf16-specific tier, so the// avx512bf16 tier maps to their avx512 kernel).enum IsaTier { ISA_SCALAR = 0, ISA_AVX2 = 1, ISA_AVX512 = 2, ISA_AVX512BF16 = 3 };@@ -1017,6 +1119,17 @@ void mxfp4_gemv_avx2(float* out, const uint8_t* blk, const uint8_t* scl, const b}#endif+fp8_block_dot_fn select_fp8_block_dot() {+ const IsaTier t = pick_isa();+#if CPU_MOE_X86+ // the avx512bf16 tier keeps the fp32 kernel: on Sapphire Rapids it is faster than widening to bf16 for dpbf16+ if (t >= ISA_AVX512) return dot_fp8_block_avx512f;+ if (t >= ISA_AVX2) return dot_fp8_block_avx2;+#endif+ (void)t;+ return dot_fp8_block_scalar;+}+mxgemv_fn select_mxgemv() {const IsaTier t = pick_isa();#if CPU_MOE_X86@@ -1229,7 +1342,8 @@ q4dot_fn select_q4dot() {return q4_0_dot_i8_scalar;}-enum WFmt { WF_BF16 = 0, WF_NVFP4 = 1, WF_MXFP4 = 2, WF_DSFP4 = 3, WF_Q4_0 = 4 };+enum WFmt { WF_BF16 = 0, WF_NVFP4 = 1, WF_MXFP4 = 2, WF_DSFP4 = 3, WF_Q4_0 = 4,+ WF_FP8_BLOCK = 5 };// Each ctor pointer arg is the address of a CPU int64 array of length// num_layers (one base address per layer, built by cpu_executor.py's@@ -1246,7 +1360,7 @@ struct CpuMoeExecutor {int H, I;int act, apply_on_input;int fmt; // WFmt- bool needs_di = false; // pre-deinterleave activations to fp32 (nvfp4/ds_fp4)+ bool needs_di = false; // pre-deinterleave activations to fp32 (nvfp4/ds_fp4/fp8_block)// Per-layer pointer tables (one base address per layer, see tbl_at). gate_up_tbl// doubles as the bf16 gate_up table and the nvfp4/mxfp4/q4_0/ds_fp4 packed-gate_up// table (down_tbl likewise for down); which reinterpretation applies is picked by@@ -1269,6 +1383,10 @@ struct CpuMoeExecutor {dsdot_fn dsdot;mxgemv_fn mxgemv;q4dot_fn q4dot;+ fp8_block_dot_fn fp8_block_dot;+ // fp8_block: bf16 scale per 128x128 weight block. These are the scale tensor's+ // trailing strides, including any padding used by the host bank layout.+ int fp8_block_gu_scale_row = 0, fp8_block_dn_scale_row = 0;// ds_fp4: the caller already FP8-round-tripped the input activations on the GPU// (same reference grid), so submit() must not repeat it on the host-callback// thread. That scalar per-element pass is single-threaded ON THE DECODE CRITICAL@@ -1287,7 +1405,7 @@ struct CpuMoeExecutor {std::vector<bf16_t> g_scratch; // [max_tokens * top_k * I] intermediatestd::vector<bf16_t> xq_scratch; // [max_tokens * H] ds_fp4 fp8-roundtripped input- // ds_fp4 activations pre-deinterleaved to fp32 (even/odd K) for the row-major dot.+ // nvfp4/ds_fp4/fp8_block activations pre-deinterleaved to fp32 (even/odd K) for the row-major dot.std::vector<float> xe_scratch, xo_scratch; // [max_tokens * H/2] (input)std::vector<float> ge_scratch, go_scratch; // [max_tokens*top_k*I/2] (intermediate)// AVX-VNNI W4A8: per-16-block int8 activations [even(8),odd(8)] + per-block scale.@@ -1315,7 +1433,7 @@ struct CpuMoeExecutor {alignas(64) std::atomic<int64_t> p1_next{0};std::atomic<int64_t> p2_next{0};- std::atomic<int64_t> prt_next{0}; // ds_fp4 intermediate fp8 round-trip phase+ std::atomic<int64_t> prt_next{0}; // intermediate-row prep phase (prep_g_row)int64_t p1_total = 0, p2_total = 0, prt_total = 0;int n_iblk = 0, n_hblk = 0;std::atomic<int> done_count{0};@@ -1373,7 +1491,8 @@ struct CpuMoeExecutor {uintptr_t gate_up_global_ptr, uintptr_t down_scale_ptr,uintptr_t down_global_ptr, uintptr_t gate_up_bias_ptr,uintptr_t down_bias_ptr, double swiglu_alpha_, double swiglu_limit_,- std::vector<int> core_ids_)+ std::vector<int> core_ids_, int fp8_block_gu_scale_stride_,+ int fp8_block_dn_scale_stride_): num_threads(num_threads_ > 0 ? num_threads_ : 1),num_layers(num_layers_),num_experts(num_experts_),@@ -1393,6 +1512,8 @@ struct CpuMoeExecutor {dn_bias_tbl(reinterpret_cast<const uint64_t*>(down_bias_ptr)),swiglu_alpha(static_cast<float>(swiglu_alpha_)),swiglu_limit(static_cast<float>(swiglu_limit_)),+ fp8_block_gu_scale_row(fp8_block_gu_scale_stride_),+ fp8_block_dn_scale_row(fp8_block_dn_scale_stride_),core_ids(std::move(core_ids_)) {DotChoice c = select_dot();dot = c.fn;@@ -1400,6 +1521,7 @@ struct CpuMoeExecutor {dsdot = select_dsdot();mxgemv = select_mxgemv();q4dot = select_q4dot();+ fp8_block_dot = select_fp8_block_dot();if (weight_format == WF_Q4_0) {if (H % 32 != 0 || I % 32 != 0)throw std::runtime_error("Q4_0 CPU MoE requires H and I to be multiples of 32");@@ -1423,8 +1545,8 @@ struct CpuMoeExecutor {// e8m0 (mxfp4 block scale) = 2^(s-127); the GPU GEMV clamps s to [0,254].for (int i = 0; i < 256; ++i) e8m0_lut[i] = std::ldexp(1.0f, std::min(i, 254) - 127);g_scratch.assign(static_cast<size_t>(max_tokens) * top_k * I, 0);- // Row-major fp4 (nvfp4/ds_fp4) pre-deinterleaves activations to fp32 even/odd.- needs_di = (fmt == WF_NVFP4 || fmt == WF_DSFP4);+ // Row-major fp4 (nvfp4/ds_fp4) and fp8_block pre-deinterleave activations to fp32 even/odd.+ needs_di = (fmt == WF_NVFP4 || fmt == WF_DSFP4 || fmt == WF_FP8_BLOCK);if (needs_di) {if (fmt == WF_DSFP4) xq_scratch.assign(static_cast<size_t>(max_tokens) * H, 0);xe_scratch.assign(static_cast<size_t>(max_tokens) * (H / 2), 0);@@ -1498,6 +1620,7 @@ struct CpuMoeExecutor {// once per task/pass by the caller via tbl_at) pick the layer's own tensors.// bf16 uses the interleaved bf16 row; nvfp4 uses the pre-split fp32 even/odd halves// (or, with AVX-VNNI, the per-16-block int8 activations).+ // fp8_block uses the halves for whole 128-blocks and the bf16 row for a ragged tail.inline float gemm1_dot(const bf16_t* gate_up_l, const uint8_t* gu_packed_l,const uint8_t* gu_scale_l, const uint16_t* gu_global_l, int e, int row,const bf16_t* x, const float* xe, const float* xo, const int8_t* xi8,@@ -1511,6 +1634,13 @@ struct CpuMoeExecutor {gu_packed_l + ((size_t)e * (2 * I) + row) * (size_t)q4_gu_row_bytes;return q4dot(w, xi8, xas, H); // W4A8: int8 activations (Q8_0), scale in xas}+ if (fmt == WF_FP8_BLOCK) {+ const uint8_t* w = gu_packed_l + ((size_t)e * (2 * I) + row) * (size_t)H;+ const bf16_t* sc = reinterpret_cast<const bf16_t*>(gu_scale_l) ++ ((size_t)e * ((2 * I + FP8_BLK - 1) / FP8_BLK) + row / FP8_BLK) *+ (size_t)fp8_block_gu_scale_row;+ return fp8_block_dot(w, xe, xo, x, sc, H, e4m3_lut);+ }const size_t r = (size_t)e * (2 * I) + row;if (use_vnni)return nvi8dot(gu_packed_l + r * (size_t)(H / 2), gu_scale_l + r * (size_t)(H / 16),@@ -1533,6 +1663,13 @@ struct CpuMoeExecutor {const uint8_t* w = dn_packed_l + ((size_t)e * H + row) * (size_t)q4_dn_row_bytes;return q4dot(w, gi8, gas, I); // W4A8: int8 activations (Q8_0), scale in gas}+ if (fmt == WF_FP8_BLOCK) {+ const uint8_t* w = dn_packed_l + ((size_t)e * H + row) * (size_t)I;+ const bf16_t* sc = reinterpret_cast<const bf16_t*>(dn_scale_l) ++ ((size_t)e * ((H + FP8_BLK - 1) / FP8_BLK) + row / FP8_BLK) *+ (size_t)fp8_block_dn_scale_row;+ return fp8_block_dot(w, ge, go, g, sc, I, e4m3_lut);+ }const size_t r = (size_t)e * H + row;if (use_vnni)return nvi8dot(dn_packed_l + r * (size_t)(I / 2), dn_scale_l + r * (size_t)(I / 16),@@ -1582,6 +1719,14 @@ struct CpuMoeExecutor {const char* isa_name() const { return isa; }+ const char* fp8_block_isa_name() const {+#if CPU_MOE_X86+ if (fp8_block_dot == dot_fp8_block_avx512f) return "avx512f";+ if (fp8_block_dot == dot_fp8_block_avx2) return "avx2";+#endif+ return "scalar";+ }+void barrier(int& local_sense) {local_sense ^= 1;if (bar_count.fetch_add(1) + 1 == num_threads) {@@ -1810,9 +1955,8 @@ struct CpuMoeExecutor {}}- // Prepare one intermediate row (token,route) for the down GEMV: ds_fp4 first FP8- // round-trips it (DSV4 act_quant), then both formats deinterleave to fp32 even/odd- // (reused across every down output row).+ // Prepare one intermediate row (token,route) for the down GEMV, reused across every down output row.+ // ds_fp4 first FP8 round-trips it (DSV4 act_quant); nvfp4/ds_fp4/fp8_block deinterleave it to fp32 even/odd; q4_0 Q8_0-quantizes it.void prep_g_row(int64_t r) {bf16_t* g = g_scratch.data() + (size_t)r * I;if (use_q4a8) { // q4_0 W4A8: Q8_0-quantize the intermediate row for the down GEMV.@@ -1865,9 +2009,8 @@ struct CpuMoeExecutor {do_pass1(t, p);}barrier(local_sense);- // Row-major fp4: prepare the intermediate rows (per token,route) before the down- // GEMV -- ds_fp4 FP8 round-trips (DSV4 act_quant), both deinterleave to fp32; q4_0- // W4A8 Q8_0-quantizes. Needs all of pass1 done (a full row spans every iblk).+ // Prepare the intermediate rows (per token,route) for the down GEMV, see prep_g_row.+ // Needs all of pass1 done (a full row spans every iblk).if (needs_di || use_q4a8) {for (;;) {int64_t r = prt_next.fetch_add(1, std::memory_order_relaxed);@@ -1938,8 +2081,8 @@ struct CpuMoeExecutor {done_count.store(0, std::memory_order_relaxed);bar_count.store(0, std::memory_order_relaxed);bar_sense.store(0, std::memory_order_relaxed);- // ds_fp4: FP8 round-trip the per-token input once, up front (single-threaded;- // tiny for decode, and done before the workers are woken below).+ // Deinterleave the per-token input once, up front (ds_fp4 FP8-round-trips it first).+ // Single-threaded: tiny for decode, and done before the workers are woken below.if (needs_di) {const size_t xn = static_cast<size_t>(t->num_tokens) * H;if (xn / 2 > xe_scratch.size()) {@@ -2160,7 +2303,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {py::class_<CpuMoeExecutor>(m, "CpuMoeExecutor").def(py::init<int, int, int, int, int, int, int, int, int, int, uintptr_t, uintptr_t,uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t,- double, double, std::vector<int>>(),+ double, double, std::vector<int>, int, int>(),py::arg("num_threads"), py::arg("num_layers"), py::arg("num_experts"),py::arg("top_k"), py::arg("hidden_size"), py::arg("inter_size"),py::arg("max_tokens"), py::arg("activation_id"),@@ -2169,7 +2312,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {py::arg("gate_up_global_ptr"), py::arg("down_scale_ptr"),py::arg("down_global_ptr"), py::arg("gate_up_bias_ptr"),py::arg("down_bias_ptr"), py::arg("swiglu_alpha"), py::arg("swiglu_limit"),- py::arg("core_ids"))+ py::arg("core_ids"), py::arg("fp8_block_gu_scale_stride") = 0,+ py::arg("fp8_block_dn_scale_stride") = 0).def("create_task", &CpuMoeExecutor::create_task, py::arg("layer_id"),py::arg("num_tokens"), py::arg("x_ptr"), py::arg("ids_ptr"), py::arg("w_ptr"),py::arg("y_ptr"))@@ -2192,6 +2336,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {[](CpuMoeExecutor& e, int v) { e.act_block = v; },py::arg("value")).def("isa_name", &CpuMoeExecutor::isa_name)+ .def("fp8_block_isa_name", &CpuMoeExecutor::fp8_block_isa_name).def("set_spin_wait",[](CpuMoeExecutor& e, bool v) { e.spin_wait.store(v, std::memory_order_relaxed); },py::arg("enabled"))@@ -2208,4 +2353,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {// (act_apply falls through to gelu_tanh); the probe turns a stale extension// into a loud rebuild instruction instead of wrong model outputs.m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLU_CLAMP); });+ // Weight-format ABI probe. A stale prebuilt .so accepts a newer WFmt id without+ // complaint and then falls through to the wrong dequant branch -- silently wrong+ // numbers, not a crash. The Python executor refuses to build against an extension+ // that predates the format it was asked for.+ m.def("max_weight_format_id", []() { return static_cast<int>(WF_FP8_BLOCK); });}diff --git a/python/freetoken/layers/quantization/moe/fp8_block.py b/python/freetoken/layers/quantization/moe/fp8_block.pyindex 7552978d..6cda6b1e 100644--- a/python/freetoken/layers/quantization/moe/fp8_block.py+++ b/python/freetoken/layers/quantization/moe/fp8_block.py@@ -19,9 +19,10 @@ def _pad_scale(rows: int, cols: int) -> int:class TritonFp8BlockMoEKernel(MoEKernel):name = "triton"+ cpu_format = "fp8_block"def unusable_reason(self, cfg: MoEConfig) -> str | None:- reason = self._common_reject(cfg, tp_ok=False, cpu_ok=False, plain_silu_only=False)+ reason = self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)if reason:return reasonif cfg.scheme is not None and fp8_block_size(cfg.scheme) != BLOCK:diff --git a/python/freetoken/moe/bench_profile.py b/python/freetoken/moe/bench_profile.pyindex d7b72073..5fa38b6f 100644--- a/python/freetoken/moe/bench_profile.py+++ b/python/freetoken/moe/bench_profile.py@@ -18,6 +18,9 @@ from freetoken.utils import init_loggerlogger = init_logger(__name__)+# Bump when a format gains or loses a CPU MoE path: verdicts and fetch fractions from an older profile are stale.+PROFILE_VERSION = 5+# Engine ``expert_quant`` (models/config.py) -> benchbw format key (offload_cache._BANK_SCHEMAS# / benchbw._offload_bank_specs). Only the offload-family formats with a CPU MoE weight path can# ever resolve to hybrid; anything not listed falls through unmapped and finds no profile entry@@ -101,6 +104,12 @@ def _usable_profile(return Noneif not isinstance(prof, dict):return None+ if not isinstance(prof.get("version"), int) or prof["version"] < PROFILE_VERSION:+ logger.warning(+ f"benchbw profile {src} predates this engine (version {prof.get('version')}, "+ f"need {PROFILE_VERSION}); ignoring it, rerun `ft bench bw`"+ )+ return Noneprof_gpu = (prof.get("gpu") or {}).get("name")if gpu_name and prof_gpu and prof_gpu != gpu_name:logger.warning(diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.pyindex e9d6ffe5..e70cad31 100644--- a/python/freetoken/moe/benchbw.py+++ b/python/freetoken/moe/benchbw.py@@ -56,6 +56,7 @@ from freetoken.gpu_select import (single_gpu_arg,)from freetoken.kernel.pinned import alloc_pinned_tensor+from freetoken.moe.bench_profile import PROFILE_VERSIONfrom freetoken.moe.cpu_executor import physical_core_cpus, resolve_threads_and_affinityfrom freetoken.utils import init_logger@@ -63,7 +64,7 @@ logger = init_logger(__name__)# Formats the CPU MoE C++ kernel can compute AND this bench can build banks for; anything# else is offload-only here. (The kernel also does q4_0, but this bench has no q4_0 banks.)-_CPU_MOE_FORMATS = frozenset({"bf16", "nvfp4", "mxfp4_triton", "ds_fp4"})+_CPU_MOE_FORMATS = frozenset({"bf16", "nvfp4", "mxfp4_triton", "ds_fp4", "fp8_block"})# Formats this bench can build synthetic (correctly-sized) banks for._BUILDABLE_FORMATS = frozenset({"bf16", "nvfp4", "fp8_block", "mxfp4_triton", "ds_fp4"})# Friendlier CLI/display aliases for the internal quant_format strings.@@ -342,6 +343,22 @@ def _cpu_moe_bank_sources(fmt: str, H: int, I: int, E: int) -> dict:gate_up.fill_(0.02) # uninitialized bf16 can be denormal -> x86 FP slowdowndown.fill_(0.02)return {"gate_up": gate_up, "down": down}+ if fmt == "fp8_block":+ def nb(n):+ return (n + 127) // 128+ b = {+ "gate_up": pin(E, 2 * I, H, dtype=torch.float8_e4m3fn),+ "gate_up_scale": pin(E, nb(2 * I), nb(H), dtype=torch.bfloat16),+ "down": pin(E, H, I, dtype=torch.float8_e4m3fn),+ "down_scale": pin(E, nb(H), nb(I), dtype=torch.bfloat16),+ }+ # e4m3 0x00 is a clean zero and the scales are unit-ish, so nothing lands in+ # the fp32 denormal range (which would slow the GEMV and skew the timing).+ b["gate_up"].view(torch.uint8).fill_(0x38) # 1.0+ b["down"].view(torch.uint8).fill_(0x38)+ b["gate_up_scale"].fill_(1.0)+ b["down_scale"].fill_(1.0)+ return bif fmt == "nvfp4":b = {"gate_up_packed": pin(E, 2 * I, H // 2, dtype=torch.uint8),@@ -649,7 +666,7 @@ def _bench_format(fmt: str, wl: Workload, device: torch.device, threshold: floatif fmt == "nvfp4": # WF_NVFP4 is the only use_vnni format_note(entry, "isa tier labels are nominal: this W4A8 kernel rides AVX-VNNI ""regardless of the forced tier")- elif fmt in ("mxfp4_triton", "ds_fp4"):+ elif fmt in ("mxfp4_triton", "ds_fp4", "fp8_block"):_note(entry, "isa tier labels are nominal: avx512bf16 == avx512f here ""(3 real kernels: scalar/avx2/avx512, no bf16/VNNI variant)")entry["expert_bytes"] = entry["expert_bytes"] or c["expert_bytes"]@@ -781,7 +798,7 @@ def run_benchbw(_prog("done")result = {- "version": 4,+ "version": PROFILE_VERSION,"timestamp": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"),"epoch": int(time.time()),"host": socket.gethostname(),diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex c4809075..f519687e 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -71,7 +71,8 @@ _ACT_IDS = {}# Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp.-_WFMT_IDS = {"bf16": 0, "nvfp4": 1, "mxfp4_triton": 2, "ds_fp4": 3, "q4_0": 4}+_WFMT_IDS = {"bf16": 0, "nvfp4": 1, "mxfp4_triton": 2, "ds_fp4": 3, "q4_0": 4,+ "fp8_block": 5}def compiled_extension_supports(activation: str) -> bool:@@ -156,7 +157,7 @@ def resolve_threads_and_affinity(requested: int) -> tuple[int, list[int]]:class CpuMoeExecutor:"""Decode-time CPU expert compute over an ``OffloadMoeCache``'s host banks- (bf16, nvfp4, mxfp4_triton, ds_fp4 or q4_0 — see ``_WFMT_IDS`` / ``_resolve_banks``)."""+ (bf16, nvfp4, mxfp4_triton, ds_fp4, q4_0 or fp8_block — see ``_WFMT_IDS`` / ``_resolve_banks``)."""def __init__(self,@@ -189,6 +190,14 @@ class CpuMoeExecutor:# error and silently computes the wrong activation in the generic# epilogue -- fail loudly with the rebuild instruction instead. (mxfp4# handles its act inside the kernel and predates the marker.)+ # A prebuilt .so that predates this format would accept the id and then take+ # the wrong dequant branch -- silently wrong output rather than a crash.+ if _WFMT_IDS[fmt] > getattr(_cpu_moe, "max_weight_format_id", lambda: 4)():+ raise RuntimeError(+ f"the compiled _cpu_moe extension predates the {fmt!r} expert format; "+ "rebuild it with `python setup.py build_ext --inplace` (or reinstall "+ "the wheel) before serving this checkpoint on the cpu/hybrid backend."+ )if _ACT_IDS[activation] >= 3 and fmt != "mxfp4_triton":supported = getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()if _ACT_IDS[activation] > supported:@@ -359,6 +368,32 @@ class CpuMoeExecutor:self._banks.extend(layers)return table+ def _resolve_fp8_block_banks(self, banks: dict) -> tuple[dict, tuple[int, int]]:+ """Block-FP8 schema: row-major e4m3 + one bf16 scale per 128x128 block; the scale banks pad their last dim, so C++ gets the real row stride."""+ gu, gus = banks["gate_up"], banks["gate_up_scale"]+ dn, dns = banks["down"], banks["down_scale"]+ assert gu[0].element_size() == 1 and dn[0].element_size() == 1, (gu[0].dtype, dn[0].dtype)+ assert gus[0].dtype == torch.bfloat16 and dns[0].dtype == torch.bfloat16, (gus[0].dtype, dns[0].dtype)+ I = int(gu[0].shape[1] // 2)+ H = int(gu[0].shape[2])+ assert gu[0].shape[1] == 2 * I+ assert tuple(dn[0].shape[1:]) == (H, I), (dn[0].shape, H, I)+ assert gus[0].shape[1] == (2 * I + 127) // 128 and gus[0].shape[2] >= (H + 127) // 128, (gus[0].shape, H, I)+ assert dns[0].shape[1] == (H + 127) // 128 and dns[0].shape[2] >= (I + 127) // 128, (dns[0].shape, H, I)+ ptrs = dict(+ gate_up_ptr=self._make_table(gu).data_ptr(),+ down_ptr=self._make_table(dn).data_ptr(),+ gate_up_scale_ptr=self._make_table(gus).data_ptr(),+ gate_up_global_ptr=0,+ down_scale_ptr=self._make_table(dns).data_ptr(),+ down_global_ptr=0,+ gate_up_bias_ptr=0,+ down_bias_ptr=0,+ fp8_block_gu_scale_stride=int(gus[0].stride(1)),+ fp8_block_dn_scale_stride=int(dns[0].stride(1)),+ )+ return ptrs, (H, I)+def _resolve_banks(self, banks: dict, fmt: str) -> tuple[dict, tuple[int, int]]:"""Return (pointer kwargs for the C++ ctor, (H, I)) for the given format.@@ -400,6 +435,9 @@ class CpuMoeExecutor:if fmt == "ds_fp4":return self._resolve_dsfp4_banks(banks)+ if fmt == "fp8_block":+ return self._resolve_fp8_block_banks(banks)+# nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp16 row globals.gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"]dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"]@@ -515,11 +553,16 @@ class CpuMoeExecutor:def _io_for(self, bs: int) -> dict[str, torch.Tensor]:io = self._io.get(bs)if io is None:+ alloc_io = (+ (lambda *shape, dtype: torch.empty(*shape, dtype=dtype))+ if self.device.type == "cpu"+ else alloc_pinned_tensor+ )io = {- "x": alloc_pinned_tensor(bs, self.H, dtype=torch.bfloat16),- "ids": alloc_pinned_tensor(bs, self.top_k, dtype=torch.int32),- "w": alloc_pinned_tensor(bs, self.top_k, dtype=torch.float32),- "y": alloc_pinned_tensor(bs, self.H, dtype=torch.bfloat16),+ "x": alloc_io(bs, self.H, dtype=torch.bfloat16),+ "ids": alloc_io(bs, self.top_k, dtype=torch.int32),+ "w": alloc_io(bs, self.top_k, dtype=torch.float32),+ "y": alloc_io(bs, self.H, dtype=torch.bfloat16),}self._io[bs] = ioreturn iodiff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex 5e6caa75..7c0ed9b0 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -751,7 +751,7 @@ def parse_args("only, where CUDA pinned memory is capped: it locks just enough head+tail layers ""for the banks over the pin budget. Any value, 'auto' included, commits to CPU ""decode before the model is built, so the expert format must have a CPU executor "- "path (bf16, nvfp4, mxfp4); do not pass it on Linux. Unset = every layer on the "+ "path (bf16, fp8_block, nvfp4, mxfp4); do not pass it on Linux. Unset = every layer on the ""GPU; a boot whose banks exceed a known pin budget stops and asks for this flag."),)diff --git a/tests/models/test_quant_config.py b/tests/models/test_quant_config.pyindex bfcfb876..0f31d6de 100644--- a/tests/models/test_quant_config.py+++ b/tests/models/test_quant_config.py@@ -26,7 +26,6 @@ from freetoken.layers import set_rope_devicefrom freetoken.layers.quantization import (CompressedTensorsConfig,Fp8BlockConfig,- KernelSelectionError,ModelOptConfig,Mxfp4Config,NoQuantConfig,@@ -352,7 +351,7 @@ CASES = [# hybrid decodes on the CPU executor too, so the expert kernel must have a CPU formatCase("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {"model.layers.{lin}.mlp.experts": (Nvfp4MoEMethod, "triton")}, strategy="hybrid"),Case("Qwen3.6-35B-A3B", NoQuantConfig, {"model.layers.{lin}.mlp.experts": (UnquantizedMoEMethod, "fused")}, strategy="hybrid"),- Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {}, strategy="hybrid", raises=KernelSelectionError),+ Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {"model.layers.{lin}.mlp.experts": (Fp8BlockMoEMethod, "triton")}, strategy="hybrid"),]diff --git a/tests/moe/test_cpu_moe_fp8_block.py b/tests/moe/test_cpu_moe_fp8_block.pynew file mode 100644index 00000000..06f954d8--- /dev/null+++ b/tests/moe/test_cpu_moe_fp8_block.py@@ -0,0 +1,268 @@+"""Block-FP8 CPU MoE GEMV vs a float64 reference and FreeToken's Triton block-fp8 decode kernel.++Two things this is really guarding:+ * the scale is indexed [row // 128][k // 128] and *multiplies*, despite upstream naming it ``weight_scale_inv``. Inverting it, or transposing the two axes, still produces plausible-looking output.+ * the SIMD kernels decode e4m3 by reading each byte as fp16 bits. Subnormals carry too little weight in a full-range GEMV to show a decode error, so they get their own case.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++BLK = 128+++def _nb(n: int) -> int:+ return (n + BLK - 1) // BLK+++def _make_fp8_block_cache(L, E, H, I, seed=0):+ from freetoken.kernel.aot_models import fp8_block_scale_pad++ if torch.cuda.is_available():+ from freetoken.kernel.pinned import alloc_pinned_tensor+ else:+ def alloc_pinned_tensor(*shape, dtype):+ return torch.empty(*shape, dtype=dtype)++ torch.manual_seed(seed)+ S = L * E++ def rows(OUT, IN):+ w = alloc_pinned_tensor(S, OUT, IN, dtype=torch.float8_e4m3fn)+ w.copy_((torch.randn(S, OUT, IN) * 6.0).to(torch.float8_e4m3fn))+ s = alloc_pinned_tensor(+ S, _nb(OUT), fp8_block_scale_pad(_nb(OUT), _nb(IN)), dtype=torch.bfloat16+ )+ s.copy_((0.01 + 0.02 * torch.rand_like(s)).to(torch.bfloat16))+ return w, s++ gu, gus = rows(2 * I, H)+ dn, dns = rows(H, I)+ return SimpleNamespace(+ quant_format="fp8_block",+ bank_sources={+ "gate_up": list(gu.split(E)), "gate_up_scale": list(gus.split(E)),+ "down": list(dn.split(E)), "down_scale": list(dns.split(E)),+ },+ num_layers=L,+ num_experts=E,+ decode_target="cpu",+ cpu_executor=None,+ )+++def test_cpu_fp8_block_accepts_qwen38_padded_scale_bank():+ from freetoken.kernel.aot_models import fp8_block_scale_pad+ from freetoken.moe.cpu_executor import CpuMoeExecutor++ H, I = 2560, 640+ cache = _make_fp8_block_cache(1, 1, H, I)+ assert cache.bank_sources["down_scale"][0].shape[1:] == (+ _nb(H), fp8_block_scale_pad(_nb(H), _nb(I))+ ) == (20, 6)+ ex = CpuMoeExecutor(+ cache,+ top_k=1,+ activation="silu",+ apply_router_weight_on_input=False,+ num_threads=1,+ max_tokens=1,+ device=torch.device("cpu"),+ )+ hidden = torch.randn(1, H, dtype=torch.bfloat16)+ io = ex._io_for(1)+ io["x"].copy_(hidden)+ io["ids"].zero_()+ io["w"].fill_(1.0)+ ex._ext.run_task(ex._task_for(0, 1))+ cpu_out = io["y"][0].float().to(torch.float64)+ banks = cache.bank_sources+ gate_up = _reference_fp8_block_gemv(banks["gate_up"][0], banks["gate_up_scale"][0], hidden[0])+ intermediate = torch.nn.functional.silu(gate_up[:I]) * gate_up[I:]+ expected = _reference_fp8_block_gemv(banks["down"][0], banks["down_scale"][0], intermediate)+ rel = (cpu_out - expected).abs().max() / (expected.abs().max() + 1e-6)+ assert rel < 5e-3, f"max relative error {rel.item()}"+++# every finite e4m3 code, and the subnormal ones (exp == 0) with both signs+_FINITE_CODES = [c for c in range(256) if c not in (0x7F, 0xFF)]+_SUBNORMAL_CODES = [c for c in range(256) if c & 0x78 == 0]+++def _make_coded_fp8_block_cache(H, I, codes, seed=0):+ from freetoken.kernel.aot_models import fp8_block_scale_pad++ if torch.cuda.is_available():+ from freetoken.kernel.pinned import alloc_pinned_tensor+ else:+ def alloc_pinned_tensor(*shape, dtype):+ return torch.empty(*shape, dtype=dtype)++ torch.manual_seed(seed)+ pool = torch.tensor(codes, dtype=torch.uint8)++ def rows(OUT, IN):+ count = OUT * IN+ picked = pool.repeat((count + pool.numel() - 1) // pool.numel())[:count]+ picked = picked[torch.randperm(count)].reshape(1, OUT, IN)+ weights = alloc_pinned_tensor(1, OUT, IN, dtype=torch.float8_e4m3fn)+ weights.copy_(picked.view(torch.float8_e4m3fn))+ scales = alloc_pinned_tensor(+ 1, _nb(OUT), fp8_block_scale_pad(_nb(OUT), _nb(IN)), dtype=torch.bfloat16+ )+ scales.copy_((0.01 + 0.09 * torch.rand_like(scales)).to(torch.bfloat16))+ return weights, scales++ gu, gus = rows(2 * I, H)+ dn, dns = rows(H, I)+ return SimpleNamespace(+ quant_format="fp8_block",+ bank_sources={+ "gate_up": [gu], "gate_up_scale": [gus],+ "down": [dn], "down_scale": [dns],+ },+ num_layers=1,+ num_experts=1,+ decode_target="cpu",+ cpu_executor=None,+ )+++def _reference_fp8_block_gemv(weights, scales, x):+ rows, cols = weights.shape[-2:]+ scale = scales[0, :_nb(rows), :_nb(cols)].to(torch.float64)+ scale = scale.repeat_interleave(BLK, 0).repeat_interleave(BLK, 1)[:rows, :cols]+ return (weights[0].to(torch.float64) * scale) @ x.to(torch.float64)++[email protected]("codes", ["finite", "subnormal"])+ "isa, expected_isa",+ [("scalar", "scalar"), ("avx2", "avx2"), ("avx512", "avx512f"), ("avx512bf16", "avx512f")],+)+def test_cpu_fp8_block_matches_float64_reference(monkeypatch, isa, expected_isa, codes):+ from freetoken.moe.cpu_executor import CpuMoeExecutor++ H, I = 259, 257 # ragged K for both GEMVs; 257 down rows exercise many reductions+ cache = _make_coded_fp8_block_cache(H, I, _FINITE_CODES if codes == "finite" else _SUBNORMAL_CODES, seed=20260904)+ monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)+ ex = CpuMoeExecutor(+ cache,+ top_k=1,+ activation="silu",+ apply_router_weight_on_input=False,+ num_threads=1,+ max_tokens=1,+ device=torch.device("cpu"),+ )+ tiers = {"scalar": 0, "avx2": 1, "avx512": 2, "avx512bf16": 3}+ if tiers[isa] > ("scalar", "avx2", "avx512f", "avx512bf16").index(ex.isa):+ pytest.skip(f"requested {isa}, host/build selected {ex.isa}")+ assert ex._ext.fp8_block_isa_name() == expected_isa++ hidden = torch.randn(1, H, dtype=torch.bfloat16)+ io = ex._io_for(1)+ io["x"].copy_(hidden)+ io["ids"].zero_()+ io["w"].fill_(1.0)+ ex._ext.run_task(ex._task_for(0, 1))+ cpu_out = io["y"][0].float().to(torch.float64)++ banks = cache.bank_sources+ gate_up = _reference_fp8_block_gemv(banks["gate_up"][0], banks["gate_up_scale"][0], hidden[0])+ intermediate = (torch.nn.functional.silu(gate_up[:I]) * gate_up[I:])+ expected = _reference_fp8_block_gemv(banks["down"][0], banks["down_scale"][0], intermediate)+ rel = (cpu_out - expected).abs().max() / (expected.abs().max() + 1e-6)+ # The executor stores bf16; measured max relative error is 3.3e-3 (finite codes) and 1.2e-3 (subnormal codes).+ # The bound leaves room for reduction order, not for a narrow accumulator.+ assert rel < 5e-3, f"max relative error {rel.item()}"++[email protected]("isa", ["scalar", "avx2", "avx512", "avx512bf16"])+def test_cpu_fp8_block_isa_override(monkeypatch, isa):+ from freetoken.moe.cpu_executor import CpuMoeExecutor++ cache = _make_fp8_block_cache(1, 1, 256, 128, seed=17)+ monkeypatch.delenv("FREETOKEN_CPU_MOE_ISA", raising=False)+ auto = CpuMoeExecutor(+ cache, top_k=1, activation="silu", apply_router_weight_on_input=False,+ num_threads=1, max_tokens=1, device=torch.device("cpu"),+ )+ names = ("scalar", "avx2", "avx512f", "avx512bf16")+ auto_rank = names.index(auto.isa)+ monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)+ forced = CpuMoeExecutor(+ cache, top_k=1, activation="silu", apply_router_weight_on_input=False,+ num_threads=1, max_tokens=1, device=torch.device("cpu"),+ )+ assert names.index(forced.isa) == min(names.index({+ "scalar": "scalar", "avx2": "avx2", "avx512": "avx512f",+ "avx512bf16": "avx512bf16",+ }[isa]), auto_rank)+++# FREETOKEN_CPU_MOE_ISA tier -> the fp8_block kernel it selects; the avx512bf16 tier selects the avx512f kernel, so it is not repeated here.+# The env var caps *down* only, so a tier the CPU or the build lacks selects a lower kernel, which turns into a skip instead of testing the same kernel twice.+_ISA_TIERS = {"scalar": "scalar", "avx2": "avx2", "avx512": "avx512f"}++[email protected](not torch.cuda.is_available(), reason="needs CUDA")[email protected]("isa", list(_ISA_TIERS))[email protected]("bs", [1, 2, 5])+def test_cpu_fp8_block_matches_gpu_decode_kernel(bs, isa, monkeypatch):+ from freetoken.moe.cpu_executor import CpuMoeExecutor+ from freetoken.moe.fused_fp8_block import fused_experts_decode_fp8_block++ L, E, H, I, top_k = 2, 8, 256, 128, 4+ layer = 1+ dev = torch.device("cuda")+ cache = _make_fp8_block_cache(L, E, H, I, seed=bs)++ # Set before construction: the executor latches its dot kernel there.+ monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)+ ex = CpuMoeExecutor(+ cache,+ top_k=top_k,+ activation="silu",+ apply_router_weight_on_input=False,+ num_threads=0,+ max_tokens=bs,+ device=dev,+ )+ got = ex._ext.fp8_block_isa_name()+ if got != _ISA_TIERS[isa]:+ pytest.skip(f"asked for {isa}, kernel selected {got}: not supported here")++ hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16)+ ids = torch.stack([torch.randperm(E, device=dev)[:top_k] for _ in range(bs)]).to(torch.int32)+ w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)++ cpu_out = ex.decode(layer, hidden, w, ids).float()+ torch.cuda.synchronize()++ gpu_out = fused_experts_decode_fp8_block(+ hidden,+ cache.bank_sources["gate_up"][layer].to(dev),+ cache.bank_sources["gate_up_scale"][layer].to(dev),+ cache.bank_sources["down"][layer].to(dev),+ cache.bank_sources["down_scale"][layer].to(dev),+ w, ids.clone(), "silu", False,+ ).float()++ rel = (cpu_out - gpu_out).abs().max() / (gpu_out.abs().max() + 1e-6)+ assert rel < 2e-2, f"bs={bs} isa={got} rel err {rel.item()}"+++def test_fp16_bits_carry_every_e4m3_byte():+ """A byte b read as fp16 bits (b & 0x80) << 8 | (b & 0x7F) << 7 is its e4m3 value times 2^-8, which the SIMD kernels rely on."""+ codes = torch.arange(256, dtype=torch.int32)+ ref = codes.to(torch.uint8).view(torch.float8_e4m3fn).float()+ bits = ((codes & 0x80) << 8) | ((codes & 0x7F) << 7)+ got = torch.where(bits >= 0x8000, bits - 0x10000, bits).to(torch.int16).view(torch.float16).float() * 256+ finite = torch.isfinite(ref)+ assert torch.equal(got[finite], ref[finite])+ assert torch.equal(got[~finite].abs(), torch.full((2,), 480.0)) # 0x7F / 0xFF, as e4m3_decodediff --git a/tests/moe/test_hybrid_fetch.py b/tests/moe/test_hybrid_fetch.pyindex 169f8e2b..a26630b8 100644--- a/tests/moe/test_hybrid_fetch.py+++ b/tests/moe/test_hybrid_fetch.py@@ -11,7 +11,7 @@ import osimport pytestimport torch-from freetoken.moe.bench_profile import default_profile_path, load_backend_recommendation, load_hybrid_fetch_fraction+from freetoken.moe.bench_profile import PROFILE_VERSION, default_profile_path, load_backend_recommendation, load_hybrid_fetch_fractionfrom freetoken.moe.offload_cache import OffloadMoeCacheQ = 1 << 16@@ -42,6 +42,7 @@ def test_balanced_fetch_tracks_fraction():def test_load_hybrid_fetch_fraction(tmp_path):prof = {+ "version": PROFILE_VERSION,"gpu": {"name": "FAKE GPU"},"dtype_kernels": {"bf16": {"cpu_moe_gbs": 100.0, "pcie_gather_gbs": 40.0},@@ -64,6 +65,9 @@ def test_load_hybrid_fetch_fraction(tmp_path):assert load_hybrid_fetch_fraction("nvfp4", path=str(path)) is None# a profile from different hardware is ignoredassert load_hybrid_fetch_fraction("bf16", gpu_name="OTHER", path=str(path)) is None+ # so is one written before the current set of CPU formats+ path.write_text(json.dumps({**prof, "version": PROFILE_VERSION - 1}))+ assert load_hybrid_fetch_fraction("bf16", path=str(path)) is Nonedef test_profile_lookup_prefers_the_gpu_uuid_file(tmp_path, monkeypatch):@@ -74,7 +78,7 @@ def test_profile_lookup_prefers_the_gpu_uuid_file(tmp_path, monkeypatch):def write(path, name, verdict):os.makedirs(os.path.dirname(path), exist_ok=True)with open(path, "w") as f:- json.dump({"gpu": {"name": name}, "dtypes": {"bf16": verdict}}, f)+ json.dump({"version": PROFILE_VERSION, "gpu": {"name": name}, "dtypes": {"bf16": verdict}}, f)# legacy single file only: used when the name matches, ignored otherwisewrite(default_profile_path(), "FAKE GPU", "hybrid")