Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(cpu-moe): serve block-fp8 experts on the cpu and hybrid backends (#662)

Xiaoze Fan95ac86f21567
diff --git a/docs/cli.md b/docs/cli.md
index a15a34b7..2166e9e8 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -87,7 +87,7 @@ See [models.md](models.md#moe-strategies) for what each strategy does.
| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family strategies) |
| `--kv-reserve-tokens` | 8192 | KV token floor reserved before `--moe-cache-auto` fills experts |
| `--moe-cpu-threads` | physical cores | CPU worker threads for the cpu/hybrid executor |
-| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, nvfp4, mxfp4), so fp8 experts cannot use it |
+| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, fp8_block, nvfp4, mxfp4) |
| `--moe-hybrid-max-fetch` | auto | With `hybrid`: max experts fetched over PCIe per layer per step; rest computed on CPU |
| `--moe-prefill-hit-d2d` | off | Prefill: copy cache-hit experts device-side, stream only misses (CUDA >= 13) |
| `--disable-moe-prefill-overlap` | overlap on | Disable the two-buffer prefill copy overlap |
diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
index 86765013..6f85971e 100644
--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
@@ -10,7 +10,7 @@
// sync() blocks the host-func thread until the pool drains. The heavy GEMV runs
// on the persistent worker threads, not the host-func thread.
//
-// Weight formats: bf16, NVFP4, MXFP4, ds_fp4 and Q4_0 expert banks (see WFmt and
+// Weight formats: bf16, NVFP4, MXFP4, ds_fp4, Q4_0 and block-FP8 expert banks (see WFmt and
// the per-format bank schemas). Compute is FP32-accumulate; the intermediate is
// stored bf16 to match the GPU decode path. ISA is chosen once at construction
// (AVX-512-BF16 dpbf16 -> AVX-512F widening -> AVX2+FMA -> scalar).
@@ -271,6 +271,108 @@ inline float e4m3_decode(uint8_t v) {
return sign * (1.0f + man / 8.0f) * std::ldexp(1.0f, (int)exp - 7);
}
+// ------------------------ block-FP8 (W8A16) dequant ------------------------
+// fp8-e4m3 weights row-major [rows, K] plus one bf16 scale per 128x128 block. The scale tensor is named weight_scale_inv but multiplies, as in kernel/triton/fp8_blockscale_moe.py.
+// Whole 128-blocks read the activations as fp32 even/odd halves (xe[m] = x[2m], xo[m] = x[2m+1]); a ragged tail reads the bf16 row x.
+// 0x7F/0xFF (NaN in torch float8_e4m3fn) decode to +-480 in every kernel, as e4m3_decode does.
+using fp8_block_dot_fn = float (*)(const uint8_t*, const float*, const float*, const bf16_t*,
+ const bf16_t*, int, const float*);
+
+constexpr int FP8_BLK = 128; // K elements covered by one weight scale
+
+inline float fp8_block_tail(const uint8_t* w, const bf16_t* x, int k0, int K, const float* e4m3) {
+ float blk = 0.0f;
+ for (int k = k0; k < K; ++k) blk += e4m3[w[k]] * bf16_to_f32(x[k]);
+ return blk;
+}
+
+float dot_fp8_block_scalar(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,
+ const bf16_t* s, int K, const float* e4m3) {
+ float acc = 0.0f;
+ int b = 0, k0 = 0;
+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {
+ // 8 independent sums: a single running sum is bound by the add latency
+ float a[8] = {};
+ const uint8_t* wb = w + k0;
+ const float* pe = xe + k0 / 2;
+ const float* po = xo + k0 / 2;
+ for (int j = 0; j < FP8_BLK / 2; j += 4)
+ for (int l = 0; l < 4; ++l) {
+ a[2 * l] += e4m3[wb[2 * (j + l)]] * pe[j + l];
+ a[2 * l + 1] += e4m3[wb[2 * (j + l) + 1]] * po[j + l];
+ }
+ acc += (((a[0] + a[1]) + (a[2] + a[3])) + ((a[4] + a[5]) + (a[6] + a[7]))) *
+ bf16_to_f32(s[b]);
+ }
+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);
+ return acc;
+}
+
+#if CPU_MOE_X86
+// A byte b read as fp16 bits (b & 0x80) << 8 | (b & 0x7F) << 7 is its e4m3 value times 2^-8, subnormals included, so the block scale carries the 256.
+// In a 16-bit lane holding bytes (2m, 2m+1), srai 1 then & 0xBF80 builds that for byte 2m+1; a left shift by 8 first does byte 2m. Every AVX2 CPU has F16C.
+__attribute__((target("avx2,fma,f16c")))
+float dot_fp8_block_avx2(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,
+ const bf16_t* s, int K, const float* e4m3) {
+ const __m256i keep = _mm256_set1_epi16((short)0xBF80);
+ __m256 total = _mm256_setzero_ps();
+ int b = 0, k0 = 0;
+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {
+ _mm_prefetch(reinterpret_cast<const char*>(w + k0) + PF_AHEAD, _MM_HINT_T0);
+ __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
+ __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();
+ for (int j = 0; j < FP8_BLK; j += 32) {
+ const __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j));
+ const __m256i odd = _mm256_and_si256(_mm256_srai_epi16(v, 1), keep);
+ const __m256i even = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v, 8), 1), keep);
+ const float* pe = xe + (k0 + j) / 2;
+ const float* po = xo + (k0 + j) / 2;
+ a0 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_castsi256_si128(even)), _mm256_loadu_ps(pe), a0);
+ a1 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_extracti128_si256(even, 1)), _mm256_loadu_ps(pe + 8), a1);
+ a2 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_castsi256_si128(odd)), _mm256_loadu_ps(po), a2);
+ a3 = _mm256_fmadd_ps(_mm256_cvtph_ps(_mm256_extracti128_si256(odd, 1)), _mm256_loadu_ps(po + 8), a3);
+ }
+ const __m256 blk = _mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3));
+ total = _mm256_fmadd_ps(blk, _mm256_set1_ps(bf16_to_f32(s[b]) * 256.0f), total);
+ }
+ float acc = hsum256(total);
+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);
+ return acc;
+}
+
+__attribute__((target("avx512f,avx2,fma,f16c")))
+float dot_fp8_block_avx512f(const uint8_t* w, const float* xe, const float* xo, const bf16_t* x,
+ const bf16_t* s, int K, const float* e4m3) {
+ const __m256i keep = _mm256_set1_epi16((short)0xBF80);
+ __m512 total = _mm512_setzero_ps();
+ int b = 0, k0 = 0;
+ for (; k0 + FP8_BLK <= K; k0 += FP8_BLK, ++b) {
+ _mm_prefetch(reinterpret_cast<const char*>(w + k0) + PF_AHEAD, _MM_HINT_T0);
+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
+ for (int j = 0; j < FP8_BLK; j += 64) {
+ const __m256i v0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j));
+ const __m256i v1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + k0 + j + 32));
+ const __m256i o0 = _mm256_and_si256(_mm256_srai_epi16(v0, 1), keep);
+ const __m256i e0 = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v0, 8), 1), keep);
+ const __m256i o1 = _mm256_and_si256(_mm256_srai_epi16(v1, 1), keep);
+ const __m256i e1 = _mm256_and_si256(_mm256_srai_epi16(_mm256_slli_epi16(v1, 8), 1), keep);
+ const float* pe = xe + (k0 + j) / 2;
+ const float* po = xo + (k0 + j) / 2;
+ a0 = _mm512_fmadd_ps(_mm512_cvtph_ps(e0), _mm512_loadu_ps(pe), a0);
+ a1 = _mm512_fmadd_ps(_mm512_cvtph_ps(o0), _mm512_loadu_ps(po), a1);
+ a2 = _mm512_fmadd_ps(_mm512_cvtph_ps(e1), _mm512_loadu_ps(pe + 16), a2);
+ a3 = _mm512_fmadd_ps(_mm512_cvtph_ps(o1), _mm512_loadu_ps(po + 16), a3);
+ }
+ const __m512 blk = _mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3));
+ total = _mm512_fmadd_ps(blk, _mm512_set1_ps(bf16_to_f32(s[b]) * 256.0f), total);
+ }
+ float acc = _mm512_reduce_add_ps(total);
+ if (k0 < K) acc += fp8_block_tail(w, x, k0, K, e4m3) * bf16_to_f32(s[b]);
+ return acc;
+}
+#endif // CPU_MOE_X86
+
// Activations pre-deinterleaved to fp32 (xe[m]=x[2m], xo[m]=x[2m+1]); see the
// ds_fp4 dot below for why (drops the hot loop to ~1.5 shuffle ops / 16 weights).
using nvdot_fn = float (*)(const uint8_t*, const uint8_t*, float, const float*, const float*,
@@ -677,7 +779,7 @@ struct DotChoice {
};
// SIMD tiers, ascending. Each format picks the highest tier <= the one chosen by
-// pick_isa() that it implements (fp4 formats have no bf16-specific tier, so the
+// pick_isa() that it implements (fp4 and fp8_block formats have no bf16-specific tier, so the
// avx512bf16 tier maps to their avx512 kernel).
enum IsaTier { ISA_SCALAR = 0, ISA_AVX2 = 1, ISA_AVX512 = 2, ISA_AVX512BF16 = 3 };
@@ -1017,6 +1119,17 @@ void mxfp4_gemv_avx2(float* out, const uint8_t* blk, const uint8_t* scl, const b
}
#endif
+fp8_block_dot_fn select_fp8_block_dot() {
+ const IsaTier t = pick_isa();
+#if CPU_MOE_X86
+ // the avx512bf16 tier keeps the fp32 kernel: on Sapphire Rapids it is faster than widening to bf16 for dpbf16
+ if (t >= ISA_AVX512) return dot_fp8_block_avx512f;
+ if (t >= ISA_AVX2) return dot_fp8_block_avx2;
+#endif
+ (void)t;
+ return dot_fp8_block_scalar;
+}
+
mxgemv_fn select_mxgemv() {
const IsaTier t = pick_isa();
#if CPU_MOE_X86
@@ -1229,7 +1342,8 @@ q4dot_fn select_q4dot() {
return q4_0_dot_i8_scalar;
}
-enum WFmt { WF_BF16 = 0, WF_NVFP4 = 1, WF_MXFP4 = 2, WF_DSFP4 = 3, WF_Q4_0 = 4 };
+enum WFmt { WF_BF16 = 0, WF_NVFP4 = 1, WF_MXFP4 = 2, WF_DSFP4 = 3, WF_Q4_0 = 4,
+ WF_FP8_BLOCK = 5 };
// Each ctor pointer arg is the address of a CPU int64 array of length
// num_layers (one base address per layer, built by cpu_executor.py's
@@ -1246,7 +1360,7 @@ struct CpuMoeExecutor {
int H, I;
int act, apply_on_input;
int fmt; // WFmt
- bool needs_di = false; // pre-deinterleave activations to fp32 (nvfp4/ds_fp4)
+ bool needs_di = false; // pre-deinterleave activations to fp32 (nvfp4/ds_fp4/fp8_block)
// Per-layer pointer tables (one base address per layer, see tbl_at). gate_up_tbl
// doubles as the bf16 gate_up table and the nvfp4/mxfp4/q4_0/ds_fp4 packed-gate_up
// table (down_tbl likewise for down); which reinterpretation applies is picked by
@@ -1269,6 +1383,10 @@ struct CpuMoeExecutor {
dsdot_fn dsdot;
mxgemv_fn mxgemv;
q4dot_fn q4dot;
+ fp8_block_dot_fn fp8_block_dot;
+ // fp8_block: bf16 scale per 128x128 weight block. These are the scale tensor's
+ // trailing strides, including any padding used by the host bank layout.
+ int fp8_block_gu_scale_row = 0, fp8_block_dn_scale_row = 0;
// ds_fp4: the caller already FP8-round-tripped the input activations on the GPU
// (same reference grid), so submit() must not repeat it on the host-callback
// thread. That scalar per-element pass is single-threaded ON THE DECODE CRITICAL
@@ -1287,7 +1405,7 @@ struct CpuMoeExecutor {
std::vector<bf16_t> g_scratch; // [max_tokens * top_k * I] intermediate
std::vector<bf16_t> xq_scratch; // [max_tokens * H] ds_fp4 fp8-roundtripped input
- // ds_fp4 activations pre-deinterleaved to fp32 (even/odd K) for the row-major dot.
+ // nvfp4/ds_fp4/fp8_block activations pre-deinterleaved to fp32 (even/odd K) for the row-major dot.
std::vector<float> xe_scratch, xo_scratch; // [max_tokens * H/2] (input)
std::vector<float> ge_scratch, go_scratch; // [max_tokens*top_k*I/2] (intermediate)
// AVX-VNNI W4A8: per-16-block int8 activations [even(8),odd(8)] + per-block scale.
@@ -1315,7 +1433,7 @@ struct CpuMoeExecutor {
alignas(64) std::atomic<int64_t> p1_next{0};
std::atomic<int64_t> p2_next{0};
- std::atomic<int64_t> prt_next{0}; // ds_fp4 intermediate fp8 round-trip phase
+ std::atomic<int64_t> prt_next{0}; // intermediate-row prep phase (prep_g_row)
int64_t p1_total = 0, p2_total = 0, prt_total = 0;
int n_iblk = 0, n_hblk = 0;
std::atomic<int> done_count{0};
@@ -1373,7 +1491,8 @@ struct CpuMoeExecutor {
uintptr_t gate_up_global_ptr, uintptr_t down_scale_ptr,
uintptr_t down_global_ptr, uintptr_t gate_up_bias_ptr,
uintptr_t down_bias_ptr, double swiglu_alpha_, double swiglu_limit_,
- std::vector<int> core_ids_)
+ std::vector<int> core_ids_, int fp8_block_gu_scale_stride_,
+ int fp8_block_dn_scale_stride_)
: num_threads(num_threads_ > 0 ? num_threads_ : 1),
num_layers(num_layers_),
num_experts(num_experts_),
@@ -1393,6 +1512,8 @@ struct CpuMoeExecutor {
dn_bias_tbl(reinterpret_cast<const uint64_t*>(down_bias_ptr)),
swiglu_alpha(static_cast<float>(swiglu_alpha_)),
swiglu_limit(static_cast<float>(swiglu_limit_)),
+ fp8_block_gu_scale_row(fp8_block_gu_scale_stride_),
+ fp8_block_dn_scale_row(fp8_block_dn_scale_stride_),
core_ids(std::move(core_ids_)) {
DotChoice c = select_dot();
dot = c.fn;
@@ -1400,6 +1521,7 @@ struct CpuMoeExecutor {
dsdot = select_dsdot();
mxgemv = select_mxgemv();
q4dot = select_q4dot();
+ fp8_block_dot = select_fp8_block_dot();
if (weight_format == WF_Q4_0) {
if (H % 32 != 0 || I % 32 != 0)
throw std::runtime_error("Q4_0 CPU MoE requires H and I to be multiples of 32");
@@ -1423,8 +1545,8 @@ struct CpuMoeExecutor {
// e8m0 (mxfp4 block scale) = 2^(s-127); the GPU GEMV clamps s to [0,254].
for (int i = 0; i < 256; ++i) e8m0_lut[i] = std::ldexp(1.0f, std::min(i, 254) - 127);
g_scratch.assign(static_cast<size_t>(max_tokens) * top_k * I, 0);
- // Row-major fp4 (nvfp4/ds_fp4) pre-deinterleaves activations to fp32 even/odd.
- needs_di = (fmt == WF_NVFP4 || fmt == WF_DSFP4);
+ // Row-major fp4 (nvfp4/ds_fp4) and fp8_block pre-deinterleave activations to fp32 even/odd.
+ needs_di = (fmt == WF_NVFP4 || fmt == WF_DSFP4 || fmt == WF_FP8_BLOCK);
if (needs_di) {
if (fmt == WF_DSFP4) xq_scratch.assign(static_cast<size_t>(max_tokens) * H, 0);
xe_scratch.assign(static_cast<size_t>(max_tokens) * (H / 2), 0);
@@ -1498,6 +1620,7 @@ struct CpuMoeExecutor {
// once per task/pass by the caller via tbl_at) pick the layer's own tensors.
// bf16 uses the interleaved bf16 row; nvfp4 uses the pre-split fp32 even/odd halves
// (or, with AVX-VNNI, the per-16-block int8 activations).
+ // fp8_block uses the halves for whole 128-blocks and the bf16 row for a ragged tail.
inline float gemm1_dot(const bf16_t* gate_up_l, const uint8_t* gu_packed_l,
const uint8_t* gu_scale_l, const uint16_t* gu_global_l, int e, int row,
const bf16_t* x, const float* xe, const float* xo, const int8_t* xi8,
@@ -1511,6 +1634,13 @@ struct CpuMoeExecutor {
gu_packed_l + ((size_t)e * (2 * I) + row) * (size_t)q4_gu_row_bytes;
return q4dot(w, xi8, xas, H); // W4A8: int8 activations (Q8_0), scale in xas
}
+ if (fmt == WF_FP8_BLOCK) {
+ const uint8_t* w = gu_packed_l + ((size_t)e * (2 * I) + row) * (size_t)H;
+ const bf16_t* sc = reinterpret_cast<const bf16_t*>(gu_scale_l) +
+ ((size_t)e * ((2 * I + FP8_BLK - 1) / FP8_BLK) + row / FP8_BLK) *
+ (size_t)fp8_block_gu_scale_row;
+ return fp8_block_dot(w, xe, xo, x, sc, H, e4m3_lut);
+ }
const size_t r = (size_t)e * (2 * I) + row;
if (use_vnni)
return nvi8dot(gu_packed_l + r * (size_t)(H / 2), gu_scale_l + r * (size_t)(H / 16),
@@ -1533,6 +1663,13 @@ struct CpuMoeExecutor {
const uint8_t* w = dn_packed_l + ((size_t)e * H + row) * (size_t)q4_dn_row_bytes;
return q4dot(w, gi8, gas, I); // W4A8: int8 activations (Q8_0), scale in gas
}
+ if (fmt == WF_FP8_BLOCK) {
+ const uint8_t* w = dn_packed_l + ((size_t)e * H + row) * (size_t)I;
+ const bf16_t* sc = reinterpret_cast<const bf16_t*>(dn_scale_l) +
+ ((size_t)e * ((H + FP8_BLK - 1) / FP8_BLK) + row / FP8_BLK) *
+ (size_t)fp8_block_dn_scale_row;
+ return fp8_block_dot(w, ge, go, g, sc, I, e4m3_lut);
+ }
const size_t r = (size_t)e * H + row;
if (use_vnni)
return nvi8dot(dn_packed_l + r * (size_t)(I / 2), dn_scale_l + r * (size_t)(I / 16),
@@ -1582,6 +1719,14 @@ struct CpuMoeExecutor {
const char* isa_name() const { return isa; }
+ const char* fp8_block_isa_name() const {
+#if CPU_MOE_X86
+ if (fp8_block_dot == dot_fp8_block_avx512f) return "avx512f";
+ if (fp8_block_dot == dot_fp8_block_avx2) return "avx2";
+#endif
+ return "scalar";
+ }
+
void barrier(int& local_sense) {
local_sense ^= 1;
if (bar_count.fetch_add(1) + 1 == num_threads) {
@@ -1810,9 +1955,8 @@ struct CpuMoeExecutor {
}
}
- // Prepare one intermediate row (token,route) for the down GEMV: ds_fp4 first FP8
- // round-trips it (DSV4 act_quant), then both formats deinterleave to fp32 even/odd
- // (reused across every down output row).
+ // Prepare one intermediate row (token,route) for the down GEMV, reused across every down output row.
+ // ds_fp4 first FP8 round-trips it (DSV4 act_quant); nvfp4/ds_fp4/fp8_block deinterleave it to fp32 even/odd; q4_0 Q8_0-quantizes it.
void prep_g_row(int64_t r) {
bf16_t* g = g_scratch.data() + (size_t)r * I;
if (use_q4a8) { // q4_0 W4A8: Q8_0-quantize the intermediate row for the down GEMV.
@@ -1865,9 +2009,8 @@ struct CpuMoeExecutor {
do_pass1(t, p);
}
barrier(local_sense);
- // Row-major fp4: prepare the intermediate rows (per token,route) before the down
- // GEMV -- ds_fp4 FP8 round-trips (DSV4 act_quant), both deinterleave to fp32; q4_0
- // W4A8 Q8_0-quantizes. Needs all of pass1 done (a full row spans every iblk).
+ // Prepare the intermediate rows (per token,route) for the down GEMV, see prep_g_row.
+ // Needs all of pass1 done (a full row spans every iblk).
if (needs_di || use_q4a8) {
for (;;) {
int64_t r = prt_next.fetch_add(1, std::memory_order_relaxed);
@@ -1938,8 +2081,8 @@ struct CpuMoeExecutor {
done_count.store(0, std::memory_order_relaxed);
bar_count.store(0, std::memory_order_relaxed);
bar_sense.store(0, std::memory_order_relaxed);
- // ds_fp4: FP8 round-trip the per-token input once, up front (single-threaded;
- // tiny for decode, and done before the workers are woken below).
+ // Deinterleave the per-token input once, up front (ds_fp4 FP8-round-trips it first).
+ // Single-threaded: tiny for decode, and done before the workers are woken below.
if (needs_di) {
const size_t xn = static_cast<size_t>(t->num_tokens) * H;
if (xn / 2 > xe_scratch.size()) {
@@ -2160,7 +2303,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
py::class_<CpuMoeExecutor>(m, "CpuMoeExecutor")
.def(py::init<int, int, int, int, int, int, int, int, int, int, uintptr_t, uintptr_t,
uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t,
- double, double, std::vector<int>>(),
+ double, double, std::vector<int>, int, int>(),
py::arg("num_threads"), py::arg("num_layers"), py::arg("num_experts"),
py::arg("top_k"), py::arg("hidden_size"), py::arg("inter_size"),
py::arg("max_tokens"), py::arg("activation_id"),
@@ -2169,7 +2312,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
py::arg("gate_up_global_ptr"), py::arg("down_scale_ptr"),
py::arg("down_global_ptr"), py::arg("gate_up_bias_ptr"),
py::arg("down_bias_ptr"), py::arg("swiglu_alpha"), py::arg("swiglu_limit"),
- py::arg("core_ids"))
+ py::arg("core_ids"), py::arg("fp8_block_gu_scale_stride") = 0,
+ py::arg("fp8_block_dn_scale_stride") = 0)
.def("create_task", &CpuMoeExecutor::create_task, py::arg("layer_id"),
py::arg("num_tokens"), py::arg("x_ptr"), py::arg("ids_ptr"), py::arg("w_ptr"),
py::arg("y_ptr"))
@@ -2192,6 +2336,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
[](CpuMoeExecutor& e, int v) { e.act_block = v; },
py::arg("value"))
.def("isa_name", &CpuMoeExecutor::isa_name)
+ .def("fp8_block_isa_name", &CpuMoeExecutor::fp8_block_isa_name)
.def("set_spin_wait",
[](CpuMoeExecutor& e, bool v) { e.spin_wait.store(v, std::memory_order_relaxed); },
py::arg("enabled"))
@@ -2208,4 +2353,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
// (act_apply falls through to gelu_tanh); the probe turns a stale extension
// into a loud rebuild instruction instead of wrong model outputs.
m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLU_CLAMP); });
+ // Weight-format ABI probe. A stale prebuilt .so accepts a newer WFmt id without
+ // complaint and then falls through to the wrong dequant branch -- silently wrong
+ // numbers, not a crash. The Python executor refuses to build against an extension
+ // that predates the format it was asked for.
+ m.def("max_weight_format_id", []() { return static_cast<int>(WF_FP8_BLOCK); });
}
diff --git a/python/freetoken/layers/quantization/moe/fp8_block.py b/python/freetoken/layers/quantization/moe/fp8_block.py
index 7552978d..6cda6b1e 100644
--- a/python/freetoken/layers/quantization/moe/fp8_block.py
+++ b/python/freetoken/layers/quantization/moe/fp8_block.py
@@ -19,9 +19,10 @@ def _pad_scale(rows: int, cols: int) -> int:
class TritonFp8BlockMoEKernel(MoEKernel):
name = "triton"
+ cpu_format = "fp8_block"
def unusable_reason(self, cfg: MoEConfig) -> str | None:
- reason = self._common_reject(cfg, tp_ok=False, cpu_ok=False, plain_silu_only=False)
+ reason = self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)
if reason:
return reason
if cfg.scheme is not None and fp8_block_size(cfg.scheme) != BLOCK:
diff --git a/python/freetoken/moe/bench_profile.py b/python/freetoken/moe/bench_profile.py
index d7b72073..5fa38b6f 100644
--- a/python/freetoken/moe/bench_profile.py
+++ b/python/freetoken/moe/bench_profile.py
@@ -18,6 +18,9 @@ from freetoken.utils import init_logger
logger = init_logger(__name__)
+# Bump when a format gains or loses a CPU MoE path: verdicts and fetch fractions from an older profile are stale.
+PROFILE_VERSION = 5
+
# Engine ``expert_quant`` (models/config.py) -> benchbw format key (offload_cache._BANK_SCHEMAS
# / benchbw._offload_bank_specs). Only the offload-family formats with a CPU MoE weight path can
# ever resolve to hybrid; anything not listed falls through unmapped and finds no profile entry
@@ -101,6 +104,12 @@ def _usable_profile(
return None
if not isinstance(prof, dict):
return None
+ if not isinstance(prof.get("version"), int) or prof["version"] < PROFILE_VERSION:
+ logger.warning(
+ f"benchbw profile {src} predates this engine (version {prof.get('version')}, "
+ f"need {PROFILE_VERSION}); ignoring it, rerun `ft bench bw`"
+ )
+ return None
prof_gpu = (prof.get("gpu") or {}).get("name")
if gpu_name and prof_gpu and prof_gpu != gpu_name:
logger.warning(
diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.py
index e9d6ffe5..e70cad31 100644
--- a/python/freetoken/moe/benchbw.py
+++ b/python/freetoken/moe/benchbw.py
@@ -56,6 +56,7 @@ from freetoken.gpu_select import (
single_gpu_arg,
)
from freetoken.kernel.pinned import alloc_pinned_tensor
+from freetoken.moe.bench_profile import PROFILE_VERSION
from freetoken.moe.cpu_executor import physical_core_cpus, resolve_threads_and_affinity
from freetoken.utils import init_logger
@@ -63,7 +64,7 @@ logger = init_logger(__name__)
# Formats the CPU MoE C++ kernel can compute AND this bench can build banks for; anything
# else is offload-only here. (The kernel also does q4_0, but this bench has no q4_0 banks.)
-_CPU_MOE_FORMATS = frozenset({"bf16", "nvfp4", "mxfp4_triton", "ds_fp4"})
+_CPU_MOE_FORMATS = frozenset({"bf16", "nvfp4", "mxfp4_triton", "ds_fp4", "fp8_block"})
# Formats this bench can build synthetic (correctly-sized) banks for.
_BUILDABLE_FORMATS = frozenset({"bf16", "nvfp4", "fp8_block", "mxfp4_triton", "ds_fp4"})
# Friendlier CLI/display aliases for the internal quant_format strings.
@@ -342,6 +343,22 @@ def _cpu_moe_bank_sources(fmt: str, H: int, I: int, E: int) -> dict:
gate_up.fill_(0.02) # uninitialized bf16 can be denormal -> x86 FP slowdown
down.fill_(0.02)
return {"gate_up": gate_up, "down": down}
+ if fmt == "fp8_block":
+ def nb(n):
+ return (n + 127) // 128
+ b = {
+ "gate_up": pin(E, 2 * I, H, dtype=torch.float8_e4m3fn),
+ "gate_up_scale": pin(E, nb(2 * I), nb(H), dtype=torch.bfloat16),
+ "down": pin(E, H, I, dtype=torch.float8_e4m3fn),
+ "down_scale": pin(E, nb(H), nb(I), dtype=torch.bfloat16),
+ }
+ # e4m3 0x00 is a clean zero and the scales are unit-ish, so nothing lands in
+ # the fp32 denormal range (which would slow the GEMV and skew the timing).
+ b["gate_up"].view(torch.uint8).fill_(0x38) # 1.0
+ b["down"].view(torch.uint8).fill_(0x38)
+ b["gate_up_scale"].fill_(1.0)
+ b["down_scale"].fill_(1.0)
+ return b
if fmt == "nvfp4":
b = {
"gate_up_packed": pin(E, 2 * I, H // 2, dtype=torch.uint8),
@@ -649,7 +666,7 @@ def _bench_format(fmt: str, wl: Workload, device: torch.device, threshold: float
if fmt == "nvfp4": # WF_NVFP4 is the only use_vnni format
_note(entry, "isa tier labels are nominal: this W4A8 kernel rides AVX-VNNI "
"regardless of the forced tier")
- elif fmt in ("mxfp4_triton", "ds_fp4"):
+ elif fmt in ("mxfp4_triton", "ds_fp4", "fp8_block"):
_note(entry, "isa tier labels are nominal: avx512bf16 == avx512f here "
"(3 real kernels: scalar/avx2/avx512, no bf16/VNNI variant)")
entry["expert_bytes"] = entry["expert_bytes"] or c["expert_bytes"]
@@ -781,7 +798,7 @@ def run_benchbw(
_prog("done")
result = {
- "version": 4,
+ "version": PROFILE_VERSION,
"timestamp": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"),
"epoch": int(time.time()),
"host": socket.gethostname(),
diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.py
index c4809075..f519687e 100644
--- a/python/freetoken/moe/cpu_executor.py
+++ b/python/freetoken/moe/cpu_executor.py
@@ -71,7 +71,8 @@ _ACT_IDS = {
}
# Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp.
-_WFMT_IDS = {"bf16": 0, "nvfp4": 1, "mxfp4_triton": 2, "ds_fp4": 3, "q4_0": 4}
+_WFMT_IDS = {"bf16": 0, "nvfp4": 1, "mxfp4_triton": 2, "ds_fp4": 3, "q4_0": 4,
+ "fp8_block": 5}
def compiled_extension_supports(activation: str) -> bool:
@@ -156,7 +157,7 @@ def resolve_threads_and_affinity(requested: int) -> tuple[int, list[int]]:
class CpuMoeExecutor:
"""Decode-time CPU expert compute over an ``OffloadMoeCache``'s host banks
- (bf16, nvfp4, mxfp4_triton, ds_fp4 or q4_0 — see ``_WFMT_IDS`` / ``_resolve_banks``)."""
+ (bf16, nvfp4, mxfp4_triton, ds_fp4, q4_0 or fp8_block — see ``_WFMT_IDS`` / ``_resolve_banks``)."""
def __init__(
self,
@@ -189,6 +190,14 @@ class CpuMoeExecutor:
# error and silently computes the wrong activation in the generic
# epilogue -- fail loudly with the rebuild instruction instead. (mxfp4
# handles its act inside the kernel and predates the marker.)
+ # A prebuilt .so that predates this format would accept the id and then take
+ # the wrong dequant branch -- silently wrong output rather than a crash.
+ if _WFMT_IDS[fmt] > getattr(_cpu_moe, "max_weight_format_id", lambda: 4)():
+ raise RuntimeError(
+ f"the compiled _cpu_moe extension predates the {fmt!r} expert format; "
+ "rebuild it with `python setup.py build_ext --inplace` (or reinstall "
+ "the wheel) before serving this checkpoint on the cpu/hybrid backend."
+ )
if _ACT_IDS[activation] >= 3 and fmt != "mxfp4_triton":
supported = getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()
if _ACT_IDS[activation] > supported:
@@ -359,6 +368,32 @@ class CpuMoeExecutor:
self._banks.extend(layers)
return table
+ def _resolve_fp8_block_banks(self, banks: dict) -> tuple[dict, tuple[int, int]]:
+ """Block-FP8 schema: row-major e4m3 + one bf16 scale per 128x128 block; the scale banks pad their last dim, so C++ gets the real row stride."""
+ gu, gus = banks["gate_up"], banks["gate_up_scale"]
+ dn, dns = banks["down"], banks["down_scale"]
+ assert gu[0].element_size() == 1 and dn[0].element_size() == 1, (gu[0].dtype, dn[0].dtype)
+ assert gus[0].dtype == torch.bfloat16 and dns[0].dtype == torch.bfloat16, (gus[0].dtype, dns[0].dtype)
+ I = int(gu[0].shape[1] // 2)
+ H = int(gu[0].shape[2])
+ assert gu[0].shape[1] == 2 * I
+ assert tuple(dn[0].shape[1:]) == (H, I), (dn[0].shape, H, I)
+ assert gus[0].shape[1] == (2 * I + 127) // 128 and gus[0].shape[2] >= (H + 127) // 128, (gus[0].shape, H, I)
+ assert dns[0].shape[1] == (H + 127) // 128 and dns[0].shape[2] >= (I + 127) // 128, (dns[0].shape, H, I)
+ ptrs = dict(
+ gate_up_ptr=self._make_table(gu).data_ptr(),
+ down_ptr=self._make_table(dn).data_ptr(),
+ gate_up_scale_ptr=self._make_table(gus).data_ptr(),
+ gate_up_global_ptr=0,
+ down_scale_ptr=self._make_table(dns).data_ptr(),
+ down_global_ptr=0,
+ gate_up_bias_ptr=0,
+ down_bias_ptr=0,
+ fp8_block_gu_scale_stride=int(gus[0].stride(1)),
+ fp8_block_dn_scale_stride=int(dns[0].stride(1)),
+ )
+ return ptrs, (H, I)
+
def _resolve_banks(self, banks: dict, fmt: str) -> tuple[dict, tuple[int, int]]:
"""Return (pointer kwargs for the C++ ctor, (H, I)) for the given format.
@@ -400,6 +435,9 @@ class CpuMoeExecutor:
if fmt == "ds_fp4":
return self._resolve_dsfp4_banks(banks)
+ if fmt == "fp8_block":
+ return self._resolve_fp8_block_banks(banks)
+
# nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp16 row globals.
gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"]
dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"]
@@ -515,11 +553,16 @@ class CpuMoeExecutor:
def _io_for(self, bs: int) -> dict[str, torch.Tensor]:
io = self._io.get(bs)
if io is None:
+ alloc_io = (
+ (lambda *shape, dtype: torch.empty(*shape, dtype=dtype))
+ if self.device.type == "cpu"
+ else alloc_pinned_tensor
+ )
io = {
- "x": alloc_pinned_tensor(bs, self.H, dtype=torch.bfloat16),
- "ids": alloc_pinned_tensor(bs, self.top_k, dtype=torch.int32),
- "w": alloc_pinned_tensor(bs, self.top_k, dtype=torch.float32),
- "y": alloc_pinned_tensor(bs, self.H, dtype=torch.bfloat16),
+ "x": alloc_io(bs, self.H, dtype=torch.bfloat16),
+ "ids": alloc_io(bs, self.top_k, dtype=torch.int32),
+ "w": alloc_io(bs, self.top_k, dtype=torch.float32),
+ "y": alloc_io(bs, self.H, dtype=torch.bfloat16),
}
self._io[bs] = io
return io
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index 5e6caa75..7c0ed9b0 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -751,7 +751,7 @@ def parse_args(
"only, where CUDA pinned memory is capped: it locks just enough head+tail layers "
"for the banks over the pin budget. Any value, 'auto' included, commits to CPU "
"decode before the model is built, so the expert format must have a CPU executor "
- "path (bf16, nvfp4, mxfp4); do not pass it on Linux. Unset = every layer on the "
+ "path (bf16, fp8_block, nvfp4, mxfp4); do not pass it on Linux. Unset = every layer on the "
"GPU; a boot whose banks exceed a known pin budget stops and asks for this flag."
),
)
diff --git a/tests/models/test_quant_config.py b/tests/models/test_quant_config.py
index bfcfb876..0f31d6de 100644
--- a/tests/models/test_quant_config.py
+++ b/tests/models/test_quant_config.py
@@ -26,7 +26,6 @@ from freetoken.layers import set_rope_device
from freetoken.layers.quantization import (
CompressedTensorsConfig,
Fp8BlockConfig,
- KernelSelectionError,
ModelOptConfig,
Mxfp4Config,
NoQuantConfig,
@@ -352,7 +351,7 @@ CASES = [
# hybrid decodes on the CPU executor too, so the expert kernel must have a CPU format
Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {"model.layers.{lin}.mlp.experts": (Nvfp4MoEMethod, "triton")}, strategy="hybrid"),
Case("Qwen3.6-35B-A3B", NoQuantConfig, {"model.layers.{lin}.mlp.experts": (UnquantizedMoEMethod, "fused")}, strategy="hybrid"),
- Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {}, strategy="hybrid", raises=KernelSelectionError),
+ Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {"model.layers.{lin}.mlp.experts": (Fp8BlockMoEMethod, "triton")}, strategy="hybrid"),
]
diff --git a/tests/moe/test_cpu_moe_fp8_block.py b/tests/moe/test_cpu_moe_fp8_block.py
new file mode 100644
index 00000000..06f954d8
--- /dev/null
+++ b/tests/moe/test_cpu_moe_fp8_block.py
@@ -0,0 +1,268 @@
+"""Block-FP8 CPU MoE GEMV vs a float64 reference and FreeToken's Triton block-fp8 decode kernel.
+
+Two things this is really guarding:
+ * the scale is indexed [row // 128][k // 128] and *multiplies*, despite upstream naming it ``weight_scale_inv``. Inverting it, or transposing the two axes, still produces plausible-looking output.
+ * the SIMD kernels decode e4m3 by reading each byte as fp16 bits. Subnormals carry too little weight in a full-range GEMV to show a decode error, so they get their own case.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+BLK = 128
+
+
+def _nb(n: int) -> int:
+ return (n + BLK - 1) // BLK
+
+
+def _make_fp8_block_cache(L, E, H, I, seed=0):
+ from freetoken.kernel.aot_models import fp8_block_scale_pad
+
+ if torch.cuda.is_available():
+ from freetoken.kernel.pinned import alloc_pinned_tensor
+ else:
+ def alloc_pinned_tensor(*shape, dtype):
+ return torch.empty(*shape, dtype=dtype)
+
+ torch.manual_seed(seed)
+ S = L * E
+
+ def rows(OUT, IN):
+ w = alloc_pinned_tensor(S, OUT, IN, dtype=torch.float8_e4m3fn)
+ w.copy_((torch.randn(S, OUT, IN) * 6.0).to(torch.float8_e4m3fn))
+ s = alloc_pinned_tensor(
+ S, _nb(OUT), fp8_block_scale_pad(_nb(OUT), _nb(IN)), dtype=torch.bfloat16
+ )
+ s.copy_((0.01 + 0.02 * torch.rand_like(s)).to(torch.bfloat16))
+ return w, s
+
+ gu, gus = rows(2 * I, H)
+ dn, dns = rows(H, I)
+ return SimpleNamespace(
+ quant_format="fp8_block",
+ bank_sources={
+ "gate_up": list(gu.split(E)), "gate_up_scale": list(gus.split(E)),
+ "down": list(dn.split(E)), "down_scale": list(dns.split(E)),
+ },
+ num_layers=L,
+ num_experts=E,
+ decode_target="cpu",
+ cpu_executor=None,
+ )
+
+
+def test_cpu_fp8_block_accepts_qwen38_padded_scale_bank():
+ from freetoken.kernel.aot_models import fp8_block_scale_pad
+ from freetoken.moe.cpu_executor import CpuMoeExecutor
+
+ H, I = 2560, 640
+ cache = _make_fp8_block_cache(1, 1, H, I)
+ assert cache.bank_sources["down_scale"][0].shape[1:] == (
+ _nb(H), fp8_block_scale_pad(_nb(H), _nb(I))
+ ) == (20, 6)
+ ex = CpuMoeExecutor(
+ cache,
+ top_k=1,
+ activation="silu",
+ apply_router_weight_on_input=False,
+ num_threads=1,
+ max_tokens=1,
+ device=torch.device("cpu"),
+ )
+ hidden = torch.randn(1, H, dtype=torch.bfloat16)
+ io = ex._io_for(1)
+ io["x"].copy_(hidden)
+ io["ids"].zero_()
+ io["w"].fill_(1.0)
+ ex._ext.run_task(ex._task_for(0, 1))
+ cpu_out = io["y"][0].float().to(torch.float64)
+ banks = cache.bank_sources
+ gate_up = _reference_fp8_block_gemv(banks["gate_up"][0], banks["gate_up_scale"][0], hidden[0])
+ intermediate = torch.nn.functional.silu(gate_up[:I]) * gate_up[I:]
+ expected = _reference_fp8_block_gemv(banks["down"][0], banks["down_scale"][0], intermediate)
+ rel = (cpu_out - expected).abs().max() / (expected.abs().max() + 1e-6)
+ assert rel < 5e-3, f"max relative error {rel.item()}"
+
+
+# every finite e4m3 code, and the subnormal ones (exp == 0) with both signs
+_FINITE_CODES = [c for c in range(256) if c not in (0x7F, 0xFF)]
+_SUBNORMAL_CODES = [c for c in range(256) if c & 0x78 == 0]
+
+
+def _make_coded_fp8_block_cache(H, I, codes, seed=0):
+ from freetoken.kernel.aot_models import fp8_block_scale_pad
+
+ if torch.cuda.is_available():
+ from freetoken.kernel.pinned import alloc_pinned_tensor
+ else:
+ def alloc_pinned_tensor(*shape, dtype):
+ return torch.empty(*shape, dtype=dtype)
+
+ torch.manual_seed(seed)
+ pool = torch.tensor(codes, dtype=torch.uint8)
+
+ def rows(OUT, IN):
+ count = OUT * IN
+ picked = pool.repeat((count + pool.numel() - 1) // pool.numel())[:count]
+ picked = picked[torch.randperm(count)].reshape(1, OUT, IN)
+ weights = alloc_pinned_tensor(1, OUT, IN, dtype=torch.float8_e4m3fn)
+ weights.copy_(picked.view(torch.float8_e4m3fn))
+ scales = alloc_pinned_tensor(
+ 1, _nb(OUT), fp8_block_scale_pad(_nb(OUT), _nb(IN)), dtype=torch.bfloat16
+ )
+ scales.copy_((0.01 + 0.09 * torch.rand_like(scales)).to(torch.bfloat16))
+ return weights, scales
+
+ gu, gus = rows(2 * I, H)
+ dn, dns = rows(H, I)
+ return SimpleNamespace(
+ quant_format="fp8_block",
+ bank_sources={
+ "gate_up": [gu], "gate_up_scale": [gus],
+ "down": [dn], "down_scale": [dns],
+ },
+ num_layers=1,
+ num_experts=1,
+ decode_target="cpu",
+ cpu_executor=None,
+ )
+
+
+def _reference_fp8_block_gemv(weights, scales, x):
+ rows, cols = weights.shape[-2:]
+ scale = scales[0, :_nb(rows), :_nb(cols)].to(torch.float64)
+ scale = scale.repeat_interleave(BLK, 0).repeat_interleave(BLK, 1)[:rows, :cols]
+ return (weights[0].to(torch.float64) * scale) @ x.to(torch.float64)
+
+
[email protected]("codes", ["finite", "subnormal"])
+ "isa, expected_isa",
+ [("scalar", "scalar"), ("avx2", "avx2"), ("avx512", "avx512f"), ("avx512bf16", "avx512f")],
+)
+def test_cpu_fp8_block_matches_float64_reference(monkeypatch, isa, expected_isa, codes):
+ from freetoken.moe.cpu_executor import CpuMoeExecutor
+
+ H, I = 259, 257 # ragged K for both GEMVs; 257 down rows exercise many reductions
+ cache = _make_coded_fp8_block_cache(H, I, _FINITE_CODES if codes == "finite" else _SUBNORMAL_CODES, seed=20260904)
+ monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)
+ ex = CpuMoeExecutor(
+ cache,
+ top_k=1,
+ activation="silu",
+ apply_router_weight_on_input=False,
+ num_threads=1,
+ max_tokens=1,
+ device=torch.device("cpu"),
+ )
+ tiers = {"scalar": 0, "avx2": 1, "avx512": 2, "avx512bf16": 3}
+ if tiers[isa] > ("scalar", "avx2", "avx512f", "avx512bf16").index(ex.isa):
+ pytest.skip(f"requested {isa}, host/build selected {ex.isa}")
+ assert ex._ext.fp8_block_isa_name() == expected_isa
+
+ hidden = torch.randn(1, H, dtype=torch.bfloat16)
+ io = ex._io_for(1)
+ io["x"].copy_(hidden)
+ io["ids"].zero_()
+ io["w"].fill_(1.0)
+ ex._ext.run_task(ex._task_for(0, 1))
+ cpu_out = io["y"][0].float().to(torch.float64)
+
+ banks = cache.bank_sources
+ gate_up = _reference_fp8_block_gemv(banks["gate_up"][0], banks["gate_up_scale"][0], hidden[0])
+ intermediate = (torch.nn.functional.silu(gate_up[:I]) * gate_up[I:])
+ expected = _reference_fp8_block_gemv(banks["down"][0], banks["down_scale"][0], intermediate)
+ rel = (cpu_out - expected).abs().max() / (expected.abs().max() + 1e-6)
+ # The executor stores bf16; measured max relative error is 3.3e-3 (finite codes) and 1.2e-3 (subnormal codes).
+ # The bound leaves room for reduction order, not for a narrow accumulator.
+ assert rel < 5e-3, f"max relative error {rel.item()}"
+
+
[email protected]("isa", ["scalar", "avx2", "avx512", "avx512bf16"])
+def test_cpu_fp8_block_isa_override(monkeypatch, isa):
+ from freetoken.moe.cpu_executor import CpuMoeExecutor
+
+ cache = _make_fp8_block_cache(1, 1, 256, 128, seed=17)
+ monkeypatch.delenv("FREETOKEN_CPU_MOE_ISA", raising=False)
+ auto = CpuMoeExecutor(
+ cache, top_k=1, activation="silu", apply_router_weight_on_input=False,
+ num_threads=1, max_tokens=1, device=torch.device("cpu"),
+ )
+ names = ("scalar", "avx2", "avx512f", "avx512bf16")
+ auto_rank = names.index(auto.isa)
+ monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)
+ forced = CpuMoeExecutor(
+ cache, top_k=1, activation="silu", apply_router_weight_on_input=False,
+ num_threads=1, max_tokens=1, device=torch.device("cpu"),
+ )
+ assert names.index(forced.isa) == min(names.index({
+ "scalar": "scalar", "avx2": "avx2", "avx512": "avx512f",
+ "avx512bf16": "avx512bf16",
+ }[isa]), auto_rank)
+
+
+# FREETOKEN_CPU_MOE_ISA tier -> the fp8_block kernel it selects; the avx512bf16 tier selects the avx512f kernel, so it is not repeated here.
+# The env var caps *down* only, so a tier the CPU or the build lacks selects a lower kernel, which turns into a skip instead of testing the same kernel twice.
+_ISA_TIERS = {"scalar": "scalar", "avx2": "avx2", "avx512": "avx512f"}
+
+
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
[email protected]("isa", list(_ISA_TIERS))
[email protected]("bs", [1, 2, 5])
+def test_cpu_fp8_block_matches_gpu_decode_kernel(bs, isa, monkeypatch):
+ from freetoken.moe.cpu_executor import CpuMoeExecutor
+ from freetoken.moe.fused_fp8_block import fused_experts_decode_fp8_block
+
+ L, E, H, I, top_k = 2, 8, 256, 128, 4
+ layer = 1
+ dev = torch.device("cuda")
+ cache = _make_fp8_block_cache(L, E, H, I, seed=bs)
+
+ # Set before construction: the executor latches its dot kernel there.
+ monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)
+ ex = CpuMoeExecutor(
+ cache,
+ top_k=top_k,
+ activation="silu",
+ apply_router_weight_on_input=False,
+ num_threads=0,
+ max_tokens=bs,
+ device=dev,
+ )
+ got = ex._ext.fp8_block_isa_name()
+ if got != _ISA_TIERS[isa]:
+ pytest.skip(f"asked for {isa}, kernel selected {got}: not supported here")
+
+ hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16)
+ ids = torch.stack([torch.randperm(E, device=dev)[:top_k] for _ in range(bs)]).to(torch.int32)
+ w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)
+
+ cpu_out = ex.decode(layer, hidden, w, ids).float()
+ torch.cuda.synchronize()
+
+ gpu_out = fused_experts_decode_fp8_block(
+ hidden,
+ cache.bank_sources["gate_up"][layer].to(dev),
+ cache.bank_sources["gate_up_scale"][layer].to(dev),
+ cache.bank_sources["down"][layer].to(dev),
+ cache.bank_sources["down_scale"][layer].to(dev),
+ w, ids.clone(), "silu", False,
+ ).float()
+
+ rel = (cpu_out - gpu_out).abs().max() / (gpu_out.abs().max() + 1e-6)
+ assert rel < 2e-2, f"bs={bs} isa={got} rel err {rel.item()}"
+
+
+def test_fp16_bits_carry_every_e4m3_byte():
+ """A byte b read as fp16 bits (b & 0x80) << 8 | (b & 0x7F) << 7 is its e4m3 value times 2^-8, which the SIMD kernels rely on."""
+ codes = torch.arange(256, dtype=torch.int32)
+ ref = codes.to(torch.uint8).view(torch.float8_e4m3fn).float()
+ bits = ((codes & 0x80) << 8) | ((codes & 0x7F) << 7)
+ got = torch.where(bits >= 0x8000, bits - 0x10000, bits).to(torch.int16).view(torch.float16).float() * 256
+ finite = torch.isfinite(ref)
+ assert torch.equal(got[finite], ref[finite])
+ assert torch.equal(got[~finite].abs(), torch.full((2,), 480.0)) # 0x7F / 0xFF, as e4m3_decode
diff --git a/tests/moe/test_hybrid_fetch.py b/tests/moe/test_hybrid_fetch.py
index 169f8e2b..a26630b8 100644
--- a/tests/moe/test_hybrid_fetch.py
+++ b/tests/moe/test_hybrid_fetch.py
@@ -11,7 +11,7 @@ import os
import pytest
import torch
-from freetoken.moe.bench_profile import default_profile_path, load_backend_recommendation, load_hybrid_fetch_fraction
+from freetoken.moe.bench_profile import PROFILE_VERSION, default_profile_path, load_backend_recommendation, load_hybrid_fetch_fraction
from freetoken.moe.offload_cache import OffloadMoeCache
Q = 1 << 16
@@ -42,6 +42,7 @@ def test_balanced_fetch_tracks_fraction():
def test_load_hybrid_fetch_fraction(tmp_path):
prof = {
+ "version": PROFILE_VERSION,
"gpu": {"name": "FAKE GPU"},
"dtype_kernels": {
"bf16": {"cpu_moe_gbs": 100.0, "pcie_gather_gbs": 40.0},
@@ -64,6 +65,9 @@ def test_load_hybrid_fetch_fraction(tmp_path):
assert load_hybrid_fetch_fraction("nvfp4", path=str(path)) is None
# a profile from different hardware is ignored
assert load_hybrid_fetch_fraction("bf16", gpu_name="OTHER", path=str(path)) is None
+ # so is one written before the current set of CPU formats
+ path.write_text(json.dumps({**prof, "version": PROFILE_VERSION - 1}))
+ assert load_hybrid_fetch_fraction("bf16", path=str(path)) is None
def test_profile_lookup_prefers_the_gpu_uuid_file(tmp_path, monkeypatch):
@@ -74,7 +78,7 @@ def test_profile_lookup_prefers_the_gpu_uuid_file(tmp_path, monkeypatch):
def write(path, name, verdict):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
- json.dump({"gpu": {"name": name}, "dtypes": {"bf16": verdict}}, f)
+ json.dump({"version": PROFILE_VERSION, "gpu": {"name": name}, "dtypes": {"bf16": verdict}}, f)
# legacy single file only: used when the name matches, ignored otherwise
write(default_profile_path(), "FAKE GPU", "hybrid")