FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(kernel): unbreak the nightly kernel-cache wheel build (#310)
Xiaoze Fan3a20a7903833
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex c9c2fb98..6268ae33 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -63,6 +63,13 @@ class AotModel:arch_aliases: tuple[str, ...] = ()+def fp8_block_scale_pad(rows: int, cols: int) -> int:+ """Trailing scale-bank dim padded so per-expert row bytes are 16B-aligned (fused copy)."""+ while (rows * cols * 2) % 16:+ cols += 1+ return cols++def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int) -> dict[str, int]:"""Per-expert row bytes for each offload bank a format registers.@@ -77,8 +84,6 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: intif fmt == "fp8_block":# qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block# scales, trailing scale dim 16B-padded (same helper as the loader)- from freetoken.moe.offload_cache import fp8_block_scale_pad-B = 128return {"gate_up": 2 * I * H,@@ -417,7 +422,8 @@ def aggregate_fast_index_copy_feature_sizes() -> tuple[int, ...]:sizes: set[int] = set(TEST_FEATURE_SIZES)for model in SUPPORTED_MODELS:sizes.update(fast_index_copy_feature_sizes(model))- return tuple(sorted(sizes))+ # the per-bank kernel copies rows in fixed 128-byte steps; other sizes cannot compile+ return tuple(sorted(size for size in sizes if size % 128 == 0))__all__ = [diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex 33a47553..e1f20dd2 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -77,11 +77,8 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {"ds_fp4": ("gate_up_packed", "gate_up_scale", "down_packed", "down_scale"),}-def fp8_block_scale_pad(rows: int, cols: int) -> int:- """Trailing scale-bank dim padded so per-expert row bytes are 16B-aligned (fused copy)."""- while (rows * cols * 2) % 16:- cols += 1- return cols+# lives in kernel/aot_models.py: the AOT row table shares it and must stay importable in the torch-only kernel-cache build env, which cannot import freetoken.moe+from freetoken.kernel.aot_models import fp8_block_scale_pad# bytes per (expert, layer) as f(hidden, moe_intermediate), from the bank shapes above; keep in sync with _BANK_SCHEMAS@@ -350,6 +347,19 @@ class OffloadMoeCache:self._init_prefill_overlap_buffers()def _build_copy_plan(self) -> None:+ self._build_fused_copy_plan()+ if self._copy_fused_ok or self.device.type != "cuda" or not self.banks:+ return+ for name in self.bank_schema:+ cache = self.bank_caches[name]+ feat = math.prod(cache.shape[1:]) * cache.element_size()+ if feat % 128:+ raise RuntimeError(+ f"MoE bank {name!r} rows are {feat} bytes (not a multiple of 128): "+ f"only the fused multi-bank copy can move them, but it is disabled"+ )++ def _build_fused_copy_plan(self) -> None:"""Precompute the fused multi-bank copy descriptor (base addrs + per-row bytes).Built once here (and on :meth:`rebuild`, which reallocates the slot caches);