FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(checkpoint): write the Qwen3.8-Flash-Next PLE table next to the FTW (#420)
Xiaoze Fan3d919e9bd94f
diff --git a/python/freetoken/checkpoint/convert.py b/python/freetoken/checkpoint/convert.pyindex 9ead434f..a875d77f 100644--- a/python/freetoken/checkpoint/convert.py+++ b/python/freetoken/checkpoint/convert.py@@ -226,6 +226,15 @@ def convert_checkpoint(dense_bytes += tensor.numel() * tensor.element_size()_progress("dense", dense_bytes, 0)+ # files the model reads directly from the checkpoint dir, not through FTW entries (Qwen3.8-Flash-Next PLE table)+ from freetoken.models.register import _load_attr, get_model_spec++ try:+ side_hook = _load_attr(get_model_spec(mc.architectures[0]).module, "ftw_side_files")+ except (AttributeError, KeyError, ValueError):+ side_hook = None+ side_files = side_hook(model_path, out_dir) if side_hook is not None else []+# 2) offload expert banks (post-repack) + alpha scales (slow path auto-picks parallel/serial)quant_format = Nonenum_layers = None@@ -308,6 +317,7 @@ def convert_checkpoint("expert_bank_num_layers": num_layers,"counts": {"weight": n_weight, "experts_bank": n_bank + n_alpha},"copied_metadata": copied,+ "side_files": side_files,})return indexdiff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.pyindex 1a74d4ae..6715e67e 100644--- a/python/freetoken/models/qwen4_exp/__init__.py+++ b/python/freetoken/models/qwen4_exp/__init__.py@@ -12,6 +12,7 @@ Contracts shared across modules (do not rename):from .config import parse_configfrom .model import Qwen4ExpForCausalLMfrom .weight import (+ ftw_side_files,nvfp4_expert_spec,iter_weights,load_ple_table,@@ -22,6 +23,7 @@ from .weight import (from freetoken.models.qwen3_5_moe.weight import iter_expert_pieces__all__ = [+ "ftw_side_files","nvfp4_expert_spec","Qwen4ExpForCausalLM","iter_weights",diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.pyindex da4bea48..2ce7462b 100644--- a/python/freetoken/models/qwen4_exp/weight.py+++ b/python/freetoken/models/qwen4_exp/weight.py@@ -53,6 +53,7 @@ _PLE_SHARD_RE = re.compile(r"\.ple\.ple_embedding\.ngram_embedding\.shard_(?P<shard>\d+)\.weight$")_PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale"+_PLE_FILE_BYTES = 4 << 30 # ple-table-*.safetensors written by ftw_side_files# Zero-centered Qwen4ExpTextRMSNorm weights, loaded RAW: GroupedPlusOneRMSNorm / GemmaPlusOneRMSNorm# and the vendored grouped_gemma_rmsnorm all apply (1+w) at runtime in fp32, so folding the +1 into@@ -222,6 +223,39 @@ def _ple_table_files(folder: str) -> list[str]:return sorted(os.path.join(folder, shard) for shard in files)+def ftw_side_files(model_path: str, out_dir: str) -> list[str]:+ """Write the PLE n-gram table tensors, and only those, into ``ple-table-*.safetensors`` next to an FTW checkpoint.++ The table is served from safetensors files in the checkpoint dir (see load_ple_table), not from FTW entries."""+ from safetensors.torch import save_file++ folder = download_hf_weight(model_path)+ written: list[str] = []+ batch: dict[str, torch.Tensor] = {}+ size = 0++ def flush():+ nonlocal batch, size+ if batch:+ name = f"ple-table-{len(written):05d}.safetensors"+ save_file(batch, os.path.join(out_dir, name))+ written.append(name)+ batch, size = {}, 0++ for path in _ple_table_files(folder):+ with safetensors.safe_open(path, framework="pt", device="cpu") as f:+ for key in f.keys():+ if _PLE_TABLE_INFIX not in key:+ continue+ t = f.get_tensor(key)+ batch[key] = t+ size += t.numel() * t.element_size()+ if size >= _PLE_FILE_BYTES:+ flush()+ flush()+ return written++def load_ple_table(model_path: str, qwen4_args, *, pin: bool = True,workers: int = 8, chunk: int = 8 << 20) -> PleTable:"""Concatenate the checkpoint's ``ngram_embedding.shard_<i>`` tensors into one pinned host bank.