Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(checkpoint): write the Qwen3.8-Flash-Next PLE table next to the FTW (#420)

Xiaoze Fan3d919e9bd94f
diff --git a/python/freetoken/checkpoint/convert.py b/python/freetoken/checkpoint/convert.py
index 9ead434f..a875d77f 100644
--- a/python/freetoken/checkpoint/convert.py
+++ b/python/freetoken/checkpoint/convert.py
@@ -226,6 +226,15 @@ def convert_checkpoint(
dense_bytes += tensor.numel() * tensor.element_size()
_progress("dense", dense_bytes, 0)
+ # files the model reads directly from the checkpoint dir, not through FTW entries (Qwen3.8-Flash-Next PLE table)
+ from freetoken.models.register import _load_attr, get_model_spec
+
+ try:
+ side_hook = _load_attr(get_model_spec(mc.architectures[0]).module, "ftw_side_files")
+ except (AttributeError, KeyError, ValueError):
+ side_hook = None
+ side_files = side_hook(model_path, out_dir) if side_hook is not None else []
+
# 2) offload expert banks (post-repack) + alpha scales (slow path auto-picks parallel/serial)
quant_format = None
num_layers = None
@@ -308,6 +317,7 @@ def convert_checkpoint(
"expert_bank_num_layers": num_layers,
"counts": {"weight": n_weight, "experts_bank": n_bank + n_alpha},
"copied_metadata": copied,
+ "side_files": side_files,
})
return index
diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.py
index 1a74d4ae..6715e67e 100644
--- a/python/freetoken/models/qwen4_exp/__init__.py
+++ b/python/freetoken/models/qwen4_exp/__init__.py
@@ -12,6 +12,7 @@ Contracts shared across modules (do not rename):
from .config import parse_config
from .model import Qwen4ExpForCausalLM
from .weight import (
+ ftw_side_files,
nvfp4_expert_spec,
iter_weights,
load_ple_table,
@@ -22,6 +23,7 @@ from .weight import (
from freetoken.models.qwen3_5_moe.weight import iter_expert_pieces
__all__ = [
+ "ftw_side_files",
"nvfp4_expert_spec",
"Qwen4ExpForCausalLM",
"iter_weights",
diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.py
index da4bea48..2ce7462b 100644
--- a/python/freetoken/models/qwen4_exp/weight.py
+++ b/python/freetoken/models/qwen4_exp/weight.py
@@ -53,6 +53,7 @@ _PLE_SHARD_RE = re.compile(
r"\.ple\.ple_embedding\.ngram_embedding\.shard_(?P<shard>\d+)\.weight$"
)
_PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale"
+_PLE_FILE_BYTES = 4 << 30 # ple-table-*.safetensors written by ftw_side_files
# Zero-centered Qwen4ExpTextRMSNorm weights, loaded RAW: GroupedPlusOneRMSNorm / GemmaPlusOneRMSNorm
# and the vendored grouped_gemma_rmsnorm all apply (1+w) at runtime in fp32, so folding the +1 into
@@ -222,6 +223,39 @@ def _ple_table_files(folder: str) -> list[str]:
return sorted(os.path.join(folder, shard) for shard in files)
+def ftw_side_files(model_path: str, out_dir: str) -> list[str]:
+ """Write the PLE n-gram table tensors, and only those, into ``ple-table-*.safetensors`` next to an FTW checkpoint.
+
+ The table is served from safetensors files in the checkpoint dir (see load_ple_table), not from FTW entries."""
+ from safetensors.torch import save_file
+
+ folder = download_hf_weight(model_path)
+ written: list[str] = []
+ batch: dict[str, torch.Tensor] = {}
+ size = 0
+
+ def flush():
+ nonlocal batch, size
+ if batch:
+ name = f"ple-table-{len(written):05d}.safetensors"
+ save_file(batch, os.path.join(out_dir, name))
+ written.append(name)
+ batch, size = {}, 0
+
+ for path in _ple_table_files(folder):
+ with safetensors.safe_open(path, framework="pt", device="cpu") as f:
+ for key in f.keys():
+ if _PLE_TABLE_INFIX not in key:
+ continue
+ t = f.get_tensor(key)
+ batch[key] = t
+ size += t.numel() * t.element_size()
+ if size >= _PLE_FILE_BYTES:
+ flush()
+ flush()
+ return written
+
+
def load_ple_table(model_path: str, qwen4_args, *, pin: bool = True,
workers: int = 8, chunk: int = 8 << 20) -> PleTable:
"""Concatenate the checkpoint's ``ngram_embedding.shard_<i>`` tensors into one pinned host bank.