FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(checkpoint): serve and repair FTW checkpoints around the vision encoder (#486)
Xiaoze Fancade1a9e452c
diff --git a/docs/ftw-hotfix.md b/docs/ftw-hotfix.mdindex 77362adb..299d522d 100644--- a/docs/ftw-hotfix.md+++ b/docs/ftw-hotfix.md@@ -14,6 +14,9 @@ python scripts/ftw_hotfix.py --ftw ~/models/DeepSeek-V4-Flash-0731-FTW# old Qwen3.8-Flash-Next FTW: add the 47.7 GiB PLE table from a local copy of the checkpoint, into a new dirpython scripts/ftw_hotfix.py --ftw ~/models/Qwen3.8-Flash-Next-NVFP4-FTW --source ~/models/Qwen3.8-Flash-Next-NVFP4 --out ~/models/Qwen3.8-Flash-Next-NVFP4-FTW-fixed+# Qwen3-VL FTW converted before image input: add the vision encoder (about 1 GiB) from the Hub, in place+python scripts/ftw_hotfix.py --ftw ~/models/Qwen3-VL-8B-Instruct-FTW --repo Qwen/Qwen3-VL-8B-Instruct+# just show what would changepython scripts/ftw_hotfix.py --ftw ~/models/GLM-5.2-NVFP4-FTW --dry-run```@@ -29,6 +32,7 @@ byte range, never the whole checkpoint; `--revision` pins the Hub revision to re| `KeyError: 'model.embed.weight'` | deepseek-ai/DeepSeek-V4-Flash-0731 | rename the index entries | none || `RuntimeError: Unexpected keys ... .weight_scale` | nvidia/GLM-5.2-NVFP4 | dequantize the old runtime-fp8 weights back to bf16 | none || `PLE shard indices are not contiguous 0..N-1: []` | Qwen3.8-Flash-Next (FTWs converted before #420) | write the PLE table as `ple-table-*.safetensors` | 47.7 GiB |+| `ValueError: ... holds no vision encoder tensors` | image-input families (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next, Gemma-4, GLM-5.3-Flash, Muse-Glimmer, MiniMax-M3) converted before the family served images | add the vision encoder: the tower's checkpoint tensors are fetched and the family's own weight reader emits them as the converter would | 1 to 4 GiB |The script decides by itself: it builds the current model from the FTW's `config.json`, comparesthe FTW index with the tensors the model declares, and applies only the repairs that are needed.diff --git a/docs/models.md b/docs/models.mdindex 16bedb41..548a363c 100644--- a/docs/models.md+++ b/docs/models.md@@ -54,6 +54,9 @@ These families accept image input by default; pass `--text-model-only` to skip tFreeToken's fast-load format, and `ft serve --model` auto-detects the result.- FTW files converted by builds before the quantization refactor may fail to load;see [ftw-hotfix.md](ftw-hotfix.md) for the affected checkpoints and the repair tool.+- An FTW converted before its family served images holds no vision encoder: `ft serve`+ refuses it unless started with `--text-model-only` (or `--mm-disable vision`); reconvert it+ with `ft checkpoint`, or add the encoder in place with [scripts/ftw_hotfix.py](ftw-hotfix.md).- DeepSeek-V4 checkpoints must keep the `inference/config.json` subdir — theauthoritative model args are read from there.- Qwen3.8-Flash-Next keeps a 47.7 GiB PLE n-gram table pinned in host RAM.diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.pyindex 207765b7..e2bd0de2 100644--- a/python/freetoken/checkpoint/ftw.py+++ b/python/freetoken/checkpoint/ftw.py@@ -38,6 +38,7 @@ import mmapimport osimport reimport threading+from collections.abc import Callablefrom concurrent.futures import ThreadPoolExecutorimport torch@@ -106,6 +107,13 @@ def is_ftw_checkpoint(path: str) -> bool:return os.path.isfile(os.path.join(path, INDEX_NAME))+def ftw_tensor_names(path: str, *kinds: str) -> list[str]:+ """Names the FTW index lists for ``kinds`` (every kind when none is given)."""+ keep = set(kinds)+ with open(os.path.join(path, INDEX_NAME)) as f:+ return [t["name"] for t in json.load(f)["tensors"] if not keep or t["kind"] in keep]++def ftw_quant_format(path: str) -> str | None:"""The ``quant_format`` an FTW checkpoint's expert banks were packed for; None when ``path`` is not an FTW checkpoint or holds no banks."""if not is_ftw_checkpoint(path):@@ -354,12 +362,13 @@ def _transient_buffer(nbytes: int) -> mmap.mmap:return mmap.mmap(-1, _align_up(nbytes))-def iter_ftw_weights(path: str, *, kinds=("weight",), workers: int = 8,- chunk: int = _DEFAULT_CHUNK, prefetch: int = 2):+def iter_ftw_weights(path: str, *, kinds=("weight",), keep: Callable[[str], bool] | None = None,+ workers: int = 8, chunk: int = _DEFAULT_CHUNK, prefetch: int = 2):"""Yield ``(name, host_tensor)`` for the requested kinds, reading each tensor viachunked O_DIRECT. A background thread prefetches the next ``prefetch`` tensors so thedisk stays busy while the consumer copies the current one to the GPU. Transient buffers- are freed as the consumer advances (peak host mem ~ prefetch+1 tensors)."""+ are freed as the consumer advances (peak host mem ~ prefetch+1 tensors). An entry whose+ name ``keep`` rejects is dropped before any of its bytes are read."""import queueimport threading@@ -367,6 +376,8 @@ def iter_ftw_weights(path: str, *, kinds=("weight",), workers: int = 8,reader = FTWReader(path)entries = reader.entries(*kinds)+ if keep is not None:+ entries = [e for e in entries if keep(e["name"])]q: queue.Queue = queue.Queue(maxsize=max(1, prefetch))_DONE = object()err: list[BaseException] = []@@ -650,6 +661,6 @@ def load_ftw_banks(__all__ = ["INDEX_NAME", "FORMAT_TAG", "FORMAT_VERSION", "ALIGN", "DEFAULT_SHARD_LIMIT",- "is_ftw_checkpoint", "FTWWriter", "FTWReader",+ "is_ftw_checkpoint", "ftw_tensor_names", "FTWWriter", "FTWReader","iter_ftw_weights", "load_ftw_banks", "layer_bank_entry_name",]diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 3848fc70..06a74ee5 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -16,6 +16,7 @@ from freetoken.layers.quantization import LayerKind, QuantBackend, finalize_quanfrom freetoken.moe.offload_cache import iter_offload_moe_layersfrom freetoken.mm.config import ENCODER_SECTIONSfrom freetoken.models import create_model, load_weight+from freetoken.models.weight import ftw_lacks_visionfrom freetoken.moe import is_offload_moe_strategyfrom freetoken.moe.expert_banks import load_expert_banksfrom freetoken.moe.host_banks import PinFailed@@ -500,6 +501,12 @@ class Engine:model_state = self.model.state_dict()if config.use_dummy_weight:return _make_dummy_weight_state_dict(model_state, device=self.device)+ if config.active_encoders and ftw_lacks_vision(config.model_path):+ raise ValueError(+ f"{config.model_path} holds no vision encoder tensors: it was converted by a build before this "+ "family served images. Reconvert it with `ft checkpoint`, add the encoder in place with "+ "scripts/ftw_hotfix.py (docs/ftw-hotfix.md), or start with --text-model-only"+ )# _materialize casts each loaded tensor to its model-param dtype (model_state), so# models declaring per-tensor dtypes (e.g. DSV4's mixed fp8/fp32/bf16) are preserved;# offload models exclude experts (served from the offload cache, not dense weights).diff --git a/python/freetoken/models/gemma4/__init__.py b/python/freetoken/models/gemma4/__init__.pyindex 925bbdb1..9f41f080 100644--- a/python/freetoken/models/gemma4/__init__.py+++ b/python/freetoken/models/gemma4/__init__.py@@ -11,6 +11,7 @@ from .moe import Gemma4Routerfrom .vision import Gemma4MultimodalEmbedder, Gemma4UnifiedVisionEmbedder, Gemma4VisionModelfrom .weight import (nvfp4_expert_spec,+ iter_vision_weights,iter_weights,iter_weights_parallel,)@@ -26,6 +27,7 @@ __all__ = ["Gemma4Router","Gemma4VisionModel","parse_config",+ "iter_vision_weights","iter_weights","iter_weights_parallel","parse_gguf_config",diff --git a/python/freetoken/models/gemma4/weight.py b/python/freetoken/models/gemma4/weight.pyindex 19b23ed7..ba990c00 100644--- a/python/freetoken/models/gemma4/weight.py+++ b/python/freetoken/models/gemma4/weight.py@@ -109,6 +109,27 @@ def _emit_nvfp4_dense_mlp(reader: ShardReader, base: str, raw_base: str, buf: direturn out+def _rename_vision_key(raw_name: str) -> str | None:+ """Encoder checkpoint key -> state-dict key, or None for anything else; the tower's clipped-linear wrapper is dropped."""+ if raw_name.startswith("model.vision_tower."):+ return ("vision_tower." + raw_name[len("model.vision_tower.") :]).replace(".linear.", ".")+ if raw_name.startswith("model.embed_vision."):+ return "embed_vision." + raw_name[len("model.embed_vision.") :]+ if raw_name.startswith("model.vision_embedder."):+ return "vision_embedder." + raw_name[len("model.vision_embedder.") :]+ return None+++def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:+ """The vision encoder alone, named as iter_weights names it."""+ for file in iter_weight_files(model_path):+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:+ for raw_name in f.keys():+ name = _rename_vision_key(raw_name)+ if name is not None:+ yield name, f.get_tensor(raw_name)++def _rename_language_key(raw_name: str) -> str:name = raw_name.removeprefix("model.language_model.")name = "model." + name.removeprefix("language_model.")@@ -140,16 +161,7 @@ def iter_weights(return _rename_language_key(raw_name)if raw_name.startswith("language_model."):return _rename_language_key(raw_name)- if raw_name.startswith("model.vision_tower."):- return ("vision_tower." + raw_name[len("model.vision_tower.") :]).replace(- ".linear.",- ".",- )- if raw_name.startswith("model.embed_vision."):- return "embed_vision." + raw_name[len("model.embed_vision.") :]- if raw_name.startswith("model.vision_embedder."):- return "vision_embedder." + raw_name[len("model.vision_embedder.") :]- return None+ return _rename_vision_key(raw_name)def merge_info(key: str) -> tuple[str, MergeRule] | None:for suffix, rule in _MERGE_RULES.items():diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.pyindex 9417286c..7eee33df 100644--- a/python/freetoken/models/glm5_next/__init__.py+++ b/python/freetoken/models/glm5_next/__init__.py@@ -1,7 +1,7 @@from .config import VisionConfig, parse_config, parse_vision_configfrom .model import Glm5NextForCausalLM, Glm5NextForConditionalGenerationfrom .vision import Glm5NextVisionModel-from .weight import iter_expert_pieces, iter_weights, nvfp4_expert_spec+from .weight import iter_expert_pieces, iter_vision_weights, iter_weights, nvfp4_expert_spec__all__ = ["nvfp4_expert_spec",@@ -11,6 +11,7 @@ __all__ = ["VisionConfig","parse_config","parse_vision_config",+ "iter_vision_weights","iter_weights","iter_expert_pieces",]diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.pyindex 60d97e90..6f967dd8 100644--- a/python/freetoken/models/glm5_next/weight.py+++ b/python/freetoken/models/glm5_next/weight.py@@ -238,6 +238,18 @@ def iter_weights(reader.close()+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:+ """The vision tower alone, named as iter_weights names it."""+ folder = download_hf_weight(model_path)+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:+ weight_map = json.load(f)["weight_map"]+ reader = _ShardReader(folder, weight_map, device)+ try:+ yield from _iter_vision(reader, weight_map)+ finally:+ reader.close()++def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):"""Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their ``_scale`` companions; other kinds use the generic readers."""if kind is not QuantKind.FP8_BLOCK:diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.pyindex ca66b680..4abf9c87 100644--- a/python/freetoken/models/minimax_m3/__init__.py+++ b/python/freetoken/models/minimax_m3/__init__.py@@ -3,6 +3,7 @@ from .model import MiniMaxM3ForCausalLM, MiniMaxM3ForConditionalGenerationfrom .vision import MiniMaxM3VisionModelfrom .weight import (nvfp4_expert_spec,+ iter_vision_weights,iter_weights,)@@ -14,5 +15,6 @@ __all__ = ["VisionConfig","parse_config","parse_vision_config",+ "iter_vision_weights","iter_weights",]diff --git a/python/freetoken/models/minimax_m3/weight.py b/python/freetoken/models/minimax_m3/weight.pyindex b3c928e2..29a729d7 100644--- a/python/freetoken/models/minimax_m3/weight.py+++ b/python/freetoken/models/minimax_m3/weight.py@@ -260,6 +260,21 @@ def iter_weights(reader.close()+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:+ """The vision tower alone, named as iter_weights names it."""+ config = parse_config(cached_load_hf_config(model_path))+ if config.vision_config is None:+ return+ folder = download_hf_weight(model_path)+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:+ weight_map = json.load(f)["weight_map"]+ reader = _ShardReader(folder, weight_map, device)+ try:+ yield from _iter_vision(reader, weight_map, config.vision_config.num_layers)+ finally:+ reader.close()++def nvfp4_expert_spec(model_path: str, config):return _NVFP4_SOURCE_SPECdiff --git a/python/freetoken/models/muse_glimmer/__init__.py b/python/freetoken/models/muse_glimmer/__init__.pyindex 6f08348e..4ee14808 100644--- a/python/freetoken/models/muse_glimmer/__init__.py+++ b/python/freetoken/models/muse_glimmer/__init__.py@@ -2,7 +2,7 @@ from .attention import MuseGlimmerAttentionfrom .config import VisionConfig, parse_config, parse_vision_configfrom .model import MuseGlimmerForCausalLM, MuseGlimmerForConditionalGenerationfrom .vision import MuseGlimmerVisionModel-from .weight import iter_weights+from .weight import iter_vision_weights, iter_weights__all__ = ["MuseGlimmerAttention",@@ -12,5 +12,6 @@ __all__ = ["VisionConfig","parse_config","parse_vision_config",+ "iter_vision_weights","iter_weights",]diff --git a/python/freetoken/models/muse_glimmer/weight.py b/python/freetoken/models/muse_glimmer/weight.pyindex b9ce56f3..8b9bb213 100644--- a/python/freetoken/models/muse_glimmer/weight.py+++ b/python/freetoken/models/muse_glimmer/weight.py@@ -115,6 +115,18 @@ def iter_weights(assert not vision_buf, f"Incomplete vision qkv fusions: {list(vision_buf.keys())}"+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:+ """The vision tower alone, named as iter_weights names it."""+ vision_buf: dict = {}+ for file in iter_weight_files(model_path):+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:+ for raw_name in f.keys():+ vision = _vision_name(raw_name)+ if vision is not None:+ yield from _vision_tensors(vision, f.get_tensor(raw_name), vision_buf)+ assert not vision_buf, f"Incomplete vision qkv fusions: {list(vision_buf.keys())}"++def _iter_weights_compressed_tensors(model_path: str, device: torch.device, include_vision: bool) -> Iterator[tuple[str, torch.Tensor]]:diff --git a/python/freetoken/models/qwen3_5_moe/__init__.py b/python/freetoken/models/qwen3_5_moe/__init__.pyindex 9acc5aac..018b1c1e 100644--- a/python/freetoken/models/qwen3_5_moe/__init__.py+++ b/python/freetoken/models/qwen3_5_moe/__init__.py@@ -5,7 +5,7 @@ from .model import (Qwen3_5MoeForCausalLM,Qwen3_5MoeForConditionalGeneration,)-from .weight import iter_expert_pieces, iter_weights, iter_weights_parallel, nvfp4_expert_spec+from .weight import iter_expert_pieces, iter_vision_weights, iter_weights, iter_weights_parallel, nvfp4_expert_spec__all__ = ["Qwen3_5ForCausalLM",@@ -13,6 +13,7 @@ __all__ = ["Qwen3_5MoeForCausalLM","Qwen3_5MoeForConditionalGeneration","parse_config",+ "iter_vision_weights","iter_weights","iter_weights_parallel","iter_expert_pieces",diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.pyindex 69501ad7..1bac8227 100644--- a/python/freetoken/models/qwen3_5_moe/weight.py+++ b/python/freetoken/models/qwen3_5_moe/weight.py@@ -286,6 +286,16 @@ def _iter_shards(model_path: str, device: torch.device, reader: _DenseReader | Nraise ValueError(f"checkpoint is missing tensors the quant config declares for {len(lines)} modules:\n {shown}")+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:+ """The vision tower alone, named as iter_weights names it."""+ for file in iter_weight_files(model_path):+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:+ for raw_name in f.keys():+ name = _rename(raw_name)+ if name is not None and name.startswith(VISION_KEY_PREFIXES):+ yield name, f.get_tensor(raw_name)++def iter_weights_parallel(model_path: str,device: torch.device,diff --git a/python/freetoken/models/qwen3_vl/__init__.py b/python/freetoken/models/qwen3_vl/__init__.pyindex d79c838f..62640d43 100644--- a/python/freetoken/models/qwen3_vl/__init__.py+++ b/python/freetoken/models/qwen3_vl/__init__.py@@ -5,7 +5,7 @@ from .model import (deepstack_add,)from .vision import Qwen3VLVisionModel-from .weight import iter_weights, iter_weights_parallel, rename_vl_prefix+from .weight import iter_vision_weights, iter_weights, iter_weights_parallel, rename_vl_prefix__all__ = ["Qwen3VLForConditionalGeneration",@@ -13,6 +13,7 @@ __all__ = ["Qwen3VLVisionModel","VisionConfig","deepstack_add",+ "iter_vision_weights","iter_weights","iter_weights_parallel","parse_config",diff --git a/python/freetoken/models/qwen3_vl/weight.py b/python/freetoken/models/qwen3_vl/weight.pyindex 3a7df174..d522151a 100644--- a/python/freetoken/models/qwen3_vl/weight.py+++ b/python/freetoken/models/qwen3_vl/weight.py@@ -91,6 +91,16 @@ def iter_weights(yield from iter_merged_tensors(tensors(), _MERGE_RULES, model_name="qwen3_vl")+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:+ """The vision tower alone, named as iter_weights names it."""+ for file in iter_weight_files(model_path):+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:+ for raw_name in f.keys():+ name = rename_vl_prefix(raw_name)+ if name.startswith(VISION_KEY_PREFIXES):+ yield name, f.get_tensor(raw_name)++def iter_weights_parallel(model_path: str,device: torch.device,diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.pyindex b063da6a..0fe4a4ff 100644--- a/python/freetoken/models/qwen4_exp/__init__.py+++ b/python/freetoken/models/qwen4_exp/__init__.py@@ -14,6 +14,7 @@ from .model import Qwen4ExpForCausalLM, Qwen4ExpForConditionalGenerationfrom .weight import (ftw_side_files,nvfp4_expert_spec,+ iter_vision_weights,iter_weights,load_ple_table,)@@ -27,6 +28,7 @@ __all__ = ["nvfp4_expert_spec","Qwen4ExpForCausalLM","Qwen4ExpForConditionalGeneration",+ "iter_vision_weights","iter_weights","load_ple_table","parse_config",diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.pyindex 0ad04011..691651d4 100644--- a/python/freetoken/models/qwen4_exp/weight.py+++ b/python/freetoken/models/qwen4_exp/weight.py@@ -234,6 +234,16 @@ def iter_weights(assert not fuser.buf, f"Incomplete projection fusions: {sorted(k[0] + k[1] for k in fuser.buf)}"+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:+ """The vision tower alone, named as iter_weights names it."""+ for file in iter_weight_files(model_path):+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:+ for raw_name in f.keys():+ name = _rename(raw_name)+ if name is not None and name.startswith(VISION_KEY_PREFIXES):+ yield name, f.get_tensor(raw_name)++# ======================================================================================# PLE n-gram table# ======================================================================================diff --git a/python/freetoken/models/weight.py b/python/freetoken/models/weight.pyindex 5165d149..e3382b1b 100644--- a/python/freetoken/models/weight.py+++ b/python/freetoken/models/weight.py@@ -221,8 +221,10 @@ def load_weight(from freetoken.checkpoint.ftw import is_ftw_checkpoint, iter_ftw_weightsfrom freetoken.models.config import VISION_KEY_PREFIXES+ # a text-only engine never built the tower, so its tensors are not even read+ keep = None if include_vision else (lambda name: not name.startswith(VISION_KEY_PREFIXES))if is_ftw_checkpoint(model_path):- weights = iter_ftw_weights(model_path)+ weights = iter_ftw_weights(model_path, keep=keep)else:_config, spec = _spec_for_model_path(model_path)iter_weights = _load_attr(spec.module, spec.iter_weights)@@ -236,12 +238,30 @@ def load_weight(**kwargs,)for name, tensor in weights:- # the FTW replay has no reader to skip the tower; a text-only engine never built it- if not include_vision and name.startswith(VISION_KEY_PREFIXES):+ if keep is not None and not keep(name):continueyield name, tensor+def load_vision_weight(model_path: str, device: torch.device) -> Iterator[Tuple[str, torch.Tensor]]:+ """The vision encoder tensors alone, named as load_weight names them, read by the family's encoder-only reader."""+ _config, spec = _spec_for_model_path(model_path)+ reader = _model_override(spec, "iter_vision_weights")+ if reader is None:+ raise ValueError(f"{spec.module} has no encoder-only weight reader")+ return reader(model_path, device)+++def ftw_lacks_vision(model_path: str) -> bool:+ """True for an FTW that holds no vision encoder tensors: converted by a build before the family served images."""+ from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint+ from freetoken.models.config import VISION_KEY_PREFIXES++ if not is_ftw_checkpoint(model_path):+ return False+ return not any(name.startswith(VISION_KEY_PREFIXES) for name in ftw_tensor_names(model_path, "weight"))++def load_q4_0_moe_expert_sources(model_path: str,model_config,@@ -262,6 +282,8 @@ def load_q4_0_moe_expert_sources(__all__ = ["load_weight",+ "load_vision_weight",+ "ftw_lacks_vision","load_q4_0_moe_expert_sources","iter_expert_tensors_parallel",]diff --git a/scripts/ftw_hotfix.py b/scripts/ftw_hotfix.pyindex 7e324ed8..f9cb62d1 100644--- a/scripts/ftw_hotfix.py+++ b/scripts/ftw_hotfix.py@@ -9,6 +9,7 @@ FTW index against the tensors that model declares, and repairs the differences:- renames (DeepSeek-V4: the dense tree moved under ``model.`` and ``.scale`` became ``.weight_scale_inv``)- fp8 weights the current model declares as bf16 (old runtime fp8 residue) are dequantized, their ``.weight_scale`` dropped- tensors the model declares but the FTW lacks (``input_scale``) are fetched from the HF repo by byte range+- an FTW converted before its family served images gets its vision encoder: the tower's checkpoint tensors are fetched into a scratch checkpoint dir and the family's own weight reader emits them under the names the FTW needs- a Qwen3.8-Flash-Next FTW without the PLE n-gram table gets it written as ``ple-table-*.safetensors``Before anything is written the FTW is checked against the model: shapes, dtypes, byte counts and@@ -33,10 +34,12 @@ import reimport shutilimport structimport sys+import tempfileimport torchfrom freetoken.checkpoint.ftw import ALIGN, FORMAT_TAG, INDEX_NAME, _align_up, _dtype_of, _dtype_str, _SHARD_FMT+from freetoken.models.config import VISION_KEY_PREFIXESfrom freetoken.distributed.info import set_tp_info, try_get_tp_infofrom freetoken.engine.config import EngineConfigfrom freetoken.engine.engine import _decode_target@@ -61,7 +64,7 @@ _PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale"_PLE_FILE_RE = re.compile(r"^ple-table-\d{5}\.safetensors$")_PLE_FILE_BYTES = 4 << 30# tensors an old FTW may carry that the text model does not declare; they are kept and never count as errors-_IGNORED_PREFIXES = ("vision_tower.", "embed_vision.")+_IGNORED_PREFIXES = VISION_KEY_PREFIXES_VERBOSE = False@@ -247,6 +250,46 @@ def resolve_fetches(fetches, source: TensorSource) -> tuple[dict[str, tuple[listreturn resolved, errors+# ------------------------------------------------------------------ vision encoder+_TOWER_SEGMENTS = ("multi_modal_projector", "patch_merge_mlp", "aligner")+++def is_vision(name: str) -> bool:+ return name.startswith(VISION_KEY_PREFIXES)+++def is_checkpoint_tower_name(name: str) -> bool:+ """A checkpoint tensor of the vision encoder: its first path segment under the wrapper root names the tower, its embedder or its projector."""+ head = name.split(".")+ if head[0] == "model" and len(head) > 1:+ head = head[1:]+ return "vision" in head[0] or "visual" in head[0] or head[0] in _TOWER_SEGMENTS+++def read_tower(source: TensorSource, ftw_dir: str, checkpoint_names: list[str]) -> dict[str, torch.Tensor]:+ """The encoder tensors as the family's encoder-only reader emits them: the tower's checkpoint tensors go into a scratch checkpoint dir next to the FTW's config files, and the reader renames and fuses them the way the converter did."""+ from safetensors.torch import save_file++ from freetoken.models.weight import load_vision_weight++ tmp = tempfile.mkdtemp(prefix="ftw-hotfix-tower-")+ try:+ copy_side_files(ftw_dir, tmp, lambda f: f.endswith(".safetensors") or f == "model.safetensors.index.json")+ b = bar(sum(source.nbytes(n) for n in checkpoint_names), "Fetching the vision encoder")+ tensors: dict[str, torch.Tensor] = {}+ for n in checkpoint_names:+ tensors[n] = source.get(n).contiguous()+ tick(b, tensors[n].numel() * tensors[n].element_size())+ done(b)+ save_file(tensors, os.path.join(tmp, "tower.safetensors"))+ with open(os.path.join(tmp, "model.safetensors.index.json"), "w") as f:+ json.dump({"metadata": {}, "weight_map": {n: "tower.safetensors" for n in checkpoint_names}}, f)+ del tensors+ return {n: t for n, t in load_vision_weight(tmp, torch.device("cpu")) if is_vision(n)}+ finally:+ shutil.rmtree(tmp, ignore_errors=True)++def dequantize_rows(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:return (weight.float() * scale.float()[:, None]).to(torch.bfloat16)@@ -764,6 +807,9 @@ def main(argv: list[str] | None = None) -> int:log(f"{arch}: model declares {len(expected)} dense tensors")source = TensorSource(ns.repo, ns.source, ns.revision) if (ns.repo or ns.source) else Nonerenames, dequants, fetches, drops, leftovers = plan(arch, index["tensors"], expected, name_map, quant)+ # the encoder comes from the family reader, not from by-name fetches+ vision_missing = sorted(n for n, *_ in fetches if is_vision(n))+ fetches = [f for f in fetches if not is_vision(f[0])]dequant_names = {n for n, _, _ in dequants}ple = PleSpec(ns.ftw) if arch.startswith("Qwen4Exp") else Nonechk = check_ftw(ns.ftw, index, expected, renames, dequant_names, ple)@@ -774,6 +820,7 @@ def main(argv: list[str] | None = None) -> int:print(f"{arch}: {len(expected)} expected dense tensors, FTW has {sum(e['kind'] == 'weight' for e in index['tensors'])}")print(f" renames {len(renames)} dequantize {len(dequants)} fetch {len(fetches)} drop {len(drops)} leftover {len(leftovers)}"+ + (f" vision encoder: {len(vision_missing)} tensors" if vision_missing else "")+ (f" PLE table: {ple_status}" + (f" ({ple_detail})" if ple_detail else "") if need_ple else "")+ (f" dead bytes in {len(dirty)} shard(s)" if dirty else ""))for n, parts, _ in fetches[:8]:@@ -799,9 +846,16 @@ def main(argv: list[str] | None = None) -> int:bad = Trueif len(problems) > 10:print(f"ERROR: ... {len(problems) - 10} more", file=sys.stderr)- if (fetches or need_ple) and source is None:+ if (fetches or need_ple or vision_missing) and source is None:print("ERROR: tensors must be fetched but neither --repo nor --source was given", file=sys.stderr)return 2+ tower_names = [n for n in source.weight_map if is_checkpoint_tower_name(n)] if vision_missing else []+ if vision_missing and not tower_names:+ print("ERROR: the source checkpoint has no vision encoder tensors", file=sys.stderr)+ bad = True+ elif vision_missing:+ print(f" vision encoder: {len(vision_missing)} tensors from the family reader over {len(tower_names)} checkpoint tensors "+ f"({sum(source.nbytes(n) for n in tower_names) / 2**30:.2f} GiB)")fetch_srcs: dict[str, tuple[list[str], bool]] = {}if fetches:fetch_srcs, errors = resolve_fetches(fetches, source)@@ -833,7 +887,19 @@ def main(argv: list[str] | None = None) -> int:if bad:return 2- replaced = dequant_names | set(fetch_srcs)+ tower: dict[str, torch.Tensor] = {}+ if vision_missing and not ns.dry_run:+ tower = read_tower(source, ns.ftw, tower_names)+ unproduced = [n for n in vision_missing if n not in tower]+ wrong = [n for n in vision_missing if n in tower and tuple(tower[n].shape) != expected[n][0]]+ if unproduced or wrong:+ for n in unproduced[:5]:+ print(f"ERROR: the family reader did not produce {n} from the source's encoder tensors", file=sys.stderr)+ for n in wrong[:5]:+ print(f"ERROR: {n}: reader shape {list(tower[n].shape)} != model shape {list(expected[n][0])}", file=sys.stderr)+ return 2++ replaced = dequant_names | set(fetch_srcs) | set(vision_missing)keep: list[tuple[dict, str]] = []for e in index["tensors"]:name = renames.get(e["name"], e["name"]) if e["kind"] == "weight" else e["name"]@@ -845,7 +911,7 @@ def main(argv: list[str] | None = None) -> int:# disk: the new entries and the PLE table, plus either one compacted shard next to its original# (in place) or the whole compact copy (--out)- new_bytes = sum(_align_up(tensor_bytes(*expected[n])) for n in replaced)+ new_bytes = sum(_align_up(tensor_bytes(tower[n].shape, tower[n].dtype) if n in tower else tensor_bytes(*expected[n])) for n in replaced)table_bytes = sum(source.nbytes(n) for n in source.weight_map if _PLE_INFIX in n) if need_ple else 0work = ns.out or ns.ftwif ns.out:@@ -860,7 +926,7 @@ def main(argv: list[str] | None = None) -> int:print(f" disk: about {need / 2**30:.1f} GiB needed under {work} ({free / 2**30:.1f} GiB free)")if ns.dry_run:return 0- if not (renames or dequants or fetch_srcs or drops or need_ple or dirty or ns.out):+ if not (renames or dequants or fetch_srcs or vision_missing or drops or need_ple or dirty or ns.out):print("nothing to do; the FTW loads as is")return 0if free < need + (1 << 30):@@ -875,6 +941,9 @@ def main(argv: list[str] | None = None) -> int:tick(b, e["nbytes"])done(b)items = list(fetch_srcs.items())+ for n in (vision_missing if _VERBOSE or not vision_missing else count_bar(vision_missing, "Writing the vision encoder")):+ log(f" encoder {n} {list(tower[n].shape)}")+ w.add_tensor(n, tower[n])for n, (srcs, reciprocal) in (items if _VERBOSE or not items else count_bar(items, "Fetching tensors")):log(f" fetch {n} <- {', '.join(srcs)}" + (" (reciprocal)" if reciprocal else ""))vals = [source.get(c).reshape(()).float() for c in srcs]@@ -885,7 +954,7 @@ def main(argv: list[str] | None = None) -> int:w.add_tensor(n, torch.stack(vals).max().reshape(()))hotfix = {"from": os.path.abspath(ns.ftw), "renamed": len(renames), "dequantized": len(dequants),- "fetched": len(fetch_srcs), "dropped": len(drops), "compacted": 0, "source": ns.repo or ns.source}+ "fetched": len(fetch_srcs) + len(vision_missing), "dropped": len(drops), "compacted": 0, "source": ns.repo or ns.source}compacted = 0if ns.out:log(f"--out: writing a fresh FTW -> {work}")@@ -919,7 +988,7 @@ def main(argv: list[str] | None = None) -> int:os.remove(os.path.join(work, f))new_entries: list[dict] = []new_shards: list[dict] = []- if dequants or fetch_srcs:+ if dequants or fetch_srcs or vision_missing:log(f"in place: appending shard {_SHARD_FMT.format(next_num)} -> {work}")w = ShardWriter(work, index["shard_limit"], first_shard=next_num, global_off=index["total_bytes"])try:diff --git a/tests/checkpoint/test_ftw_hotfix_vision.py b/tests/checkpoint/test_ftw_hotfix_vision.pynew file mode 100644index 00000000..d4a3eac6--- /dev/null+++ b/tests/checkpoint/test_ftw_hotfix_vision.py@@ -0,0 +1,78 @@+"""The hotfix tool's vision repair: picking the encoder's checkpoint tensors and letting the family reader name them."""++import importlib.util+import json+import os+import pathlib+import shutil++import pytest++from freetoken.models.config import VISION_KEY_PREFIXES++_SCRIPT = pathlib.Path(__file__).resolve().parents[2] / "scripts" / "ftw_hotfix.py"++# env var -> local checkpoint, as tests/README.md lists them+_CHECKPOINTS = ("FREETOKEN_QWEN3VL_MODEL", "FREETOKEN_GEMMA4_MODEL", "FREETOKEN_GEMMA4_UNIFIED_MODEL",+ "FREETOKEN_GLM53_MODEL", "FREETOKEN_MUSE_MODEL", "FREETOKEN_MINIMAX_M3_MODEL")++[email protected](scope="module")+def hotfix():+ spec = importlib.util.spec_from_file_location("ftw_hotfix", _SCRIPT)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module++[email protected]("name, tower", [+ ("model.visual.blocks.0.attn.qkv.weight", True),+ ("model.vision_tower.encoder.layers.0.self_attn.q_proj.linear.weight", True),+ ("model.embed_vision.embedding_projection.weight", True),+ ("model.vision_embedder.patch_embedding.weight", True),+ ("model.vision_adapter.fc1.weight", True),+ ("vision_tower.vision_model.embeddings.patch_embedding.weight", True),+ ("multi_modal_projector.linear_1.weight", True),+ ("patch_merge_mlp.linear_2.bias", True),+ ("model.embed_audio.embedding_projection.weight", False),+ ("model.language_model.layers.0.self_attn.q_proj.weight", False),+ ("model.layers.3.mlp.experts.0.gate_proj.weight", False),+ ("lm_head.weight", False),+])+def test_checkpoint_tower_names(hotfix, name, tower):+ assert hotfix.is_checkpoint_tower_name(name) is tower+++def _config_dir(hotfix, checkpoint, tmp_path):+ """An FTW-shaped dir holding only the checkpoint's config files and an empty index, enough to build the model on the meta device."""+ for f in os.listdir(checkpoint):+ if f.endswith((".json", ".py", ".jinja")) and not f.startswith("model"):+ shutil.copy(os.path.join(checkpoint, f), tmp_path)+ index = {"format": hotfix.FORMAT_TAG, "version": 1, "align": hotfix.ALIGN, "shard_limit": 8 << 30, "total_bytes": 0, "tensors": [], "shards": []}+ (tmp_path / hotfix.INDEX_NAME).write_text(json.dumps(index))+ return str(tmp_path)++[email protected]_weights[email protected]("env", _CHECKPOINTS)+def test_reader_emits_every_declared_encoder_tensor(hotfix, env, tmp_path):+ checkpoint = os.environ.get(env)+ if not checkpoint:+ pytest.skip(f"{env} not set")+ ftw_like = _config_dir(hotfix, checkpoint, tmp_path)+ _, expected, _, _ = hotfix.expected_tensors(ftw_like, resident_experts=False)+ declared = {n: shape for n, (shape, _) in expected.items() if n.startswith(VISION_KEY_PREFIXES)}+ assert declared, "the family declares no encoder tensors"+ source = hotfix.TensorSource(None, checkpoint)+ tower_names = [n for n in source.weight_map if hotfix.is_checkpoint_tower_name(n)]+ got = hotfix.read_tower(source, ftw_like, tower_names)+ assert sorted(set(declared) - set(got)) == []+ assert [n for n in declared if tuple(got[n].shape) != declared[n]] == []+++def test_every_family_with_an_encoder_has_an_encoder_only_reader():+ from freetoken.models.register import _MODEL_REGISTRY, _load_attr++ for spec in _MODEL_REGISTRY.values():+ if spec.encoders:+ assert callable(_load_attr(spec.module, "iter_vision_weights")), spec.modulediff --git a/tests/checkpoint/test_ftw_weights.py b/tests/checkpoint/test_ftw_weights.pynew file mode 100644index 00000000..b3b3236b--- /dev/null+++ b/tests/checkpoint/test_ftw_weights.py@@ -0,0 +1,56 @@+"""FTW replay: dropping entries by name before their bytes are read, and the vision-tower presence check."""++import torch++from freetoken.checkpoint.ftw import FTWReader, FTWWriter, ftw_tensor_names, iter_ftw_weights+from freetoken.models.weight import ftw_lacks_vision, load_weight+++def _write_ftw(out_dir, names):+ writer = FTWWriter(str(out_dir))+ tensors = {name: torch.full((4, 8), float(i), dtype=torch.bfloat16) for i, name in enumerate(names)}+ for name, tensor in tensors.items():+ writer.add_tensor(name, tensor)+ writer.finalize({})+ return tensors+++def test_keep_drops_entries_before_their_bytes_are_read(tmp_path, monkeypatch):+ tensors = _write_ftw(tmp_path, ["model.a.weight", "visual.b.weight", "model.c.weight"])+ read = []+ original = FTWReader.read_into++ def spy(self, dest, entry, **kwargs):+ read.append(entry["name"])+ return original(self, dest, entry, **kwargs)++ monkeypatch.setattr(FTWReader, "read_into", spy)+ got = dict(iter_ftw_weights(str(tmp_path), keep=lambda name: not name.startswith("visual.")))+ assert list(got) == ["model.a.weight", "model.c.weight"]+ assert read == ["model.a.weight", "model.c.weight"]+ for name, tensor in got.items():+ assert torch.equal(tensor, tensors[name])+++def test_no_keep_replays_every_entry(tmp_path):+ tensors = _write_ftw(tmp_path, ["model.a.weight", "visual.b.weight"])+ got = dict(iter_ftw_weights(str(tmp_path)))+ assert list(got) == list(tensors)+ assert torch.equal(got["visual.b.weight"], tensors["visual.b.weight"])+++def test_load_weight_text_only_skips_the_tower(tmp_path):+ names = ["model.a.weight", "visual.b.weight", "vision_tower.c.weight"]+ _write_ftw(tmp_path, names)+ cpu = torch.device("cpu")+ assert [n for n, _ in load_weight(str(tmp_path), cpu, include_vision=False)] == ["model.a.weight"]+ assert [n for n, _ in load_weight(str(tmp_path), cpu)] == names+++def test_ftw_lacks_vision(tmp_path):+ _write_ftw(tmp_path / "text", ["model.a.weight"])+ _write_ftw(tmp_path / "vl", ["model.a.weight", "visual.b.weight"])+ assert ftw_lacks_vision(str(tmp_path / "text"))+ assert not ftw_lacks_vision(str(tmp_path / "vl"))+ assert not ftw_lacks_vision(str(tmp_path))+ assert ftw_tensor_names(str(tmp_path / "vl"), "weight") == ["model.a.weight", "visual.b.weight"]