Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(checkpoint): serve and repair FTW checkpoints around the vision encoder (#486)

Xiaoze Fancade1a9e452c
diff --git a/docs/ftw-hotfix.md b/docs/ftw-hotfix.md
index 77362adb..299d522d 100644
--- a/docs/ftw-hotfix.md
+++ b/docs/ftw-hotfix.md
@@ -14,6 +14,9 @@ python scripts/ftw_hotfix.py --ftw ~/models/DeepSeek-V4-Flash-0731-FTW
# old Qwen3.8-Flash-Next FTW: add the 47.7 GiB PLE table from a local copy of the checkpoint, into a new dir
python scripts/ftw_hotfix.py --ftw ~/models/Qwen3.8-Flash-Next-NVFP4-FTW --source ~/models/Qwen3.8-Flash-Next-NVFP4 --out ~/models/Qwen3.8-Flash-Next-NVFP4-FTW-fixed
+# Qwen3-VL FTW converted before image input: add the vision encoder (about 1 GiB) from the Hub, in place
+python scripts/ftw_hotfix.py --ftw ~/models/Qwen3-VL-8B-Instruct-FTW --repo Qwen/Qwen3-VL-8B-Instruct
+
# just show what would change
python scripts/ftw_hotfix.py --ftw ~/models/GLM-5.2-NVFP4-FTW --dry-run
```
@@ -29,6 +32,7 @@ byte range, never the whole checkpoint; `--revision` pins the Hub revision to re
| `KeyError: 'model.embed.weight'` | deepseek-ai/DeepSeek-V4-Flash-0731 | rename the index entries | none |
| `RuntimeError: Unexpected keys ... .weight_scale` | nvidia/GLM-5.2-NVFP4 | dequantize the old runtime-fp8 weights back to bf16 | none |
| `PLE shard indices are not contiguous 0..N-1: []` | Qwen3.8-Flash-Next (FTWs converted before #420) | write the PLE table as `ple-table-*.safetensors` | 47.7 GiB |
+| `ValueError: ... holds no vision encoder tensors` | image-input families (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next, Gemma-4, GLM-5.3-Flash, Muse-Glimmer, MiniMax-M3) converted before the family served images | add the vision encoder: the tower's checkpoint tensors are fetched and the family's own weight reader emits them as the converter would | 1 to 4 GiB |
The script decides by itself: it builds the current model from the FTW's `config.json`, compares
the FTW index with the tensors the model declares, and applies only the repairs that are needed.
diff --git a/docs/models.md b/docs/models.md
index 16bedb41..548a363c 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -54,6 +54,9 @@ These families accept image input by default; pass `--text-model-only` to skip t
FreeToken's fast-load format, and `ft serve --model` auto-detects the result.
- FTW files converted by builds before the quantization refactor may fail to load;
see [ftw-hotfix.md](ftw-hotfix.md) for the affected checkpoints and the repair tool.
+- An FTW converted before its family served images holds no vision encoder: `ft serve`
+ refuses it unless started with `--text-model-only` (or `--mm-disable vision`); reconvert it
+ with `ft checkpoint`, or add the encoder in place with [scripts/ftw_hotfix.py](ftw-hotfix.md).
- DeepSeek-V4 checkpoints must keep the `inference/config.json` subdir — the
authoritative model args are read from there.
- Qwen3.8-Flash-Next keeps a 47.7 GiB PLE n-gram table pinned in host RAM.
diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.py
index 207765b7..e2bd0de2 100644
--- a/python/freetoken/checkpoint/ftw.py
+++ b/python/freetoken/checkpoint/ftw.py
@@ -38,6 +38,7 @@ import mmap
import os
import re
import threading
+from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
import torch
@@ -106,6 +107,13 @@ def is_ftw_checkpoint(path: str) -> bool:
return os.path.isfile(os.path.join(path, INDEX_NAME))
+def ftw_tensor_names(path: str, *kinds: str) -> list[str]:
+ """Names the FTW index lists for ``kinds`` (every kind when none is given)."""
+ keep = set(kinds)
+ with open(os.path.join(path, INDEX_NAME)) as f:
+ return [t["name"] for t in json.load(f)["tensors"] if not keep or t["kind"] in keep]
+
+
def ftw_quant_format(path: str) -> str | None:
"""The ``quant_format`` an FTW checkpoint's expert banks were packed for; None when ``path`` is not an FTW checkpoint or holds no banks."""
if not is_ftw_checkpoint(path):
@@ -354,12 +362,13 @@ def _transient_buffer(nbytes: int) -> mmap.mmap:
return mmap.mmap(-1, _align_up(nbytes))
-def iter_ftw_weights(path: str, *, kinds=("weight",), workers: int = 8,
- chunk: int = _DEFAULT_CHUNK, prefetch: int = 2):
+def iter_ftw_weights(path: str, *, kinds=("weight",), keep: Callable[[str], bool] | None = None,
+ workers: int = 8, chunk: int = _DEFAULT_CHUNK, prefetch: int = 2):
"""Yield ``(name, host_tensor)`` for the requested kinds, reading each tensor via
chunked O_DIRECT. A background thread prefetches the next ``prefetch`` tensors so the
disk stays busy while the consumer copies the current one to the GPU. Transient buffers
- are freed as the consumer advances (peak host mem ~ prefetch+1 tensors)."""
+ are freed as the consumer advances (peak host mem ~ prefetch+1 tensors). An entry whose
+ name ``keep`` rejects is dropped before any of its bytes are read."""
import queue
import threading
@@ -367,6 +376,8 @@ def iter_ftw_weights(path: str, *, kinds=("weight",), workers: int = 8,
reader = FTWReader(path)
entries = reader.entries(*kinds)
+ if keep is not None:
+ entries = [e for e in entries if keep(e["name"])]
q: queue.Queue = queue.Queue(maxsize=max(1, prefetch))
_DONE = object()
err: list[BaseException] = []
@@ -650,6 +661,6 @@ def load_ftw_banks(
__all__ = [
"INDEX_NAME", "FORMAT_TAG", "FORMAT_VERSION", "ALIGN", "DEFAULT_SHARD_LIMIT",
- "is_ftw_checkpoint", "FTWWriter", "FTWReader",
+ "is_ftw_checkpoint", "ftw_tensor_names", "FTWWriter", "FTWReader",
"iter_ftw_weights", "load_ftw_banks", "layer_bank_entry_name",
]
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index 3848fc70..06a74ee5 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -16,6 +16,7 @@ from freetoken.layers.quantization import LayerKind, QuantBackend, finalize_quan
from freetoken.moe.offload_cache import iter_offload_moe_layers
from freetoken.mm.config import ENCODER_SECTIONS
from freetoken.models import create_model, load_weight
+from freetoken.models.weight import ftw_lacks_vision
from freetoken.moe import is_offload_moe_strategy
from freetoken.moe.expert_banks import load_expert_banks
from freetoken.moe.host_banks import PinFailed
@@ -500,6 +501,12 @@ class Engine:
model_state = self.model.state_dict()
if config.use_dummy_weight:
return _make_dummy_weight_state_dict(model_state, device=self.device)
+ if config.active_encoders and ftw_lacks_vision(config.model_path):
+ raise ValueError(
+ f"{config.model_path} holds no vision encoder tensors: it was converted by a build before this "
+ "family served images. Reconvert it with `ft checkpoint`, add the encoder in place with "
+ "scripts/ftw_hotfix.py (docs/ftw-hotfix.md), or start with --text-model-only"
+ )
# _materialize casts each loaded tensor to its model-param dtype (model_state), so
# models declaring per-tensor dtypes (e.g. DSV4's mixed fp8/fp32/bf16) are preserved;
# offload models exclude experts (served from the offload cache, not dense weights).
diff --git a/python/freetoken/models/gemma4/__init__.py b/python/freetoken/models/gemma4/__init__.py
index 925bbdb1..9f41f080 100644
--- a/python/freetoken/models/gemma4/__init__.py
+++ b/python/freetoken/models/gemma4/__init__.py
@@ -11,6 +11,7 @@ from .moe import Gemma4Router
from .vision import Gemma4MultimodalEmbedder, Gemma4UnifiedVisionEmbedder, Gemma4VisionModel
from .weight import (
nvfp4_expert_spec,
+ iter_vision_weights,
iter_weights,
iter_weights_parallel,
)
@@ -26,6 +27,7 @@ __all__ = [
"Gemma4Router",
"Gemma4VisionModel",
"parse_config",
+ "iter_vision_weights",
"iter_weights",
"iter_weights_parallel",
"parse_gguf_config",
diff --git a/python/freetoken/models/gemma4/weight.py b/python/freetoken/models/gemma4/weight.py
index 19b23ed7..ba990c00 100644
--- a/python/freetoken/models/gemma4/weight.py
+++ b/python/freetoken/models/gemma4/weight.py
@@ -109,6 +109,27 @@ def _emit_nvfp4_dense_mlp(reader: ShardReader, base: str, raw_base: str, buf: di
return out
+def _rename_vision_key(raw_name: str) -> str | None:
+ """Encoder checkpoint key -> state-dict key, or None for anything else; the tower's clipped-linear wrapper is dropped."""
+ if raw_name.startswith("model.vision_tower."):
+ return ("vision_tower." + raw_name[len("model.vision_tower.") :]).replace(".linear.", ".")
+ if raw_name.startswith("model.embed_vision."):
+ return "embed_vision." + raw_name[len("model.embed_vision.") :]
+ if raw_name.startswith("model.vision_embedder."):
+ return "vision_embedder." + raw_name[len("model.vision_embedder.") :]
+ return None
+
+
+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
+ """The vision encoder alone, named as iter_weights names it."""
+ for file in iter_weight_files(model_path):
+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
+ for raw_name in f.keys():
+ name = _rename_vision_key(raw_name)
+ if name is not None:
+ yield name, f.get_tensor(raw_name)
+
+
def _rename_language_key(raw_name: str) -> str:
name = raw_name.removeprefix("model.language_model.")
name = "model." + name.removeprefix("language_model.")
@@ -140,16 +161,7 @@ def iter_weights(
return _rename_language_key(raw_name)
if raw_name.startswith("language_model."):
return _rename_language_key(raw_name)
- if raw_name.startswith("model.vision_tower."):
- return ("vision_tower." + raw_name[len("model.vision_tower.") :]).replace(
- ".linear.",
- ".",
- )
- if raw_name.startswith("model.embed_vision."):
- return "embed_vision." + raw_name[len("model.embed_vision.") :]
- if raw_name.startswith("model.vision_embedder."):
- return "vision_embedder." + raw_name[len("model.vision_embedder.") :]
- return None
+ return _rename_vision_key(raw_name)
def merge_info(key: str) -> tuple[str, MergeRule] | None:
for suffix, rule in _MERGE_RULES.items():
diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.py
index 9417286c..7eee33df 100644
--- a/python/freetoken/models/glm5_next/__init__.py
+++ b/python/freetoken/models/glm5_next/__init__.py
@@ -1,7 +1,7 @@
from .config import VisionConfig, parse_config, parse_vision_config
from .model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration
from .vision import Glm5NextVisionModel
-from .weight import iter_expert_pieces, iter_weights, nvfp4_expert_spec
+from .weight import iter_expert_pieces, iter_vision_weights, iter_weights, nvfp4_expert_spec
__all__ = [
"nvfp4_expert_spec",
@@ -11,6 +11,7 @@ __all__ = [
"VisionConfig",
"parse_config",
"parse_vision_config",
+ "iter_vision_weights",
"iter_weights",
"iter_expert_pieces",
]
diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.py
index 60d97e90..6f967dd8 100644
--- a/python/freetoken/models/glm5_next/weight.py
+++ b/python/freetoken/models/glm5_next/weight.py
@@ -238,6 +238,18 @@ def iter_weights(
reader.close()
+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
+ """The vision tower alone, named as iter_weights names it."""
+ folder = download_hf_weight(model_path)
+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:
+ weight_map = json.load(f)["weight_map"]
+ reader = _ShardReader(folder, weight_map, device)
+ try:
+ yield from _iter_vision(reader, weight_map)
+ finally:
+ reader.close()
+
+
def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
"""Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their ``_scale`` companions; other kinds use the generic readers."""
if kind is not QuantKind.FP8_BLOCK:
diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.py
index ca66b680..4abf9c87 100644
--- a/python/freetoken/models/minimax_m3/__init__.py
+++ b/python/freetoken/models/minimax_m3/__init__.py
@@ -3,6 +3,7 @@ from .model import MiniMaxM3ForCausalLM, MiniMaxM3ForConditionalGeneration
from .vision import MiniMaxM3VisionModel
from .weight import (
nvfp4_expert_spec,
+ iter_vision_weights,
iter_weights,
)
@@ -14,5 +15,6 @@ __all__ = [
"VisionConfig",
"parse_config",
"parse_vision_config",
+ "iter_vision_weights",
"iter_weights",
]
diff --git a/python/freetoken/models/minimax_m3/weight.py b/python/freetoken/models/minimax_m3/weight.py
index b3c928e2..29a729d7 100644
--- a/python/freetoken/models/minimax_m3/weight.py
+++ b/python/freetoken/models/minimax_m3/weight.py
@@ -260,6 +260,21 @@ def iter_weights(
reader.close()
+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
+ """The vision tower alone, named as iter_weights names it."""
+ config = parse_config(cached_load_hf_config(model_path))
+ if config.vision_config is None:
+ return
+ folder = download_hf_weight(model_path)
+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:
+ weight_map = json.load(f)["weight_map"]
+ reader = _ShardReader(folder, weight_map, device)
+ try:
+ yield from _iter_vision(reader, weight_map, config.vision_config.num_layers)
+ finally:
+ reader.close()
+
+
def nvfp4_expert_spec(model_path: str, config):
return _NVFP4_SOURCE_SPEC
diff --git a/python/freetoken/models/muse_glimmer/__init__.py b/python/freetoken/models/muse_glimmer/__init__.py
index 6f08348e..4ee14808 100644
--- a/python/freetoken/models/muse_glimmer/__init__.py
+++ b/python/freetoken/models/muse_glimmer/__init__.py
@@ -2,7 +2,7 @@ from .attention import MuseGlimmerAttention
from .config import VisionConfig, parse_config, parse_vision_config
from .model import MuseGlimmerForCausalLM, MuseGlimmerForConditionalGeneration
from .vision import MuseGlimmerVisionModel
-from .weight import iter_weights
+from .weight import iter_vision_weights, iter_weights
__all__ = [
"MuseGlimmerAttention",
@@ -12,5 +12,6 @@ __all__ = [
"VisionConfig",
"parse_config",
"parse_vision_config",
+ "iter_vision_weights",
"iter_weights",
]
diff --git a/python/freetoken/models/muse_glimmer/weight.py b/python/freetoken/models/muse_glimmer/weight.py
index b9ce56f3..8b9bb213 100644
--- a/python/freetoken/models/muse_glimmer/weight.py
+++ b/python/freetoken/models/muse_glimmer/weight.py
@@ -115,6 +115,18 @@ def iter_weights(
assert not vision_buf, f"Incomplete vision qkv fusions: {list(vision_buf.keys())}"
+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
+ """The vision tower alone, named as iter_weights names it."""
+ vision_buf: dict = {}
+ for file in iter_weight_files(model_path):
+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
+ for raw_name in f.keys():
+ vision = _vision_name(raw_name)
+ if vision is not None:
+ yield from _vision_tensors(vision, f.get_tensor(raw_name), vision_buf)
+ assert not vision_buf, f"Incomplete vision qkv fusions: {list(vision_buf.keys())}"
+
+
def _iter_weights_compressed_tensors(
model_path: str, device: torch.device, include_vision: bool
) -> Iterator[tuple[str, torch.Tensor]]:
diff --git a/python/freetoken/models/qwen3_5_moe/__init__.py b/python/freetoken/models/qwen3_5_moe/__init__.py
index 9acc5aac..018b1c1e 100644
--- a/python/freetoken/models/qwen3_5_moe/__init__.py
+++ b/python/freetoken/models/qwen3_5_moe/__init__.py
@@ -5,7 +5,7 @@ from .model import (
Qwen3_5MoeForCausalLM,
Qwen3_5MoeForConditionalGeneration,
)
-from .weight import iter_expert_pieces, iter_weights, iter_weights_parallel, nvfp4_expert_spec
+from .weight import iter_expert_pieces, iter_vision_weights, iter_weights, iter_weights_parallel, nvfp4_expert_spec
__all__ = [
"Qwen3_5ForCausalLM",
@@ -13,6 +13,7 @@ __all__ = [
"Qwen3_5MoeForCausalLM",
"Qwen3_5MoeForConditionalGeneration",
"parse_config",
+ "iter_vision_weights",
"iter_weights",
"iter_weights_parallel",
"iter_expert_pieces",
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index 69501ad7..1bac8227 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -286,6 +286,16 @@ def _iter_shards(model_path: str, device: torch.device, reader: _DenseReader | N
raise ValueError(f"checkpoint is missing tensors the quant config declares for {len(lines)} modules:\n {shown}")
+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
+ """The vision tower alone, named as iter_weights names it."""
+ for file in iter_weight_files(model_path):
+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
+ for raw_name in f.keys():
+ name = _rename(raw_name)
+ if name is not None and name.startswith(VISION_KEY_PREFIXES):
+ yield name, f.get_tensor(raw_name)
+
+
def iter_weights_parallel(
model_path: str,
device: torch.device,
diff --git a/python/freetoken/models/qwen3_vl/__init__.py b/python/freetoken/models/qwen3_vl/__init__.py
index d79c838f..62640d43 100644
--- a/python/freetoken/models/qwen3_vl/__init__.py
+++ b/python/freetoken/models/qwen3_vl/__init__.py
@@ -5,7 +5,7 @@ from .model import (
deepstack_add,
)
from .vision import Qwen3VLVisionModel
-from .weight import iter_weights, iter_weights_parallel, rename_vl_prefix
+from .weight import iter_vision_weights, iter_weights, iter_weights_parallel, rename_vl_prefix
__all__ = [
"Qwen3VLForConditionalGeneration",
@@ -13,6 +13,7 @@ __all__ = [
"Qwen3VLVisionModel",
"VisionConfig",
"deepstack_add",
+ "iter_vision_weights",
"iter_weights",
"iter_weights_parallel",
"parse_config",
diff --git a/python/freetoken/models/qwen3_vl/weight.py b/python/freetoken/models/qwen3_vl/weight.py
index 3a7df174..d522151a 100644
--- a/python/freetoken/models/qwen3_vl/weight.py
+++ b/python/freetoken/models/qwen3_vl/weight.py
@@ -91,6 +91,16 @@ def iter_weights(
yield from iter_merged_tensors(tensors(), _MERGE_RULES, model_name="qwen3_vl")
+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
+ """The vision tower alone, named as iter_weights names it."""
+ for file in iter_weight_files(model_path):
+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
+ for raw_name in f.keys():
+ name = rename_vl_prefix(raw_name)
+ if name.startswith(VISION_KEY_PREFIXES):
+ yield name, f.get_tensor(raw_name)
+
+
def iter_weights_parallel(
model_path: str,
device: torch.device,
diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.py
index b063da6a..0fe4a4ff 100644
--- a/python/freetoken/models/qwen4_exp/__init__.py
+++ b/python/freetoken/models/qwen4_exp/__init__.py
@@ -14,6 +14,7 @@ from .model import Qwen4ExpForCausalLM, Qwen4ExpForConditionalGeneration
from .weight import (
ftw_side_files,
nvfp4_expert_spec,
+ iter_vision_weights,
iter_weights,
load_ple_table,
)
@@ -27,6 +28,7 @@ __all__ = [
"nvfp4_expert_spec",
"Qwen4ExpForCausalLM",
"Qwen4ExpForConditionalGeneration",
+ "iter_vision_weights",
"iter_weights",
"load_ple_table",
"parse_config",
diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.py
index 0ad04011..691651d4 100644
--- a/python/freetoken/models/qwen4_exp/weight.py
+++ b/python/freetoken/models/qwen4_exp/weight.py
@@ -234,6 +234,16 @@ def iter_weights(
assert not fuser.buf, f"Incomplete projection fusions: {sorted(k[0] + k[1] for k in fuser.buf)}"
+def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
+ """The vision tower alone, named as iter_weights names it."""
+ for file in iter_weight_files(model_path):
+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
+ for raw_name in f.keys():
+ name = _rename(raw_name)
+ if name is not None and name.startswith(VISION_KEY_PREFIXES):
+ yield name, f.get_tensor(raw_name)
+
+
# ======================================================================================
# PLE n-gram table
# ======================================================================================
diff --git a/python/freetoken/models/weight.py b/python/freetoken/models/weight.py
index 5165d149..e3382b1b 100644
--- a/python/freetoken/models/weight.py
+++ b/python/freetoken/models/weight.py
@@ -221,8 +221,10 @@ def load_weight(
from freetoken.checkpoint.ftw import is_ftw_checkpoint, iter_ftw_weights
from freetoken.models.config import VISION_KEY_PREFIXES
+ # a text-only engine never built the tower, so its tensors are not even read
+ keep = None if include_vision else (lambda name: not name.startswith(VISION_KEY_PREFIXES))
if is_ftw_checkpoint(model_path):
- weights = iter_ftw_weights(model_path)
+ weights = iter_ftw_weights(model_path, keep=keep)
else:
_config, spec = _spec_for_model_path(model_path)
iter_weights = _load_attr(spec.module, spec.iter_weights)
@@ -236,12 +238,30 @@ def load_weight(
**kwargs,
)
for name, tensor in weights:
- # the FTW replay has no reader to skip the tower; a text-only engine never built it
- if not include_vision and name.startswith(VISION_KEY_PREFIXES):
+ if keep is not None and not keep(name):
continue
yield name, tensor
+def load_vision_weight(model_path: str, device: torch.device) -> Iterator[Tuple[str, torch.Tensor]]:
+ """The vision encoder tensors alone, named as load_weight names them, read by the family's encoder-only reader."""
+ _config, spec = _spec_for_model_path(model_path)
+ reader = _model_override(spec, "iter_vision_weights")
+ if reader is None:
+ raise ValueError(f"{spec.module} has no encoder-only weight reader")
+ return reader(model_path, device)
+
+
+def ftw_lacks_vision(model_path: str) -> bool:
+ """True for an FTW that holds no vision encoder tensors: converted by a build before the family served images."""
+ from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint
+ from freetoken.models.config import VISION_KEY_PREFIXES
+
+ if not is_ftw_checkpoint(model_path):
+ return False
+ return not any(name.startswith(VISION_KEY_PREFIXES) for name in ftw_tensor_names(model_path, "weight"))
+
+
def load_q4_0_moe_expert_sources(
model_path: str,
model_config,
@@ -262,6 +282,8 @@ def load_q4_0_moe_expert_sources(
__all__ = [
"load_weight",
+ "load_vision_weight",
+ "ftw_lacks_vision",
"load_q4_0_moe_expert_sources",
"iter_expert_tensors_parallel",
]
diff --git a/scripts/ftw_hotfix.py b/scripts/ftw_hotfix.py
index 7e324ed8..f9cb62d1 100644
--- a/scripts/ftw_hotfix.py
+++ b/scripts/ftw_hotfix.py
@@ -9,6 +9,7 @@ FTW index against the tensors that model declares, and repairs the differences:
- renames (DeepSeek-V4: the dense tree moved under ``model.`` and ``.scale`` became ``.weight_scale_inv``)
- fp8 weights the current model declares as bf16 (old runtime fp8 residue) are dequantized, their ``.weight_scale`` dropped
- tensors the model declares but the FTW lacks (``input_scale``) are fetched from the HF repo by byte range
+- an FTW converted before its family served images gets its vision encoder: the tower's checkpoint tensors are fetched into a scratch checkpoint dir and the family's own weight reader emits them under the names the FTW needs
- a Qwen3.8-Flash-Next FTW without the PLE n-gram table gets it written as ``ple-table-*.safetensors``
Before anything is written the FTW is checked against the model: shapes, dtypes, byte counts and
@@ -33,10 +34,12 @@ import re
import shutil
import struct
import sys
+import tempfile
import torch
from freetoken.checkpoint.ftw import ALIGN, FORMAT_TAG, INDEX_NAME, _align_up, _dtype_of, _dtype_str, _SHARD_FMT
+from freetoken.models.config import VISION_KEY_PREFIXES
from freetoken.distributed.info import set_tp_info, try_get_tp_info
from freetoken.engine.config import EngineConfig
from freetoken.engine.engine import _decode_target
@@ -61,7 +64,7 @@ _PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale"
_PLE_FILE_RE = re.compile(r"^ple-table-\d{5}\.safetensors$")
_PLE_FILE_BYTES = 4 << 30
# tensors an old FTW may carry that the text model does not declare; they are kept and never count as errors
-_IGNORED_PREFIXES = ("vision_tower.", "embed_vision.")
+_IGNORED_PREFIXES = VISION_KEY_PREFIXES
_VERBOSE = False
@@ -247,6 +250,46 @@ def resolve_fetches(fetches, source: TensorSource) -> tuple[dict[str, tuple[list
return resolved, errors
+# ------------------------------------------------------------------ vision encoder
+_TOWER_SEGMENTS = ("multi_modal_projector", "patch_merge_mlp", "aligner")
+
+
+def is_vision(name: str) -> bool:
+ return name.startswith(VISION_KEY_PREFIXES)
+
+
+def is_checkpoint_tower_name(name: str) -> bool:
+ """A checkpoint tensor of the vision encoder: its first path segment under the wrapper root names the tower, its embedder or its projector."""
+ head = name.split(".")
+ if head[0] == "model" and len(head) > 1:
+ head = head[1:]
+ return "vision" in head[0] or "visual" in head[0] or head[0] in _TOWER_SEGMENTS
+
+
+def read_tower(source: TensorSource, ftw_dir: str, checkpoint_names: list[str]) -> dict[str, torch.Tensor]:
+ """The encoder tensors as the family's encoder-only reader emits them: the tower's checkpoint tensors go into a scratch checkpoint dir next to the FTW's config files, and the reader renames and fuses them the way the converter did."""
+ from safetensors.torch import save_file
+
+ from freetoken.models.weight import load_vision_weight
+
+ tmp = tempfile.mkdtemp(prefix="ftw-hotfix-tower-")
+ try:
+ copy_side_files(ftw_dir, tmp, lambda f: f.endswith(".safetensors") or f == "model.safetensors.index.json")
+ b = bar(sum(source.nbytes(n) for n in checkpoint_names), "Fetching the vision encoder")
+ tensors: dict[str, torch.Tensor] = {}
+ for n in checkpoint_names:
+ tensors[n] = source.get(n).contiguous()
+ tick(b, tensors[n].numel() * tensors[n].element_size())
+ done(b)
+ save_file(tensors, os.path.join(tmp, "tower.safetensors"))
+ with open(os.path.join(tmp, "model.safetensors.index.json"), "w") as f:
+ json.dump({"metadata": {}, "weight_map": {n: "tower.safetensors" for n in checkpoint_names}}, f)
+ del tensors
+ return {n: t for n, t in load_vision_weight(tmp, torch.device("cpu")) if is_vision(n)}
+ finally:
+ shutil.rmtree(tmp, ignore_errors=True)
+
+
def dequantize_rows(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
return (weight.float() * scale.float()[:, None]).to(torch.bfloat16)
@@ -764,6 +807,9 @@ def main(argv: list[str] | None = None) -> int:
log(f"{arch}: model declares {len(expected)} dense tensors")
source = TensorSource(ns.repo, ns.source, ns.revision) if (ns.repo or ns.source) else None
renames, dequants, fetches, drops, leftovers = plan(arch, index["tensors"], expected, name_map, quant)
+ # the encoder comes from the family reader, not from by-name fetches
+ vision_missing = sorted(n for n, *_ in fetches if is_vision(n))
+ fetches = [f for f in fetches if not is_vision(f[0])]
dequant_names = {n for n, _, _ in dequants}
ple = PleSpec(ns.ftw) if arch.startswith("Qwen4Exp") else None
chk = check_ftw(ns.ftw, index, expected, renames, dequant_names, ple)
@@ -774,6 +820,7 @@ def main(argv: list[str] | None = None) -> int:
print(f"{arch}: {len(expected)} expected dense tensors, FTW has {sum(e['kind'] == 'weight' for e in index['tensors'])}")
print(f" renames {len(renames)} dequantize {len(dequants)} fetch {len(fetches)} drop {len(drops)} leftover {len(leftovers)}"
+ + (f" vision encoder: {len(vision_missing)} tensors" if vision_missing else "")
+ (f" PLE table: {ple_status}" + (f" ({ple_detail})" if ple_detail else "") if need_ple else "")
+ (f" dead bytes in {len(dirty)} shard(s)" if dirty else ""))
for n, parts, _ in fetches[:8]:
@@ -799,9 +846,16 @@ def main(argv: list[str] | None = None) -> int:
bad = True
if len(problems) > 10:
print(f"ERROR: ... {len(problems) - 10} more", file=sys.stderr)
- if (fetches or need_ple) and source is None:
+ if (fetches or need_ple or vision_missing) and source is None:
print("ERROR: tensors must be fetched but neither --repo nor --source was given", file=sys.stderr)
return 2
+ tower_names = [n for n in source.weight_map if is_checkpoint_tower_name(n)] if vision_missing else []
+ if vision_missing and not tower_names:
+ print("ERROR: the source checkpoint has no vision encoder tensors", file=sys.stderr)
+ bad = True
+ elif vision_missing:
+ print(f" vision encoder: {len(vision_missing)} tensors from the family reader over {len(tower_names)} checkpoint tensors "
+ f"({sum(source.nbytes(n) for n in tower_names) / 2**30:.2f} GiB)")
fetch_srcs: dict[str, tuple[list[str], bool]] = {}
if fetches:
fetch_srcs, errors = resolve_fetches(fetches, source)
@@ -833,7 +887,19 @@ def main(argv: list[str] | None = None) -> int:
if bad:
return 2
- replaced = dequant_names | set(fetch_srcs)
+ tower: dict[str, torch.Tensor] = {}
+ if vision_missing and not ns.dry_run:
+ tower = read_tower(source, ns.ftw, tower_names)
+ unproduced = [n for n in vision_missing if n not in tower]
+ wrong = [n for n in vision_missing if n in tower and tuple(tower[n].shape) != expected[n][0]]
+ if unproduced or wrong:
+ for n in unproduced[:5]:
+ print(f"ERROR: the family reader did not produce {n} from the source's encoder tensors", file=sys.stderr)
+ for n in wrong[:5]:
+ print(f"ERROR: {n}: reader shape {list(tower[n].shape)} != model shape {list(expected[n][0])}", file=sys.stderr)
+ return 2
+
+ replaced = dequant_names | set(fetch_srcs) | set(vision_missing)
keep: list[tuple[dict, str]] = []
for e in index["tensors"]:
name = renames.get(e["name"], e["name"]) if e["kind"] == "weight" else e["name"]
@@ -845,7 +911,7 @@ def main(argv: list[str] | None = None) -> int:
# disk: the new entries and the PLE table, plus either one compacted shard next to its original
# (in place) or the whole compact copy (--out)
- new_bytes = sum(_align_up(tensor_bytes(*expected[n])) for n in replaced)
+ new_bytes = sum(_align_up(tensor_bytes(tower[n].shape, tower[n].dtype) if n in tower else tensor_bytes(*expected[n])) for n in replaced)
table_bytes = sum(source.nbytes(n) for n in source.weight_map if _PLE_INFIX in n) if need_ple else 0
work = ns.out or ns.ftw
if ns.out:
@@ -860,7 +926,7 @@ def main(argv: list[str] | None = None) -> int:
print(f" disk: about {need / 2**30:.1f} GiB needed under {work} ({free / 2**30:.1f} GiB free)")
if ns.dry_run:
return 0
- if not (renames or dequants or fetch_srcs or drops or need_ple or dirty or ns.out):
+ if not (renames or dequants or fetch_srcs or vision_missing or drops or need_ple or dirty or ns.out):
print("nothing to do; the FTW loads as is")
return 0
if free < need + (1 << 30):
@@ -875,6 +941,9 @@ def main(argv: list[str] | None = None) -> int:
tick(b, e["nbytes"])
done(b)
items = list(fetch_srcs.items())
+ for n in (vision_missing if _VERBOSE or not vision_missing else count_bar(vision_missing, "Writing the vision encoder")):
+ log(f" encoder {n} {list(tower[n].shape)}")
+ w.add_tensor(n, tower[n])
for n, (srcs, reciprocal) in (items if _VERBOSE or not items else count_bar(items, "Fetching tensors")):
log(f" fetch {n} <- {', '.join(srcs)}" + (" (reciprocal)" if reciprocal else ""))
vals = [source.get(c).reshape(()).float() for c in srcs]
@@ -885,7 +954,7 @@ def main(argv: list[str] | None = None) -> int:
w.add_tensor(n, torch.stack(vals).max().reshape(()))
hotfix = {"from": os.path.abspath(ns.ftw), "renamed": len(renames), "dequantized": len(dequants),
- "fetched": len(fetch_srcs), "dropped": len(drops), "compacted": 0, "source": ns.repo or ns.source}
+ "fetched": len(fetch_srcs) + len(vision_missing), "dropped": len(drops), "compacted": 0, "source": ns.repo or ns.source}
compacted = 0
if ns.out:
log(f"--out: writing a fresh FTW -> {work}")
@@ -919,7 +988,7 @@ def main(argv: list[str] | None = None) -> int:
os.remove(os.path.join(work, f))
new_entries: list[dict] = []
new_shards: list[dict] = []
- if dequants or fetch_srcs:
+ if dequants or fetch_srcs or vision_missing:
log(f"in place: appending shard {_SHARD_FMT.format(next_num)} -> {work}")
w = ShardWriter(work, index["shard_limit"], first_shard=next_num, global_off=index["total_bytes"])
try:
diff --git a/tests/checkpoint/test_ftw_hotfix_vision.py b/tests/checkpoint/test_ftw_hotfix_vision.py
new file mode 100644
index 00000000..d4a3eac6
--- /dev/null
+++ b/tests/checkpoint/test_ftw_hotfix_vision.py
@@ -0,0 +1,78 @@
+"""The hotfix tool's vision repair: picking the encoder's checkpoint tensors and letting the family reader name them."""
+
+import importlib.util
+import json
+import os
+import pathlib
+import shutil
+
+import pytest
+
+from freetoken.models.config import VISION_KEY_PREFIXES
+
+_SCRIPT = pathlib.Path(__file__).resolve().parents[2] / "scripts" / "ftw_hotfix.py"
+
+# env var -> local checkpoint, as tests/README.md lists them
+_CHECKPOINTS = ("FREETOKEN_QWEN3VL_MODEL", "FREETOKEN_GEMMA4_MODEL", "FREETOKEN_GEMMA4_UNIFIED_MODEL",
+ "FREETOKEN_GLM53_MODEL", "FREETOKEN_MUSE_MODEL", "FREETOKEN_MINIMAX_M3_MODEL")
+
+
[email protected](scope="module")
+def hotfix():
+ spec = importlib.util.spec_from_file_location("ftw_hotfix", _SCRIPT)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
[email protected]("name, tower", [
+ ("model.visual.blocks.0.attn.qkv.weight", True),
+ ("model.vision_tower.encoder.layers.0.self_attn.q_proj.linear.weight", True),
+ ("model.embed_vision.embedding_projection.weight", True),
+ ("model.vision_embedder.patch_embedding.weight", True),
+ ("model.vision_adapter.fc1.weight", True),
+ ("vision_tower.vision_model.embeddings.patch_embedding.weight", True),
+ ("multi_modal_projector.linear_1.weight", True),
+ ("patch_merge_mlp.linear_2.bias", True),
+ ("model.embed_audio.embedding_projection.weight", False),
+ ("model.language_model.layers.0.self_attn.q_proj.weight", False),
+ ("model.layers.3.mlp.experts.0.gate_proj.weight", False),
+ ("lm_head.weight", False),
+])
+def test_checkpoint_tower_names(hotfix, name, tower):
+ assert hotfix.is_checkpoint_tower_name(name) is tower
+
+
+def _config_dir(hotfix, checkpoint, tmp_path):
+ """An FTW-shaped dir holding only the checkpoint's config files and an empty index, enough to build the model on the meta device."""
+ for f in os.listdir(checkpoint):
+ if f.endswith((".json", ".py", ".jinja")) and not f.startswith("model"):
+ shutil.copy(os.path.join(checkpoint, f), tmp_path)
+ index = {"format": hotfix.FORMAT_TAG, "version": 1, "align": hotfix.ALIGN, "shard_limit": 8 << 30, "total_bytes": 0, "tensors": [], "shards": []}
+ (tmp_path / hotfix.INDEX_NAME).write_text(json.dumps(index))
+ return str(tmp_path)
+
+
[email protected]("env", _CHECKPOINTS)
+def test_reader_emits_every_declared_encoder_tensor(hotfix, env, tmp_path):
+ checkpoint = os.environ.get(env)
+ if not checkpoint:
+ pytest.skip(f"{env} not set")
+ ftw_like = _config_dir(hotfix, checkpoint, tmp_path)
+ _, expected, _, _ = hotfix.expected_tensors(ftw_like, resident_experts=False)
+ declared = {n: shape for n, (shape, _) in expected.items() if n.startswith(VISION_KEY_PREFIXES)}
+ assert declared, "the family declares no encoder tensors"
+ source = hotfix.TensorSource(None, checkpoint)
+ tower_names = [n for n in source.weight_map if hotfix.is_checkpoint_tower_name(n)]
+ got = hotfix.read_tower(source, ftw_like, tower_names)
+ assert sorted(set(declared) - set(got)) == []
+ assert [n for n in declared if tuple(got[n].shape) != declared[n]] == []
+
+
+def test_every_family_with_an_encoder_has_an_encoder_only_reader():
+ from freetoken.models.register import _MODEL_REGISTRY, _load_attr
+
+ for spec in _MODEL_REGISTRY.values():
+ if spec.encoders:
+ assert callable(_load_attr(spec.module, "iter_vision_weights")), spec.module
diff --git a/tests/checkpoint/test_ftw_weights.py b/tests/checkpoint/test_ftw_weights.py
new file mode 100644
index 00000000..b3b3236b
--- /dev/null
+++ b/tests/checkpoint/test_ftw_weights.py
@@ -0,0 +1,56 @@
+"""FTW replay: dropping entries by name before their bytes are read, and the vision-tower presence check."""
+
+import torch
+
+from freetoken.checkpoint.ftw import FTWReader, FTWWriter, ftw_tensor_names, iter_ftw_weights
+from freetoken.models.weight import ftw_lacks_vision, load_weight
+
+
+def _write_ftw(out_dir, names):
+ writer = FTWWriter(str(out_dir))
+ tensors = {name: torch.full((4, 8), float(i), dtype=torch.bfloat16) for i, name in enumerate(names)}
+ for name, tensor in tensors.items():
+ writer.add_tensor(name, tensor)
+ writer.finalize({})
+ return tensors
+
+
+def test_keep_drops_entries_before_their_bytes_are_read(tmp_path, monkeypatch):
+ tensors = _write_ftw(tmp_path, ["model.a.weight", "visual.b.weight", "model.c.weight"])
+ read = []
+ original = FTWReader.read_into
+
+ def spy(self, dest, entry, **kwargs):
+ read.append(entry["name"])
+ return original(self, dest, entry, **kwargs)
+
+ monkeypatch.setattr(FTWReader, "read_into", spy)
+ got = dict(iter_ftw_weights(str(tmp_path), keep=lambda name: not name.startswith("visual.")))
+ assert list(got) == ["model.a.weight", "model.c.weight"]
+ assert read == ["model.a.weight", "model.c.weight"]
+ for name, tensor in got.items():
+ assert torch.equal(tensor, tensors[name])
+
+
+def test_no_keep_replays_every_entry(tmp_path):
+ tensors = _write_ftw(tmp_path, ["model.a.weight", "visual.b.weight"])
+ got = dict(iter_ftw_weights(str(tmp_path)))
+ assert list(got) == list(tensors)
+ assert torch.equal(got["visual.b.weight"], tensors["visual.b.weight"])
+
+
+def test_load_weight_text_only_skips_the_tower(tmp_path):
+ names = ["model.a.weight", "visual.b.weight", "vision_tower.c.weight"]
+ _write_ftw(tmp_path, names)
+ cpu = torch.device("cpu")
+ assert [n for n, _ in load_weight(str(tmp_path), cpu, include_vision=False)] == ["model.a.weight"]
+ assert [n for n, _ in load_weight(str(tmp_path), cpu)] == names
+
+
+def test_ftw_lacks_vision(tmp_path):
+ _write_ftw(tmp_path / "text", ["model.a.weight"])
+ _write_ftw(tmp_path / "vl", ["model.a.weight", "visual.b.weight"])
+ assert ftw_lacks_vision(str(tmp_path / "text"))
+ assert not ftw_lacks_vision(str(tmp_path / "vl"))
+ assert not ftw_lacks_vision(str(tmp_path))
+ assert ftw_tensor_names(str(tmp_path / "vl"), "weight") == ["model.a.weight", "visual.b.weight"]