FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Diagnose why FreeToken does not load an HF safetensors checkpoint, without reading its weights.
python scripts/checkpoint_diagnose/diagnose.py <ckpt dir or HF repo[@revision]> ... [--out report.jsonl] [--text-model-only]
A repo id is fetched as a header-only stub (``fetch_headers.py``) into ~/.cache/freetoken/checkpoint-stubs first. GGUF and FTW checkpoints are pointed elsewhere.
Per checkpoint, through the code the server runs, on the CPU and the meta device:
- config: the architecture is registered and the quantization_config parses
- build: the model builds with a quant method for every layer
- dense load: the family's weight reader yields exactly the model's state dict (names and shapes)
- experts: every MoE layer's expert banks fill, packed by the layer's expert method
The readers get meta tensors with the header's shapes and dtypes; a reader that needs values gets a second run on the real bytes (zeros in a stub).
On a failure it also checks the tensor names against transformers' own model and its checkpoint conversion rules, and lints the files for the known non-standard forms.
The failure is then put in one category (see README.md), with the fixes the fix scripts can apply; ``--out`` keeps them for ``--from-report``.
"""
from __future__ import annotations
import argparse
import concurrent.futures as cf
import contextlib
import functools
import json
import logging
import multiprocessing as mp
import os
import re
import struct
import sys
import time
import traceback
import warnings
from collections import Counter
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
STUB_DIR = os.path.expanduser("~/.cache/freetoken/checkpoint-stubs")
CATEGORIES = {
"arch": "architecture not served",
"quant_method": "quantization method not supported",
"quant_model": "quantization method supported, but not for this model or layer",
"loader": "model supported, the loader does not read this checkpoint",
"packing": "the checkpoint packs tensors in a form the loader does not read",
"checkpoint": "the checkpoint itself is non-standard",
}
FIX_SCRIPTS = {"config": "fix_config.py", "rename": "rename_tensors.py"}
# tensors stored next to a quantized weight; they fold into the module's .weight when the names are compared with transformers
QUANT_COMPANIONS = (
"weight_scale", "weight_scale_2", "weight_scale_inv", "weight_global_scale", "input_scale", "input_global_scale",
"weight_zero_point", "k_scale", "v_scale", "q_scale", "prob_scale", "output_scale", "weight_shape",
)
_QUANT_DTYPES = ("U8", "I8", "F8_E4M3", "F8_E5M2", "I32", "F4")
_DIGITS = re.compile(r"\.\d+(?=\.|$)")
# known non-standard prefixes: (pattern, replacement, what is wrong, checkpoint root the family expects)
PREFIX_RULES = (
(r"^model\.language_model\.(?:language_model\.)+", "model.language_model.", "the text tower is nested more than once under language_model", "model.language_model"),
(r"^model\.language_model\.visual\.", "model.visual.", "the vision tower is stored under the text tower", "model.visual"),
(r"^language_model\.model\.", "model.language_model.", "the text tower uses the older language_model.model. layout", "model.language_model"),
(r"^language_model\.lm_head\.", "lm_head.", "lm_head uses the older language_model.lm_head layout", "model.language_model"),
)
# fp8 KV calibration scales under the projection's name; vLLM reads them as attn.k_scale, which FreeToken drops
KV_SCALE_RULE = (r"\.([kv])_proj\.output_scale$", r".\1_scale")
# the families whose weight reader drops .k_scale / .v_scale tensors; for the others the rename only moves the unexpected key
KV_SCALE_DROPPERS = ("freetoken.models.qwen3_5_moe",)
_PER_EXPERT = re.compile(r"^(?P<pre>.*\.layers\.)(?P<layer>\d+)(?P<mid>\..*\.experts\.)(?P<e>\d+)\.(?P<proj>[^.]+)\.")
# compressed-tensors targets written with vLLM's internal module names
TARGET_SWAPS = ((r"language_model\[\.\]model\[\.\]", "model[.]language_model[.]"), (r"language_model\\\.model\\\.", r"model\\.language_model\\."))
# per-module tensors a quant method's reader produces next to the weight
_SCALE_ROLES = (".weight_scale", ".weight_scale_inv", ".weight_scale_2", ".weight_global", ".input_scale")
def pattern_of(name: str) -> str:
return _DIGITS.sub(".N", name)
def group_names(names, limit: int = 4) -> str:
"""Names grouped by their pattern (layer and expert numbers as N), most common first."""
c = Counter(pattern_of(n) for n in names)
parts = [f"{p} x{n}" if n > 1 else p for p, n in c.most_common(limit)]
return ", ".join(parts) + (f", ... ({len(c) - limit} more patterns)" if len(c) > limit else "")
def shard_files(path: str) -> list[str]:
"""The root shards the loaders read: consolidated.safetensors only when it is the only one (loader.iter_weight_files)."""
files = sorted(f for f in os.listdir(path) if f.endswith(".safetensors"))
return [f for f in files if f != "consolidated.safetensors"] or files
def read_tensor_table(path: str) -> dict[str, dict]:
"""name -> {dtype, shape, shard} from every shard header."""
table = {}
for f in shard_files(path):
for k, v in _header(os.path.join(path, f)).items():
if k != "__metadata__":
table[k] = {"dtype": v["dtype"], "shape": v["shape"], "shard": f}
return table
_ITEM_BITS = {"F64": 64, "I64": 64, "U64": 64, "F32": 32, "I32": 32, "U32": 32, "F16": 16, "BF16": 16, "I16": 16, "U16": 16,
"I8": 8, "U8": 8, "BOOL": 8, "F8_E4M3": 8, "F8_E5M2": 8, "F8_E8M0": 8, "F4": 4}
def check_files(path: str) -> list[str]:
"""What safetensors and the loaders would refuse in the files themselves: data offsets that leave gaps or do not cover the file, a tensor whose bytes do not match its shape, an index that disagrees with the shard headers."""
problems, names = [], {}
for f in shard_files(path):
p = os.path.join(path, f)
with open(p, "rb") as fh:
(n,) = struct.unpack("<Q", fh.read(8))
header = _header(p)
spans = sorted((v["data_offsets"][0], v["data_offsets"][1], k) for k, v in header.items() if k != "__metadata__")
pos = 0
for start, end, k in spans:
if start != pos:
problems.append(f"{f}: tensor {k} starts at byte {start} of the data, expected {pos} (gap or overlap)")
break
bits = _ITEM_BITS.get(header[k]["dtype"])
numel = 1
for d in header[k]["shape"]:
numel *= d
if bits is not None and end - start != numel * bits // 8:
problems.append(f"{f}: tensor {k} has {end - start} bytes, its shape {header[k]['shape']} {header[k]['dtype']} needs {numel * bits // 8}")
break
pos = end
size = os.path.getsize(p) - 8 - n
if not problems and pos != size:
problems.append(f"{f}: the tensors cover {pos} data bytes but the file has {size} (an incomplete download?)")
names.update({k: f for k in header if k != "__metadata__"})
index = os.path.join(path, "model.safetensors.index.json")
if os.path.isfile(index):
with open(index) as fh:
wm = json.load(fh).get("weight_map", {})
read = set(shard_files(path))
# entries for shards the loaders skip (a subdir, an unread consolidated.safetensors) say nothing about the read ones
wm = {k: v for k, v in wm.items() if "/" not in v and (v in read or v != "consolidated.safetensors")}
absent = [k for k in wm if k not in names]
unlisted = [k for k in names if k not in wm]
moved = [k for k in wm if k in names and wm[k] != names[k]]
if absent:
problems.append(f"the index lists {len(absent)} tensors no shard holds: {group_names(absent)}")
if unlisted:
problems.append(f"the shards hold {len(unlisted)} tensors the index does not list: {group_names(unlisted)}")
if moved:
problems.append(f"the index puts {len(moved)} tensors in another shard than the one that holds them: {group_names(moved)}")
return problems
def split_module(name: str) -> tuple[str, str]:
"""(module, leaf) of a tensor name: model.layers.0.mlp.gate.weight_scale -> (model.layers.0.mlp.gate, weight_scale)."""
module, _, leaf = name.rpartition(".")
return module, leaf
def stored_modules(table: dict) -> dict[str, tuple[bool, str]]:
"""module -> (stored quantized, weight dtype) for every module with a 2-D weight or a packed weight."""
out = {}
by_module: dict[str, dict[str, dict]] = {}
for name, meta in table.items():
module, leaf = split_module(name)
by_module.setdefault(module, {})[leaf] = meta
for module, leaves in by_module.items():
w = leaves.get("weight_packed") or leaves.get("weight")
if w is None or (len(w["shape"]) != 2 and "weight_packed" not in leaves):
continue
quantized = "weight_packed" in leaves or w["dtype"] in _QUANT_DTYPES
out[module] = (quantized, w["dtype"])
return out
# --- format routing (main process, no torch) ---
def route(path: str) -> dict | None:
"""A record for a checkpoint this script does not check (GGUF, FTW, no safetensors), else None."""
files = os.listdir(path)
if "freetoken_weight.json" in files:
return {"path": path, "status": "skip", "format": "ftw",
"hint": "an FTW checkpoint converted by `ft checkpoint`: diagnose its source HF checkpoint; docs/ftw-hotfix.md covers FTW repairs"}
stub = _stub_record(path)
gguf = any(f.endswith(".gguf") for f in files) or bool(stub and stub.get("gguf"))
shards = [f for f in files if f.endswith(".safetensors")]
if gguf and not shards:
return {"path": path, "status": "skip", "format": "gguf",
"hint": "a GGUF checkpoint: check it with scripts/gguf_check/ (README.md there); the GGUF section of this README sorts its failures"}
code = _missing_config_code(path) if stub else []
if code:
return {"path": path, "status": "skip", "format": "remote code",
"hint": f"the repo ships its own config code ({', '.join(code)}), which a header-only stub does not fetch and FreeToken runs with trust_remote_code; download the repo and diagnose the local directory"}
if not shards:
other = sorted({os.path.splitext(f)[1] for f in files if f.endswith((".bin", ".pt", ".pth", ".npz"))} | set((stub or {}).get("other_weights") or ()))
return {"path": path, "status": "skip", "format": "other",
"hint": f"no root *.safetensors shard ({', '.join(other) or 'no weights found'}); FreeToken reads safetensors and GGUF only"}
return None
def _missing_config_code(path: str) -> list[str]:
"""The .py files the config's auto_map names for AutoConfig that are not in ``path``."""
try:
with open(os.path.join(path, "config.json")) as f:
cfg = json.load(f)
except (OSError, ValueError):
return []
files = []
for c in (cfg, cfg.get("text_config") or {}):
ref = (c.get("auto_map") or {}).get("AutoConfig") if isinstance(c, dict) else None
if isinstance(ref, str) and "--" not in ref:
files.append(ref.rsplit(".", 1)[0] + ".py")
return [f for f in dict.fromkeys(files) if not os.path.isfile(os.path.join(path, f))]
def _stub_record(path: str) -> dict | None:
p = os.path.join(path, ".stub.json")
if not os.path.isfile(p):
return None
with open(p) as f:
return json.load(f)
# --- meta-device readers (worker) ---
@functools.cache
def _header(path: str) -> dict:
with open(path, "rb") as f:
(n,) = struct.unpack("<Q", f.read(8))
return json.loads(f.read(n))
def _meta_tensor(meta: dict):
import torch
from freetoken.models.weight import _ST_DTYPE
return torch.empty(meta["shape"], dtype=_ST_DTYPE[meta["dtype"]], device="meta")
class _MetaSlice:
def __init__(self, meta: dict):
self.meta = meta
def get_shape(self):
return list(self.meta["shape"])
def get_dtype(self):
return self.meta["dtype"]
def __getitem__(self, idx):
return _meta_tensor(self.meta)[idx]
class _MetaSafeOpen:
"""safetensors.safe_open whose tensors are meta tensors with the header's shape and dtype."""
def __init__(self, filename, framework="pt", device="cpu"):
self._h = _header(os.fspath(filename))
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def keys(self):
return [k for k in self._h if k != "__metadata__"]
def offset_keys(self):
return self.keys()
def metadata(self):
return self._h.get("__metadata__")
def get_tensor(self, name):
return _meta_tensor(self._h[name])
def get_slice(self, name):
return _MetaSlice(self._h[name])
def _meta_iter_expert_tensors_parallel(model_path, is_expert, **_kwargs):
from freetoken.models.loader import safetensors_weight_map
from freetoken.utils.hf import download_hf_weight
folder = download_hf_weight(model_path)
for name, shard in safetensors_weight_map(folder).items():
if is_expert(name):
yield name, _meta_tensor(_header(os.path.join(folder, shard))[name])
@functools.cache
def _meta_wins():
"""A torch function mode that moves CPU operands to meta when an op mixes the two."""
import torch
from torch.overrides import TorchFunctionMode
from torch.utils._pytree import tree_flatten, tree_unflatten
class MetaWins(TorchFunctionMode):
def __torch_function__(self, func, types, args=(), kwargs=None):
flat, spec = tree_flatten((args, kwargs or {}))
if any(isinstance(x, torch.Tensor) and x.is_meta for x in flat):
flat = [x.to("meta") if isinstance(x, torch.Tensor) and not x.is_meta else x for x in flat]
args, kwargs = tree_unflatten(flat, spec)
return func(*args, **kwargs)
return MetaWins
@contextlib.contextmanager
def meta_reads(enabled: bool):
"""Point safetensors.safe_open and every binding of iter_expert_tensors_parallel at the meta readers."""
if not enabled:
yield
return
import safetensors
import freetoken.models.weight as weight
real_open, real_parallel = safetensors.safe_open, weight.iter_expert_tensors_parallel
swapped = []
safetensors.safe_open = _MetaSafeOpen
for module in list(sys.modules.values()):
if getattr(module, "__name__", "").startswith("freetoken.") and getattr(module, "iter_expert_tensors_parallel", None) is real_parallel:
module.iter_expert_tensors_parallel = _meta_iter_expert_tensors_parallel
swapped.append(module)
try:
with _meta_wins()():
yield
finally:
safetensors.safe_open = real_open
for module in swapped:
module.iter_expert_tensors_parallel = real_parallel
def _needs_values(exc: BaseException) -> bool:
return "meta" in f"{type(exc).__name__}: {exc}".lower()
# --- load stages (worker) ---
def _init_worker() -> None:
os.environ["CUDA_VISIBLE_DEVICES"] = ""
# the readers' progress bars; errors reach the main process in the record
sys.stderr = open(os.devnull, "w")
warnings.filterwarnings("ignore")
logging.disable(logging.WARNING)
import torch
from freetoken.distributed import set_tp_info
from freetoken.layers import rotary
set_tp_info(rank=0, size=1)
rotary.set_rope_device(torch.device("cpu"))
def _engine_config(path: str, text_only: bool):
import torch
from freetoken.distributed import try_get_tp_info
from freetoken.engine.config import EngineConfig
from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig
mm = MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS) if text_only else frozenset())
return EngineConfig(model_path=path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy="offload", mm=mm)
def _build_model(config):
import torch
from freetoken.engine.engine import _decode_target
from freetoken.layers.quantization import QuantBackend, set_quant_backend
from freetoken.models import create_model
from freetoken.utils.torch_utils import torch_dtype
set_quant_backend(QuantBackend.parse(config.quant_backend))
object.__setattr__(config.model_config, "moe_strategy", "offload")
object.__setattr__(config.model_config, "decode_target", _decode_target(config))
with torch.device("meta"), torch_dtype(torch.bfloat16):
return create_model(config.model_config)
class _Recorder(dict):
"""The state dict model.load_state_dict consumes: it records the keys a layer asks for and does not find, and the shapes that differ, instead of stopping at the first.
The layers' own load_state_dict decide what they take (a tied lm_head pops lm_head.weight itself); what is left at the end is unexpected."""
def __init__(self, expected: dict):
super().__init__()
self.expected, self.missing, self.shapes = expected, [], []
def pop(self, key, *default):
import torch
if key not in self:
if default:
return default[0]
self.missing.append(key)
return torch.empty_like(self.expected[key], device="meta")
t = super().pop(key)
ref = self.expected.get(key)
if ref is None:
return t
if tuple(t.shape) != tuple(ref.shape):
self.shapes.append(f"{key}: checkpoint {tuple(t.shape)}, model {tuple(ref.shape)}")
return torch.empty_like(ref, device="meta")
return t.to(dtype=ref.dtype)
_LOSSY = {("float8_e4m3fn", "bfloat16"), ("float8_e5m2", "bfloat16"), ("bfloat16", "float8_e4m3fn"), ("float32", "float8_e4m3fn"), ("bfloat16", "uint8")}
def dense_load(path: str, config, model, *, meta: bool) -> tuple[list[str], list[str], dict]:
"""(problems, warnings, facts) of the engine's dense weight load: what load_weight yields against the model's state dict."""
import torch
from freetoken.models import load_weight
expected = model.state_dict()
is_moe = bool(getattr(config.model_config, "is_moe", False))
loaded, dups, casts = _Recorder(expected), [], Counter()
with meta_reads(meta):
for key, t in load_weight(path, torch.device("cpu"), include_moe_experts=not is_moe, include_vision=bool(config.active_encoders)):
ref = expected.get(key)
# as _materialize_loaded_weight_state_dict: drop an input_scale the layer declares none for, cast to the parameter dtype
if ref is None and key.endswith(".input_scale"):
continue
if key in loaded:
dups.append(key)
if ref is not None and (str(t.dtype).removeprefix("torch."), str(ref.dtype).removeprefix("torch.")) in _LOSSY:
casts[f"{t.dtype} -> {ref.dtype}"] += 1
# the check keeps no tensor bytes
dict.__setitem__(loaded, key, t if t.is_meta else torch.empty_like(t, device="meta"))
try:
model.load_state_dict(loaded)
except RuntimeError as e:
if not str(e).startswith("Unexpected keys"):
raise
unexpected, missing, shapes = sorted(loaded), sorted(loaded.missing), loaded.shapes
problems = []
if unexpected:
problems.append(f"{len(unexpected)} unexpected tensors: {group_names(unexpected)}")
if missing:
problems.append(f"{len(missing)} model tensors never loaded: {group_names(missing)}")
if shapes:
problems.append(f"{len(shapes)} shape mismatches, e.g. {shapes[0]}")
warns = [f"{n} tensors cast {k} at load (lossy)" for k, n in casts.items()]
if dups:
warns.append(f"{len(dups)} tensors are read twice and the engine keeps the last: {group_names(dups)}")
facts = {"missing_only_scales": bool(missing) and not unexpected and all(m.endswith(_SCALE_ROLES) for m in missing)}
return problems, warns, facts
def expert_load(path: str, config, model, *, meta: bool) -> list[str]:
"""Problems of filling every MoE layer's expert banks, as build_expert_banks fills them; nothing is kept."""
import torch
from freetoken.engine.engine import offload_layer_methods
from freetoken.moe.expert_banks import method_groups
from freetoken.models.weight import experts_scattered
from freetoken.moe.expert_pieces import iter_expert_pieces
methods = offload_layer_methods(model)
if methods is None:
return []
mc = config.model_config
num_layers, num_experts = mc.num_moe_layers, mc.num_experts
written = torch.zeros(num_layers, num_experts, dtype=torch.int32)
groups = method_groups(methods)
device = torch.device("meta" if meta else "cpu")
# the reader load_expert_banks picks: the parallel one (names from the index) for experts stored as many small tensors
parallel = experts_scattered(path)
def fill(method, layers, par: bool) -> str | None:
keep = set(layers)
layout = {role: spec for role, spec in method.layout().items() if not spec.resident}
pieces = iter_expert_pieces(path, mc, method.kind, layers=layers if len(groups) > 1 else None, parallel=par)
for layer, e0, e1, piece in pieces:
if not (0 <= layer < num_layers and 0 <= e0 < e1 <= num_experts):
return f"expert piece out of range: layer {layer}, experts {e0}:{e1} of {num_layers} x {num_experts}"
if layer not in keep:
continue
if written[layer, e0:e1].any():
return f"expert rows read more than once: layer {layer}, experts {e0}:{e1}"
written[layer, e0:e1] = 1
out = {role: torch.empty((e1 - e0, *spec.shape), dtype=spec.dtype, device=device) for role, spec in layout.items()}
method.pack(piece, out)
return None
with meta_reads(meta):
for method, layers in groups:
try:
problem = fill(method, layers, parallel)
except NotImplementedError:
# load_expert_banks rebuilds serially when a quant has no parallel reader
if not parallel:
raise
written[layers] = 0
problem = fill(method, layers, False)
if problem:
return [problem]
missing = (written == 0).nonzero().tolist()
if missing:
return [f"expert banks were not filled: {len(missing)} (layer, expert) rows missing (first {missing[:4]})"]
return []
# --- transformers cross-check (worker) ---
def hf_check(path: str, table: dict) -> dict:
"""Whether transformers' own model takes these tensor names, through its checkpoint conversion rules.
Quantization companions fold into their module's ``.weight``. Returns the names it does not map, the model keys nothing maps to, and the names a WeightConverter repacks."""
import torch
import transformers
from transformers.conversion_mapping import get_model_conversion_mapping
from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
transformers.utils.logging.disable_progress_bar()
cfg = transformers.AutoConfig.from_pretrained(path)
arch = (getattr(cfg, "architectures", None) or [None])[0]
cls = getattr(transformers, arch or "", None)
if cls is None:
return {"skipped": f"transformers has no class {arch}"}
with torch.device("meta"):
model = cls._from_config(cfg)
meta_sd = model.state_dict()
transforms = get_model_conversion_mapping(model)
renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
converters = [t for t in transforms if isinstance(t, WeightConverter)]
ignore_unexpected = [re.compile(p) for p in (getattr(model, "_keys_to_ignore_on_load_unexpected", None) or [])]
ignore_missing = [re.compile(p) for p in (getattr(model, "_keys_to_ignore_on_load_missing", None) or [])]
names = set()
for name in table:
module, leaf = split_module(name)
if leaf in QUANT_COMPANIONS:
continue
names.add(module + ".weight" if leaf == "weight_packed" else name)
mapped, converted = {}, set()
for name in names:
new, conv = rename_source_key(name, renamings, converters, model.base_model_prefix, meta_sd)
mapped[name] = new
if conv is not None:
converted.add(name)
unmapped = sorted(n for n, k in mapped.items() if k not in meta_sd and not any(rx.search(k) for rx in ignore_unexpected))
tied = bool(getattr(cfg.get_text_config(), "tie_word_embeddings", False))
targets = set(mapped.values())
missing = sorted(k for k in meta_sd if k not in targets and not any(rx.search(k) for rx in ignore_missing)
and not (tied and k.endswith("lm_head.weight")))
renamed = sorted(n for n, k in mapped.items() if k != n and n not in converted and k in meta_sd)
return {"arch": arch, "unmapped": unmapped, "missing": missing, "converted": sorted(converted), "renamed": renamed}
# --- lints (worker) ---
def _skip_entry(dialect: str, modules: list[str], all_modules) -> list[str]:
"""Skip-list entries in the dialect's syntax that cover exactly ``modules`` of ``all_modules``.
A pattern (layer numbers as a wildcard) stands for its modules only when every module of ``all_modules`` with that pattern is in ``modules``; the others are listed by name."""
chosen = set(modules)
by_pattern: dict[str, list[str]] = {}
for m in all_modules:
by_pattern.setdefault(pattern_of(m), []).append(m)
entries = []
for p in sorted({pattern_of(m) for m in chosen}):
whole = all(m in chosen for m in by_pattern.get(p, ())) and "N" in p.split(".")
if whole and dialect == "modelopt":
entries.append(p.replace(".N", ".*"))
elif whole and dialect == "compressed-tensors":
entries.append("re:^" + re.escape(p).replace(r"\.N", r"\.\d+") + "$")
else:
entries += sorted(m for m in by_pattern.get(p, [p]) if m in chosen)
return entries
def _entry_matches(dialect: str, entry: str, name: str) -> bool:
from freetoken.layers.quantization.names import ct_set, name_set, substr_set
if dialect == "compressed-tensors":
return ct_set((entry,), class_names=False)(name)
if dialect == "fp8":
return name_set((entry,))(name) or substr_set((entry,))(name)
return name_set((entry,))(name)
def _raw_skip_entries(path: str) -> list[str]:
from ckpt_edit import SKIP_KEYS, Checkpoint
entries = []
for _, q in Checkpoint(path).quant_configs():
for key in SKIP_KEYS:
entries += [e for e in q.get(key) or () if isinstance(e, str)]
return list(dict.fromkeys(entries))
def lint_quant(path: str, table: dict, spec, hf_config) -> list[dict]:
"""Modules whose stored form contradicts what the quantization_config declares for them, with the skip-list edits that make the two agree."""
from freetoken.models.register import checkpoint_quant_config
quant = checkpoint_quant_config(path, hf_config, spec)
dialect = getattr(quant, "dialect", "none")
if dialect in ("none", "gguf"):
return []
declared_q_stored_f, declared_f_stored_q, exempt_dense = [], [], []
stored = stored_modules(table)
for module, (quantized, _) in stored.items():
# embedding tables (and PLE n-gram tables) are read dense or by their own method whatever the config says
if any("embed" in seg for seg in module.split(".")) or module.endswith("conv1d"):
continue
declared = quant.scheme_for_name(module) is not None
if quant.unquantized(module):
if declared and not quantized:
exempt_dense.append(module)
elif declared and not quantized:
declared_q_stored_f.append(module)
elif quantized and not declared:
declared_f_stored_q.append(module)
findings = []
if exempt_dense:
findings.append({
"id": "quant_unlisted_dense_router", "level": "config", "category": "loader",
"message": f"{len(exempt_dense)} modules FreeToken serves in bf16 anyway are stored bf16 and not listed as skipped: {group_names(exempt_dense)}; listing them works around a reader that wants the stored form to match the config",
"fix": {"op": "skip_add", "entries": _skip_entry(dialect, exempt_dense, stored), "auto": True},
})
targets = _lint_ct_targets(path, table, dialect)
if declared_q_stored_f:
findings.append({
"id": "quant_declared_but_dense", "level": "config", "category": "checkpoint",
"message": f"{len(declared_q_stored_f)} modules are stored unquantized but the quantization_config does not list them as skipped: {group_names(declared_q_stored_f)}",
"fix": {"op": "skip_add", "entries": _skip_entry(dialect, declared_q_stored_f, stored), "auto": True},
})
# targets that name no module explain the quantized modules nothing covers; recheck after fixing them
if declared_f_stored_q and not targets:
entries = _raw_skip_entries(path)
remove, add = [], []
for e in entries:
hits = [m for m in declared_f_stored_q if _entry_matches(dialect, e, m)]
if not hits:
continue
remove.append(e)
# an entry naming a container also covers its dense children: list those alone instead
dense = [m for m, (q, _) in stored.items() if not q and _entry_matches(dialect, e, m)]
add += _skip_entry(dialect, dense, stored) if dense else []
msg = f"{len(declared_f_stored_q)} modules are stored quantized but the quantization_config skips them or no target covers them: {group_names(declared_f_stored_q)}"
finding = {"id": "quant_skipped_but_quantized", "level": "config", "category": "checkpoint", "message": msg}
if remove:
finding["message"] += f"; skip entries that cover them: {remove[:6]}"
finding["fixes"] = [{"op": "skip_remove", "entries": remove, "auto": True}] + ([{"op": "skip_add", "entries": add, "auto": True}] if add else [])
findings.append(finding)
return findings + targets
def _lint_ct_targets(path: str, table: dict, dialect: str) -> list[dict]:
if dialect != "compressed-tensors":
return []
from ckpt_edit import Checkpoint
modules = list(stored_modules(table))
dead: dict[tuple[str, str], list[str]] = {}
for _, q in Checkpoint(path).quant_configs():
for group in (q.get("config_groups") or {}).values():
for target in group.get("targets") or ():
if target == "Linear" or any(_entry_matches(dialect, target, m) for m in modules):
continue
for pat, repl in TARGET_SWAPS:
swapped = re.sub(pat, repl, target)
if swapped != target and any(_entry_matches(dialect, swapped, m) for m in modules):
dead.setdefault((pat, repl), []).append(target)
break
return [{
"id": "ct_target_vllm_names", "level": "config", "category": "checkpoint",
"message": f"{len(ts)} compressed-tensors targets name modules as vLLM does (language_model.model.) and match no tensor, e.g. {ts[0]!r}",
"fix": {"op": "quant_sub", "pattern": pat, "repl": repl, "auto": True},
} for (pat, repl), ts in dead.items()]
def rename_rules(findings: list[dict]) -> list[tuple[str, str]]:
return [tuple(r) for f in findings if f["level"] == "rename" for r in f["fix"]["rules"]]
def renamed_table(table: dict, rules) -> dict:
"""The tensor table as it reads after the rename rules."""
out = {}
for name, meta in table.items():
for pat, repl in rules:
name = re.sub(pat, repl, name)
out[name] = meta
return out
def lint_kv_scales(table: dict, spec) -> list[dict]:
hits = [n for n in table if re.search(KV_SCALE_RULE[0], n)]
if not hits or spec.module not in KV_SCALE_DROPPERS:
return []
return [{
"id": "kv_output_scale", "level": "rename", "category": "loader",
"message": f"{len(hits)} fp8 KV-cache scales stored as <k|v>_proj.output_scale ({group_names(hits, 2)}); vLLM reads them as k_scale / v_scale, FreeToken drops k_scale / v_scale but not output_scale",
"fix": {"op": "rename", "rules": [list(KV_SCALE_RULE)], "auto": True},
}]
def lint_prefixes(table: dict, spec) -> list[dict]:
roots = {ckpt for _, ckpt in spec.checkpoint_roots}
names = list(table)
findings = []
for pat, repl, why, root in PREFIX_RULES:
if root not in roots:
continue
rx = re.compile(pat)
hits = [n for n in names if rx.search(n)]
if not hits:
continue
renamed = {rx.sub(repl, n) for n in hits}
if renamed & (set(names) - set(hits)):
continue
findings.append({
"id": "prefix", "level": "rename", "category": "checkpoint",
"message": f"{why}: {len(hits)} tensors {rx.search(hits[0]).group(0)} -> {repl}",
"fix": {"op": "rename", "rules": [[pat, repl]], "quant_too": True, "auto": True},
})
return findings
def lint_experts(table: dict, num_experts: int | None) -> list[dict]:
"""Per-expert tensors (experts.E.<proj>) where some layer lacks some expert's projection: a checkpoint that lost tensors, not a packing form."""
if not num_experts:
return []
seen: dict[tuple[str, str, str], set[int]] = {}
for name in table:
m = _PER_EXPERT.match(name)
if m:
seen.setdefault((m["pre"] + m["layer"], m["mid"], m["proj"]), set()).add(int(m["e"]))
gaps = [(f"{pre}{mid}N.{proj}", sorted(set(range(num_experts)) - es)) for (pre, mid, proj), es in seen.items() if len(es) < num_experts]
if not gaps:
return []
shown = "; ".join(f"{where} lacks experts {missing[:6]}" for where, missing in gaps[:3])
return [{"id": "expert_tensors_incomplete", "level": "none", "category": "checkpoint",
"message": f"{len(gaps)} (layer, projection) groups of per-expert tensors are incomplete: {shown}; the checkpoint is missing tensors (download it again, or ask its uploader)"}]
def lint_vision(path: str, table: dict, spec, text_only: bool) -> list[dict]:
from freetoken.models.config import VISION_KEY_PREFIXES
if text_only or not spec.encoders:
return []
# checkpoint names put the tower under a root (model.visual., model.vision_tower.)
segments = tuple("." + p for p in VISION_KEY_PREFIXES)
if not any(("." + n).count(s) for n in table for s in segments):
return [{"id": "no_vision_tower", "level": "runtime", "category": "checkpoint",
"message": "the architecture has a vision tower but the checkpoint stores none: a text-only checkpoint under a multimodal architecture; serve it with --text-model-only (and diagnose it with --text-model-only)"}]
if spec.mm_processor and not any(os.path.isfile(os.path.join(path, f)) for f in ("preprocessor_config.json", "processor_config.json")):
return [{"id": "no_processor_config", "level": "file", "category": "checkpoint",
"message": "the checkpoint has a vision tower but no preprocessor_config.json, so image requests fail with a 400; copy that file from the base model the checkpoint was tuned from"}]
return []
def lint_misc(table: dict, hf_config) -> list[dict]:
from freetoken.layers.quantization.configs import quantization_config_of
out = []
dtypes = Counter(m["dtype"] for m in table.values())
if quantization_config_of(hf_config) is None and (dtypes["F8_E4M3"] or dtypes["F8_E5M2"]):
out.append({"id": "fp8_without_config", "level": "none", "category": "checkpoint",
"message": f"{dtypes['F8_E4M3'] + dtypes['F8_E5M2']} fp8 tensors but no quantization_config says how to read them (scales, block size)"})
if any(n.endswith((".scales", ".biases")) for n in table) and any(m["dtype"] == "U32" for m in table.values()):
out.append({"id": "mlx", "level": "none", "category": "quant_method", "message": "MLX-quantized weights (.scales / .biases, U32 packing); FreeToken does not read MLX"})
return out
def expert_layout(table: dict) -> str | None:
per_expert = any(re.search(r"\.experts\.\d+\.(gate|up|down|w[123])", n) for n in table)
stacked = any(re.search(r"\.experts\.(gate_up_proj|down_proj|w13_weight|w2_weight)$", n) for n in table)
if per_expert and stacked:
return "mixed: some layers per expert, some stacked"
return "per expert (experts.N.<proj>)" if per_expert else "stacked (experts.gate_up_proj / down_proj)" if stacked else None
# --- classification ---
_QUANT_METHOD = re.compile(r"quantization method .* is not supported|quant_algo .* is not supported|compressed-tensors .*scheme .* is not supported|MIXED_PRECISION without|fp8 checkpoint with weight_block_size|fp8 block size .* is not supported|target class .* is not supported")
_QUANT_MODEL = re.compile(r"no quant method for|provides no (expert reader|nvfp4_expert_spec|iter_weights_parallel)|needs in/out sizes divisible|is not supported for|cannot use the .* kernel")
def classify(rec: dict) -> str:
"""The category of a failed record (README.md, "Safetensors: six kinds of failure"), first match wins."""
first = (rec["problems"] or [""])[0]
findings = rec["findings"]
hf = rec.get("transformers") or {}
if rec["stage"] == "files":
return "checkpoint"
if rec["stage"] == "config" and "architecture" in first.lower() and "not supported" in first:
return "arch"
if any(f["category"] == "quant_method" for f in findings) or _QUANT_METHOD.search(first):
return "quant_method"
# a broken prefix misnames every module under it, so whatever fails next is its symptom
if any(f["id"] == "prefix" and f["category"] == "checkpoint" for f in findings):
return "checkpoint"
if _QUANT_MODEL.search(first) or rec.get("missing_only_scales"):
return "quant_model"
if hf.get("unmapped") or hf.get("missing"):
return "checkpoint"
if any(f["category"] == "checkpoint" for f in findings):
return "checkpoint"
if rec["stage"] == "experts" and hf.get("converted"):
return "packing"
return "loader"
def _fix_list(findings: list[dict]) -> list[dict]:
fixes = []
for f in findings:
for fix in f.get("fixes") or ([f["fix"]] if "fix" in f else []):
fixes.append({"level": f["level"], "finding": f["id"], **fix})
return fixes
def run_lints(path: str, table: dict, spec, hf_config, text_only: bool, warns: list[str], num_experts: int | None = None) -> list[dict]:
"""Every lint; the quantization lints see the names as they read after the rename fixes, which a rename applies first."""
def safe(fn, *args):
try:
return fn(*args)
except Exception as e:
warns.append(f"lint {fn.__name__} failed: {type(e).__name__}: {str(e)[:200]}")
return []
if spec is None:
return safe(lint_misc, table, hf_config) if hf_config is not None else []
findings = safe(lint_prefixes, table, spec) + safe(lint_kv_scales, table, spec)
after = renamed_table(table, rename_rules(findings))
findings += safe(lint_experts, after, num_experts) + safe(lint_vision, path, after, spec, text_only) + safe(lint_misc, after, hf_config)
return findings + safe(lint_quant, path, after, spec, hf_config)
class _Stop(Exception):
"""A stage found problems of its own; the later stages would only repeat them."""
def check(path: str, text_only: bool) -> dict:
rec = {"path": path, "stage": "config", "status": "fail", "problems": [], "warnings": [], "findings": []}
stub = _stub_record(path)
if stub:
rec["repo"], rec["revision"] = stub["repo"], stub["revision"]
t0 = time.time()
hf_config = spec = table = None
try:
from freetoken.models.register import get_model_spec
from freetoken.utils import cached_load_hf_config
rec["stage"] = "files"
table = read_tensor_table(path)
file_problems = check_files(path)
if file_problems:
rec["problems"] += file_problems
raise _Stop
rec["stage"] = "config"
hf_config = cached_load_hf_config(path)
rec["arch"] = (getattr(hf_config, "architectures", None) or [None])[0]
spec = get_model_spec(rec["arch"])
rec["stage"] = "quant"
config = _engine_config(path, text_only)
mc = config.model_config
rec["dialect"] = getattr(getattr(mc, "quant", None), "dialect", None)
rec["moe"] = {"layers": mc.num_moe_layers, "experts": mc.num_experts, "layout": expert_layout(table)} if getattr(mc, "is_moe", False) else None
rec["encoders"] = [e.kind for e in config.active_encoders]
rec["stage"] = "build"
model = _build_model(config)
for stage, fn in (("dense", dense_load), ("experts", expert_load)):
rec["stage"] = stage
try:
out = fn(path, config, model, meta=True)
except Exception as e:
if not _needs_values(e):
raise
rec.setdefault("real_reads", []).append(stage)
out = fn(path, config, model, meta=False)
problems, warns, facts = out if stage == "dense" else (out, [], {})
rec["warnings"] += warns
rec.update(facts)
if problems:
rec["problems"] += problems
break
else:
rec["stage"], rec["status"] = "done", "pass"
except _Stop:
pass
except Exception as e:
rec["problems"].append(f"{type(e).__name__}: {str(e)[:600]}")
rec["traceback"] = traceback.format_exc()[-2500:]
if table is not None:
rec["findings"] = run_lints(path, table, spec, hf_config, text_only, rec["warnings"], (rec.get("moe") or {}).get("experts"))
if rec["status"] == "fail" and table is not None and rec["stage"] in ("build", "dense", "experts"):
try:
rec["transformers"] = hf_check(path, table)
except Exception as e:
rec["transformers"] = {"skipped": f"{type(e).__name__}: {str(e)[:200]}"}
hf = rec["transformers"]
if "skipped" not in hf and not hf["unmapped"] and not hf["missing"]:
# transformers reads these names, so a rename is a workaround for the loader rather than a repair
for f in rec["findings"]:
if f["id"] == "prefix":
f["category"] = "loader"
if rec["status"] == "fail":
rec["category"] = classify(rec)
# nothing to repair in the files of a checkpoint whose format or architecture is not served
fixes = [] if rec.get("category") in ("arch", "quant_method") else _fix_list(rec["findings"])
# a remote repo is only diagnosed: fixes are made on a local download, diagnosed again
rec["fixes"] = [] if stub else fixes
rec["fixable_locally"] = bool(stub and fixes)
rec["seconds"] = round(time.time() - t0, 1)
return rec
# --- main process ---
def resolve(target: str, stub_dir: str) -> tuple[str | None, dict | None]:
"""(local path, None), or (None, error record) when a repo cannot be fetched."""
if os.path.isdir(target):
return os.path.abspath(target), None
from fetch_headers import fetch_repo
try:
return fetch_repo(target, stub_dir)["stub"], None
except Exception as e:
return None, {"path": target, "status": "skip", "format": "unknown", "hint": f"not a local directory, and fetching it from HF failed: {type(e).__name__}: {str(e)[:300]}"}
def _short(rec: dict) -> str:
return rec.get("repo") or rec["path"]
def print_record(rec: dict, report_path: str | None) -> None:
print(f"\n== {_short(rec)}" + (" (header-only stub)" if rec.get("repo") else ""))
if rec["status"] == "skip":
print(f" {rec['hint']}")
return
bits = [rec.get("arch") or "?", f"quant {rec.get('dialect') or 'none'}"]
if rec.get("moe"):
bits.append(f"MoE {rec['moe']['layers']} layers x {rec['moe']['experts']} experts, {rec['moe']['layout'] or 'experts not found by name'}")
print(" " + ", ".join(bits))
if rec["status"] == "pass":
print(" PASS: config, build, dense load" + (", expert banks" if rec.get("moe") else ""))
else:
cat = rec["category"]
print(f" FAIL at {rec['stage']}: [{cat}] {CATEGORIES[cat]}")
for p in rec["problems"]:
print(f" - {p}")
hf = rec.get("transformers") or {}
if hf.get("skipped"):
print(f" transformers check skipped: {hf['skipped']}")
elif hf:
if hf["unmapped"] or hf["missing"]:
parts = []
if hf["unmapped"]:
parts.append(f"{len(hf['unmapped'])} checkpoint names it cannot map ({group_names(hf['unmapped'], 3)})")
if hf["missing"]:
parts.append(f"{len(hf['missing'])} model keys nothing fills ({group_names(hf['missing'], 3)})")
print(f" transformers {hf['arch']} would not load it either: " + "; ".join(parts))
else:
extra = []
if hf["renamed"]:
extra.append(f"renames {len(hf['renamed'])} ({group_names(hf['renamed'], 2)})")
if hf["converted"]:
extra.append(f"repacks {len(hf['converted'])} ({group_names(hf['converted'], 2)})")
print(" transformers loads these names" + (": " + "; ".join(extra) if extra else ""))
for w in rec["warnings"]:
print(f" warning: {w}")
for f in rec["findings"]:
print(f" {'note' if rec['status'] == 'pass' else 'finding'} [{f['level']}, {f['category']}]: {f['message']}")
if rec.get("fixable_locally") and rec["status"] == "fail":
print(" fixable: download the checkpoint and run diagnose.py on the local directory to get the fix commands (a remote repo is only diagnosed)")
fixes = rec.get("fixes") or []
if not fixes or rec["status"] != "fail":
return
target, report_arg = rec["path"], report_path or "<report.jsonl>"
# one level per round: a rename changes the names the config findings are about, so diagnose again after it
level = "rename" if any(f["level"] == "rename" and f.get("auto") for f in fixes) else "config"
if any(f["level"] == level and f.get("auto") for f in fixes):
if not report_path:
print(" fixes: rerun with --out <report.jsonl> to get the fix command")
return
print(f" next step ({level} fix; it prints a plan and asks before writing):")
print(f" python scripts/checkpoint_diagnose/{FIX_SCRIPTS[level]} {target} --out <new dir> --from-report {report_arg}")
print(" then diagnose <new dir> again")
def summary(records: list[dict]) -> None:
status = Counter(r["status"] for r in records)
print(f"\n{status['pass']} pass, {status['fail']} fail, {status['skip']} not checked")
cats = Counter(r["category"] for r in records if r["status"] == "fail")
for cat in CATEGORIES:
if cats[cat]:
print(f" {cats[cat]:4d} [{cat}] {CATEGORIES[cat]}")
for r in records:
if r["status"] == "fail" and r["category"] == cat:
fixable = [lv for lv in FIX_SCRIPTS if any(f["level"] == lv and f.get("auto") for f in r.get("fixes", []))]
note = f" (fixes: {', '.join(fixable)})" if fixable else " (fixable on a local download)" if r.get("fixable_locally") else ""
print(f" {_short(r)}{note}")
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("targets", nargs="+", help="checkpoint directories or HF repo ids")
ap.add_argument("--out", metavar="JSONL", help="append the records here; the fix scripts read them with --from-report")
ap.add_argument("--text-model-only", action="store_true", help="check as `ft serve --text-model-only` loads: no vision tower")
args = ap.parse_args()
records = []
out = open(args.out, "a") if args.out else None
ctx = mp.get_context("spawn")
pool = None
for target in args.targets:
path, rec = resolve(target, STUB_DIR)
rec = rec or route(path)
if rec is None:
# one worker process, reused; a crash takes only the checkpoint it was checking
pool = pool or cf.ProcessPoolExecutor(1, mp_context=ctx, initializer=_init_worker)
try:
rec = pool.submit(check, path, args.text_model_only).result()
except cf.process.BrokenProcessPool:
pool = None
rec = {"path": path, "status": "fail", "stage": "?", "category": "loader", "problems": ["the check process died (out of host memory, or a crash)"],
"warnings": [], "findings": [], "fixes": []}
records.append(rec)
print_record(rec, args.out)
if out:
out.write(json.dumps(rec) + "\n")
out.flush()
if pool:
pool.shutdown()
if out:
out.close()
if len(records) > 1:
summary(records)
return 0 if all(r["status"] != "fail" for r in records) else 1
if __name__ == "__main__":
sys.exit(main())