Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): add scripts that check which gguf files load (#756)

Xiaoze Fan5596e6b3904e
diff --git a/scripts/gguf_check/README.md b/scripts/gguf_check/README.md
new file mode 100644
index 00000000..a1f369f7
--- /dev/null
+++ b/scripts/gguf_check/README.md
@@ -0,0 +1,42 @@
+# gguf_check
+
+Checks for GGUF support of a model family: does FreeToken load most of the GGUF files people publish for it, and does it read them right.
+Run from the repo root with `PYTHONPATH=python` and transformers 5.18.
+
+| Script | Needs | Checks |
+|---|---|---|
+| `fetch_headers.py` | network | nothing, writes header-only stubs of HF GGUF files |
+| `probe_kernels.py` | GPU | which ggml ops this build runs per ggml type |
+| `check.py` | CPU | config, dry load, expert banks, kernel coverage, tokenizer, chat template per file |
+| `tok_compare.py` | CPU | GGUF tokenizer vs the model's HF `tokenizer.json` |
+
+## Coverage: most published files load
+
+```bash
+# header-only stubs of every GGUF in the repos (about 15 MB each, weights read as zeros)
+python scripts/gguf_check/fetch_headers.py ~/gguf-stubs --search GLM-4.5 --search GLM-4.7 unsloth/GLM-4.7-GGUF > fetch.jsonl
+
+# once per build and GPU architecture
+python scripts/gguf_check/probe_kernels.py kernels.json
+
+python scripts/gguf_check/check.py --kernels kernels.json --out glm.jsonl ~/gguf-stubs
+python scripts/gguf_check/check.py --report glm.jsonl
+```
+
+`check.py` builds the model on the meta device and runs the family's weight reader on meta matrices, so a worker needs no RAM for the weights; it falls back to real reads only for a reader that needs the values.
+It prints `passed/total` per served GGUF architecture, the files of architectures FreeToken does not serve (draft models, ...), and the failures grouped by their first problem.
+Every failure should be one of:
+
+- the family's loader refuses the file with a clear error (a type or layout we do not serve), or
+- the file is broken.
+
+List the refused kinds under "Not covered" in the PR.
+Passing means the server builds the model and finds every tensor and kernel. It does not read the weights or check the model fits in memory.
+
+## Correctness: the files load right
+
+On one or two real files per family, smallest and most common quant:
+
+- `tok_compare.py <gguf> <HF model dir>`: 0 differing texts. The reference is the model's own tokenizer, not llama.cpp.
+- `ft serve --model-path <file>`: a short eval (e.g. AIME problems) against llama.cpp on the same file, greedy outputs against llama.cpp, 48 staggered requests at concurrency 16 with no empty or corrupted output, and bs=1 tok/s.
+- `pytest tests/models tests/moe tests/kernels/test_ggml.py tests/server`.
diff --git a/scripts/gguf_check/check.py b/scripts/gguf_check/check.py
new file mode 100644
index 00000000..390a52b4
--- /dev/null
+++ b/scripts/gguf_check/check.py
@@ -0,0 +1,593 @@
+"""Check that FreeToken loads GGUF files, on the CPU and without reading their weights.
+
+ python scripts/gguf_check/check.py --kernels kernels.json --out results.jsonl <gguf file or dir> ...
+ python scripts/gguf_check/check.py --report results.jsonl
+
+Per file, through the code the server runs:
+
+- config: the family's ``parse_gguf_config`` accepts the file (unsupported files must fail here with a clear error)
+- dry load: the model builds on the meta device, and the family's weight reader yields exactly its state dict (names, shapes, packed vs float); the expert banks build for every MoE layer with the shapes the model needs
+- kernels: every quantized ggml type a GGUF linear, embedding or expert bank uses has the ops it needs in the ``probe_kernels.py`` table
+- tokenizer: it builds, the chat template renders, and the file names an EOS token
+
+The weight reader runs on meta tensors, so no weight bytes are read or copied. A reader that needs real values (a numpy dequant) gets a second run on the real bytes, once enough RAM is free.
+The files can be real or ``fetch_headers.py`` stubs. Passing does not prove the weights are right or that the model fits in memory.
+Results append to ``--out``; a rerun skips the files already there, except crashed ones (and failed ones with ``--retry-failed``).
+"""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import contextlib
+import functools
+import glob
+import hashlib
+import json
+import logging
+import multiprocessing as mp
+import os
+import re
+import struct
+import sys
+import time
+import traceback
+import warnings
+from collections import Counter
+
+MOE_TOKENS = ("mmid1", "mmid8", "mmid16", "mmid512")
+# ops each layer kind runs over the token counts the server sends; fused-glu experts also need glu<T> up to the type's mmvq_max_tokens
+REQUIRED_OPS = {
+ "linear": ("mm1", "mm8", "mm16", "mm512"),
+ "embedding": ("get_rows",),
+ "experts gate_up": MOE_TOKENS,
+ "experts down": MOE_TOKENS,
+}
+GLU_TOKENS = (1, 8)
+_SPLIT = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$")
+_SCALAR = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
+_STRING, _ARRAY = 8, 9
+_UINT32 = 4
+
+
+class _Field:
+ __slots__ = ("value",)
+
+ def __init__(self, value):
+ self.value = value
+
+ def contents(self):
+ return self.value
+
+
+class _Tensor:
+ __slots__ = ("name", "shape", "tensor_type", "data_offset", "n_bytes")
+
+
+class FastReader:
+ """The parts of gguf-py's ``GGUFReader`` FreeToken reads, parsed in pure Python, with the same checks.
+
+ gguf-py builds numpy views for every field and takes seconds and about 0.7 GB per large file, which a scan over many files cannot afford."""
+
+ def __init__(self, path: str):
+ import numpy as np
+
+ from freetoken.third_party.gguf import GGML_QUANT_SIZES, GGMLQuantizationType
+
+ self.data = data = np.memmap(path, dtype=np.uint8, mode="r")
+ pos = 0
+
+ def unpack(fmt):
+ nonlocal pos
+ vals = struct.unpack_from("<" + fmt, data, pos)
+ pos += struct.calcsize("<" + fmt)
+ return vals
+
+ def string():
+ nonlocal pos
+ (n,) = unpack("Q")
+ pos += n
+ return data[pos - n : pos].tobytes().decode("utf-8")
+
+ def value(kind):
+ if kind in _SCALAR:
+ return unpack(_SCALAR[kind])[0]
+ if kind == _STRING:
+ return string()
+ if kind != _ARRAY:
+ raise ValueError(f"{path}: unknown GGUF value type {kind}")
+ elem, n = unpack("I")[0], unpack("Q")[0]
+ if elem in _SCALAR:
+ return list(unpack(f"{n}{_SCALAR[elem]}"))
+ return [value(elem) for _ in range(n)]
+
+ if data[:4].tobytes() != b"GGUF":
+ raise ValueError(f"{path} is not a GGUF file")
+ pos = 4
+ _, n_tensors, n_kv = unpack("IQQ")
+ self.fields = {}
+ align = 32
+ for _ in range(n_kv):
+ key = string()
+ if key in self.fields:
+ raise KeyError(f"{path}: duplicate GGUF key {key}")
+ kind = unpack("I")[0]
+ self.fields[key] = _Field(value(kind))
+ if key == "general.alignment":
+ align = self.fields[key].value
+ if kind != _UINT32 or align <= 0 or align & (align - 1):
+ raise ValueError(f"{path}: bad general.alignment {align}")
+ infos = []
+ for _ in range(n_tensors):
+ name = string()
+ (nd,) = unpack("I")
+ dims = unpack(f"{nd}Q")
+ kind, offset = unpack("IQ")
+ infos.append((name, dims, kind, offset))
+ start = (pos + align - 1) // align * align
+ self.tensors, names = [], set()
+ for name, dims, kind, offset in infos:
+ if name in names:
+ raise ValueError(f"{path}: duplicate tensor {name}")
+ names.add(name)
+ block, size = GGML_QUANT_SIZES[GGMLQuantizationType(kind)]
+ t = _Tensor()
+ t.name, t.shape, t.tensor_type, t.data_offset = name, list(dims), kind, start + offset
+ t.n_bytes = _prod(dims) // block * size
+ if t.data_offset + t.n_bytes > len(data):
+ raise ValueError(f"{path}: tensor {name} runs past the end of the file")
+ self.tensors.append(t)
+
+
+def _prod(dims) -> int:
+ n = 1
+ for d in dims:
+ n *= d
+ return n
+
+
+def _clear_caches() -> None:
+ """Drop the caches that hold a file's metadata, config or rope tables."""
+ import freetoken.models.gguf.reader as reader
+ from freetoken.layers import rotary
+ from freetoken.utils import hf
+
+ for fn in (reader._reader, reader.load_gguf_metadata, reader.gguf_split_paths, reader.gguf_tensor_index,
+ rotary.get_rope, rotary._cos_sin_table, hf._load_hf_config):
+ fn.cache_clear()
+
+
+_KERNELS: dict | None = None
+_TOKENIZERS: dict = {}
+_IN_FLIGHT = None
+_MIN_FREE_GB = 32.0
+
+
+def _init_worker(kernels: dict | None, in_flight, min_free_gb: float) -> None:
+ os.environ["CUDA_VISIBLE_DEVICES"] = ""
+ warnings.filterwarnings("ignore")
+ logging.disable(logging.WARNING)
+ import torch
+
+ import freetoken.models.gguf.reader as reader
+ from freetoken.distributed import set_tp_info
+ from freetoken.layers import rotary
+
+ reader._reader = functools.cache(FastReader)
+ set_tp_info(rank=0, size=1)
+ rotary.set_rope_device(torch.device("cpu"))
+ global _KERNELS, _IN_FLIGHT, _MIN_FREE_GB
+ _KERNELS, _IN_FLIGHT, _MIN_FREE_GB = kernels, in_flight, min_free_gb
+
+
+def _build_model(path: str):
+ """The engine's config and the model it builds for ``path`` (experts offloaded), on the meta device."""
+ import torch
+
+ from freetoken.distributed import try_get_tp_info
+ from freetoken.engine.config import EngineConfig
+ from freetoken.engine.engine import _decode_target
+ from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig
+ from freetoken.models import create_model
+ from freetoken.utils.torch_utils import torch_dtype
+
+ config = EngineConfig(model_path=path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy="offload",
+ mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)))
+ object.__setattr__(config.model_config, "moe_strategy", "offload")
+ object.__setattr__(config.model_config, "decode_target", _decode_target(config))
+ with torch.device("meta"), torch_dtype(torch.bfloat16):
+ return config, create_model(config.model_config)
+
+
+def _gguf_layers(model) -> tuple[list, bool]:
+ """(kind, ggml types) of every GGUF linear and embedding, and whether any GGUF MoE layer runs the fused-glu kernel."""
+ from freetoken.layers.base import BaseOP
+ from freetoken.layers.quantization import QuantKind
+ from freetoken.layers.quantization.moe.gguf import fused_glu
+ from freetoken.layers.quantization.registry import LayerKind
+ from freetoken.layers.quantization.scheme import gguf_types
+
+ kinds = {LayerKind.LINEAR: "linear", LayerKind.EMBEDDING: "embedding"}
+ seen, out, glu = set(), [], False
+
+ def walk(op):
+ nonlocal glu
+ if id(op) in seen:
+ return
+ seen.add(id(op))
+ method = getattr(op, "quant_method", None)
+ if method is not None and method.kind is QuantKind.GGUF:
+ if method.layer_kind in kinds:
+ out.append((kinds[method.layer_kind], gguf_types(method.scheme)))
+ elif method.layer_kind is LayerKind.MOE:
+ glu = glu or fused_glu(op) is not None
+ for value in vars(op).values():
+ for item in value if isinstance(value, (list, tuple)) else (value,):
+ if isinstance(item, BaseOP):
+ walk(item)
+
+ walk(model)
+ return out, glu
+
+
+def _check_kernels(layers, tied: bool, glu: bool, expert_tags, problems: list[str]) -> dict:
+ from freetoken.layers.quantization.ggml_types import is_quantized, type_name
+ from freetoken.third_party.gguf import GGMLQuantizationType
+
+ used: dict[str, Counter] = {}
+ if tied:
+ layers = layers + [("linear", types) for kind, types in layers if kind == "embedding"]
+ for kind, types in layers:
+ for t in types:
+ used.setdefault(kind, Counter())[type_name(t)] += 1
+ for _, gate_up, down in expert_tags:
+ used.setdefault("experts gate_up", Counter())[gate_up] += 1
+ used.setdefault("experts down", Counter())[down] += 1
+ if _KERNELS is not None:
+ for kind, names in used.items():
+ for name in names:
+ if not is_quantized(int(GGMLQuantizationType[name])):
+ continue
+ probe = _KERNELS.get(name, {})
+ need = list(REQUIRED_OPS[kind])
+ if kind == "experts gate_up" and glu:
+ need += [f"glu{t}" for t in GLU_TOKENS if t <= probe.get("mmvq_max_tokens", 0)]
+ missing = [op for op in need if probe.get(op) != "ok"]
+ if missing:
+ problems.append(f"{kind} {name}: no {', '.join(missing)} kernel")
+ return {k: dict(v) for k, v in used.items()}
+
+
+def _meta_readers():
+ """The GGUF tensor readers, and stand-ins that give matrices as meta tensors."""
+ import torch
+
+ from freetoken.models.gguf import tensors
+ from freetoken.models.gguf.tensors import _FLOAT_DTYPES
+
+ real_float = tensors.float_tensor
+
+ def float_tensor(info):
+ # 1-D tensors (norms, biases, rope_freqs) are small, and config parses read their values
+ if len(info.ne) == 1:
+ return real_float(info)
+ return torch.empty(tuple(reversed(info.ne)), dtype=_FLOAT_DTYPES[info.ggml_type], device="meta")
+
+ def packed_rows(info):
+ rows = _prod(info.ne[1:])
+ return torch.empty(rows, info.nbytes // rows, dtype=torch.uint8, device="meta")
+
+ fakes = {"float_tensor": float_tensor, "packed_rows": packed_rows}
+ real = {name: getattr(tensors, name) for name in fakes}
+ return real, fakes
+
+
+def _meta_wins():
+ """A torch function mode that moves CPU operands to meta when an op mixes the two, so a reader can combine meta weights with the small tensors it builds."""
+ import torch
+ from torch.overrides import TorchFunctionMode
+ from torch.utils._pytree import tree_flatten, tree_unflatten
+
+ class MetaWins(TorchFunctionMode):
+ def __torch_function__(self, func, types, args=(), kwargs=None):
+ flat, spec = tree_flatten((args, kwargs or {}))
+ if any(isinstance(x, torch.Tensor) and x.is_meta for x in flat):
+ flat = [x.to("meta") if isinstance(x, torch.Tensor) and not x.is_meta else x for x in flat]
+ args, kwargs = tree_unflatten(flat, spec)
+ return func(*args, **kwargs)
+
+ return MetaWins
+
+
+def _swap_readers(use_fakes: bool) -> None:
+ """Point every loaded binding of the readers, including the family modules' imported names, at the fakes or back."""
+ real, fakes = _meta_readers()
+ src, dst = (real, fakes) if use_fakes else (fakes, real)
+ for module in list(sys.modules.values()):
+ if getattr(module, "__name__", "").startswith("freetoken.models"):
+ for name, fn in src.items():
+ if getattr(module, name, None) is fn:
+ setattr(module, name, dst[name])
+
+
+def _wait_for_ram(gb: float, timeout: float = 600.0) -> bool:
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ with open("/proc/meminfo") as f:
+ avail = next(int(line.split()[1]) for line in f if line.startswith("MemAvailable:")) / 2**20
+ if avail >= gb:
+ return True
+ time.sleep(2)
+ return False
+
+
+def _dry_load(path: str, config, model, *, meta: bool) -> list[str]:
+ import torch
+
+ from freetoken.models.register import _load_attr
+
+ spec = config.model_spec
+ expected = model.state_dict()
+ problems, seen = [], set()
+ _swap_readers(meta)
+ try:
+ with _meta_wins()() if meta else contextlib.nullcontext():
+ for key, t in _load_attr(spec.module, spec.iter_weights)(path, torch.device("cpu"), include_moe_experts=False, include_non_moe=True):
+ ref = expected.get(key)
+ if ref is None:
+ problems.append(f"unexpected tensor {key}")
+ continue
+ if key in seen:
+ problems.append(f"tensor {key} yielded twice")
+ seen.add(key)
+ if tuple(t.shape) != tuple(ref.shape) or (t.dtype == torch.uint8) != (ref.dtype == torch.uint8):
+ problems.append(f"{key}: got {tuple(t.shape)} {t.dtype}, model has {tuple(ref.shape)} {ref.dtype}")
+ finally:
+ _swap_readers(False)
+ missing = sorted(set(expected) - seen)
+ if missing:
+ problems.append(f"{len(missing)} tensors never loaded, e.g. {missing[:3]}")
+ return problems
+
+
+def _expert_banks(path: str, cfg, problems: list[str]) -> list:
+ """Build the expert banks on the meta device and check their shapes; returns each MoE layer's (format, gate_up type, down type)."""
+ import torch
+
+ if not cfg.num_experts:
+ return []
+ from freetoken.layers.quantization.ggml_types import float_dtype, row_bytes
+ from freetoken.models.gguf.experts import load_gguf_expert_banks
+ from freetoken.third_party.gguf import GGMLQuantizationType
+
+ banks = load_gguf_expert_banks(path, cfg.num_moe_layers, first_layer=cfg.first_k_dense_replace, dummy=True, device=torch.device("meta"))
+ e, inter, hidden = cfg.num_experts, cfg.moe_intermediate_size, cfg.hidden_size
+ for layer, (_, gate_up, down) in enumerate(banks.layer_tags):
+ for role, name, rows, k in (("gate_up", gate_up, 2 * inter, hidden), ("down", down, hidden, inter)):
+ t = int(GGMLQuantizationType[name])
+ want = (e, rows, k if float_dtype(t) is not None else row_bytes(t, k))
+ got = tuple(banks.sources[role][layer].shape)
+ if got != want:
+ problems.append(f"MoE layer {layer} {role} bank is {got}, the model needs {want}")
+ return banks.layer_tags
+
+
+def _tokenizer_key(meta: dict) -> str:
+ blob = json.dumps(sorted((k, v) for k, v in meta.items() if k.startswith("tokenizer.")))
+ return hashlib.sha1(blob.encode()).hexdigest()[:12]
+
+
+def _check_tokenizer(path: str) -> dict:
+ from freetoken.models.gguf.tokenizer import gguf_eos_token_ids, load_gguf_tokenizer
+
+ problems = []
+ tok = load_gguf_tokenizer(path)
+ eos = sorted(gguf_eos_token_ids(path))
+ if not eos:
+ problems.append("tokenizer: no EOS token")
+ tail = None
+ if tok.chat_template is None:
+ problems.append("tokenizer: no chat template")
+ else:
+ try:
+ tail = tok.apply_chat_template([{"role": "user", "content": "hi"}], tokenize=False, add_generation_prompt=True)[-60:]
+ except Exception as e:
+ problems.append(f"chat template: {type(e).__name__}: {str(e)[:150]}")
+ return {"problems": problems, "eos": eos, "template_tail": tail}
+
+
+def check(path: str) -> dict:
+ from freetoken.models.gguf.config import GGUF_ARCH_TO_REGISTRY
+ from freetoken.models.gguf.reader import gguf_architecture, load_gguf_metadata
+
+ if _IN_FLIGHT is not None:
+ _IN_FLIGHT[os.getpid()] = path
+ t0 = time.time()
+ out = {"path": path, "problems": []}
+ problems = out["problems"]
+ try:
+ out["arch"] = gguf_architecture(path)
+ if out["arch"] not in GGUF_ARCH_TO_REGISTRY:
+ out["status"] = "unsupported"
+ return out
+ config, model = _build_model(path)
+ cfg = config.model_config
+ out["config"] = {"layers": cfg.num_layers, "hidden": cfg.hidden_size, "experts": cfg.num_experts, "vocab": cfg.vocab_size, "tied": cfg.tie_word_embeddings}
+ out["reads"] = "meta"
+ try:
+ load_problems = _dry_load(path, config, model, meta=True)
+ except Exception:
+ out["reads"] = "real"
+ if not _wait_for_ram(_MIN_FREE_GB):
+ raise MemoryError(f"the weight reader needs real bytes and free RAM stayed under {_MIN_FREE_GB} GB") from None
+ load_problems = _dry_load(path, config, model, meta=False)
+ problems.extend(load_problems)
+ tags = _expert_banks(path, cfg, problems)
+ layers, glu = _gguf_layers(model)
+ out["types"] = _check_kernels(layers, cfg.tie_word_embeddings, glu, tags, problems)
+ meta = load_gguf_metadata(path)
+ key = _tokenizer_key(meta)
+ if key not in _TOKENIZERS:
+ try:
+ _TOKENIZERS[key] = _check_tokenizer(path)
+ except Exception as e:
+ _TOKENIZERS[key] = {"problems": [f"tokenizer: {type(e).__name__}: {str(e)[:200]}"]}
+ tok = _TOKENIZERS[key]
+ problems.extend(tok["problems"])
+ out["tokenizer"] = {"hash": key, "pre": meta.get("tokenizer.ggml.pre"), "eos": tok.get("eos"), "template_tail": tok.get("template_tail")}
+ except Exception as e:
+ problems.append(f"{type(e).__name__}: {str(e)[:300]}")
+ out["traceback"] = traceback.format_exc()[-1500:]
+ finally:
+ _clear_caches()
+ out["seconds"] = round(time.time() - t0, 1)
+ if _IN_FLIGHT is not None:
+ _IN_FLIGHT.pop(os.getpid(), None)
+ out.setdefault("status", "fail" if problems else "pass")
+ return out
+
+
+def gguf_files(paths: list[str]) -> list[str]:
+ """Every model GGUF under ``paths``, once; a split model is checked through its first file."""
+ files = []
+ for p in paths:
+ found = sorted(glob.glob(os.path.join(p, "**", "*.gguf"), recursive=True)) if os.path.isdir(p) else [p]
+ for f in found:
+ m = _SPLIT.search(f)
+ if os.path.isfile(f) and (m is None or m.group(1) == "00001") and not any(s in os.path.basename(f).lower() for s in ("mmproj", "imatrix")):
+ files.append(os.path.abspath(f))
+ return list(dict.fromkeys(files))
+
+
+def read_results(path: str) -> list[dict]:
+ """The latest record per file; a line cut short by a killed run is dropped."""
+ latest = {}
+ if os.path.exists(path):
+ with open(path) as f:
+ for line in f:
+ try:
+ rec = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ latest[rec["path"]] = rec
+ return list(latest.values())
+
+
+def _problem_kind(problem: str) -> str:
+ """A problem with tensor names, layer numbers and sizes masked, to group the same failure across files."""
+ problem = re.sub(r"blk\.\d+", "blk.N", problem)
+ problem = re.sub(r"model\.layers\.\d+", "model.layers.N", problem)
+ problem = re.sub(r"/\S+\.gguf", "<file>", problem)
+ return re.sub(r"\b\d+\b", "N", problem)[:160]
+
+
+def report(results: list[dict]) -> None:
+ supported = [r for r in results if r["status"] != "unsupported"]
+ by_arch: dict[str, list[int]] = {}
+ for r in supported:
+ stats = by_arch.setdefault(r.get("arch") or "?", [0, 0])
+ stats[0] += r["status"] == "pass"
+ stats[1] += 1
+ print(f"{sum(r['status'] == 'pass' for r in supported)}/{len(supported)} files of served architectures pass")
+ for arch, (ok, n) in sorted(by_arch.items()):
+ print(f" {arch:16s} {ok}/{n}")
+ unsupported = Counter(r.get("arch") for r in results if r["status"] == "unsupported")
+ if unsupported:
+ print("not checked, architecture not served: " + ", ".join(f"{a} {n}" for a, n in unsupported.most_common()))
+ real = sum(r.get("reads") == "real" for r in supported)
+ if real:
+ print(f"{real} files needed real weight reads")
+ kinds = Counter(_problem_kind(r["problems"][0]) for r in supported if r["status"] != "pass")
+ if kinds:
+ print("failures, by first problem:")
+ for kind, n in kinds.most_common():
+ print(f" {n:5d} {kind}")
+
+
+class _Pool:
+ """Worker processes that check files, and know which files were running when a worker died."""
+
+ def __init__(self, kernels: dict | None, min_free_gb: float):
+ self.ctx = mp.get_context("spawn")
+ self.manager = self.ctx.Manager()
+ self.in_flight = self.manager.dict()
+ self.args = (kernels, self.in_flight, min_free_gb)
+
+ def run(self, files: list[str], workers: int):
+ """Yields (path, record), or (path, None) for a file that was running when a worker died."""
+ pending = list(files)
+ while pending:
+ self.in_flight.clear()
+ # no max_tasks_per_child: on Python 3.12 the pool hangs once every worker has retired
+ with cf.ProcessPoolExecutor(min(workers, len(pending)), mp_context=self.ctx, initializer=_init_worker, initargs=self.args) as pool:
+ futures = {pool.submit(check, f): f for f in pending}
+ pending, broken = [], False
+ for fut in cf.as_completed(futures):
+ try:
+ yield futures[fut], fut.result()
+ except cf.process.BrokenProcessPool:
+ broken = True
+ pending.append(futures[fut])
+ if broken:
+ # a worker that died before taking a file leaves none in flight: blame every file left
+ running = set(self.in_flight.values()) & set(pending) or set(pending)
+ for path in running:
+ yield path, None
+ pending = [p for p in pending if p not in running]
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("paths", nargs="*", help="GGUF files or directories to search")
+ ap.add_argument("--out", help="results JSONL, appended to")
+ ap.add_argument("--kernels", help="kernel table from probe_kernels.py; without it the kernel check is skipped")
+ ap.add_argument("--workers", type=int, default=8)
+ ap.add_argument("--min-free-gb", type=float, default=32.0, help="free RAM a worker waits for before reading real weight bytes")
+ ap.add_argument("--retry-failed", action="store_true", help="check again the files that failed in --out")
+ ap.add_argument("--report", metavar="JSONL", help="only print the summary of an earlier run")
+ args = ap.parse_args()
+
+ if args.report:
+ report(read_results(args.report))
+ return 0
+ if not args.out or not args.paths:
+ ap.error("give --out and at least one path")
+ kernels = None
+ if args.kernels:
+ with open(args.kernels) as f:
+ kernels = json.load(f)["types"]
+ else:
+ print("no --kernels table: the kernel check is skipped", file=sys.stderr)
+ keep = {"pass", "unsupported"} if args.retry_failed else {"pass", "unsupported", "fail"}
+ done = {r["path"] for r in read_results(args.out) if r["status"] in keep}
+ files = [f for f in gguf_files(args.paths) if f not in done]
+ print(f"{len(files)} files to check, {len(done)} already in {args.out}", file=sys.stderr, flush=True)
+
+ pool = _Pool(kernels, args.min_free_gb)
+ with open(args.out, "a") as out:
+
+ def write(rec):
+ mark = {"pass": "ok ", "unsupported": "skip"}.get(rec["status"], "FAIL")
+ print(f"{mark} {rec['path']}" + (f" {rec['problems'][0][:120]}" if rec["problems"] else ""), file=sys.stderr, flush=True)
+ out.write(json.dumps(rec) + "\n")
+ out.flush()
+
+ crashed = []
+ for path, rec in pool.run(files, args.workers):
+ if rec is None:
+ crashed.append(path)
+ else:
+ write(rec)
+ # each file that was running when a worker died runs alone, to find the one that kills it
+ for path in crashed:
+ for _, rec in pool.run([path], 1):
+ write(rec or {"path": path, "status": "crash", "problems": ["worker process died (out of memory or a crash)"]})
+ results = read_results(args.out)
+ report(results)
+ return 0 if all(r["status"] in ("pass", "unsupported") for r in results) else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/gguf_check/fetch_headers.py b/scripts/gguf_check/fetch_headers.py
new file mode 100644
index 00000000..fdcbd337
--- /dev/null
+++ b/scripts/gguf_check/fetch_headers.py
@@ -0,0 +1,193 @@
+"""Write header-only stubs of the GGUF files in HF repos: the KV metadata and tensor table, in a sparse file of the real size.
+
+ python scripts/gguf_check/fetch_headers.py <out_dir> [<repo> ...] [--search <query>]
+
+The weights stay holes that read as zeros, so a stub takes a few MB of disk and ``check.py`` reads it like the real file.
+Stubs land at ``<out_dir>/<repo>/<path in repo>``; a stub already there with the right size is kept, so a rerun only fetches what is missing.
+Prints one JSON line per file. Gated repos read ``HF_TOKEN``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import json
+import os
+import struct
+import sys
+import threading
+
+import requests
+from huggingface_hub import HfApi, hf_hub_url
+from huggingface_hub.utils import build_hf_headers
+
+_FIRST_READ, _NEXT_READ = 24 << 20, 16 << 20
+_SCALAR = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
+_STRING, _ARRAY = 8, 9
+_F32 = 0
+# tensors whose values a parse_gguf_config reads, so their bytes go into the stub
+VALUE_TENSORS = ("rope_freqs.weight",)
+
+_local = threading.local()
+
+
+def _get_range(url: str, start: int, n: int) -> requests.Response:
+ """Bytes [start, start + n) of ``url``, or fewer at the end of the file; refuses a reply that is not exactly that range."""
+ r = _session().get(url, headers={"Range": f"bytes={start}-{start + n - 1}"}, timeout=300)
+ r.raise_for_status()
+ got = r.headers.get("Content-Range", "")
+ if r.status_code != 206 or not got.startswith(f"bytes {start}-"):
+ raise ValueError(f"{url}: asked for bytes {start}-{start + n - 1}, got status {r.status_code} {got!r}")
+ end = int(got.split("-", 1)[1].split("/", 1)[0])
+ if len(r.content) != end - start + 1:
+ raise ValueError(f"{url}: short read at byte {start}")
+ return r
+
+
+def _session() -> requests.Session:
+ if not hasattr(_local, "session"):
+ _local.session = requests.Session()
+ _local.session.headers.update(build_hf_headers())
+ return _local.session
+
+
+class _Remote:
+ """Reads a remote file front to back through range requests, keeping every byte read so far."""
+
+ def __init__(self, url: str):
+ self.url, self.buf, self.pos, self.size = url, bytearray(), 0, None
+
+ def _need(self, n: int) -> None:
+ while len(self.buf) < self.pos + n:
+ start = len(self.buf)
+ if self.size is not None and start >= self.size:
+ raise EOFError(f"{self.url}: header runs past the end of the file")
+ r = _get_range(self.url, start, max(_FIRST_READ if start == 0 else _NEXT_READ, self.pos + n - start))
+ if self.size is None:
+ self.size = int(r.headers["Content-Range"].rsplit("/", 1)[1])
+ self.buf += r.content
+
+ def read(self, n: int) -> bytes:
+ self._need(n)
+ self.pos += n
+ return bytes(self.buf[self.pos - n : self.pos])
+
+ def unpack(self, fmt: str) -> tuple:
+ return struct.unpack("<" + fmt, self.read(struct.calcsize("<" + fmt)))
+
+ def string(self) -> str:
+ (n,) = self.unpack("Q")
+ return self.read(n).decode("utf-8", "replace")
+
+ def skip_value(self, kind: int) -> None:
+ if kind in _SCALAR:
+ self.read(struct.calcsize(_SCALAR[kind]))
+ elif kind == _STRING:
+ self.string()
+ elif kind == _ARRAY:
+ elem, n = self.unpack("I")[0], self.unpack("Q")[0]
+ if elem in _SCALAR:
+ self.read(struct.calcsize(_SCALAR[elem]) * n)
+ else:
+ for _ in range(n):
+ self.skip_value(elem)
+ else:
+ raise ValueError(f"{self.url}: unknown GGUF value type {kind}")
+
+
+def fetch(repo: str, path: str, size: int, out_dir: str) -> dict:
+ dst = os.path.join(out_dir, repo, path)
+ rec = {"repo": repo, "path": path, "stub": dst, "size": size, "value_tensors": list(VALUE_TENSORS)}
+ # the sidecar marks a finished stub and which tensors it holds the values of
+ sidecar = dst + ".json"
+ if os.path.isfile(dst) and os.path.getsize(dst) == size and os.path.isfile(sidecar):
+ with open(sidecar) as f:
+ if json.load(f).get("value_tensors") == rec["value_tensors"]:
+ return {**rec, "ok": True, "cached": True}
+ url = hf_hub_url(repo, path)
+ try:
+ f = _Remote(url)
+ if f.read(4) != b"GGUF":
+ raise ValueError("not a GGUF file")
+ _, n_tensors, n_kv = f.unpack("IQQ")
+ align = 32
+ for _ in range(n_kv):
+ key, kind = f.string(), f.unpack("I")[0]
+ if key == "general.alignment" and kind in (4, 5):
+ align = f.unpack("I")[0]
+ else:
+ f.skip_value(kind)
+ tensors = {}
+ for _ in range(n_tensors):
+ name = f.string()
+ (nd,) = f.unpack("I")
+ dims = f.unpack(f"{nd}Q")
+ kind, offset = f.unpack("IQ")
+ tensors[name] = (dims, kind, offset)
+ header_end = f.pos
+ data_start = (header_end + align - 1) // align * align
+ tmp = dst + ".part"
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ with open(tmp, "wb") as out:
+ out.write(f.buf[:header_end])
+ out.truncate(f.size)
+ for name in VALUE_TENSORS:
+ if name not in tensors:
+ continue
+ dims, kind, offset = tensors[name]
+ if kind != _F32 or len(dims) != 1:
+ raise ValueError(f"{name} is type {kind} with dims {dims}, expected 1-D F32")
+ start, n = data_start + offset, dims[0] * 4
+ r = _get_range(url, start, n)
+ if len(r.content) != n:
+ raise ValueError(f"{name} runs past the end of the file")
+ out.seek(start)
+ out.write(r.content)
+ os.replace(tmp, dst)
+ rec.update(header_bytes=header_end, tensors=n_tensors)
+ with open(sidecar, "w") as f:
+ json.dump(rec, f)
+ return {**rec, "ok": True}
+ except Exception as e:
+ return {**rec, "ok": False, "error": f"{type(e).__name__}: {e}"[:300]}
+
+
+def list_gguf(api: HfApi, repo: str) -> list[tuple[str, int]]:
+ """(path, size) of every model GGUF in the repo; vision projectors and imatrix files are left out."""
+ files = []
+ for entry in api.list_repo_tree(repo, recursive=True):
+ path = entry.path
+ if path.endswith(".gguf") and not any(s in path.lower() for s in ("mmproj", "imatrix")):
+ files.append((path, entry.size))
+ return files
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("out_dir")
+ ap.add_argument("repos", nargs="*")
+ ap.add_argument("--search", action="append", default=[], help="also take every HF GGUF repo whose name matches this query (repeatable)")
+ ap.add_argument("--workers", type=int, default=12)
+ args = ap.parse_args()
+
+ api = HfApi()
+ repos = list(args.repos)
+ for query in args.search:
+ repos += [m.id for m in api.list_models(search=query, filter="gguf") if m.id not in repos]
+ jobs = []
+ for repo in repos:
+ try:
+ jobs += [(repo, path, size) for path, size in list_gguf(api, repo)]
+ except Exception as e:
+ print(json.dumps({"repo": repo, "ok": False, "error": f"{type(e).__name__}: {e}"[:300]}), flush=True)
+ print(f"{len(jobs)} GGUF files in {len(repos)} repos", file=sys.stderr, flush=True)
+ failed = 0
+ with cf.ThreadPoolExecutor(args.workers) as ex:
+ for rec in ex.map(lambda job: fetch(*job, args.out_dir), jobs):
+ failed += not rec["ok"]
+ print(json.dumps(rec), flush=True)
+ return 1 if failed else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/gguf_check/probe_kernels.py b/scripts/gguf_check/probe_kernels.py
new file mode 100644
index 00000000..4b53cd06
--- /dev/null
+++ b/scripts/gguf_check/probe_kernels.py
@@ -0,0 +1,111 @@
+"""Record which ggml ops this build runs for each quantized ggml type, as the kernel table ``check.py --kernels`` reads.
+
+ python scripts/gguf_check/probe_kernels.py <out.json>
+
+Runs every op on random packed weights on the current GPU, at the token counts that pick different kernels (vector, small-batch and MMQ).
+An op counts as covered when it runs; the values are not checked. The table holds for this build and GPU architecture only.
+Each type runs in its own process, so a CUDA fault in one type leaves the others' results intact.
+"""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import json
+import multiprocessing as mp
+import sys
+
+K, N, EXPERTS, TOP_K = 1024, 256, 8, 2
+TOKENS = (1, 8, 16, 512)
+GLU_TOKENS = (1, 8)
+# not weight types: plain integers and doubles, and the q8 blocks ggml quantizes activations to
+_NOT_WEIGHTS = ("F64", "I8", "I16", "I32", "I64", "Q8_1", "Q8_K")
+
+
+def _run(fn) -> str:
+ import torch
+
+ try:
+ fn()
+ torch.cuda.synchronize()
+ return "ok"
+ except Exception as e:
+ return f"{type(e).__name__}: {e}"[:200]
+
+
+def probe(ggml_type: int) -> dict:
+ import torch
+
+ from freetoken.kernel import ggml
+ from freetoken.layers.quantization.ggml_types import row_bytes
+
+ gen = torch.Generator(device="cuda").manual_seed(0)
+ row = row_bytes(ggml_type, K)
+ w = torch.randint(0, 256, (N, row), dtype=torch.uint8, device="cuda", generator=gen)
+ banks = torch.randint(0, 256, (EXPERTS, N, row), dtype=torch.uint8, device="cuda", generator=gen)
+ out = {"mmvq_max_tokens": ggml.mmvq_max_tokens(ggml_type)}
+ for t in TOKENS:
+ x = torch.randn(t, K, dtype=torch.bfloat16, device="cuda", generator=gen)
+ # distinct experts per token, as a router picks them: MMQ faults on a token that routes to one expert twice
+ ids = torch.rand(t, EXPERTS, device="cuda", generator=gen).argsort(dim=1)[:, :TOP_K].to(torch.int32).contiguous()
+ out[f"mm{t}"] = _run(lambda: ggml.mul_mat(w, ggml_type, x))
+ out[f"mmid{t}"] = _run(lambda: ggml.mul_mat_id(banks, ggml_type, x, ids))
+ if t in GLU_TOKENS:
+ out[f"glu{t}"] = _run(lambda: ggml.mul_mat_id_glu(banks, ggml_type, x, ids, "swiglu"))
+ ids = torch.arange(4, dtype=torch.int32, device="cuda")
+ out["get_rows"] = _run(lambda: ggml.get_rows(w, ggml_type, K, ids))
+ out["dequant"] = _run(lambda: ggml.dequantize(w, ggml_type, K))
+ return out
+
+
+def weight_types() -> list[str]:
+ from freetoken.layers.quantization.ggml_types import is_quantized, row_bytes
+ from freetoken.third_party.gguf import GGML_QUANT_SIZES, GGMLQuantizationType
+
+ names = []
+ for qtype in GGMLQuantizationType:
+ if qtype not in GGML_QUANT_SIZES or not is_quantized(int(qtype)) or qtype.name in _NOT_WEIGHTS:
+ continue
+ try:
+ row_bytes(int(qtype), K)
+ except ValueError:
+ continue
+ names.append(qtype.name)
+ return names
+
+
+def _probe_by_name(name: str) -> dict:
+ from freetoken.third_party.gguf import GGMLQuantizationType
+
+ return probe(int(GGMLQuantizationType[name]))
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("out")
+ args = ap.parse_args()
+
+ import torch
+
+ from freetoken.kernel import ggml
+
+ # build the module once up front: a failed build would otherwise show up as every op missing
+ ggml.load_module()
+ ctx = mp.get_context("spawn")
+ table = {}
+ for name in weight_types():
+ with cf.ProcessPoolExecutor(1, mp_context=ctx) as pool:
+ try:
+ table[name] = pool.submit(_probe_by_name, name).result()
+ except cf.process.BrokenProcessPool:
+ table[name] = {"error": "probe process died"}
+ bad = sorted(op for op, r in table[name].items() if op != "mmvq_max_tokens" and r != "ok")
+ print(f"{name:10s} {'ok' if not bad else 'missing ' + ' '.join(bad)}", flush=True)
+ cc = torch.cuda.get_device_capability()
+ with open(args.out, "w") as f:
+ json.dump({"gpu": torch.cuda.get_device_name(), "sm": f"{cc[0]}{cc[1]}", "types": table}, f, indent=1)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/gguf_check/tok_compare.py b/scripts/gguf_check/tok_compare.py
new file mode 100644
index 00000000..4635fb12
--- /dev/null
+++ b/scripts/gguf_check/tok_compare.py
@@ -0,0 +1,90 @@
+"""Compare the tokenizer FreeToken builds from a GGUF with the model's own HF tokenizer.
+
+ python scripts/gguf_check/tok_compare.py <gguf> <HF model dir or tokenizer.json> [--combos 20000]
+
+Encodes, without special tokens, a set of edge strings, every normal vocabulary token, and random joins of 2-4 vocabulary tokens, and counts the texts whose ids differ.
+The HF tokenizer is the reference: a stub works, since only the GGUF metadata is read.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import sys
+
+EDGE = [
+ "Hello world",
+ " leading spaces",
+ "trailing spaces ",
+ "tabs\t\tand\nnewlines\n\n\n",
+ "I'm you're it's we'll they'd DON'T",
+ "1234567890 3.14159 1,000,000",
+ "def f(x):\n return x ** 2 # square\n",
+ "数学题:求 x 的值。日本語のテキスト。한국어 문장.",
+ "emoji 😀👍🏽 and combining é ä",
+ "Ünïcödé àccents fi ligature",
+ "<|im_start|> <s> </s> [INST] literal special-looking text",
+ "$$\\frac{a}{b} + \\sqrt{x}$$",
+ "URL https://example.com/a?b=c&d=e",
+]
+_NORMAL = 1
+
+
+def vocab_words(meta: dict) -> list[str]:
+ """The text of every normal vocabulary token, for byte-level BPE and SentencePiece vocabularies."""
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
+
+ tokens, types = meta["tokenizer.ggml.tokens"], meta.get("tokenizer.ggml.token_type") or [_NORMAL] * len(meta["tokenizer.ggml.tokens"])
+ model = meta.get("tokenizer.ggml.model")
+ byte_of = {c: b for b, c in bytes_to_unicode().items()}
+ words = []
+ for token, kind in zip(tokens, types):
+ if kind != _NORMAL:
+ continue
+ if model == "gpt2":
+ try:
+ words.append(bytes(byte_of[c] for c in token).decode())
+ except (KeyError, UnicodeDecodeError):
+ pass
+ elif model == "llama":
+ words.append(token.replace("\u2581", " "))
+ else:
+ words.append(token)
+ return words
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("gguf")
+ ap.add_argument("hf", help="HF model dir or its tokenizer.json")
+ ap.add_argument("--combos", type=int, default=20000)
+ ap.add_argument("--examples", type=int, default=3)
+ args = ap.parse_args()
+
+ from tokenizers import Tokenizer
+
+ from freetoken.models.gguf.reader import load_gguf_metadata
+ from freetoken.models.gguf.tokenizer import load_gguf_tokenizer
+
+ hf = Tokenizer.from_file(os.path.join(args.hf, "tokenizer.json") if os.path.isdir(args.hf) else args.hf)
+ ours = load_gguf_tokenizer(args.gguf).backend_tokenizer
+ meta = load_gguf_metadata(args.gguf)
+ words = vocab_words(meta)
+ rng = random.Random(0)
+ combos = [rng.choice(["", " ", "\n"]).join(rng.choice(words) for _ in range(rng.randint(2, 4))) for _ in range(args.combos)]
+ out = {"gguf": args.gguf, "pre": meta.get("tokenizer.ggml.pre")}
+ differ = 0
+ for name, texts in (("edge", EDGE), ("vocab", words), ("combos", combos)):
+ ref = [e.ids for e in hf.encode_batch(texts, add_special_tokens=False)]
+ got = [e.ids for e in ours.encode_batch(texts, add_special_tokens=False)]
+ bad = [(t, r, g) for t, r, g in zip(texts, ref, got) if r != g]
+ differ += len(bad)
+ out[name] = {"texts": len(texts), "differ": len(bad), "examples": [{"text": t, "hf": r[:16], "gguf": g[:16]} for t, r, g in bad[: args.examples]]}
+ print(json.dumps(out, ensure_ascii=False, indent=1))
+ return 1 if differ else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())