FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(gguf): add scripts that check which gguf files load (#756)
Xiaoze Fan5596e6b3904e
diff --git a/scripts/gguf_check/README.md b/scripts/gguf_check/README.mdnew file mode 100644index 00000000..a1f369f7--- /dev/null+++ b/scripts/gguf_check/README.md@@ -0,0 +1,42 @@+# gguf_check++Checks for GGUF support of a model family: does FreeToken load most of the GGUF files people publish for it, and does it read them right.+Run from the repo root with `PYTHONPATH=python` and transformers 5.18.++| Script | Needs | Checks |+|---|---|---|+| `fetch_headers.py` | network | nothing, writes header-only stubs of HF GGUF files |+| `probe_kernels.py` | GPU | which ggml ops this build runs per ggml type |+| `check.py` | CPU | config, dry load, expert banks, kernel coverage, tokenizer, chat template per file |+| `tok_compare.py` | CPU | GGUF tokenizer vs the model's HF `tokenizer.json` |++## Coverage: most published files load++```bash+# header-only stubs of every GGUF in the repos (about 15 MB each, weights read as zeros)+python scripts/gguf_check/fetch_headers.py ~/gguf-stubs --search GLM-4.5 --search GLM-4.7 unsloth/GLM-4.7-GGUF > fetch.jsonl++# once per build and GPU architecture+python scripts/gguf_check/probe_kernels.py kernels.json++python scripts/gguf_check/check.py --kernels kernels.json --out glm.jsonl ~/gguf-stubs+python scripts/gguf_check/check.py --report glm.jsonl+```++`check.py` builds the model on the meta device and runs the family's weight reader on meta matrices, so a worker needs no RAM for the weights; it falls back to real reads only for a reader that needs the values.+It prints `passed/total` per served GGUF architecture, the files of architectures FreeToken does not serve (draft models, ...), and the failures grouped by their first problem.+Every failure should be one of:++- the family's loader refuses the file with a clear error (a type or layout we do not serve), or+- the file is broken.++List the refused kinds under "Not covered" in the PR.+Passing means the server builds the model and finds every tensor and kernel. It does not read the weights or check the model fits in memory.++## Correctness: the files load right++On one or two real files per family, smallest and most common quant:++- `tok_compare.py <gguf> <HF model dir>`: 0 differing texts. The reference is the model's own tokenizer, not llama.cpp.+- `ft serve --model-path <file>`: a short eval (e.g. AIME problems) against llama.cpp on the same file, greedy outputs against llama.cpp, 48 staggered requests at concurrency 16 with no empty or corrupted output, and bs=1 tok/s.+- `pytest tests/models tests/moe tests/kernels/test_ggml.py tests/server`.diff --git a/scripts/gguf_check/check.py b/scripts/gguf_check/check.pynew file mode 100644index 00000000..390a52b4--- /dev/null+++ b/scripts/gguf_check/check.py@@ -0,0 +1,593 @@+"""Check that FreeToken loads GGUF files, on the CPU and without reading their weights.++ python scripts/gguf_check/check.py --kernels kernels.json --out results.jsonl <gguf file or dir> ...+ python scripts/gguf_check/check.py --report results.jsonl++Per file, through the code the server runs:++- config: the family's ``parse_gguf_config`` accepts the file (unsupported files must fail here with a clear error)+- dry load: the model builds on the meta device, and the family's weight reader yields exactly its state dict (names, shapes, packed vs float); the expert banks build for every MoE layer with the shapes the model needs+- kernels: every quantized ggml type a GGUF linear, embedding or expert bank uses has the ops it needs in the ``probe_kernels.py`` table+- tokenizer: it builds, the chat template renders, and the file names an EOS token++The weight reader runs on meta tensors, so no weight bytes are read or copied. A reader that needs real values (a numpy dequant) gets a second run on the real bytes, once enough RAM is free.+The files can be real or ``fetch_headers.py`` stubs. Passing does not prove the weights are right or that the model fits in memory.+Results append to ``--out``; a rerun skips the files already there, except crashed ones (and failed ones with ``--retry-failed``).+"""++from __future__ import annotations++import argparse+import concurrent.futures as cf+import contextlib+import functools+import glob+import hashlib+import json+import logging+import multiprocessing as mp+import os+import re+import struct+import sys+import time+import traceback+import warnings+from collections import Counter++MOE_TOKENS = ("mmid1", "mmid8", "mmid16", "mmid512")+# ops each layer kind runs over the token counts the server sends; fused-glu experts also need glu<T> up to the type's mmvq_max_tokens+REQUIRED_OPS = {+ "linear": ("mm1", "mm8", "mm16", "mm512"),+ "embedding": ("get_rows",),+ "experts gate_up": MOE_TOKENS,+ "experts down": MOE_TOKENS,+}+GLU_TOKENS = (1, 8)+_SPLIT = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$")+_SCALAR = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}+_STRING, _ARRAY = 8, 9+_UINT32 = 4+++class _Field:+ __slots__ = ("value",)++ def __init__(self, value):+ self.value = value++ def contents(self):+ return self.value+++class _Tensor:+ __slots__ = ("name", "shape", "tensor_type", "data_offset", "n_bytes")+++class FastReader:+ """The parts of gguf-py's ``GGUFReader`` FreeToken reads, parsed in pure Python, with the same checks.++ gguf-py builds numpy views for every field and takes seconds and about 0.7 GB per large file, which a scan over many files cannot afford."""++ def __init__(self, path: str):+ import numpy as np++ from freetoken.third_party.gguf import GGML_QUANT_SIZES, GGMLQuantizationType++ self.data = data = np.memmap(path, dtype=np.uint8, mode="r")+ pos = 0++ def unpack(fmt):+ nonlocal pos+ vals = struct.unpack_from("<" + fmt, data, pos)+ pos += struct.calcsize("<" + fmt)+ return vals++ def string():+ nonlocal pos+ (n,) = unpack("Q")+ pos += n+ return data[pos - n : pos].tobytes().decode("utf-8")++ def value(kind):+ if kind in _SCALAR:+ return unpack(_SCALAR[kind])[0]+ if kind == _STRING:+ return string()+ if kind != _ARRAY:+ raise ValueError(f"{path}: unknown GGUF value type {kind}")+ elem, n = unpack("I")[0], unpack("Q")[0]+ if elem in _SCALAR:+ return list(unpack(f"{n}{_SCALAR[elem]}"))+ return [value(elem) for _ in range(n)]++ if data[:4].tobytes() != b"GGUF":+ raise ValueError(f"{path} is not a GGUF file")+ pos = 4+ _, n_tensors, n_kv = unpack("IQQ")+ self.fields = {}+ align = 32+ for _ in range(n_kv):+ key = string()+ if key in self.fields:+ raise KeyError(f"{path}: duplicate GGUF key {key}")+ kind = unpack("I")[0]+ self.fields[key] = _Field(value(kind))+ if key == "general.alignment":+ align = self.fields[key].value+ if kind != _UINT32 or align <= 0 or align & (align - 1):+ raise ValueError(f"{path}: bad general.alignment {align}")+ infos = []+ for _ in range(n_tensors):+ name = string()+ (nd,) = unpack("I")+ dims = unpack(f"{nd}Q")+ kind, offset = unpack("IQ")+ infos.append((name, dims, kind, offset))+ start = (pos + align - 1) // align * align+ self.tensors, names = [], set()+ for name, dims, kind, offset in infos:+ if name in names:+ raise ValueError(f"{path}: duplicate tensor {name}")+ names.add(name)+ block, size = GGML_QUANT_SIZES[GGMLQuantizationType(kind)]+ t = _Tensor()+ t.name, t.shape, t.tensor_type, t.data_offset = name, list(dims), kind, start + offset+ t.n_bytes = _prod(dims) // block * size+ if t.data_offset + t.n_bytes > len(data):+ raise ValueError(f"{path}: tensor {name} runs past the end of the file")+ self.tensors.append(t)+++def _prod(dims) -> int:+ n = 1+ for d in dims:+ n *= d+ return n+++def _clear_caches() -> None:+ """Drop the caches that hold a file's metadata, config or rope tables."""+ import freetoken.models.gguf.reader as reader+ from freetoken.layers import rotary+ from freetoken.utils import hf++ for fn in (reader._reader, reader.load_gguf_metadata, reader.gguf_split_paths, reader.gguf_tensor_index,+ rotary.get_rope, rotary._cos_sin_table, hf._load_hf_config):+ fn.cache_clear()+++_KERNELS: dict | None = None+_TOKENIZERS: dict = {}+_IN_FLIGHT = None+_MIN_FREE_GB = 32.0+++def _init_worker(kernels: dict | None, in_flight, min_free_gb: float) -> None:+ os.environ["CUDA_VISIBLE_DEVICES"] = ""+ warnings.filterwarnings("ignore")+ logging.disable(logging.WARNING)+ import torch++ import freetoken.models.gguf.reader as reader+ from freetoken.distributed import set_tp_info+ from freetoken.layers import rotary++ reader._reader = functools.cache(FastReader)+ set_tp_info(rank=0, size=1)+ rotary.set_rope_device(torch.device("cpu"))+ global _KERNELS, _IN_FLIGHT, _MIN_FREE_GB+ _KERNELS, _IN_FLIGHT, _MIN_FREE_GB = kernels, in_flight, min_free_gb+++def _build_model(path: str):+ """The engine's config and the model it builds for ``path`` (experts offloaded), on the meta device."""+ import torch++ from freetoken.distributed import try_get_tp_info+ from freetoken.engine.config import EngineConfig+ from freetoken.engine.engine import _decode_target+ from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig+ from freetoken.models import create_model+ from freetoken.utils.torch_utils import torch_dtype++ config = EngineConfig(model_path=path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy="offload",+ mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)))+ object.__setattr__(config.model_config, "moe_strategy", "offload")+ object.__setattr__(config.model_config, "decode_target", _decode_target(config))+ with torch.device("meta"), torch_dtype(torch.bfloat16):+ return config, create_model(config.model_config)+++def _gguf_layers(model) -> tuple[list, bool]:+ """(kind, ggml types) of every GGUF linear and embedding, and whether any GGUF MoE layer runs the fused-glu kernel."""+ from freetoken.layers.base import BaseOP+ from freetoken.layers.quantization import QuantKind+ from freetoken.layers.quantization.moe.gguf import fused_glu+ from freetoken.layers.quantization.registry import LayerKind+ from freetoken.layers.quantization.scheme import gguf_types++ kinds = {LayerKind.LINEAR: "linear", LayerKind.EMBEDDING: "embedding"}+ seen, out, glu = set(), [], False++ def walk(op):+ nonlocal glu+ if id(op) in seen:+ return+ seen.add(id(op))+ method = getattr(op, "quant_method", None)+ if method is not None and method.kind is QuantKind.GGUF:+ if method.layer_kind in kinds:+ out.append((kinds[method.layer_kind], gguf_types(method.scheme)))+ elif method.layer_kind is LayerKind.MOE:+ glu = glu or fused_glu(op) is not None+ for value in vars(op).values():+ for item in value if isinstance(value, (list, tuple)) else (value,):+ if isinstance(item, BaseOP):+ walk(item)++ walk(model)+ return out, glu+++def _check_kernels(layers, tied: bool, glu: bool, expert_tags, problems: list[str]) -> dict:+ from freetoken.layers.quantization.ggml_types import is_quantized, type_name+ from freetoken.third_party.gguf import GGMLQuantizationType++ used: dict[str, Counter] = {}+ if tied:+ layers = layers + [("linear", types) for kind, types in layers if kind == "embedding"]+ for kind, types in layers:+ for t in types:+ used.setdefault(kind, Counter())[type_name(t)] += 1+ for _, gate_up, down in expert_tags:+ used.setdefault("experts gate_up", Counter())[gate_up] += 1+ used.setdefault("experts down", Counter())[down] += 1+ if _KERNELS is not None:+ for kind, names in used.items():+ for name in names:+ if not is_quantized(int(GGMLQuantizationType[name])):+ continue+ probe = _KERNELS.get(name, {})+ need = list(REQUIRED_OPS[kind])+ if kind == "experts gate_up" and glu:+ need += [f"glu{t}" for t in GLU_TOKENS if t <= probe.get("mmvq_max_tokens", 0)]+ missing = [op for op in need if probe.get(op) != "ok"]+ if missing:+ problems.append(f"{kind} {name}: no {', '.join(missing)} kernel")+ return {k: dict(v) for k, v in used.items()}+++def _meta_readers():+ """The GGUF tensor readers, and stand-ins that give matrices as meta tensors."""+ import torch++ from freetoken.models.gguf import tensors+ from freetoken.models.gguf.tensors import _FLOAT_DTYPES++ real_float = tensors.float_tensor++ def float_tensor(info):+ # 1-D tensors (norms, biases, rope_freqs) are small, and config parses read their values+ if len(info.ne) == 1:+ return real_float(info)+ return torch.empty(tuple(reversed(info.ne)), dtype=_FLOAT_DTYPES[info.ggml_type], device="meta")++ def packed_rows(info):+ rows = _prod(info.ne[1:])+ return torch.empty(rows, info.nbytes // rows, dtype=torch.uint8, device="meta")++ fakes = {"float_tensor": float_tensor, "packed_rows": packed_rows}+ real = {name: getattr(tensors, name) for name in fakes}+ return real, fakes+++def _meta_wins():+ """A torch function mode that moves CPU operands to meta when an op mixes the two, so a reader can combine meta weights with the small tensors it builds."""+ import torch+ from torch.overrides import TorchFunctionMode+ from torch.utils._pytree import tree_flatten, tree_unflatten++ class MetaWins(TorchFunctionMode):+ def __torch_function__(self, func, types, args=(), kwargs=None):+ flat, spec = tree_flatten((args, kwargs or {}))+ if any(isinstance(x, torch.Tensor) and x.is_meta for x in flat):+ flat = [x.to("meta") if isinstance(x, torch.Tensor) and not x.is_meta else x for x in flat]+ args, kwargs = tree_unflatten(flat, spec)+ return func(*args, **kwargs)++ return MetaWins+++def _swap_readers(use_fakes: bool) -> None:+ """Point every loaded binding of the readers, including the family modules' imported names, at the fakes or back."""+ real, fakes = _meta_readers()+ src, dst = (real, fakes) if use_fakes else (fakes, real)+ for module in list(sys.modules.values()):+ if getattr(module, "__name__", "").startswith("freetoken.models"):+ for name, fn in src.items():+ if getattr(module, name, None) is fn:+ setattr(module, name, dst[name])+++def _wait_for_ram(gb: float, timeout: float = 600.0) -> bool:+ deadline = time.time() + timeout+ while time.time() < deadline:+ with open("/proc/meminfo") as f:+ avail = next(int(line.split()[1]) for line in f if line.startswith("MemAvailable:")) / 2**20+ if avail >= gb:+ return True+ time.sleep(2)+ return False+++def _dry_load(path: str, config, model, *, meta: bool) -> list[str]:+ import torch++ from freetoken.models.register import _load_attr++ spec = config.model_spec+ expected = model.state_dict()+ problems, seen = [], set()+ _swap_readers(meta)+ try:+ with _meta_wins()() if meta else contextlib.nullcontext():+ for key, t in _load_attr(spec.module, spec.iter_weights)(path, torch.device("cpu"), include_moe_experts=False, include_non_moe=True):+ ref = expected.get(key)+ if ref is None:+ problems.append(f"unexpected tensor {key}")+ continue+ if key in seen:+ problems.append(f"tensor {key} yielded twice")+ seen.add(key)+ if tuple(t.shape) != tuple(ref.shape) or (t.dtype == torch.uint8) != (ref.dtype == torch.uint8):+ problems.append(f"{key}: got {tuple(t.shape)} {t.dtype}, model has {tuple(ref.shape)} {ref.dtype}")+ finally:+ _swap_readers(False)+ missing = sorted(set(expected) - seen)+ if missing:+ problems.append(f"{len(missing)} tensors never loaded, e.g. {missing[:3]}")+ return problems+++def _expert_banks(path: str, cfg, problems: list[str]) -> list:+ """Build the expert banks on the meta device and check their shapes; returns each MoE layer's (format, gate_up type, down type)."""+ import torch++ if not cfg.num_experts:+ return []+ from freetoken.layers.quantization.ggml_types import float_dtype, row_bytes+ from freetoken.models.gguf.experts import load_gguf_expert_banks+ from freetoken.third_party.gguf import GGMLQuantizationType++ banks = load_gguf_expert_banks(path, cfg.num_moe_layers, first_layer=cfg.first_k_dense_replace, dummy=True, device=torch.device("meta"))+ e, inter, hidden = cfg.num_experts, cfg.moe_intermediate_size, cfg.hidden_size+ for layer, (_, gate_up, down) in enumerate(banks.layer_tags):+ for role, name, rows, k in (("gate_up", gate_up, 2 * inter, hidden), ("down", down, hidden, inter)):+ t = int(GGMLQuantizationType[name])+ want = (e, rows, k if float_dtype(t) is not None else row_bytes(t, k))+ got = tuple(banks.sources[role][layer].shape)+ if got != want:+ problems.append(f"MoE layer {layer} {role} bank is {got}, the model needs {want}")+ return banks.layer_tags+++def _tokenizer_key(meta: dict) -> str:+ blob = json.dumps(sorted((k, v) for k, v in meta.items() if k.startswith("tokenizer.")))+ return hashlib.sha1(blob.encode()).hexdigest()[:12]+++def _check_tokenizer(path: str) -> dict:+ from freetoken.models.gguf.tokenizer import gguf_eos_token_ids, load_gguf_tokenizer++ problems = []+ tok = load_gguf_tokenizer(path)+ eos = sorted(gguf_eos_token_ids(path))+ if not eos:+ problems.append("tokenizer: no EOS token")+ tail = None+ if tok.chat_template is None:+ problems.append("tokenizer: no chat template")+ else:+ try:+ tail = tok.apply_chat_template([{"role": "user", "content": "hi"}], tokenize=False, add_generation_prompt=True)[-60:]+ except Exception as e:+ problems.append(f"chat template: {type(e).__name__}: {str(e)[:150]}")+ return {"problems": problems, "eos": eos, "template_tail": tail}+++def check(path: str) -> dict:+ from freetoken.models.gguf.config import GGUF_ARCH_TO_REGISTRY+ from freetoken.models.gguf.reader import gguf_architecture, load_gguf_metadata++ if _IN_FLIGHT is not None:+ _IN_FLIGHT[os.getpid()] = path+ t0 = time.time()+ out = {"path": path, "problems": []}+ problems = out["problems"]+ try:+ out["arch"] = gguf_architecture(path)+ if out["arch"] not in GGUF_ARCH_TO_REGISTRY:+ out["status"] = "unsupported"+ return out+ config, model = _build_model(path)+ cfg = config.model_config+ out["config"] = {"layers": cfg.num_layers, "hidden": cfg.hidden_size, "experts": cfg.num_experts, "vocab": cfg.vocab_size, "tied": cfg.tie_word_embeddings}+ out["reads"] = "meta"+ try:+ load_problems = _dry_load(path, config, model, meta=True)+ except Exception:+ out["reads"] = "real"+ if not _wait_for_ram(_MIN_FREE_GB):+ raise MemoryError(f"the weight reader needs real bytes and free RAM stayed under {_MIN_FREE_GB} GB") from None+ load_problems = _dry_load(path, config, model, meta=False)+ problems.extend(load_problems)+ tags = _expert_banks(path, cfg, problems)+ layers, glu = _gguf_layers(model)+ out["types"] = _check_kernels(layers, cfg.tie_word_embeddings, glu, tags, problems)+ meta = load_gguf_metadata(path)+ key = _tokenizer_key(meta)+ if key not in _TOKENIZERS:+ try:+ _TOKENIZERS[key] = _check_tokenizer(path)+ except Exception as e:+ _TOKENIZERS[key] = {"problems": [f"tokenizer: {type(e).__name__}: {str(e)[:200]}"]}+ tok = _TOKENIZERS[key]+ problems.extend(tok["problems"])+ out["tokenizer"] = {"hash": key, "pre": meta.get("tokenizer.ggml.pre"), "eos": tok.get("eos"), "template_tail": tok.get("template_tail")}+ except Exception as e:+ problems.append(f"{type(e).__name__}: {str(e)[:300]}")+ out["traceback"] = traceback.format_exc()[-1500:]+ finally:+ _clear_caches()+ out["seconds"] = round(time.time() - t0, 1)+ if _IN_FLIGHT is not None:+ _IN_FLIGHT.pop(os.getpid(), None)+ out.setdefault("status", "fail" if problems else "pass")+ return out+++def gguf_files(paths: list[str]) -> list[str]:+ """Every model GGUF under ``paths``, once; a split model is checked through its first file."""+ files = []+ for p in paths:+ found = sorted(glob.glob(os.path.join(p, "**", "*.gguf"), recursive=True)) if os.path.isdir(p) else [p]+ for f in found:+ m = _SPLIT.search(f)+ if os.path.isfile(f) and (m is None or m.group(1) == "00001") and not any(s in os.path.basename(f).lower() for s in ("mmproj", "imatrix")):+ files.append(os.path.abspath(f))+ return list(dict.fromkeys(files))+++def read_results(path: str) -> list[dict]:+ """The latest record per file; a line cut short by a killed run is dropped."""+ latest = {}+ if os.path.exists(path):+ with open(path) as f:+ for line in f:+ try:+ rec = json.loads(line)+ except json.JSONDecodeError:+ continue+ latest[rec["path"]] = rec+ return list(latest.values())+++def _problem_kind(problem: str) -> str:+ """A problem with tensor names, layer numbers and sizes masked, to group the same failure across files."""+ problem = re.sub(r"blk\.\d+", "blk.N", problem)+ problem = re.sub(r"model\.layers\.\d+", "model.layers.N", problem)+ problem = re.sub(r"/\S+\.gguf", "<file>", problem)+ return re.sub(r"\b\d+\b", "N", problem)[:160]+++def report(results: list[dict]) -> None:+ supported = [r for r in results if r["status"] != "unsupported"]+ by_arch: dict[str, list[int]] = {}+ for r in supported:+ stats = by_arch.setdefault(r.get("arch") or "?", [0, 0])+ stats[0] += r["status"] == "pass"+ stats[1] += 1+ print(f"{sum(r['status'] == 'pass' for r in supported)}/{len(supported)} files of served architectures pass")+ for arch, (ok, n) in sorted(by_arch.items()):+ print(f" {arch:16s} {ok}/{n}")+ unsupported = Counter(r.get("arch") for r in results if r["status"] == "unsupported")+ if unsupported:+ print("not checked, architecture not served: " + ", ".join(f"{a} {n}" for a, n in unsupported.most_common()))+ real = sum(r.get("reads") == "real" for r in supported)+ if real:+ print(f"{real} files needed real weight reads")+ kinds = Counter(_problem_kind(r["problems"][0]) for r in supported if r["status"] != "pass")+ if kinds:+ print("failures, by first problem:")+ for kind, n in kinds.most_common():+ print(f" {n:5d} {kind}")+++class _Pool:+ """Worker processes that check files, and know which files were running when a worker died."""++ def __init__(self, kernels: dict | None, min_free_gb: float):+ self.ctx = mp.get_context("spawn")+ self.manager = self.ctx.Manager()+ self.in_flight = self.manager.dict()+ self.args = (kernels, self.in_flight, min_free_gb)++ def run(self, files: list[str], workers: int):+ """Yields (path, record), or (path, None) for a file that was running when a worker died."""+ pending = list(files)+ while pending:+ self.in_flight.clear()+ # no max_tasks_per_child: on Python 3.12 the pool hangs once every worker has retired+ with cf.ProcessPoolExecutor(min(workers, len(pending)), mp_context=self.ctx, initializer=_init_worker, initargs=self.args) as pool:+ futures = {pool.submit(check, f): f for f in pending}+ pending, broken = [], False+ for fut in cf.as_completed(futures):+ try:+ yield futures[fut], fut.result()+ except cf.process.BrokenProcessPool:+ broken = True+ pending.append(futures[fut])+ if broken:+ # a worker that died before taking a file leaves none in flight: blame every file left+ running = set(self.in_flight.values()) & set(pending) or set(pending)+ for path in running:+ yield path, None+ pending = [p for p in pending if p not in running]+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("paths", nargs="*", help="GGUF files or directories to search")+ ap.add_argument("--out", help="results JSONL, appended to")+ ap.add_argument("--kernels", help="kernel table from probe_kernels.py; without it the kernel check is skipped")+ ap.add_argument("--workers", type=int, default=8)+ ap.add_argument("--min-free-gb", type=float, default=32.0, help="free RAM a worker waits for before reading real weight bytes")+ ap.add_argument("--retry-failed", action="store_true", help="check again the files that failed in --out")+ ap.add_argument("--report", metavar="JSONL", help="only print the summary of an earlier run")+ args = ap.parse_args()++ if args.report:+ report(read_results(args.report))+ return 0+ if not args.out or not args.paths:+ ap.error("give --out and at least one path")+ kernels = None+ if args.kernels:+ with open(args.kernels) as f:+ kernels = json.load(f)["types"]+ else:+ print("no --kernels table: the kernel check is skipped", file=sys.stderr)+ keep = {"pass", "unsupported"} if args.retry_failed else {"pass", "unsupported", "fail"}+ done = {r["path"] for r in read_results(args.out) if r["status"] in keep}+ files = [f for f in gguf_files(args.paths) if f not in done]+ print(f"{len(files)} files to check, {len(done)} already in {args.out}", file=sys.stderr, flush=True)++ pool = _Pool(kernels, args.min_free_gb)+ with open(args.out, "a") as out:++ def write(rec):+ mark = {"pass": "ok ", "unsupported": "skip"}.get(rec["status"], "FAIL")+ print(f"{mark} {rec['path']}" + (f" {rec['problems'][0][:120]}" if rec["problems"] else ""), file=sys.stderr, flush=True)+ out.write(json.dumps(rec) + "\n")+ out.flush()++ crashed = []+ for path, rec in pool.run(files, args.workers):+ if rec is None:+ crashed.append(path)+ else:+ write(rec)+ # each file that was running when a worker died runs alone, to find the one that kills it+ for path in crashed:+ for _, rec in pool.run([path], 1):+ write(rec or {"path": path, "status": "crash", "problems": ["worker process died (out of memory or a crash)"]})+ results = read_results(args.out)+ report(results)+ return 0 if all(r["status"] in ("pass", "unsupported") for r in results) else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/gguf_check/fetch_headers.py b/scripts/gguf_check/fetch_headers.pynew file mode 100644index 00000000..fdcbd337--- /dev/null+++ b/scripts/gguf_check/fetch_headers.py@@ -0,0 +1,193 @@+"""Write header-only stubs of the GGUF files in HF repos: the KV metadata and tensor table, in a sparse file of the real size.++ python scripts/gguf_check/fetch_headers.py <out_dir> [<repo> ...] [--search <query>]++The weights stay holes that read as zeros, so a stub takes a few MB of disk and ``check.py`` reads it like the real file.+Stubs land at ``<out_dir>/<repo>/<path in repo>``; a stub already there with the right size is kept, so a rerun only fetches what is missing.+Prints one JSON line per file. Gated repos read ``HF_TOKEN``.+"""++from __future__ import annotations++import argparse+import concurrent.futures as cf+import json+import os+import struct+import sys+import threading++import requests+from huggingface_hub import HfApi, hf_hub_url+from huggingface_hub.utils import build_hf_headers++_FIRST_READ, _NEXT_READ = 24 << 20, 16 << 20+_SCALAR = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}+_STRING, _ARRAY = 8, 9+_F32 = 0+# tensors whose values a parse_gguf_config reads, so their bytes go into the stub+VALUE_TENSORS = ("rope_freqs.weight",)++_local = threading.local()+++def _get_range(url: str, start: int, n: int) -> requests.Response:+ """Bytes [start, start + n) of ``url``, or fewer at the end of the file; refuses a reply that is not exactly that range."""+ r = _session().get(url, headers={"Range": f"bytes={start}-{start + n - 1}"}, timeout=300)+ r.raise_for_status()+ got = r.headers.get("Content-Range", "")+ if r.status_code != 206 or not got.startswith(f"bytes {start}-"):+ raise ValueError(f"{url}: asked for bytes {start}-{start + n - 1}, got status {r.status_code} {got!r}")+ end = int(got.split("-", 1)[1].split("/", 1)[0])+ if len(r.content) != end - start + 1:+ raise ValueError(f"{url}: short read at byte {start}")+ return r+++def _session() -> requests.Session:+ if not hasattr(_local, "session"):+ _local.session = requests.Session()+ _local.session.headers.update(build_hf_headers())+ return _local.session+++class _Remote:+ """Reads a remote file front to back through range requests, keeping every byte read so far."""++ def __init__(self, url: str):+ self.url, self.buf, self.pos, self.size = url, bytearray(), 0, None++ def _need(self, n: int) -> None:+ while len(self.buf) < self.pos + n:+ start = len(self.buf)+ if self.size is not None and start >= self.size:+ raise EOFError(f"{self.url}: header runs past the end of the file")+ r = _get_range(self.url, start, max(_FIRST_READ if start == 0 else _NEXT_READ, self.pos + n - start))+ if self.size is None:+ self.size = int(r.headers["Content-Range"].rsplit("/", 1)[1])+ self.buf += r.content++ def read(self, n: int) -> bytes:+ self._need(n)+ self.pos += n+ return bytes(self.buf[self.pos - n : self.pos])++ def unpack(self, fmt: str) -> tuple:+ return struct.unpack("<" + fmt, self.read(struct.calcsize("<" + fmt)))++ def string(self) -> str:+ (n,) = self.unpack("Q")+ return self.read(n).decode("utf-8", "replace")++ def skip_value(self, kind: int) -> None:+ if kind in _SCALAR:+ self.read(struct.calcsize(_SCALAR[kind]))+ elif kind == _STRING:+ self.string()+ elif kind == _ARRAY:+ elem, n = self.unpack("I")[0], self.unpack("Q")[0]+ if elem in _SCALAR:+ self.read(struct.calcsize(_SCALAR[elem]) * n)+ else:+ for _ in range(n):+ self.skip_value(elem)+ else:+ raise ValueError(f"{self.url}: unknown GGUF value type {kind}")+++def fetch(repo: str, path: str, size: int, out_dir: str) -> dict:+ dst = os.path.join(out_dir, repo, path)+ rec = {"repo": repo, "path": path, "stub": dst, "size": size, "value_tensors": list(VALUE_TENSORS)}+ # the sidecar marks a finished stub and which tensors it holds the values of+ sidecar = dst + ".json"+ if os.path.isfile(dst) and os.path.getsize(dst) == size and os.path.isfile(sidecar):+ with open(sidecar) as f:+ if json.load(f).get("value_tensors") == rec["value_tensors"]:+ return {**rec, "ok": True, "cached": True}+ url = hf_hub_url(repo, path)+ try:+ f = _Remote(url)+ if f.read(4) != b"GGUF":+ raise ValueError("not a GGUF file")+ _, n_tensors, n_kv = f.unpack("IQQ")+ align = 32+ for _ in range(n_kv):+ key, kind = f.string(), f.unpack("I")[0]+ if key == "general.alignment" and kind in (4, 5):+ align = f.unpack("I")[0]+ else:+ f.skip_value(kind)+ tensors = {}+ for _ in range(n_tensors):+ name = f.string()+ (nd,) = f.unpack("I")+ dims = f.unpack(f"{nd}Q")+ kind, offset = f.unpack("IQ")+ tensors[name] = (dims, kind, offset)+ header_end = f.pos+ data_start = (header_end + align - 1) // align * align+ tmp = dst + ".part"+ os.makedirs(os.path.dirname(dst), exist_ok=True)+ with open(tmp, "wb") as out:+ out.write(f.buf[:header_end])+ out.truncate(f.size)+ for name in VALUE_TENSORS:+ if name not in tensors:+ continue+ dims, kind, offset = tensors[name]+ if kind != _F32 or len(dims) != 1:+ raise ValueError(f"{name} is type {kind} with dims {dims}, expected 1-D F32")+ start, n = data_start + offset, dims[0] * 4+ r = _get_range(url, start, n)+ if len(r.content) != n:+ raise ValueError(f"{name} runs past the end of the file")+ out.seek(start)+ out.write(r.content)+ os.replace(tmp, dst)+ rec.update(header_bytes=header_end, tensors=n_tensors)+ with open(sidecar, "w") as f:+ json.dump(rec, f)+ return {**rec, "ok": True}+ except Exception as e:+ return {**rec, "ok": False, "error": f"{type(e).__name__}: {e}"[:300]}+++def list_gguf(api: HfApi, repo: str) -> list[tuple[str, int]]:+ """(path, size) of every model GGUF in the repo; vision projectors and imatrix files are left out."""+ files = []+ for entry in api.list_repo_tree(repo, recursive=True):+ path = entry.path+ if path.endswith(".gguf") and not any(s in path.lower() for s in ("mmproj", "imatrix")):+ files.append((path, entry.size))+ return files+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("out_dir")+ ap.add_argument("repos", nargs="*")+ ap.add_argument("--search", action="append", default=[], help="also take every HF GGUF repo whose name matches this query (repeatable)")+ ap.add_argument("--workers", type=int, default=12)+ args = ap.parse_args()++ api = HfApi()+ repos = list(args.repos)+ for query in args.search:+ repos += [m.id for m in api.list_models(search=query, filter="gguf") if m.id not in repos]+ jobs = []+ for repo in repos:+ try:+ jobs += [(repo, path, size) for path, size in list_gguf(api, repo)]+ except Exception as e:+ print(json.dumps({"repo": repo, "ok": False, "error": f"{type(e).__name__}: {e}"[:300]}), flush=True)+ print(f"{len(jobs)} GGUF files in {len(repos)} repos", file=sys.stderr, flush=True)+ failed = 0+ with cf.ThreadPoolExecutor(args.workers) as ex:+ for rec in ex.map(lambda job: fetch(*job, args.out_dir), jobs):+ failed += not rec["ok"]+ print(json.dumps(rec), flush=True)+ return 1 if failed else 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/gguf_check/probe_kernels.py b/scripts/gguf_check/probe_kernels.pynew file mode 100644index 00000000..4b53cd06--- /dev/null+++ b/scripts/gguf_check/probe_kernels.py@@ -0,0 +1,111 @@+"""Record which ggml ops this build runs for each quantized ggml type, as the kernel table ``check.py --kernels`` reads.++ python scripts/gguf_check/probe_kernels.py <out.json>++Runs every op on random packed weights on the current GPU, at the token counts that pick different kernels (vector, small-batch and MMQ).+An op counts as covered when it runs; the values are not checked. The table holds for this build and GPU architecture only.+Each type runs in its own process, so a CUDA fault in one type leaves the others' results intact.+"""++from __future__ import annotations++import argparse+import concurrent.futures as cf+import json+import multiprocessing as mp+import sys++K, N, EXPERTS, TOP_K = 1024, 256, 8, 2+TOKENS = (1, 8, 16, 512)+GLU_TOKENS = (1, 8)+# not weight types: plain integers and doubles, and the q8 blocks ggml quantizes activations to+_NOT_WEIGHTS = ("F64", "I8", "I16", "I32", "I64", "Q8_1", "Q8_K")+++def _run(fn) -> str:+ import torch++ try:+ fn()+ torch.cuda.synchronize()+ return "ok"+ except Exception as e:+ return f"{type(e).__name__}: {e}"[:200]+++def probe(ggml_type: int) -> dict:+ import torch++ from freetoken.kernel import ggml+ from freetoken.layers.quantization.ggml_types import row_bytes++ gen = torch.Generator(device="cuda").manual_seed(0)+ row = row_bytes(ggml_type, K)+ w = torch.randint(0, 256, (N, row), dtype=torch.uint8, device="cuda", generator=gen)+ banks = torch.randint(0, 256, (EXPERTS, N, row), dtype=torch.uint8, device="cuda", generator=gen)+ out = {"mmvq_max_tokens": ggml.mmvq_max_tokens(ggml_type)}+ for t in TOKENS:+ x = torch.randn(t, K, dtype=torch.bfloat16, device="cuda", generator=gen)+ # distinct experts per token, as a router picks them: MMQ faults on a token that routes to one expert twice+ ids = torch.rand(t, EXPERTS, device="cuda", generator=gen).argsort(dim=1)[:, :TOP_K].to(torch.int32).contiguous()+ out[f"mm{t}"] = _run(lambda: ggml.mul_mat(w, ggml_type, x))+ out[f"mmid{t}"] = _run(lambda: ggml.mul_mat_id(banks, ggml_type, x, ids))+ if t in GLU_TOKENS:+ out[f"glu{t}"] = _run(lambda: ggml.mul_mat_id_glu(banks, ggml_type, x, ids, "swiglu"))+ ids = torch.arange(4, dtype=torch.int32, device="cuda")+ out["get_rows"] = _run(lambda: ggml.get_rows(w, ggml_type, K, ids))+ out["dequant"] = _run(lambda: ggml.dequantize(w, ggml_type, K))+ return out+++def weight_types() -> list[str]:+ from freetoken.layers.quantization.ggml_types import is_quantized, row_bytes+ from freetoken.third_party.gguf import GGML_QUANT_SIZES, GGMLQuantizationType++ names = []+ for qtype in GGMLQuantizationType:+ if qtype not in GGML_QUANT_SIZES or not is_quantized(int(qtype)) or qtype.name in _NOT_WEIGHTS:+ continue+ try:+ row_bytes(int(qtype), K)+ except ValueError:+ continue+ names.append(qtype.name)+ return names+++def _probe_by_name(name: str) -> dict:+ from freetoken.third_party.gguf import GGMLQuantizationType++ return probe(int(GGMLQuantizationType[name]))+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("out")+ args = ap.parse_args()++ import torch++ from freetoken.kernel import ggml++ # build the module once up front: a failed build would otherwise show up as every op missing+ ggml.load_module()+ ctx = mp.get_context("spawn")+ table = {}+ for name in weight_types():+ with cf.ProcessPoolExecutor(1, mp_context=ctx) as pool:+ try:+ table[name] = pool.submit(_probe_by_name, name).result()+ except cf.process.BrokenProcessPool:+ table[name] = {"error": "probe process died"}+ bad = sorted(op for op, r in table[name].items() if op != "mmvq_max_tokens" and r != "ok")+ print(f"{name:10s} {'ok' if not bad else 'missing ' + ' '.join(bad)}", flush=True)+ cc = torch.cuda.get_device_capability()+ with open(args.out, "w") as f:+ json.dump({"gpu": torch.cuda.get_device_name(), "sm": f"{cc[0]}{cc[1]}", "types": table}, f, indent=1)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/gguf_check/tok_compare.py b/scripts/gguf_check/tok_compare.pynew file mode 100644index 00000000..4635fb12--- /dev/null+++ b/scripts/gguf_check/tok_compare.py@@ -0,0 +1,90 @@+"""Compare the tokenizer FreeToken builds from a GGUF with the model's own HF tokenizer.++ python scripts/gguf_check/tok_compare.py <gguf> <HF model dir or tokenizer.json> [--combos 20000]++Encodes, without special tokens, a set of edge strings, every normal vocabulary token, and random joins of 2-4 vocabulary tokens, and counts the texts whose ids differ.+The HF tokenizer is the reference: a stub works, since only the GGUF metadata is read.+"""++from __future__ import annotations++import argparse+import json+import os+import random+import sys++EDGE = [+ "Hello world",+ " leading spaces",+ "trailing spaces ",+ "tabs\t\tand\nnewlines\n\n\n",+ "I'm you're it's we'll they'd DON'T",+ "1234567890 3.14159 1,000,000",+ "def f(x):\n return x ** 2 # square\n",+ "数学题:求 x 的值。日本語のテキスト。한국어 문장.",+ "emoji 😀👍🏽 and combining é ä",+ "Ünïcödé àccents fi ligature",+ "<|im_start|> <s> </s> [INST] literal special-looking text",+ "$$\\frac{a}{b} + \\sqrt{x}$$",+ "URL https://example.com/a?b=c&d=e",+]+_NORMAL = 1+++def vocab_words(meta: dict) -> list[str]:+ """The text of every normal vocabulary token, for byte-level BPE and SentencePiece vocabularies."""+ from transformers.convert_slow_tokenizer import bytes_to_unicode++ tokens, types = meta["tokenizer.ggml.tokens"], meta.get("tokenizer.ggml.token_type") or [_NORMAL] * len(meta["tokenizer.ggml.tokens"])+ model = meta.get("tokenizer.ggml.model")+ byte_of = {c: b for b, c in bytes_to_unicode().items()}+ words = []+ for token, kind in zip(tokens, types):+ if kind != _NORMAL:+ continue+ if model == "gpt2":+ try:+ words.append(bytes(byte_of[c] for c in token).decode())+ except (KeyError, UnicodeDecodeError):+ pass+ elif model == "llama":+ words.append(token.replace("\u2581", " "))+ else:+ words.append(token)+ return words+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("gguf")+ ap.add_argument("hf", help="HF model dir or its tokenizer.json")+ ap.add_argument("--combos", type=int, default=20000)+ ap.add_argument("--examples", type=int, default=3)+ args = ap.parse_args()++ from tokenizers import Tokenizer++ from freetoken.models.gguf.reader import load_gguf_metadata+ from freetoken.models.gguf.tokenizer import load_gguf_tokenizer++ hf = Tokenizer.from_file(os.path.join(args.hf, "tokenizer.json") if os.path.isdir(args.hf) else args.hf)+ ours = load_gguf_tokenizer(args.gguf).backend_tokenizer+ meta = load_gguf_metadata(args.gguf)+ words = vocab_words(meta)+ rng = random.Random(0)+ combos = [rng.choice(["", " ", "\n"]).join(rng.choice(words) for _ in range(rng.randint(2, 4))) for _ in range(args.combos)]+ out = {"gguf": args.gguf, "pre": meta.get("tokenizer.ggml.pre")}+ differ = 0+ for name, texts in (("edge", EDGE), ("vocab", words), ("combos", combos)):+ ref = [e.ids for e in hf.encode_batch(texts, add_special_tokens=False)]+ got = [e.ids for e in ours.encode_batch(texts, add_special_tokens=False)]+ bad = [(t, r, g) for t, r, g in zip(texts, ref, got) if r != g]+ differ += len(bad)+ out[name] = {"texts": len(texts), "differ": len(bad), "examples": [{"text": t, "hf": r[:16], "gguf": g[:16]} for t, r, g in bad[: args.examples]]}+ print(json.dumps(out, ensure_ascii=False, indent=1))+ return 1 if differ else 0+++if __name__ == "__main__":+ sys.exit(main())