FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Record which ggml ops this build runs for each quantized ggml type, as the kernel table ``check.py --kernels`` reads.
python scripts/gguf_check/probe_kernels.py <out.json>
Runs every op on random packed weights on the current GPU, at the token counts that pick different kernels (vector, small-batch and MMQ).
An op counts as covered when it runs; the values are not checked. The table holds for this build and GPU architecture only.
Each type runs in its own process, so a CUDA fault in one type leaves the others' results intact.
"""
from __future__ import annotations
import argparse
import concurrent.futures as cf
import json
import multiprocessing as mp
import sys
K, N, EXPERTS, TOP_K = 1024, 256, 8, 2
TOKENS = (1, 8, 16, 512)
GLU_TOKENS = (1, 8)
# not weight types: plain integers and doubles, and the q8 blocks ggml quantizes activations to
_NOT_WEIGHTS = ("F64", "I8", "I16", "I32", "I64", "Q8_1", "Q8_K")
def _run(fn) -> str:
import torch
try:
fn()
torch.cuda.synchronize()
return "ok"
except Exception as e:
return f"{type(e).__name__}: {e}"[:200]
def probe(ggml_type: int) -> dict:
import torch
from freetoken.kernel import ggml
from freetoken.layers.quantization.ggml_types import row_bytes
gen = torch.Generator(device="cuda").manual_seed(0)
row = row_bytes(ggml_type, K)
w = torch.randint(0, 256, (N, row), dtype=torch.uint8, device="cuda", generator=gen)
banks = torch.randint(0, 256, (EXPERTS, N, row), dtype=torch.uint8, device="cuda", generator=gen)
out = {"mmvq_max_tokens": ggml.mmvq_max_tokens(ggml_type)}
for t in TOKENS:
x = torch.randn(t, K, dtype=torch.bfloat16, device="cuda", generator=gen)
# distinct experts per token, as a router picks them: MMQ faults on a token that routes to one expert twice
ids = torch.rand(t, EXPERTS, device="cuda", generator=gen).argsort(dim=1)[:, :TOP_K].to(torch.int32).contiguous()
out[f"mm{t}"] = _run(lambda: ggml.mul_mat(w, ggml_type, x))
out[f"mmid{t}"] = _run(lambda: ggml.mul_mat_id(banks, ggml_type, x, ids))
if t in GLU_TOKENS:
out[f"glu{t}"] = _run(lambda: ggml.mul_mat_id_glu(banks, ggml_type, x, ids, "swiglu"))
ids = torch.arange(4, dtype=torch.int32, device="cuda")
out["get_rows"] = _run(lambda: ggml.get_rows(w, ggml_type, K, ids))
out["dequant"] = _run(lambda: ggml.dequantize(w, ggml_type, K))
return out
def weight_types() -> list[str]:
from freetoken.layers.quantization.ggml_types import is_quantized, row_bytes
from freetoken.third_party.gguf import GGML_QUANT_SIZES, GGMLQuantizationType
names = []
for qtype in GGMLQuantizationType:
if qtype not in GGML_QUANT_SIZES or not is_quantized(int(qtype)) or qtype.name in _NOT_WEIGHTS:
continue
try:
row_bytes(int(qtype), K)
except ValueError:
continue
names.append(qtype.name)
return names
def _probe_by_name(name: str) -> dict:
from freetoken.third_party.gguf import GGMLQuantizationType
return probe(int(GGMLQuantizationType[name]))
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("out")
args = ap.parse_args()
import torch
from freetoken.kernel import ggml
# build the module once up front: a failed build would otherwise show up as every op missing
ggml.load_module()
ctx = mp.get_context("spawn")
table = {}
for name in weight_types():
with cf.ProcessPoolExecutor(1, mp_context=ctx) as pool:
try:
table[name] = pool.submit(_probe_by_name, name).result()
except cf.process.BrokenProcessPool:
table[name] = {"error": "probe process died"}
bad = sorted(op for op, r in table[name].items() if op != "mmvq_max_tokens" and r != "ok")
print(f"{name:10s} {'ok' if not bad else 'missing ' + ' '.join(bad)}", flush=True)
cc = torch.cuda.get_device_capability()
with open(args.out, "w") as f:
json.dump({"gpu": torch.cuda.get_device_name(), "sm": f"{cc[0]}{cc[1]}", "types": table}, f, indent=1)
return 0
if __name__ == "__main__":
sys.exit(main())