Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
5.7 KB
Raw Blame
"""tools/nvfp4_verify.py - prove the ModelOpt -> ggml NVFP4 repack is lossless on a real checkpoint.

    python tools/nvfp4_verify.py --model <ModelOpt NVFP4 dir> [--experts 4] [--scan-all]

llama.cpp's converter repacks ModelOpt NVFP4 by moving bits: nibbles into ggml's split-half order, and each
E4M3 block scale into a UE4M3 byte by stripping its sign bit. That is lossless only while every scale is a
finite, non-negative E4M3 - a negative scale would silently flip, a NaN would propagate. So this checks two
things independently of the converter's own code path:

  1. every weight_scale byte in the checkpoint (--scan-all): no sign bit set, no NaN encoding (0x7F / 0xFF);
  2. for a few experts: dequantize ModelOpt directly (E2M1 value x E4M3 scale x weight_scale_2) and compare
     with gguf-py's dequantization of the repacked block_nvfp4 bytes times the same weight_scale_2. They must
     agree bit for bit in float32.
"""
from __future__ import annotations

import argparse
import json
import pathlib
import sys

import numpy as np
import torch
from safetensors import safe_open

LLAMA = pathlib.Path(__file__).resolve().parents[1] / "third_party" / "llama.cpp"
sys.path.insert(0, str(LLAMA / "gguf-py"))
sys.path.insert(0, str(LLAMA))
from gguf import GGMLQuantizationType as Q, quants  # noqa: E402
from conversion.base import ModelBase  # noqa: E402

E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0])


def modelopt_dequant(weight: torch.Tensor, scale: torch.Tensor, scale2: torch.Tensor) -> np.ndarray:
    """The ModelOpt definition, written out independently of the converter."""
    rows = weight.shape[0]
    lo = (weight & 0x0F).long()
    hi = (weight >> 4).long()
    nib = torch.stack([lo, hi], dim=-1).reshape(rows, -1)          # element 2j low, 2j+1 high
    vals = E2M1[nib]
    s = scale.float().repeat_interleave(16, dim=1)
    return (vals * s * scale2.float()).numpy().astype(np.float32)


def check_scales(model: pathlib.Path) -> int:
    index = json.loads((model / "model.safetensors.index.json").read_text(encoding="utf-8"))["weight_map"]
    by_shard: dict[str, list[str]] = {}
    for name, shard in index.items():
        if name.endswith(".weight_scale"):
            by_shard.setdefault(shard, []).append(name)
    n_bytes = n_sign = n_nan = n_tensors = 0
    worst = []
    for i, (shard, names) in enumerate(sorted(by_shard.items())):
        with safe_open(model / shard, framework="pt") as f:
            for name in names:
                t = f.get_tensor(name)
                if t.dtype != torch.float8_e4m3fn or t.ndim != 2:
                    continue                                          # FP8 per-channel scales are not NVFP4
                b = t.view(torch.uint8)
                sign = int((b & 0x80).count_nonzero())
                nan = int(((b & 0x7F) == 0x7F).sum())
                n_bytes += b.numel()
                n_sign += sign
                n_nan += nan
                n_tensors += 1
                if sign or nan:
                    worst.append((name, sign, nan))
        if i % 40 == 0:
            print("  scanned %3d/%d shards, %d scale tensors" % (i + 1, len(by_shard), n_tensors), flush=True)
    print("scale bytes %.2f GiB in %d tensors: sign bit set %d, NaN %d" % (n_bytes / 2**30, n_tensors, n_sign, n_nan))
    for w in worst[:10]:
        print("  BAD", *w)
    return 1 if (n_sign or n_nan) else 0


def check_experts(model: pathlib.Path, n: int) -> int:
    index = json.loads((model / "model.safetensors.index.json").read_text(encoding="utf-8"))["weight_map"]
    picks = [name for name in sorted(index)
             if ".experts." in name and name.endswith(".weight") and ".mtp." not in name
             and name.replace(".weight", ".weight_scale") in index][:: max(1, 50000 // max(1, n))][:n * 3]
    bad = 0
    for name in picks:
        shard = model / index[name]
        with safe_open(shard, framework="pt") as f:
            w = f.get_tensor(name)
            s = f.get_tensor(name.replace(".weight", ".weight_scale"))
            s2_name = name.replace(".weight", ".weight_scale_2")
            s2 = f.get_tensor(s2_name) if s2_name in f.keys() else torch.tensor(1.0)
        ref = modelopt_dequant(w, s, s2)
        raw, shape = ModelBase._nvfp4_pack(w, s)
        got = quants.dequantize(np.ascontiguousarray(raw), Q.NVFP4).reshape(shape).astype(np.float32)
        got = (torch.from_numpy(got) * s2.float()).numpy()
        # E2M1 code 8 is -0.0; ggml's kvalues_fp4 stores both zeros as integer 0, so the repacked weights give
        # +0.0 where ModelOpt gives -0.0. That is the same value in every sum (x + -0 == x), so normalise zeros
        # (adding +0.0 turns -0.0 into +0.0) and keep the comparison bitwise for everything else.
        neg_zero = int(np.count_nonzero((ref == 0) & np.signbit(ref)))
        same = np.array_equal((ref + np.float32(0)).view(np.uint32), (got + np.float32(0)).view(np.uint32))
        diff = float(np.abs(ref - got).max())
        print("  %-70s %-12s bitwise %s  max|diff| %.3g  (-0.0 in source: %d)"
              % (name[-70:], tuple(shape), "OK " if same else "NO ", diff, neg_zero))
        bad += not same
    return 1 if bad else 0


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", required=True)
    ap.add_argument("--experts", type=int, default=4, help="how many experts to compare (3 projections each)")
    ap.add_argument("--scan-all", action="store_true", help="check every weight_scale byte in the checkpoint")
    a = ap.parse_args()
    model = pathlib.Path(a.model)
    rc = check_experts(model, a.experts)
    if a.scan_all:
        rc |= check_scales(model)
    print("RESULT:", "lossless" if rc == 0 else "NOT lossless")
    return rc


if __name__ == "__main__":
    raise SystemExit(main())