Skip to content

Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.

49.0 KB
Raw Blame
"""Evaluate NVFP4 KV-cache quality on long-context, reasoning and code work.

For each requested KV mode a single in-process vLLM engine is built and every
benchmark is scored against it, then the engine is torn down and the next mode
loads.  Results stream to per-cell JSONL and a final ``summary.md`` /
``summary.json`` table (mean per cell + delta vs the BF16 baseline).

Modes: ``bitdecoding-nvfp4`` and ``bitdecoding-mxfp4`` (CUSTOM), ``bf16``, ``fp8``,
``turboquant-4bit-nc``, ``turboquant-4bit-nc-all-layers``, and ``nvfp4``
(official vLLM NVFP4).
BitDecoding-NVFP4 automatically calibrates key smoothing and probability scaling.

    MODEL_PATH=/path/to/Qwen3-4B-Thinking-2507 \\
    python -m evaluation.run_eval --modes bitdecoding-nvfp4 --limit 20 --out-dir _out/smoke
"""
from __future__ import annotations

import argparse
import gc
import hashlib
import json
import os
import subprocess
import sys
from pathlib import Path
import time
from typing import List

from . import (
    aime,
    aime25,
    config,
    gpqa,
    lcb,
    longbench_pro,
    mrcr,
    ruler,
)

_BENCHMARKS = {
    "longbench_pro": (longbench_pro, longbench_pro.METRIC_KEY, "LongBench-Pro F1"),
    "mrcr": (mrcr, mrcr.METRIC_KEY, "MRCR ratio"),
    "aime": (aime, aime.METRIC_KEY, "AIME 2024 Avg@32"),
    "aime25": (aime25, aime25.METRIC_KEY, "AIME 2025 Avg@32"),
    "lcb": (lcb, lcb.METRIC_KEY, "LiveCodeBench v6 pass@1"),
    "ruler_niah": (ruler, ruler.METRIC_KEY, "RULER NIAH recall"),
    "gpqa": (gpqa, gpqa.METRIC_KEY, "GPQA Diamond Avg@8"),
}


def _write_jsonl(path, records):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    temporary = f"{path}.tmp.{os.getpid()}"
    with open(temporary, "w") as f:
        for r in records:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
        f.flush()
        os.fsync(f.fileno())
    os.replace(temporary, path)


def _append_jsonl(path, records):
    """Durably append a completed scheduler group to a partial result."""
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "a") as handle:
        for record in records:
            handle.write(json.dumps(record, ensure_ascii=False) + "\n")
        handle.flush()
        os.fsync(handle.fileno())


def _read_jsonl(path):
    records = []
    with open(path) as handle:
        for line_number, line in enumerate(handle, start=1):
            if not line.strip():
                continue
            try:
                records.append(json.loads(line))
            except json.JSONDecodeError as error:
                raise RuntimeError(
                    f"invalid JSONL checkpoint {path}:{line_number}: {error}"
                ) from error
    return records


def _partial_jsonl_path(path):
    stem, extension = os.path.splitext(path)
    return f"{stem}.partial{extension}"


_IDENTITY_KEYS = (
    "_source_index",
    "id",
    "bench_index",
    "task",
    "problem",
    "problem_index",
    "repeat",
    "token_length",
    "secondary_task",
    "bucket",
    "needle_name",
)


def _example_identity(example):
    return {key: example.get(key) for key in _IDENTITY_KEYS}


def _record_identity(record):
    identity = record.get("_selection_identity")
    if not isinstance(identity, dict):
        raise RuntimeError("resumable result row lacks _selection_identity")
    return identity


def _load_result_prefix(path, prepared):
    """Load only an exact deterministic prefix of the selected shard."""
    records = _read_jsonl(path)
    if len(records) > len(prepared):
        raise RuntimeError(
            f"checkpoint {path} has {len(records)} rows for a "
            f"{len(prepared)}-row shard"
        )
    for index, record in enumerate(records):
        expected = _example_identity(prepared[index]["example"])
        observed = _record_identity(record)
        if observed != expected:
            raise RuntimeError(
                f"checkpoint {path} row {index} identity {observed!r} "
                f"does not match selected shard {expected!r}"
            )
    return records


def _write_json(path, value):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    temporary = f"{path}.tmp.{os.getpid()}"
    with open(temporary, "w") as handle:
        json.dump(value, handle, ensure_ascii=False, indent=2, sort_keys=True)
        handle.write("\n")
    os.replace(temporary, path)


def _write_or_validate_json(path, value):
    """Create an immutable manifest, or prove an existing one is identical."""
    if not os.path.exists(path):
        _write_json(path, value)
        return
    with open(path) as handle:
        existing = json.load(handle)
    if existing != value:
        raise RuntimeError(
            f"existing run manifest does not match the requested contract: {path}"
        )


def _sha256(path):
    digest = hashlib.sha256()
    with open(path, "rb") as handle:
        for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def _file_identity(path):
    if not isinstance(path, str) or not os.path.isfile(path):
        return None
    return {
        "path": os.path.realpath(path),
        "size": os.path.getsize(path),
        "sha256": _sha256(path),
    }


def _selection_sha256(examples_by_benchmark):
    """Digest the rows a run selected, not merely how many it selected.

    ``bench_index`` is what identifies a row in the benchmarks that carry no
    ``id``. Without it LiveCodeBench selections hash to the same
    value for any two selections of equal size, so a shard or ``--limit``
    mismatch would pass the resume check that exists to catch exactly that.
    """
    rows = [
        {
            "benchmark": benchmark,
            "_source_index": row.get("_source_index"),
            "id": row.get("id"),
            "bench_index": row.get("bench_index"),
            "task": row.get("task"),
            "problem": row.get("problem"),
            "problem_index": row.get("problem_index"),
            "repeat": row.get("repeat"),
            "bucket": row.get("bucket"),
            "token_length": row.get("token_length"),
            "secondary_task": row.get("secondary_task"),
        }
        for benchmark in sorted(examples_by_benchmark)
        for row in examples_by_benchmark[benchmark]
    ]
    payload = json.dumps(rows, ensure_ascii=False, sort_keys=True).encode("utf-8")
    return hashlib.sha256(payload).hexdigest()


def _default_seeds(prepared, base_seed: int):
    """One stable draw per selected row when a benchmark has no seed scheme."""
    seeds = []
    for item in prepared:
        payload = json.dumps(
            _example_identity(item["example"]),
            ensure_ascii=False,
            sort_keys=True,
        ).encode("utf-8")
        offset = int.from_bytes(hashlib.sha256(payload).digest()[:4], "little")
        seeds.append((base_seed + offset) % (2**31 - 1))
    return seeds


def _select_explicit_rows(examples, indices):
    """Select audited source rows without renumbering their identities or seeds."""
    if not indices or any(type(i) is not int for i in indices) or len(set(indices)) != len(indices):
        raise ValueError("explicit selection must contain unique integer source indices")
    wanted = set(indices)
    selected = [row for row in examples if row["_source_index"] in wanted]
    if len(selected) != len(indices):
        raise ValueError("explicit selection references rows outside the selected protocol")
    return selected


def _frozen_prompt_audit(name, tokenizer, conversations, prepared, model_path, thinking):
    """Verify a schema-2 prompt freeze using the exact serving tokenizer."""
    path = os.environ.get("BITDECODING_LONG_CONTEXT_TOKEN_AUDIT")
    if not path:
        return None
    with open(path) as handle:
        audit = json.load(handle)
    if audit.get("schema") != 2:
        return None
    rows = {json.dumps(row["identity"], sort_keys=True): row
            for row in audit["rows"] if row["benchmark"] == name}
    selected = []
    for messages, item in zip(conversations, prepared, strict=True):
        expected = rows.get(json.dumps(_example_identity(item["example"]), sort_keys=True))
        rendered = tokenizer.apply_chat_template(messages, tokenize=False,
            add_generation_prompt=True, **config.chat_template_kwargs(model_path, thinking))
        tokens = tokenizer.encode(rendered, add_special_tokens=False)
        token_hash = hashlib.sha256(json.dumps(tokens, separators=(",", ":")).encode()).hexdigest()
        if expected is None or expected["token_sha256"] != token_hash:
            raise RuntimeError("prompt token IDs differ from the frozen evaluation input")
        if expected["rendered_sha256"] != hashlib.sha256(rendered.encode()).hexdigest():
            raise RuntimeError("rendered prompt differs from the frozen evaluation input")
        item["prompt_tokens"] = len(tokens)
        selected.append(expected)
    return selected

def _run_manifest(
    args, mode, mode_config, runner, examples_cache, selection_metadata
):
    audit_path = os.environ.get("BITDECODING_LONG_CONTEXT_TOKEN_AUDIT")
    engine = runner.engine_contract()
    if (mode == "bitdecoding-nvfp4" and getattr(args, "calibration_report", None)
            and config.RECIPE["underflow_aware_probability_scaling"]["enabled"]):
        tau = config.RECIPE["underflow_aware_probability_scaling"]["tau"]
        if engine.get("nvfp4_p_scale_tau") != tau:
            raise RuntimeError("the scoring engine did not load the calibrated tau")
    requested_rope = engine.get("requested_rope_scaling")
    resolved_rope = engine.get("resolved_rope_scaling")
    if requested_rope is not None:
        if not isinstance(resolved_rope, dict):
            raise RuntimeError("engine did not expose the requested YaRN config")
        requested_semantic = {
            "rope_type": requested_rope.get("rope_type", requested_rope.get("type")),
            "factor": float(requested_rope.get("factor")),
            "original_max_position_embeddings": int(
                requested_rope.get("original_max_position_embeddings")
            ),
        }
        resolved_semantic = {
            "rope_type": resolved_rope.get("rope_type", resolved_rope.get("type")),
            "factor": float(resolved_rope.get("factor")),
            "original_max_position_embeddings": int(
                resolved_rope.get("original_max_position_embeddings")
            ),
        }
        if resolved_semantic != requested_semantic:
            raise RuntimeError(
                f"resolved rope config {resolved_semantic} does not match "
                f"requested {requested_semantic}"
            )
    resolved_max_model_len = engine.get("resolved_max_model_len")
    if resolved_max_model_len is not None and resolved_max_model_len != args.max_model_len:
        raise RuntimeError(
            f"engine resolved max_model_len={resolved_max_model_len}, "
            f"requested {args.max_model_len}"
        )
    expected_cache_dtype = str(mode_config["kv_cache_dtype"])
    if engine.get("resolved_cache_dtype") != expected_cache_dtype:
        raise RuntimeError(
            f"engine resolved cache dtype {engine.get('resolved_cache_dtype')!r}, "
            f"but arm {mode!r} requires {expected_cache_dtype!r}"
        )
    layer_dtypes = engine.get("layer_cache_dtype_counts") or {}
    layer_backends = engine.get("layer_attention_backend_counts") or {}
    if not layer_dtypes or not layer_backends:
        raise RuntimeError(
            "engine did not expose per-layer KV dtype/backend attestation"
        )
    if mode in {"turboquant-4bit-nc", "turboquant-4bit-nc-all-layers"}:
        all_layers = bool(mode_config.get("turboquant_all_layers"))
        layer_count = _model_layer_count(args.model_path)
        expected_skip = [] if all_layers else [
            str(index) for index in (0, 1, layer_count - 2, layer_count - 1)
        ]
        if engine.get("kv_cache_dtype_skip_layers") != expected_skip:
            raise RuntimeError(
                "TurboQuant skip layers differ from the registered arm: "
                f"{engine.get('kv_cache_dtype_skip_layers')} != {expected_skip}"
            )
        expected_layer_dtypes = (
            {expected_cache_dtype: layer_count}
            if all_layers
            else {"auto": 4, expected_cache_dtype: layer_count - 4}
        )
        if layer_dtypes != expected_layer_dtypes:
            raise RuntimeError(
                f"TurboQuant layer KV dtypes are {layer_dtypes}, expected "
                f"{expected_layer_dtypes}"
            )
        expected_turboquant_layers = layer_count if all_layers else layer_count - 4
        expected_native_layers = 0 if all_layers else 4
        if (
            layer_backends.get("TURBOQUANT") != expected_turboquant_layers
            or sum(
                count for backend, count in layer_backends.items()
                if backend != "TURBOQUANT"
            ) != expected_native_layers
        ):
            raise RuntimeError(
                "TurboQuant resolved the wrong compressed/native layer split: "
                f"expected {expected_turboquant_layers}/"
                f"{expected_native_layers}, got {layer_backends}"
            )
    else:
        expected_layers = {expected_cache_dtype: sum(layer_dtypes.values())}
        if layer_dtypes != expected_layers:
            raise RuntimeError(
                f"arm {mode!r} resolved mixed layer KV dtypes {layer_dtypes}"
            )
        expected_backend = (
            "CUSTOM"
            if mode_config.get("custom_backend")
            else (mode_config.get("engine_kwargs") or {}).get(
                "attention_backend"
            )
        )
        if expected_backend is not None and layer_backends != {
            expected_backend: sum(layer_backends.values())
        }:
            raise RuntimeError(
                f"arm {mode!r} requires backend {expected_backend}, got "
                f"{layer_backends}"
            )
    return {
        "schema": 2,
        "mode": mode,
        "mode_label": mode_config["label"],
        "model": os.path.realpath(args.model_path),
        "model_layer_count": (
            _model_layer_count(args.model_path)
            if os.path.isfile(os.path.join(args.model_path, "config.json"))
            else None
        ),
        "model_revision": os.environ.get("BITDECODING_MODEL_REVISION"),
        "data_dir": os.path.realpath(args.data_dir),
        "dataset_manifest": _file_identity(
            os.path.join(args.data_dir, "MANIFEST.json")
        ),
        "ruler_generator": _file_identity(
            os.path.join(args.data_dir, "ruler", "generator.json")
        ),
        "benchmarks": list(args.benchmarks),
        "shard_index": args.shard_index,
        "num_shards": args.num_shards,
        "max_model_len": args.max_model_len,
        "gpu_memory_utilization": args.gpu_memory_utilization,
        "scale_mode_override": getattr(args, "scale_mode", None),
        "thinking": args.thinking,
        "seed": args.seed,
        "sampling": {
            name: _sampling_for(
                name, args.thinking, mode, max_model_len=args.max_model_len,
                model_path=args.model_path,
            )
            for name in args.benchmarks
        },
        "selection_sha256": _selection_sha256(examples_cache),
        "selected_rows": {
            name: len(rows) for name, rows in examples_cache.items()
        },
        "selection_metadata": selection_metadata,
        "prompt_budget": int(os.environ.get(
            "BITDECODING_LONGBENCH_PROMPT_BUDGET", config.LONGBENCH_PROMPT_BUDGET
        )),
        "engine": engine,
        "recipe": config.RECIPE if mode_config.get("custom_backend") else None,
        "calibration": _file_identity(getattr(args, "calibration_report", None)),
        "mode_config": mode_config,
        "sidecars": {
            key: _file_identity(mode_config.get(key))
            for key in ("key_shift",)
            if mode_config.get(key) is not None
        },
        "token_audit": _file_identity(audit_path),
        "resumable_output": {
            "schema": 1,
            "unit": "completed_scheduler_group",
            "source_files": {
                name: _file_identity(os.path.join(os.path.dirname(__file__), name))
                for name in (
                    "config.py", "gpqa.py", "longbench_pro.py", "ruler.py",
                    "run_eval.py", "vllm_runner.py", "aime.py", "aime25.py",
                    "lcb.py", "mrcr.py", "scoring.py", "longbench_pro_metrics.py",
                )
            },
        },
    }


def _mean(records, key):
    vals = [float(r[key]) for r in records if r.get(key) is not None]
    return (sum(vals) / len(vals)) if vals else None


def _validate_shard(shard_index: int, num_shards: int) -> None:
    if num_shards <= 0:
        raise ValueError("num_shards must be positive")
    if not 0 <= shard_index < num_shards:
        raise ValueError(
            f"shard_index must be in [0, {num_shards}), got {shard_index}"
        )


def _select_shard(examples, *, shard_index: int, num_shards: int):
    _validate_shard(shard_index, num_shards)
    return examples[shard_index::num_shards]


def _evenly_spaced(examples, limit: int):
    """Select deterministic coverage across an ordered benchmark."""
    if limit <= 0:
        raise ValueError("limit must be positive")
    if len(examples) <= limit:
        return list(examples)
    if limit == 1:
        return [examples[0]]
    return [
        examples[index * (len(examples) - 1) // (limit - 1)]
        for index in range(limit)
    ]


def _stratified_slice(name: str, examples, limit: int):
    """Select smoke rows spanning long-context length/problem coverage."""
    if name == "longbench_pro":
        buckets = {}
        for example in examples:
            buckets.setdefault(str(example.get("token_length")), []).append(example)
        ordered = sorted(
            buckets,
            key=lambda value: int(value[:-1]) if value.endswith("k") else value,
        )
        base, remainder = divmod(min(limit, len(examples)), len(ordered))
        selected = []
        for position, bucket in enumerate(ordered):
            take = base + (position < remainder)
            selected.extend(_evenly_spaced(buckets[bucket], take))
        # The official T4 scorer gate needs one English and one Chinese row,
        # while keeping the exact per-bucket smoke allocation unchanged.
        for language in ("English", "Chinese"):
            if any(
                str(row.get("secondary_task", "")).startswith("T4.")
                and row.get("language") == language
                for row in selected
            ):
                continue
            candidate = next(
                (
                    row for row in examples
                    if str(row.get("secondary_task", "")).startswith("T4.")
                    and row.get("language") == language
                ),
                None,
            )
            if candidate is None:
                continue
            bucket = str(candidate.get("token_length"))
            replace_at = next(
                (
                    index for index, row in enumerate(selected)
                    if str(row.get("token_length")) == bucket
                    and not str(row.get("secondary_task", "")).startswith("T4.")
                ),
                None,
            )
            if replace_at is not None:
                selected[replace_at] = candidate
        return selected
    if name == "mrcr":
        ordered = sorted(examples, key=lambda row: int(row["prompt_tokens"]))
        return _evenly_spaced(ordered, limit)
    if name == "ruler_niah":
        tasks = {}
        for example in examples:
            tasks.setdefault(example["task"], []).append(example)
        ordered = [task for task in ruler.TASKS if task in tasks]
        base, remainder = divmod(min(limit, len(examples)), len(ordered))
        selected = []
        for position, task in enumerate(ordered):
            selected.extend(
                _evenly_spaced(tasks[task], base + (position < remainder))
            )
        return selected
    return _evenly_spaced(examples, limit)


def _long_context_filter(name: str, examples, modes):
    if name != "longbench_pro":
        return examples
    examples = [
        row for row in examples
        if not str(row.get("secondary_task", "")).startswith("T4.")
    ]
    included_prefixes = tuple(
        value.strip() for value in
        os.environ.get("BITDECODING_LONG_CONTEXT_LBP_TASK_PREFIXES", "").split(",")
        if value.strip()
    )
    if included_prefixes:
        examples = [
            row for row in examples
            if str(row.get("secondary_task", "")).startswith(included_prefixes)
        ]
    excluded = {
        value.strip() for value in
        os.environ.get(
            "BITDECODING_LONG_CONTEXT_EXCLUDED_LBP_BUCKETS",
            ",".join(config.EVALUATION["longbench_excluded_buckets"]),
        ).split(",")
        if value.strip()
    }
    if not excluded:
        return examples
    return [row for row in examples if str(row.get("token_length")) not in excluded]


def build_parser():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--model-path", default=config.MODEL_PATH)
    ap.add_argument("--data-dir", default=config.DATA_DIR)
    ap.add_argument("--selection-file", default=None,
                    help="JSON mapping benchmark names to frozen source-row indices")
    ap.add_argument("--out-dir", default="_out")
    ap.add_argument("--modes", nargs="+", default=["bitdecoding-nvfp4"],
                    choices=sorted(config.KV_MODES),
                    help="bitdecoding-nvfp4 and bitdecoding-mxfp4 use CUSTOM; bf16, fp8, "
                         "both turboquant-4bit-nc arms and official nvfp4 are "
                         "baselines")
    ap.add_argument("--benchmarks", nargs="+", default=list(_BENCHMARKS),
                    choices=list(_BENCHMARKS))
    ap.add_argument("--limit", type=int, default=None,
                    help="cap examples per benchmark (smoke runs)")
    ap.add_argument("--stratified-limit", type=int, default=None,
                    help="cap examples while spanning long-context length/problem coverage")
    ap.add_argument(
        "--resume", action="store_true",
        help="resume an exact partial shard at scheduler-group boundaries",
    )
    ap.add_argument("--shard-index", type=int, default=0,
                    help="zero-based deterministic example shard")
    ap.add_argument("--num-shards", type=int, default=1,
                    help="number of deterministic example shards")
    ap.add_argument("--tp-size", type=int, default=1)
    ap.add_argument("--distributed-executor-backend", choices=("mp", "ray"))
    ap.add_argument("--gpu-memory-utilization", type=float, default=0.9)
    ap.add_argument("--max-model-len", type=int, default=config.MAX_MODEL_LEN)
    ap.add_argument("--enforce-eager", dest="enforce_eager",
                    action="store_true", default=None)
    ap.add_argument("--no-enforce-eager", dest="enforce_eager",
                    action="store_false")
    ap.add_argument("--attention-backend", default=None,
                    help="e.g. FLASHINFER (nvfp4/SM100) or TRITON_ATTN")
    ap.add_argument("--scale-mode", default=None,
                    help="CUSTOM NVFP4 block-scale candidates: a mode name "
                         "such as four_six for all three tensors, or "
                         "key=...,value=...,query=... to set them apart")
    ap.add_argument("--seed", type=int, default=42)
    ap.add_argument("--thinking", action="store_true",
                    help="run every benchmark in Qwen3 thinking mode, which "
                         "also switches LongBench-Pro and MRCR off greedy "
                         "(see config.THINKING_BENCH_OVERRIDES)")
    ap.add_argument("--score-longbench-t4", action="store_true",
                    help="post-score saved LongBench-Pro T4 generations in --out-dir")
    ap.add_argument("--embedding-model-path", default=None)
    ap.add_argument("--official-longbench-utils", default=None)
    ap.add_argument(
        "--official-evaluator-commit",
        default="a1d3fd6eab275981a75f03ebda2169b72b7b876e",
    )
    ap.add_argument(
        "--embedding-model-revision",
        default="1d8ad4ca9b3dd8059ad90a75d4983776a23d44af",
    )
    return ap


def _model_family(model_path=None):
    return config.model_family(model_path)


def _model_layer_count(model_path):
    try:
        with open(os.path.join(model_path, "config.json")) as handle:
            model_config = json.load(handle)
    except (OSError, TypeError, json.JSONDecodeError) as error:
        raise RuntimeError(
            f"cannot read model config for layer attestation: {model_path}"
        ) from error
    for candidate in (model_config, model_config.get("text_config", {})):
        for key in ("num_hidden_layers", "num_layers", "n_layer"):
            if candidate.get(key) is not None:
                return int(candidate[key])
    raise RuntimeError(f"model config has no layer count: {model_path}")


def _sampling_for(name, thinking: bool, mode: str, max_model_len=None, model_path=None) -> dict:
    defaults = dict(config.BENCH_DEFAULTS[name])
    if thinking:
        defaults.update(config.THINKING_BENCH_OVERRIDES.get(name, {}))
    window = config.SHORT_CONTEXT_MAX_MODEL_LEN if max_model_len is None else max_model_len
    if window > config.SHORT_CONTEXT_MAX_MODEL_LEN:
        defaults.update(config.LONG_CONTEXT_BENCH_OVERRIDES.get(name, {}))
    if thinking:
        family = _model_family(model_path)
        defaults.update(config.MODEL_THINKING_SAMPLING[family])
        if family == "glm":
            defaults.pop("top_k", None)
    return defaults


def _budgets(defaults, prepared, max_model_len):
    """The generation budget: one number, or one per row filling the window."""
    want = defaults["max_new_tokens"]
    if want != config.FILL_WINDOW:
        return want
    budgets = []
    for item in prepared:
        prompt_tokens = item.get("prompt_tokens")
        if prompt_tokens is None:
            raise ValueError(
                "a benchmark asking to fill the window has to report "
                "prompt_tokens for every row"
            )
        budgets.append(
            max_model_len - prompt_tokens - config.GENERATION_MARGIN
        )
    if min(budgets) <= 0:
        raise ValueError(
            f"a prompt leaves no room to answer in a {max_model_len} "
            f"window (smallest budget {min(budgets)})"
        )
    return budgets


def _templated_prompt_lengths(tokenizer, conversations, thinking, model_path=None):
    """Count the exact chat-template tokens used by a long-window engine."""
    lengths = []
    for messages in conversations:
        kwargs = {
            "tokenize": True,
            "add_generation_prompt": True,
            **config.chat_template_kwargs(model_path, thinking),
        }
        try:
            value = tokenizer.apply_chat_template(messages, **kwargs)
        except TypeError:
            kwargs.pop("enable_thinking", None)
            value = tokenizer.apply_chat_template(messages, **kwargs)
        # ``apply_chat_template(tokenize=True)`` may return a Transformers
        # BatchEncoding.  ``len(value)`` then counts fields rather than token
        # IDs, typically producing 2 and giving fill-window generation an
        # invalid budget.  Unwrap the actual input-id sequence first.
        if hasattr(value, "input_ids"):
            value = value.input_ids
        elif isinstance(value, dict) and "input_ids" in value:
            value = value["input_ids"]
        if hasattr(value, "tolist"):
            value = value.tolist()
        if value and isinstance(value[0], list):
            value = value[0]
        lengths.append(len(value))
    return lengths


def _verify_frozen_completions(audited_rows, completions):
    if audited_rows is None:
        return
    for expected, completion in zip(audited_rows, completions, strict=True):
        if (completion.get("prompt_tokens") != expected["prompt_tokens"]
                or completion.get("prompt_token_sha256") != expected["token_sha256"]):
            raise RuntimeError("engine consumed prompt token IDs that differ from the input freeze")


def _run_benchmark(name, runner, examples_cache, args, mode, *, output_path=None):
    module, metric_key, _label = _BENCHMARKS[name]
    thinking = args.thinking
    defaults = _sampling_for(
        name, args.thinking, mode, max_model_len=args.max_model_len,
        model_path=getattr(args, "model_path", None),
    )

    if name == "longbench_pro":
        conversations, prepared = module.build(
            examples_cache[name], tokenize=runner.tokenize,
            detokenize=runner.detokenize, thinking=thinking,
            reject_truncation=False,
        )
    else:
        conversations, prepared = module.build(examples_cache[name])

    # GLM's tokenizer is not Qwen's tokenizer.  The staged RULER/MRCR
    # ``prompt_tokens`` are Qwen-side measurements, so using them to fill a
    # GLM window would silently give GLM the wrong answer budget.  Count the
    # exact template that the serving tokenizer will consume after the engine
    # is built and retain those lengths in the manifest/results.
    if (
        name in {"ruler_niah", "mrcr"}
        and _model_family(getattr(args, "model_path", None)) == "glm"
    ):
        for item, prompt_tokens in zip(
            prepared,
            _templated_prompt_lengths(runner.tokenizer, conversations, thinking,
                                      getattr(args, "model_path", None)),
            strict=True,
        ):
            item["prompt_tokens"] = prompt_tokens
    frozen = _frozen_prompt_audit(name, getattr(runner, "tokenizer", None), conversations, prepared,
                                  getattr(args, "model_path", None), thinking)

    # A benchmark that samples the same prompt more than once has to say so;
    # one seed for such a batch makes the repeats identical.
    seed = (
        module.seeds(prepared, args.seed)
        if hasattr(module, "seeds")
        else _default_seeds(prepared, args.seed)
    )
    budget = _budgets(defaults, prepared, args.max_model_len)
    if frozen is not None:
        for i, row in enumerate(frozen):
            row_budget = budget[i] if isinstance(budget, list) else budget
            if row["sampling_seed"] != seed[i] or row["generation_budget"] != row_budget:
                raise RuntimeError("sampling seed or generation budget differs from the freeze")
    def attach_identity(records, selected_prepared, selected_seeds, selected_completions):
        row_seeds = (
            list(selected_seeds)
            if isinstance(selected_seeds, (list, tuple))
            else [selected_seeds] * len(selected_prepared)
        )
        for record, item, row_seed, completion in zip(
            records, selected_prepared, row_seeds, selected_completions, strict=True
        ):
            record["_selection_identity"] = _example_identity(item["example"])
            record["_sampling_seed"] = int(row_seed)
            if "output_token_ids" in completion:
                record["output_token_ids"] = completion["output_token_ids"]
            if "prompt_token_sha256" in completion:
                record["prompt_token_sha256"] = completion["prompt_token_sha256"]
        return records

    resumable = args.resume and output_path is not None
    if resumable:
        partial_path = _partial_jsonl_path(output_path)
        if os.path.exists(output_path):
            if os.path.exists(partial_path):
                raise RuntimeError(
                    f"both final and partial results exist: {output_path}, "
                    f"{partial_path}"
                )
            records = _load_result_prefix(output_path, prepared)
            if len(records) != len(prepared):
                raise RuntimeError(
                    f"final result {output_path} has {len(records)}/"
                    f"{len(prepared)} rows"
                )
            print(f"  [resume] {name}: final result already complete", flush=True)
            return records, metric_key

        records = records if os.path.exists(output_path) else (
            _load_result_prefix(partial_path, prepared)
            if os.path.exists(partial_path) else []
        )
        completed = len(records)
        if completed:
            print(
                f"  [resume] {name}: validated {completed}/{len(prepared)} "
                f"checkpoint rows",
                flush=True,
            )
        batch_size = int(
            getattr(runner, "_request_batch_size", None)
            or len(conversations) or 1
        )

        def selected(value, start, stop):
            return value[start:stop] if isinstance(value, (list, tuple)) else value

        for start in range(completed, len(prepared), batch_size):
            stop = min(start + batch_size, len(prepared))
            completions = runner.chat(
                conversations[start:stop],
                temperature=defaults["temperature"], top_p=defaults["top_p"],
                top_k=defaults.get("top_k"), min_p=defaults.get("min_p"),
                max_new_tokens=selected(budget, start, stop),
                seed=selected(seed, start, stop),
                enable_thinking=True if args.thinking else None,
            )
            _verify_frozen_completions(frozen[start:stop] if frozen is not None else None, completions)
            batch_records = attach_identity(
                module.grade(prepared[start:stop], completions),
                prepared[start:stop],
                selected(seed, start, stop),
                completions,
            )
            if len(batch_records) != stop - start:
                raise RuntimeError(
                    f"{name} graded {len(batch_records)} rows for a "
                    f"{stop - start}-row scheduler group"
                )
            _append_jsonl(partial_path, batch_records)
            records.extend(batch_records)
            print(
                f"  [checkpoint] {name}: {len(records)}/{len(prepared)} rows",
                flush=True,
            )
        if os.path.exists(partial_path):
            os.replace(partial_path, output_path)
        elif not os.path.exists(output_path):
            _write_jsonl(output_path, [])
    else:
        completions = runner.chat(
            conversations,
            temperature=defaults["temperature"], top_p=defaults["top_p"],
            top_k=defaults.get("top_k"), min_p=defaults.get("min_p"),
            max_new_tokens=budget, seed=seed,
            enable_thinking=True if args.thinking else None,
        )
        _verify_frozen_completions(frozen, completions)
        records = attach_identity(module.grade(prepared, completions), prepared, seed, completions)
    truncated = sum(1 for r in records if r.get("finish_reason") == "length")
    if truncated:
        # Loud because it is the failure mode that looks like a result: every
        # arm truncating at once reads as "the cache does not matter here".
        described = (f"{min(budget)}..{max(budget)}"
                     if isinstance(budget, list) else budget)
        print(
            f"  [truncation] {name}: {truncated}/{len(records)} hit "
            f"max_new_tokens={described}",
            flush=True,
        )
    return records, metric_key


def _prepare_calibration(args, scale_mode=None):
    """Fit the key mean and tau before the scoring process imports the kernel."""
    modes = set(args.modes) & {"bitdecoding-nvfp4", "bitdecoding-mxfp4"}
    if not modes:
        return
    fit_tau = "bitdecoding-nvfp4" in modes and config.needs_tau_calibration(config.RECIPE)
    smooth = config.RECIPE["outlier_aware_key_quantization"]["key_smoothing"]
    if not fit_tau and (not smooth or all(
        os.path.isfile(config.KV_MODES[mode].get("key_shift") or "") for mode in modes
    )):
        return

    directory = Path(args.out_dir).resolve() / "calibration"
    sidecar = directory / "key_smoothing.safetensors"
    report_path = sidecar.with_suffix(".json")
    mode = config.KV_MODES["bitdecoding-nvfp4" if fit_tau else sorted(modes)[0]]
    scales = scale_mode if scale_mode is not None else mode["scale_mode"]
    additional = {
        "nvfp4_scale_mode": scales, "nvfp4_query_terms": mode["query_terms"],
        "nvfp4_rotation": mode["rotation"], "nvfp4_key_smoothing": None,
    }
    expected = {
        "model_path": os.path.realpath(args.model_path),
        "max_model_len": args.max_model_len, "context_tokens": args.max_model_len - 1,
        "tensor_parallel_size": args.tp_size, "additional_config": additional,
    }
    if not report_path.exists():
        if sidecar.exists():
            raise ValueError(f"incomplete calibration at {sidecar}; choose a new output directory")
        directory.mkdir(parents=True, exist_ok=True)
        command = [
            sys.executable, str(Path(config.REPO_ROOT) / "tools/calibrate_key_smooth_sidecar.py"),
            "--model-path", args.model_path, "--output-path", str(sidecar),
            "--max-model-len", str(args.max_model_len),
            "--context-tokens", str(args.max_model_len - 1),
            "--tensor-parallel-size", str(args.tp_size), "--max-num-seqs", "1",
            "--gpu-memory-utilization", str(args.gpu_memory_utilization),
            "--scale-mode", json.dumps(scales),
            "--query-terms", str(mode["query_terms"]), "--rotation", mode["rotation"],
        ]
        environment = os.environ.copy()
        if fit_tau:
            command += ["--probability-multiplier", str(config.RECIPE["underflow_aware_probability_scaling"]["c"])]
            probability = config.probability_environment(config.RECIPE, calibration=True)
        else:
            probability = {"NVFP4_P_SCALE_MODE": None, "NVFP4_P_SCALE_TAU": None}
        for key, value in {**probability, "NVFP4_P_SCALE_DIVISOR": None}.items():
            if value is None:
                environment.pop(key, None)
            else:
                environment[key] = value
        environment["PYTHONPATH"] = os.pathsep.join([
            str(Path(config.REPO_ROOT) / "src"), config.REPO_ROOT,
            environment.get("PYTHONPATH", ""),
        ])
        print("Calibrating key smoothing and probability scaling before evaluation...", flush=True)
        subprocess.run(command, env=environment, check=True)

    report = json.loads(report_path.read_text())
    if not sidecar.is_file() or any(report.get(key) != value for key, value in expected.items()):
        raise ValueError(f"calibration does not match this model and recipe: {report_path}")
    if fit_tau:
        probability = report.get("probability_scaling", {})
        resolved = {**config.RECIPE, "underflow_aware_probability_scaling": {
            **config.RECIPE["underflow_aware_probability_scaling"], "tau": probability.get("tau"),
        }}
        config.probability_environment(resolved)
        if probability.get("c") != config.RECIPE["underflow_aware_probability_scaling"]["c"]:
            raise ValueError(f"calibration probability multiplier differs from the recipe: {report_path}")
        config.RECIPE["underflow_aware_probability_scaling"]["tau"] = probability["tau"]
        print(f"Calibrated tau={probability['tau']:.9g}", flush=True)
    if smooth:
        for name in modes:
            config.KV_MODES[name]["key_shift"] = str(sidecar)
    args.calibration_report = str(report_path)


def _configure_probability_scaling(mode):
    if mode == "bitdecoding-nvfp4":
        environment = config.probability_environment(config.RECIPE)
    elif mode == "bitdecoding-mxfp4":
        environment = {"NVFP4_P_SCALE_MODE": None, "NVFP4_P_SCALE_TAU": None}
    else:
        return
    environment["NVFP4_P_SCALE_DIVISOR"] = None
    for name, value in environment.items():
        if value is None:
            os.environ.pop(name, None)
        else:
            os.environ[name] = value


def main():
    args = build_parser().parse_args()
    if args.score_longbench_t4:
        if not args.embedding_model_path or not args.official_longbench_utils:
            raise SystemExit(
                "--score-longbench-t4 requires --embedding-model-path and "
                "--official-longbench-utils"
            )
        manifest = longbench_pro.score_t4_run(
            args.out_dir,
            embedding_model_path=args.embedding_model_path,
            official_utils_path=args.official_longbench_utils,
            evaluator_commit=args.official_evaluator_commit,
            embedding_revision=args.embedding_model_revision,
        )
        print(
            f"[T4] scored {manifest['t4_rows']}/{manifest['rows']} rows "
            f"under {args.out_dir}",
            flush=True,
        )
        return
    try:
        _validate_shard(args.shard_index, args.num_shards)
    except ValueError as error:
        raise SystemExit(str(error)) from error
    if args.limit is not None and args.stratified_limit is not None:
        raise SystemExit("--limit and --stratified-limit are mutually exclusive")
    if args.stratified_limit is not None and args.stratified_limit <= 0:
        raise SystemExit("--stratified-limit must be positive")
    scale_mode = None
    if args.scale_mode is not None:
        from nvfp4_decode_kernel.scale_mode import parse_scale_mode_spec

        try:
            scale_mode = parse_scale_mode_spec(args.scale_mode)
        except ValueError as error:
            raise SystemExit(str(error)) from error
    config.MODEL_PATH = args.model_path
    config.configure_data_dir(args.data_dir)

    problems = config.verify_prestaging(args.benchmarks)
    problems += config.verify_dependencies(args.benchmarks)
    if problems:
        raise SystemExit("Pre-staging incomplete:\n- " + "\n- ".join(problems))
    _prepare_calibration(args, scale_mode)
    problems = config.verify_sidecars(args.modes)
    if problems:
        raise SystemExit("Calibration incomplete:\n- " + "\n- ".join(problems))
    if "bitdecoding-nvfp4" in args.modes:
        _configure_probability_scaling("bitdecoding-nvfp4")

    # Load each dataset once (shared across modes).
    examples_cache = {}
    selection_metadata = {}
    for name in args.benchmarks:
        module = _BENCHMARKS[name][0]
        pre_sharded = False
        if (
            name == "ruler_niah"
            and args.limit is None
            and not args.selection_file
        ):
            ex = module.load_examples(
                shard_index=args.shard_index,
                num_shards=args.num_shards,
                stratified_limit=args.stratified_limit,
            )
            pre_sharded = True
        else:
            ex = module.load_examples()
        for source_index, row in enumerate(ex):
            row.setdefault("_source_index", source_index)
        ex = _long_context_filter(name, ex, args.modes)
        if args.selection_file:
            if args.limit or args.stratified_limit or pre_sharded:
                raise ValueError("explicit selection cannot be combined with a limit or pre-sharded data")
            with open(args.selection_file) as handle:
                indices = json.load(handle)[name]
            ex = _select_explicit_rows(ex, indices)
            selection_metadata.setdefault(name, {})["selection_file"] = _file_identity(args.selection_file)
        if args.limit:
            ex = ex[: args.limit]
        elif args.stratified_limit and not pre_sharded:
            ex = _stratified_slice(name, ex, args.stratified_limit)
        if not pre_sharded:
            ex = _select_shard(
                ex,
                shard_index=args.shard_index,
                num_shards=args.num_shards,
            )
        examples_cache[name] = ex
        print(
            f"[data] {name}: {len(ex)} examples "
            f"(shard {args.shard_index}/{args.num_shards})",
            flush=True,
        )

    results = {}  # (benchmark, mode) -> (mean, n)
    for mode in args.modes:
        _configure_probability_scaling(mode)
        mode_config = config.resolve_mode(mode, args.model_path)
        kv_dtype = mode_config["kv_cache_dtype"]
        print(f"\n=== KV mode: {mode} ({mode_config['label']}) "
              f"kv_cache_dtype={kv_dtype} ===", flush=True)
        from .vllm_runner import VllmRunner

        engine_started = time.monotonic()
        runner = VllmRunner(
            model_path=args.model_path, kv_cache_dtype=mode_config,
            max_model_len=args.max_model_len,
            gpu_memory_utilization=args.gpu_memory_utilization,
            tensor_parallel_size=args.tp_size,
            distributed_executor_backend=args.distributed_executor_backend,
            enforce_eager=args.enforce_eager,
            attention_backend=args.attention_backend,
            scale_mode=scale_mode,
        )
        _append_jsonl(os.path.join(args.out_dir, f"engine_startup__{mode}.jsonl"), [{
            "mode": mode, "benchmarks": args.benchmarks,
            "host": os.uname().nodename, "pid": os.getpid(),
            "started_unix": time.time() - (time.monotonic() - engine_started),
            "engine_startup_seconds": time.monotonic() - engine_started,
            "includes_tokenizer_load_and_graph_initialization": True,
            "model_path": os.path.realpath(args.model_path),
            "tensor_parallel_size": args.tp_size,
        }])
        try:
            manifest_name = (
                "manifest__" + "_".join(args.benchmarks) + f"__{mode}.json"
            )
            _write_or_validate_json(
                os.path.join(args.out_dir, manifest_name),
                _run_manifest(
                    args,
                    mode,
                    mode_config,
                    runner,
                    examples_cache,
                    selection_metadata,
                ),
            )
            for name in args.benchmarks:
                output_path = os.path.join(
                    args.out_dir, f"{name}__{mode}.jsonl"
                )
                health_path = output_path.removesuffix(".jsonl") + ".health.json"
                preserve_health = os.path.isfile(output_path) and os.path.isfile(
                    health_path
                )
                try:
                    import torch

                    torch.cuda.reset_peak_memory_stats()
                except (ImportError, RuntimeError):
                    torch = None
                started = time.monotonic()
                records, metric_key = _run_benchmark(
                    name, runner, examples_cache, args, mode,
                    output_path=output_path,
                )
                if not args.resume:
                    _write_jsonl(output_path, records)
                if not preserve_health:
                    max_gpu_memory = (
                        torch.cuda.max_memory_reserved() / 2**20
                        if torch is not None and torch.cuda.is_available()
                        else None
                    )
                    elapsed = time.monotonic() - started
                    _write_json(health_path, {
                        "schema": 1,
                        "benchmark": name,
                        "mode": mode,
                        "n": len(records),
                        "wall_clock_s": elapsed,
                        "max_gpu_mem_mib": max_gpu_memory,
                        "generated_tokens": sum(
                            int(record.get("gen_tokens") or 0)
                            for record in records
                        ),
                        "finished_on_length": sum(
                            record.get("finish_reason") == "length"
                            for record in records
                        ),
                    })
                mean = _mean(records, metric_key)
                results[(name, mode)] = (mean, len(records))
                mean_str = "n/a (deferred)" if mean is None else f"{mean:.4f}"
                print(f"  {name}: mean {metric_key} = {mean_str} (n={len(records)})",
                      flush=True)
        finally:
            runner.shutdown()
            del runner
            gc.collect()

    _write_summary(args, results)


def _write_summary(args, results):
    benchmarks = args.benchmarks
    header = ["KV mode"]
    for b in benchmarks:
        header += [b, f"{b} Δ vs bf16"]
    lines = ["| " + " | ".join(header) + " |",
             "|" + "|".join(["---"] * len(header)) + "|"]
    base = {b: (results.get((b, "bf16")) or (None, 0))[0] for b in benchmarks}
    summary_json = {"results": {}}
    for mode in args.modes:
        row = [config.KV_MODES[mode]["label"]]
        for b in benchmarks:
            mean, n = results.get((b, mode), (None, 0))
            summary_json["results"][f"{b}__{mode}"] = {"mean": mean, "n": n}
            if mean is None:
                row += ["—", "—"]
                continue
            row.append(f"{mean:.4f}")
            bm = base.get(b)
            row.append(f"{mean - bm:+.4f}" if bm is not None else "—")
        lines.append("| " + " | ".join(row) + " |")
    table = "\n".join(lines)

    os.makedirs(args.out_dir, exist_ok=True)
    md_path = os.path.join(args.out_dir, "summary.md")
    with open(md_path, "w") as f:
        f.write("# BitDecoding-FP4 — NVFP4 KV-cache evaluation\n\n")
        f.write(f"Model: `{args.model_path}`\n\n")
        f.write(table + "\n")
    with open(os.path.join(args.out_dir, "summary.json"), "w") as f:
        json.dump(summary_json, f, indent=2)
    print("\n" + table)
    print(f"\nWrote {md_path}")


if __name__ == "__main__":
    main()