Skip to content

Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.

7.1 KB
Raw Blame
#!/usr/bin/env python3
"""Grade LiveCodeBench generations with the official checker, in a sandbox.

Runs in a **new process** of the inference environment
(``_local/envs/bd_fp4``), never in a process that already holds a CUDA
context. LiveCodeBench is imported from a checkout on ``PYTHONPATH``; it is
not pip-installed, because that would replace this repository's vLLM.

Grading itself is ``lcb_runner.evaluation.codegen_metrics`` -- the same
function ``custom_evaluator`` calls. What is skipped is that CLI's benchmark
load, because it goes through a loader that downloads 4,486 MB to read the 134
that hold v6 (see tools/lcb_data.py), and its alignment step, which sorts
generations by ``str(question_id)`` and zips them against the benchmark.
Records carry ``bench_index`` instead, so alignment is positional and cannot
silently pair a solution with another problem's tests.

Isolation is a user namespace with no network and its own PID namespace, plus
an address-space limit. LCB's ``reliability_guard`` says in its own docstring
that it is "not a security sandbox" -- it nulls out some Python attributes --
and it is called with ``maximum_memory_bytes=None``, so nothing else here caps
memory.

    _local/envs/bd_fp4/bin/python tools/score_lcb.py \\
        --run-dir _local/eval_smoke --mode bitdecoding-nvfp4
"""
from __future__ import annotations

import argparse
import glob
import json
import os
import resource
import sys

REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
sys.path.insert(0, REPO_ROOT)

from tools.lcb_data import load_v6_problems  # noqa: E402
from evaluation.lcb import REPEATS  # noqa: E402

SANDBOX_MARKER = "BITDECODING_LCB_SANDBOXED"


def _reexec_sandboxed(memory_gib: int) -> None:
    """Re-run this program inside a network-less user + PID namespace."""
    os.environ[SANDBOX_MARKER] = "1"
    os.environ["BITDECODING_LCB_MEMORY_GIB"] = str(memory_gib)
    command = [
        "unshare", "--user", "--map-root-user", "--net", "--pid", "--fork",
        sys.executable, os.path.abspath(__file__), *sys.argv[1:],
    ]
    os.execvp(command[0], command)


def load_records(run_dir: str, mode: str) -> dict:
    """Every graded row for one arm, keyed by its position in the benchmark."""
    pattern = os.path.join(run_dir, "**", f"lcb__{mode}.jsonl")
    paths = sorted(glob.glob(pattern, recursive=True))
    if not paths:
        raise SystemExit(f"no lcb__{mode}.jsonl under {run_dir}")
    by_index, source = {}, {}
    for path in paths:
        with open(path) as handle:
            for line in handle:
                line = line.strip()
                if not line:
                    continue
                record = json.loads(line)
                index = (
                    record.get("problem_index", record["bench_index"]),
                    record.get("repeat", 0),
                )
                if index in by_index:
                    raise SystemExit(
                        f"row {index} appears in both "
                        f"{source[index]} and {path}"
                    )
                by_index[index] = record
                source[index] = path
    return by_index, source


def write_back(source: dict, by_index: dict) -> None:
    """Rewrite each shard file with ``pass1`` filled in, order preserved."""
    for path in sorted(set(source.values())):
        with open(path) as handle:
            rows = [json.loads(line) for line in handle if line.strip()]
        for row in rows:
            key = (
                row.get("problem_index", row["bench_index"]),
                row.get("repeat", 0),
            )
            row["pass1"] = by_index[key]["pass1"]
        with open(path, "w") as handle:
            for row in rows:
                handle.write(json.dumps(row, ensure_ascii=False) + "\n")


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--run-dir", required=True)
    parser.add_argument("--mode", required=True)
    parser.add_argument("--num-process", type=int, default=32)
    parser.add_argument("--timeout", type=int, default=6)
    parser.add_argument("--memory-gib", type=int, default=16,
                        help="address-space cap for this process and its children")
    parser.add_argument("--no-sandbox", action="store_true")
    parser.add_argument("--allow-partial", action="store_true",
                        help="grade a subset (smoke runs), instead of requiring "
                             "every problem in the slice")
    args = parser.parse_args()

    if not args.no_sandbox and not os.environ.get(SANDBOX_MARKER):
        _reexec_sandboxed(args.memory_gib)

    limit = int(os.environ.get("BITDECODING_LCB_MEMORY_GIB", args.memory_gib)) * 1024 ** 3
    resource.setrlimit(resource.RLIMIT_AS, (limit, limit))

    from lcb_runner.evaluation import codegen_metrics, extract_instance_results

    problems = load_v6_problems()
    by_index, source = load_records(args.run_dir, args.mode)
    keys = sorted(by_index)
    problem_indices = sorted({key[0] for key in keys})
    if problem_indices and (problem_indices[-1] >= len(problems)):
        raise SystemExit(
            f"problem_index {problem_indices[-1]} is outside the {len(problems)}-problem "
            "slice; the prompts and the grader disagree about the benchmark"
        )
    expected_keys = {
        (problem_index, repeat)
        for problem_index in problem_indices
        for repeat in range(REPEATS)
    }
    missing = expected_keys - set(keys)
    if missing and not args.allow_partial:
        raise SystemExit(f"missing repeat rows, for example {sorted(missing)[:3]}")
    if len(problem_indices) != len(problems) and not args.allow_partial:
        raise SystemExit(
            f"{len(problem_indices)} of {len(problems)} problems present; pass "
            "--allow-partial to grade a subset"
        )

    samples = [problems[key[0]].get_evaluation_sample() for key in keys]
    generations = [[by_index[key]["code"] or ""] for key in keys]
    empty = sum(1 for g in generations if not g[0].strip())
    print(
        f"[lcb] grading {len(keys)} rows for {args.mode} "
        f"({empty} with no extracted code)",
        flush=True,
    )

    metrics, results, _metadata = codegen_metrics(
        samples, generations,
        num_process_evaluate=args.num_process,
        timeout=args.timeout,
    )
    graded = extract_instance_results(results)

    for position, key in enumerate(keys):
        by_index[key]["pass1"] = float(bool(graded[position][0]))
    write_back(source, by_index)

    per_problem = {}
    for key in keys:
        per_problem.setdefault(key[0], []).append(by_index[key]["pass1"])
    if not args.allow_partial:
        bad = [index for index, values in per_problem.items() if len(values) != REPEATS]
        if bad:
            raise SystemExit(f"problem repeats incomplete, for example {bad[:3]}")
    problem_means = [sum(values) / len(values) for values in per_problem.values()]
    passed = sum(problem_means)
    print(
        f"[lcb] Avg@{REPEATS} = {passed / len(problem_means):.4f} "
        f"({passed:.2f}/{len(problem_means)} problems)"
    )
    print(f"[lcb] official metrics: {metrics}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())