Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.
#!/usr/bin/env python3
"""Stage every dataset BitDecoding-FP4 reads, from public sources only.
Nothing under ``datasets/`` is committed except ``MANIFEST.json``; this script
rebuilds the rest.
python tools/prepare_data.py --steps eval
python tools/prepare_data.py --steps long-context
python tools/prepare_data.py --steps verify
"""
from __future__ import annotations
import argparse
import glob
import hashlib
import json
import os
import subprocess
import sys
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
sys.path.insert(0, REPO_ROOT)
DATASETS_ROOT = os.path.join(REPO_ROOT, "datasets")
MANIFEST_PATH = os.path.join(DATASETS_ROOT, "MANIFEST.json")
LONGBENCH_PRO_DIR = os.path.join(DATASETS_ROOT, "longbench_pro")
MRCR_DIR = os.path.join(DATASETS_ROOT, "mrcr")
LONGBENCH_PRO_REPO = "caskcsg/LongBench-Pro"
MRCR_REPO = "openai/mrcr"
AIME24_REPO = "HuggingFaceH4/aime_2024"
AIME25_REPO = "math-ai/aime25"
LONG_CONTEXT_MODEL_REPO = "Qwen/Qwen3-4B-Thinking-2507"
LONG_CONTEXT_REFERENCE_TOKENIZER_REPO = "Qwen/Qwen3-8B"
LONG_CONTEXT_MODEL_DIR = os.path.join(
REPO_ROOT, "_local", "models", "Qwen3-4B-Thinking-2507"
)
LONG_CONTEXT_REFERENCE_TOKENIZER_DIR = os.path.join(
REPO_ROOT, "_local", "models", "Qwen3-8B-tokenizer"
)
LONG_CONTEXT_MRCR_BUDGET = 229_120
RULER_SOURCE = os.environ.get(
"BITDECODING_RULER_SOURCE", os.path.join(REPO_ROOT, "_local", "ruler_official")
)
RULER_REVISION = "c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a"
RULER_TASKS = (
"niah_single_1", "niah_single_2", "niah_single_3",
"niah_multikey_1", "niah_multikey_2", "niah_multikey_3",
"niah_multivalue", "niah_multiquery",
)
RULER_MAX_SEQ_LENGTH = 122_880
RULER_NUM_SAMPLES = 500
RULER_SEED = 42
RULER_NUM_SHARDS = 8
RULER_SMOKE_LIMIT = 20
NEEDLE_GROUPS = (2, 4, 8)
STEPS = (
"long-context-model", "longbench-pro", "mrcr", "long-context-mrcr",
"aime24", "aime25", "gpqa", "lcb", "ruler", "verify",
"long-context-verify",
)
EVAL_STEPS = (
"longbench-pro", "mrcr", "aime24", "aime25", "lcb", "ruler", "gpqa",
"verify",
)
LONG_CONTEXT_STEPS = (
"long-context-model", "longbench-pro", "long-context-mrcr",
"aime24", "aime25", "lcb", "gpqa", "ruler",
"long-context-verify",
)
# --------------------------------------------------------------------------- #
# hashing
# --------------------------------------------------------------------------- #
def _norm(obj):
"""Canonicalise a parquet cell so the digest ignores container types."""
import numpy as np
if isinstance(obj, np.ndarray):
return [_norm(x) for x in obj.tolist()]
if isinstance(obj, dict):
return {k: _norm(v) for k, v in sorted(obj.items())}
if isinstance(obj, (list, tuple)):
return [_norm(x) for x in obj]
if isinstance(obj, np.integer):
return int(obj)
if isinstance(obj, np.floating):
return None if np.isnan(obj) else float(obj)
if obj is not None and not isinstance(obj, (str, int, float, bool)):
return str(obj)
return obj
def content_hash_rows(rows) -> str:
"""Order-sensitive digest over row dicts (writer/version independent)."""
h = hashlib.sha256()
for row in rows:
h.update(json.dumps(_norm(row), ensure_ascii=False, sort_keys=True).encode())
h.update(b"\n")
return h.hexdigest()
def content_hash_parquet(path: str) -> str:
import pandas as pd
df = pd.read_parquet(path)
return content_hash_rows(df.to_dict(orient="records"))
def sha256_file(path: str) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def load_manifest() -> dict:
with open(MANIFEST_PATH) as f:
return json.load(f)
def _expected(manifest: dict, rel_path: str) -> dict:
return manifest["derived"].get(rel_path, {})
# --------------------------------------------------------------------------- #
# public downloads
# --------------------------------------------------------------------------- #
def step_longbench_pro(force: bool = False) -> None:
from datasets import load_dataset
target = LONGBENCH_PRO_DIR
if os.path.isfile(os.path.join(target, "dataset_info.json")) and not force:
print(f"[longbench-pro] present, skipping ({target})")
return
print(f"[longbench-pro] downloading {LONGBENCH_PRO_REPO} -> {target}")
# Upstream ships one split and has renamed it: it was ``train`` when this
# was written and is ``test`` now. Both are the same single split, so the
# name is taken from the download rather than asserted.
splits = load_dataset(
LONGBENCH_PRO_REPO,
revision=load_manifest()["upstream"][LONGBENCH_PRO_REPO]["revision"],
)
if len(splits) != 1:
raise SystemExit(
f"[longbench-pro] expected one split, got {sorted(splits)}"
)
(name,) = list(splits)
print(f"[longbench-pro] split {name!r}, {len(splits[name])} rows")
splits[name].save_to_disk(target)
def _tokenizer_vocab_sha256(tokenizer) -> str:
payload = json.dumps(
sorted(tokenizer.get_vocab().items()), ensure_ascii=False,
separators=(",", ":"),
)
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def verify_long_context_tokenizer(model_path: str = LONG_CONTEXT_MODEL_DIR) -> str:
"""Assert that the new checkpoint keeps the Qwen3-8B MRCR vocabulary."""
from transformers import AutoTokenizer
old = AutoTokenizer.from_pretrained(
LONG_CONTEXT_REFERENCE_TOKENIZER_DIR, trust_remote_code=True,
local_files_only=True,
)
new = AutoTokenizer.from_pretrained(
model_path, trust_remote_code=True, local_files_only=True,
)
old_hash = _tokenizer_vocab_sha256(old)
new_hash = _tokenizer_vocab_sha256(new)
if old_hash != new_hash:
raise SystemExit(
"[long-context-tokenizer] Qwen3-4B-Thinking-2507 vocabulary differs "
f"from Qwen3-8B: {new_hash} != {old_hash}"
)
print(f"[long-context-tokenizer] vocabulary match: {new_hash}")
return new_hash
def step_long_context_model(force: bool = False) -> None:
from huggingface_hub import snapshot_download
from transformers import AutoTokenizer
config_path = os.path.join(LONG_CONTEXT_MODEL_DIR, "config.json")
reference_config = os.path.join(
LONG_CONTEXT_REFERENCE_TOKENIZER_DIR, "tokenizer_config.json"
)
if not os.path.isfile(reference_config):
try:
reference = AutoTokenizer.from_pretrained(
LONG_CONTEXT_REFERENCE_TOKENIZER_REPO, local_files_only=True,
trust_remote_code=True,
)
except OSError:
reference = AutoTokenizer.from_pretrained(
LONG_CONTEXT_REFERENCE_TOKENIZER_REPO, trust_remote_code=True,
)
os.makedirs(LONG_CONTEXT_REFERENCE_TOKENIZER_DIR, exist_ok=True)
reference.save_pretrained(LONG_CONTEXT_REFERENCE_TOKENIZER_DIR)
if os.path.isfile(config_path) and not force:
print(f"[long-context-model] present, skipping ({LONG_CONTEXT_MODEL_DIR})")
else:
print(f"[long-context-model] downloading {LONG_CONTEXT_MODEL_REPO} -> {LONG_CONTEXT_MODEL_DIR}")
snapshot_download(
LONG_CONTEXT_MODEL_REPO, local_dir=LONG_CONTEXT_MODEL_DIR,
revision=load_manifest()["upstream"][LONG_CONTEXT_MODEL_REPO]["revision"],
)
verify_long_context_tokenizer()
def step_long_context_mrcr(force: bool = False, budget: int = LONG_CONTEXT_MRCR_BUDGET) -> None:
from huggingface_hub import snapshot_download
target = MRCR_DIR
snapshot_download(
MRCR_REPO,
repo_type="dataset",
local_dir=target,
allow_patterns=[f"{n}needle/*" for n in NEEDLE_GROUPS],
)
filename = f"_filtered_le{budget}.jsonl"
rel = f"datasets/mrcr/{filename}"
build_mrcr_cache(
LONG_CONTEXT_MODEL_DIR,
force=force,
budget=budget,
out_path=os.path.join(target, filename),
manifest_key=rel,
)
def step_mrcr(tokenizer: str, force: bool = False) -> None:
from huggingface_hub import snapshot_download
target = MRCR_DIR
print(f"[mrcr] downloading {MRCR_REPO} shards -> {target} (~1.4 GB)")
snapshot_download(
MRCR_REPO,
repo_type="dataset",
local_dir=target,
allow_patterns=[f"{n}needle/*" for n in NEEDLE_GROUPS],
)
build_mrcr_cache(tokenizer, force=force)
build_mrcr_cache(
tokenizer, force=force, budget=98048,
out_path=os.path.join(MRCR_DIR, "_filtered_le98048.jsonl"),
manifest_key="datasets/mrcr/_filtered_le98048.jsonl",
)
def build_mrcr_cache(
tokenizer: str,
force: bool = False,
*,
budget: int | None = None,
out_path: str | None = None,
manifest_key: str | None = None,
) -> None:
"""Keep MRCR rows whose prompt is <= max_prompt_tokens Qwen3 tokens."""
import pandas as pd
from transformers import AutoTokenizer
manifest = load_manifest()
manifest_key = (
manifest_key or "datasets/mrcr/_filtered_le34000.jsonl"
)
spec = _expected(manifest, manifest_key)
budget = budget if budget is not None else spec.get("max_prompt_tokens", 34000)
out_path = out_path or os.path.join(MRCR_DIR, "_filtered_le34000.jsonl")
if os.path.isfile(out_path) and not force:
print(f"[mrcr-cache] present, skipping ({out_path})")
return
tok = AutoTokenizer.from_pretrained(tokenizer)
rows = []
for n in NEEDLE_GROUPS:
shards = sorted(glob.glob(os.path.join(MRCR_DIR, f"{n}needle", "*.parquet")))
if not shards:
raise SystemExit(f"[mrcr-cache] no shards for {n}needle — run --steps mrcr first")
for shard in shards:
df = pd.read_parquet(shard)
for _, r in df.iterrows():
messages = json.loads(r["prompt"])
encoded = tok([m["content"] for m in messages], add_special_tokens=False)
n_tokens = sum(len(ids) for ids in encoded["input_ids"])
if n_tokens > budget:
continue
rows.append({
"id": None,
"n_needles": n,
"messages": messages,
"answer": r["answer"],
"random_string_to_prepend": r["random_string_to_prepend"],
"prompt_tokens": n_tokens,
})
print(f"[mrcr-cache] {n}needle: kept {sum(1 for x in rows if x['n_needles'] == n)}")
os.makedirs(os.path.dirname(out_path), exist_ok=True)
with open(out_path, "w") as f:
for row in rows:
f.write(json.dumps(row, ensure_ascii=False) + "\n")
print(f"[mrcr-cache] wrote {len(rows)} rows -> {out_path}")
_report("mrcr cache", sha256_file(out_path), spec.get("sha256"))
expected_rows = spec.get("n_rows")
if expected_rows is not None and len(rows) != expected_rows:
raise SystemExit(
f"[mrcr-cache] row-count mismatch: {len(rows)} != {expected_rows}"
)
def step_aime25(force: bool = False) -> None:
"""AIME 2025, 30 problems, one row each.
The 32 repeats Avg@32 needs are made at load time by ``evaluation.aime25`` rather
than written out, so this file stays the upstream set and cannot drift from
the constant that decides how many there are.
"""
from datasets import load_dataset
import pandas as pd
out_path = os.path.join(DATASETS_ROOT, "aime25", "aime25.parquet")
if os.path.isfile(out_path) and not force:
print(f"[aime25] present, skipping ({out_path})")
return
splits = load_dataset(
AIME25_REPO, revision=load_manifest()["upstream"][AIME25_REPO]["revision"],
)
if len(splits) != 1:
raise SystemExit(f"[aime25] expected one split, got {sorted(splits)}")
(name,) = list(splits)
frame = splits[name].to_pandas()[["id", "problem", "answer"]]
if len(frame) != 30:
raise SystemExit(f"[aime25] expected 30 problems, got {len(frame)}")
non_integer = [a for a in frame["answer"] if not str(a).strip().isdigit()]
if non_integer:
raise SystemExit(f"[aime25] non-integer answers: {non_integer[:5]}")
os.makedirs(os.path.dirname(out_path), exist_ok=True)
frame.to_parquet(out_path, index=False)
print(f"[aime25] wrote {len(frame)} problems -> {out_path}")
def step_aime24(force: bool = False) -> None:
"""AIME 2024, 30 problems. Avg@32 repeats are made at load time."""
from datasets import load_dataset
out_path = os.path.join(DATASETS_ROOT, "aime24", "aime24.parquet")
if os.path.isfile(out_path) and not force:
print(f"[aime24] present, skipping ({out_path})")
return
splits = load_dataset(
AIME24_REPO, revision=load_manifest()["upstream"][AIME24_REPO]["revision"],
)
if len(splits) != 1:
raise SystemExit(f"[aime24] expected one split, got {sorted(splits)}")
(name,) = list(splits)
frame = splits[name].to_pandas()
if not {"problem", "answer"}.issubset(frame.columns):
raise SystemExit(f"[aime24] missing problem/answer columns: {list(frame.columns)}")
frame = frame[["problem", "answer"]]
if len(frame) != 30:
raise SystemExit(f"[aime24] expected 30 problems, got {len(frame)}")
os.makedirs(os.path.dirname(out_path), exist_ok=True)
frame.to_parquet(out_path, index=False)
print(f"[aime24] wrote {len(frame)} problems -> {out_path}")
def step_gpqa(force: bool = False) -> None:
"""Stage the 198-question GPQA Diamond CSV used by simple-evals."""
import urllib.request
import pandas as pd
out_path = os.path.join(DATASETS_ROOT, "gpqa", "gpqa_diamond.parquet")
if os.path.isfile(out_path) and not force:
print(f"[gpqa] present, skipping ({out_path})")
return
url = "https://raw.githubusercontent.com/openai/simple-evals/main/data/gpqa_diamond.csv"
os.makedirs(os.path.dirname(out_path), exist_ok=True)
csv_path = out_path.removesuffix(".parquet") + ".csv"
urllib.request.urlretrieve(url, csv_path)
frame = pd.read_csv(csv_path)
if len(frame) != 198:
raise SystemExit(f"[gpqa] expected 198 Diamond rows, got {len(frame)}")
frame.to_parquet(out_path, index=False)
os.remove(csv_path)
print(f"[gpqa] wrote {len(frame)} questions -> {out_path}")
def step_lcb(force: bool = False) -> None:
"""Stage the v6 prompts and the official grader checkout."""
from tools.lcb_data import LCB_CHECKOUT, PROMPTS_PATH, export_prompts
if not os.path.isdir(os.path.join(LCB_CHECKOUT, "lcb_runner")):
os.makedirs(os.path.dirname(LCB_CHECKOUT), exist_ok=True)
subprocess.run([
"git", "clone", "https://github.com/LiveCodeBench/LiveCodeBench.git",
LCB_CHECKOUT,
], check=True)
if os.path.isfile(PROMPTS_PATH) and not force:
print(f"[lcb] present, skipping ({PROMPTS_PATH})")
return
count = export_prompts()
if count != 175:
raise SystemExit(f"[lcb] expected 175 v6 problems, got {count}")
print(f"[lcb] wrote {count} prompts -> {PROMPTS_PATH}")
def step_ruler(force: bool = False, tokenizer_path: str = LONG_CONTEXT_MODEL_REPO) -> None:
"""Generate the eight official RULER NIAH tasks at 122,880 tokens."""
prepare = os.path.join(RULER_SOURCE, "scripts", "data", "prepare.py")
revision_path = os.path.join(RULER_SOURCE, "REVISION")
if not os.path.exists(RULER_SOURCE):
os.makedirs(os.path.dirname(RULER_SOURCE), exist_ok=True)
subprocess.run([
"git", "clone", "https://github.com/NVIDIA/RULER.git", RULER_SOURCE,
], check=True)
subprocess.run([
"git", "-C", RULER_SOURCE, "checkout", "--detach", RULER_REVISION,
], check=True)
if os.path.exists(os.path.join(RULER_SOURCE, ".git")):
revision = subprocess.check_output(
["git", "-C", RULER_SOURCE, "rev-parse", "HEAD"], text=True,
).strip()
else:
with open(revision_path) as handle:
revision = handle.read().strip()
if not os.path.isfile(prepare):
raise SystemExit(f"[ruler] missing generator: {prepare}")
if revision != RULER_REVISION:
raise SystemExit(
f"[ruler] source revision {revision!r} != pinned {RULER_REVISION}"
)
out_dir = os.path.join(DATASETS_ROOT, "ruler")
out_path = os.path.join(out_dir, "niah_122880.jsonl")
provenance_path = os.path.join(out_dir, "generator.json")
if os.path.isfile(out_path) and os.path.isfile(provenance_path) and not force:
print(f"[ruler] present, skipping ({out_path})")
return
raw_dir = os.path.join(out_dir, "raw_122880")
os.makedirs(raw_dir, exist_ok=True)
env = dict(os.environ)
env["PATH"] = os.path.dirname(sys.executable) + os.pathsep + env.get("PATH", "")
for task in RULER_TASKS:
command = [
sys.executable,
prepare,
"--save_dir", raw_dir,
"--benchmark", "synthetic",
"--task", task,
"--tokenizer_path", tokenizer_path,
"--tokenizer_type", "hf",
"--max_seq_length", str(RULER_MAX_SEQ_LENGTH),
"--num_samples", str(RULER_NUM_SAMPLES),
"--random_seed", str(RULER_SEED),
"--model_template_type", "base",
]
print(f"[ruler] generating {task}")
subprocess.run(command, check=True, env=env)
task_path = os.path.join(raw_dir, task, "validation.jsonl")
if not os.path.isfile(task_path):
raise SystemExit(f"[ruler] generator did not create {task_path}")
with open(task_path) as handle:
count = sum(bool(line.strip()) for line in handle)
if count != RULER_NUM_SAMPLES:
raise SystemExit(
f"[ruler] {task} generated {count}, expected {RULER_NUM_SAMPLES}"
)
temporary = f"{out_path}.tmp.{os.getpid()}"
shard_paths = [
os.path.join(
out_dir,
f"niah_122880.shard_{index}_of_{RULER_NUM_SHARDS}.jsonl",
)
for index in range(RULER_NUM_SHARDS)
]
shard_temporaries = [f"{path}.tmp.{os.getpid()}" for path in shard_paths]
shard_handles = [open(path, "w") for path in shard_temporaries]
smoke_paths = [
os.path.join(
out_dir,
f"niah_122880.smoke_{RULER_SMOKE_LIMIT}.shard_{index}_of_"
f"{RULER_NUM_SHARDS}.jsonl",
)
for index in range(RULER_NUM_SHARDS)
]
smoke_temporaries = [f"{path}.tmp.{os.getpid()}" for path in smoke_paths]
smoke_handles = [open(path, "w") for path in smoke_temporaries]
smoke_base, smoke_remainder = divmod(RULER_SMOKE_LIMIT, len(RULER_TASKS))
smoke_source_indices = {}
for position, task in enumerate(RULER_TASKS):
take = smoke_base + (position < smoke_remainder)
if take == 1:
selected = {0}
else:
selected = {
index * (RULER_NUM_SAMPLES - 1) // (take - 1)
for index in range(take)
}
smoke_source_indices[task] = selected
bench_index = 0
smoke_index = 0
source_files = {}
try:
with open(temporary, "w") as output:
for task in RULER_TASKS:
task_path = os.path.join(raw_dir, task, "validation.jsonl")
source_files[task] = {
"path": os.path.relpath(task_path, REPO_ROOT),
"sha256": sha256_file(task_path),
"rows": RULER_NUM_SAMPLES,
}
with open(task_path) as source:
for source_index, line in enumerate(source):
if not line.strip():
continue
row = json.loads(line)
row.update({
"bench_index": bench_index,
"task": task,
"source_index": source_index,
"generator_revision": revision,
})
rendered = json.dumps(row, ensure_ascii=False) + "\n"
output.write(rendered)
shard_handles[bench_index % RULER_NUM_SHARDS].write(rendered)
if source_index in smoke_source_indices[task]:
smoke_handles[
smoke_index % RULER_NUM_SHARDS
].write(rendered)
smoke_index += 1
bench_index += 1
output.flush()
os.fsync(output.fileno())
for handle in shard_handles:
handle.flush()
os.fsync(handle.fileno())
for handle in smoke_handles:
handle.flush()
os.fsync(handle.fileno())
finally:
for handle in shard_handles + smoke_handles:
handle.close()
os.replace(temporary, out_path)
for temporary_path, shard_path in zip(
shard_temporaries, shard_paths, strict=True
):
os.replace(temporary_path, shard_path)
for temporary_path, shard_path in zip(
smoke_temporaries, smoke_paths, strict=True
):
os.replace(temporary_path, shard_path)
if bench_index != len(RULER_TASKS) * RULER_NUM_SAMPLES:
raise SystemExit(f"[ruler] combined row count is {bench_index}")
if smoke_index != RULER_SMOKE_LIMIT:
raise SystemExit(f"[ruler] smoke row count is {smoke_index}")
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_path, trust_remote_code=True
)
provenance = {
"generator": "NVIDIA/RULER scripts/data/prepare.py",
"generator_revision": revision,
"random_seed": RULER_SEED,
"tokenizer_path": (os.path.realpath(tokenizer_path)
if os.path.isdir(tokenizer_path) else tokenizer_path),
"tokenizer_vocab_sha256": _tokenizer_vocab_sha256(tokenizer),
"max_seq_length": RULER_MAX_SEQ_LENGTH,
"num_samples_per_task": RULER_NUM_SAMPLES,
"tasks": list(RULER_TASKS),
"source_files": source_files,
"evaluation_shards": [
{
"path": os.path.relpath(path, REPO_ROOT),
"sha256": sha256_file(path),
"rows": bench_index // RULER_NUM_SHARDS,
}
for path in shard_paths
],
"smoke_shards": [
{
"path": os.path.relpath(path, REPO_ROOT),
"sha256": sha256_file(path),
"rows": (
RULER_SMOKE_LIMIT + RULER_NUM_SHARDS - 1 - index
) // RULER_NUM_SHARDS,
}
for index, path in enumerate(smoke_paths)
],
"combined": {
"path": os.path.relpath(out_path, REPO_ROOT),
"sha256": sha256_file(out_path),
"rows": bench_index,
},
}
os.makedirs(os.path.dirname(provenance_path), exist_ok=True)
with open(provenance_path, "w") as handle:
json.dump(provenance, handle, indent=2, sort_keys=True)
handle.write("\n")
print(f"[ruler] wrote {bench_index} prompts -> {out_path}")
# --------------------------------------------------------------------------- #
# verification
# --------------------------------------------------------------------------- #
def _report(label: str, got: str, expected: str) -> bool:
if not expected:
print(f" {label}: {got} (no expectation recorded)")
return True
ok = got == expected
print(f" {'OK ' if ok else 'FAIL'} {label}: {got}{'' if ok else f' != {expected}'}")
return ok
def _jsonl_identity(path: str) -> tuple[int, str]:
"""Count non-empty rows and hash one large JSONL in a single pass."""
digest = hashlib.sha256()
rows = 0
with open(path, "rb") as handle:
for line in handle:
digest.update(line)
rows += bool(line.strip())
return rows, digest.hexdigest()
def step_verify() -> bool:
"""Hash every staged evaluation input and validate its registered size."""
manifest = load_manifest()
print("[verify] checking staged artifacts against datasets/MANIFEST.json")
ok, missing = True, 0
if os.path.isfile(os.path.join(LONGBENCH_PRO_DIR, "dataset_info.json")):
from datasets import load_from_disk
dataset = load_from_disk(LONGBENCH_PRO_DIR)
if hasattr(dataset, "values"):
values = list(dataset.values())
if len(values) != 1:
raise SystemExit(
f"[verify] LongBench-Pro has {len(values)} splits, expected one"
)
dataset = values[0]
spec = _expected(manifest, "datasets/longbench_pro")
ok &= _report(
"LongBench-Pro rows", str(len(dataset)), str(spec.get("n_rows", ""))
)
ok &= _report(
"LongBench-Pro content sha256",
content_hash_rows(dataset),
spec.get("content_sha256"),
)
else:
print(" MISS datasets/longbench_pro")
missing += 1
for rel, path in [
("datasets/mrcr/_filtered_le34000.jsonl",
os.path.join(MRCR_DIR, "_filtered_le34000.jsonl")),
("datasets/mrcr/_filtered_le98048.jsonl",
os.path.join(MRCR_DIR, "_filtered_le98048.jsonl")),
("datasets/aime24/aime24.parquet",
os.path.join(DATASETS_ROOT, "aime24", "aime24.parquet")),
("datasets/aime25/aime25.parquet",
os.path.join(DATASETS_ROOT, "aime25", "aime25.parquet")),
("datasets/gpqa/gpqa_diamond.parquet",
os.path.join(DATASETS_ROOT, "gpqa", "gpqa_diamond.parquet")),
("datasets/lcb_v6/prompts.jsonl",
os.path.join(DATASETS_ROOT, "lcb_v6", "prompts.jsonl")),
("datasets/ruler/niah_122880.jsonl",
os.path.join(DATASETS_ROOT, "ruler", "niah_122880.jsonl")),
]:
spec = _expected(manifest, rel)
if not os.path.exists(path):
print(f" MISS {rel}")
missing += 1
continue
if path.endswith(".parquet"):
import pandas as pd
rows = len(pd.read_parquet(path))
ok &= _report(f"{rel} rows", str(rows), str(spec.get("n_rows", "")))
if spec.get("content_sha256"):
ok &= _report(
f"{rel} content sha256",
content_hash_parquet(path),
spec["content_sha256"],
)
else:
ok &= _report(f"{rel} sha256", sha256_file(path), spec.get("sha256"))
else:
rows, digest = _jsonl_identity(path)
ok &= _report(f"{rel} rows", str(rows), str(spec.get("n_rows", "")))
ok &= _report(f"{rel} sha256", digest, spec.get("sha256"))
ruler_provenance = os.path.join(DATASETS_ROOT, "ruler", "generator.json")
if not os.path.isfile(ruler_provenance):
print(" MISS datasets/ruler/generator.json")
missing += 1
else:
spec = _expected(manifest, "datasets/ruler/generator.json")
ok &= _report(
"datasets/ruler/generator.json sha256",
sha256_file(ruler_provenance),
spec.get("sha256"),
)
with open(ruler_provenance) as handle:
provenance = json.load(handle)
for identity in (
provenance.get("evaluation_shards", [])
+ provenance.get("smoke_shards", [])
):
path = os.path.join(REPO_ROOT, identity["path"])
if not os.path.isfile(path):
print(f" MISS {identity['path']}")
missing += 1
continue
rows, digest = _jsonl_identity(path)
ok &= _report(
f"{identity['path']} rows", str(rows), str(identity.get("rows", ""))
)
ok &= _report(
f"{identity['path']} sha256", digest, identity.get("sha256")
)
if missing:
print(f"[verify] {missing} artifact(s) not staged — run --steps eval")
return ok and missing == 0
def step_long_context_verify() -> bool:
"""Verify the long-context model, tokenizer, and MRCR cache."""
config_path = os.path.join(LONG_CONTEXT_MODEL_DIR, "config.json")
index_path = os.path.join(
LONG_CONTEXT_MODEL_DIR, "model.safetensors.index.json"
)
cache_path = os.path.join(MRCR_DIR, "_filtered_le229120.jsonl")
if not os.path.isfile(config_path):
print(f" MISS {config_path}")
return False
with open(config_path) as handle:
model_config = json.load(handle)
native_context = int(model_config.get("max_position_embeddings", 0))
if native_context < 262_144:
print(f" FAIL native context: {native_context} < 262144")
return False
manifest = load_manifest()
model_spec = manifest["upstream"].get(LONG_CONTEXT_MODEL_REPO, {})
ok = True
ok &= _report(
"long-context model config sha256",
sha256_file(config_path),
model_spec.get("config_sha256"),
)
if not os.path.isfile(index_path):
print(f" MISS {index_path}")
return False
ok &= _report(
"long-context model index sha256",
sha256_file(index_path),
model_spec.get("model_index_sha256"),
)
vocab_hash = verify_long_context_tokenizer()
ok &= _report(
"long-context tokenizer vocabulary sha256",
vocab_hash,
model_spec.get("tokenizer_vocab_sha256"),
)
if not os.path.isfile(cache_path):
print(f" MISS {cache_path}")
return False
count = 0
maximum = 0
with open(cache_path) as handle:
for line in handle:
if not line.strip():
continue
row = json.loads(line)
count += 1
maximum = max(maximum, int(row["prompt_tokens"]))
spec = _expected(load_manifest(), "datasets/mrcr/_filtered_le229120.jsonl")
expected_rows = spec.get("n_rows")
ok &= expected_rows in (None, count) and maximum <= LONG_CONTEXT_MRCR_BUDGET
print(
f" {'OK ' if ok else 'FAIL'} long-context MRCR: "
f"rows={count}, max_prompt_tokens={maximum}"
)
expected_hash = spec.get("sha256")
if expected_hash:
ok &= _report("long-context MRCR sha256", sha256_file(cache_path), expected_hash)
return ok
def main() -> int:
default_tokenizer = os.environ.get("BITDECODING_TOKENIZER") or os.environ.get(
"MODEL_PATH") or LONG_CONTEXT_MODEL_REPO
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument(
"--steps", nargs="+", default=["eval"],
choices=list(STEPS) + ["all", "eval", "long-context"],
help="'eval' = all seven paper datasets; "
"'long-context' = native-262k checkpoint and cache",
)
parser.add_argument("--tokenizer", default=default_tokenizer,
help="Qwen3 tokenizer for the MRCR length filter")
parser.add_argument(
"--long-context-mrcr-budget", type=int, default=LONG_CONTEXT_MRCR_BUDGET,
help="long-context evaluation MRCR prompt cap (the 131k fallback uses 98048)",
)
parser.add_argument("--force", action="store_true",
help="rebuild artifacts that already exist")
args = parser.parse_args()
aliases = {
"all": list(STEPS),
"eval": list(EVAL_STEPS),
"long-context": list(LONG_CONTEXT_STEPS),
}
steps = []
for s in args.steps:
steps.extend(aliases.get(s, [s]))
seen, ordered = set(), []
for s in STEPS:
if s in steps and s not in seen:
seen.add(s)
ordered.append(s)
for step in ordered:
if step == "long-context-model":
step_long_context_model(args.force)
elif step == "longbench-pro":
step_longbench_pro(args.force)
elif step == "mrcr":
step_mrcr(args.tokenizer, args.force)
elif step == "long-context-mrcr":
step_long_context_mrcr(args.force, args.long_context_mrcr_budget)
elif step == "aime24":
step_aime24(args.force)
elif step == "aime25":
step_aime25(args.force)
elif step == "gpqa":
step_gpqa(args.force)
elif step == "lcb":
step_lcb(args.force)
elif step == "ruler":
step_ruler(args.force, args.tokenizer)
elif step == "verify":
if not step_verify():
print("\n[verify] MISMATCH — staged data differs from MANIFEST.json.")
return 1
elif step == "long-context-verify":
if not step_long_context_verify():
print("\n[long-context-verify] MISMATCH")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())