Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.
"""RULER needle retrieval at the registered 122,880-token stress point.
The staged rows are generated by NVIDIA/RULER at a pinned commit. We keep the
official prompts and reference lists, but intentionally do not prepend
RULER's answer prefix: Qwen3-Thinking reasons before its final answer, so the
prefix cannot occupy the position upstream designed it for. The thinking
block is removed before applying RULER's recall metric so a value considered
and rejected during reasoning does not count as retrieved.
"""
from __future__ import annotations
import json
import os
from typing import List
from . import config
from .scoring import strip_thinking
METRIC_KEY = "recall"
TASKS = (
"niah_single_1",
"niah_single_2",
"niah_single_3",
"niah_multikey_1",
"niah_multikey_2",
"niah_multikey_3",
"niah_multivalue",
"niah_multiquery",
)
def load_examples(
path: str = None, *, shard_index: int = None, num_shards: int = None,
stratified_limit: int = None,
) -> List[dict]:
path = path or config.RULER_NIAH_PROMPTS
if shard_index is not None:
if num_shards is None:
raise ValueError("num_shards is required with shard_index")
stem, extension = path.rsplit(".", 1)
if stratified_limit is None:
staged_shard = (
f"{stem}.shard_{shard_index}_of_{num_shards}.{extension}"
)
else:
staged_shard = (
f"{stem}.smoke_{stratified_limit}.shard_{shard_index}_of_"
f"{num_shards}.{extension}"
)
if not os.path.isfile(staged_shard):
raise FileNotFoundError(
f"staged RULER shard is missing: {staged_shard}; rebuild with "
"tools/prepare_data.py --steps ruler --force"
)
path = staged_shard
rows = []
with open(path) as handle:
for line in handle:
if line.strip():
row = json.loads(line)
if row.get("task") not in TASKS:
raise ValueError(f"unexpected RULER task {row.get('task')!r}")
rows.append(row)
rows.sort(key=lambda row: row["bench_index"])
return rows
def build(examples):
conversations = [
[{"role": "user", "content": example["input"]}]
for example in examples
]
prepared = [
{"example": example, "prompt_tokens": example.get("prompt_tokens")}
for example in examples
]
return conversations, prepared
def recall(response: str, references: List[str]) -> float:
"""RULER's ``string_match_all`` for one row, on the final answer only."""
answer = strip_thinking(response).lower()
if not references:
return 0.0
return sum(reference.lower() in answer for reference in references) / len(references)
def grade(prepared, completions) -> List[dict]:
records = []
for item, completion in zip(prepared, completions, strict=True):
example = item["example"]
raw = str(completion.get("text") or "")
records.append({
"bench_index": example["bench_index"],
"task": example["task"],
"source_index": example.get("source_index"),
"references": example["outputs"],
"recall": recall(raw, example["outputs"]),
"prompt_tokens": completion.get(
"prompt_tokens", item.get("prompt_tokens")
),
"generator_length": example.get("length"),
"answer_token_position": example.get("token_position_answer"),
"raw_output": raw,
"gen_tokens": completion.get("gen_tokens"),
"finish_reason": completion.get("finish_reason"),
})
return records