Skip to content

Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.

3.7 KB
Raw Blame
"""RULER needle retrieval at the registered 122,880-token stress point.

The staged rows are generated by NVIDIA/RULER at a pinned commit.  We keep the
official prompts and reference lists, but intentionally do not prepend
RULER's answer prefix: Qwen3-Thinking reasons before its final answer, so the
prefix cannot occupy the position upstream designed it for.  The thinking
block is removed before applying RULER's recall metric so a value considered
and rejected during reasoning does not count as retrieved.
"""
from __future__ import annotations

import json
import os
from typing import List

from . import config
from .scoring import strip_thinking


METRIC_KEY = "recall"
TASKS = (
    "niah_single_1",
    "niah_single_2",
    "niah_single_3",
    "niah_multikey_1",
    "niah_multikey_2",
    "niah_multikey_3",
    "niah_multivalue",
    "niah_multiquery",
)


def load_examples(
    path: str = None, *, shard_index: int = None, num_shards: int = None,
    stratified_limit: int = None,
) -> List[dict]:
    path = path or config.RULER_NIAH_PROMPTS
    if shard_index is not None:
        if num_shards is None:
            raise ValueError("num_shards is required with shard_index")
        stem, extension = path.rsplit(".", 1)
        if stratified_limit is None:
            staged_shard = (
                f"{stem}.shard_{shard_index}_of_{num_shards}.{extension}"
            )
        else:
            staged_shard = (
                f"{stem}.smoke_{stratified_limit}.shard_{shard_index}_of_"
                f"{num_shards}.{extension}"
            )
        if not os.path.isfile(staged_shard):
            raise FileNotFoundError(
                f"staged RULER shard is missing: {staged_shard}; rebuild with "
                "tools/prepare_data.py --steps ruler --force"
            )
        path = staged_shard
    rows = []
    with open(path) as handle:
        for line in handle:
            if line.strip():
                row = json.loads(line)
                if row.get("task") not in TASKS:
                    raise ValueError(f"unexpected RULER task {row.get('task')!r}")
                rows.append(row)
    rows.sort(key=lambda row: row["bench_index"])
    return rows


def build(examples):
    conversations = [
        [{"role": "user", "content": example["input"]}]
        for example in examples
    ]
    prepared = [
        {"example": example, "prompt_tokens": example.get("prompt_tokens")}
        for example in examples
    ]
    return conversations, prepared


def recall(response: str, references: List[str]) -> float:
    """RULER's ``string_match_all`` for one row, on the final answer only."""
    answer = strip_thinking(response).lower()
    if not references:
        return 0.0
    return sum(reference.lower() in answer for reference in references) / len(references)


def grade(prepared, completions) -> List[dict]:
    records = []
    for item, completion in zip(prepared, completions, strict=True):
        example = item["example"]
        raw = str(completion.get("text") or "")
        records.append({
            "bench_index": example["bench_index"],
            "task": example["task"],
            "source_index": example.get("source_index"),
            "references": example["outputs"],
            "recall": recall(raw, example["outputs"]),
            "prompt_tokens": completion.get(
                "prompt_tokens", item.get("prompt_tokens")
            ),
            "generator_length": example.get("length"),
            "answer_token_position": example.get("token_position_answer"),
            "raw_output": raw,
            "gen_tokens": completion.get("gen_tokens"),
            "finish_reason": completion.get("finish_reason"),
        })
    return records